API output without Persian diacritics or written ezafe ی (company decision); tutor answers too

plain_persian() strips kasra/fatha/damma/shadda/sukun/hamza-above and the written «ه‌ی» ezafe, keeping tanwin. Applied
at output, not in data: PlainPersianJSONRenderer runs one byte-level pass over every JSON response (~5 ms per MB
locally), and to_plain() cleans tutor answers including the stream. Rewriting titles in the DB/seed would have broken
question binding by title path and SeedRecord keys. Tests updated to expect the cleaned text; config/tests_plain_persian.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-14 16:57:17 +03:30
parent 7b241c1aac
commit 1c7c4b8eca
16 changed files with 181 additions and 64 deletions

View File

@@ -10,6 +10,8 @@ import json
from types import SimpleNamespace
from unittest.mock import patch
from content.plain_math import plain_persian
from django.core.cache import cache
from django.test import SimpleTestCase, TestCase, override_settings
from rest_framework.test import APIClient
@@ -109,7 +111,8 @@ class RefusalReplacementTests(SimpleTestCase):
result = {}
with patch('rag.services.clients.get_openai_client', return_value=_client_answering(text, usage)):
out = ''.join(stream_answer('۲+۳؟', None, 'هشتم', result=result))
self.assertEqual(out, text)
# «بی‌تغییر» جز پاک شدنِ اعراب/«ی»ِ اضافه (content.plain_math.plain_persian)، که روی هر پاسخ اعمال می‌شود.
self.assertEqual(out, plain_persian(text))
self.assertFalse(result['refused'])
self.assertEqual(result['usage'], {'promptTokens': 1200, 'completionTokens': 80})
@@ -141,7 +144,7 @@ class GuardrailThroughTheApiTests(TestCase):
done = json.loads(events[-1][1])
else:
text, done = response.json()['answer'], response.json()
self.assertEqual(text, guardrails.REFUSAL)
self.assertEqual(text, plain_persian(guardrails.REFUSAL))
self.assertEqual(done['usedChunks'], [])
self.assertEqual(Message.objects.filter(conversation__user_id=self.user.id, role='user').count(), 2)
self.assertEqual(
@@ -153,7 +156,7 @@ class GuardrailThroughTheApiTests(TestCase):
patch('rag.services.clients.get_openai_client', return_value=_client_answering(MARKER)):
response = self.client.post('/api/ai/ask', {'userQuestion': 'حالم خوب نیست، با دوستم دعوام شده'},
format='json')
self.assertEqual(response.json()['answer'], guardrails.REFUSAL)
self.assertEqual(response.json()['answer'], plain_persian(guardrails.REFUSAL))
self.assertEqual(response.json()['usedChunks'], [])
self.assertEqual(Message.objects.get(role='assistant').text, guardrails.REFUSAL)