API output without Persian diacritics or written ezafe ی (company decision); tutor answers too
plain_persian() strips kasra/fatha/damma/shadda/sukun/hamza-above and the written «هی» ezafe, keeping tanwin. Applied at output, not in data: PlainPersianJSONRenderer runs one byte-level pass over every JSON response (~5 ms per MB locally), and to_plain() cleans tutor answers including the stream. Rewriting titles in the DB/seed would have broken question binding by title path and SeedRecord keys. Tests updated to expect the cleaned text; config/tests_plain_persian.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,8 @@ import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from content.plain_math import plain_persian
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.test import SimpleTestCase, TestCase, override_settings
|
||||
from rest_framework.test import APIClient
|
||||
@@ -109,7 +111,8 @@ class RefusalReplacementTests(SimpleTestCase):
|
||||
result = {}
|
||||
with patch('rag.services.clients.get_openai_client', return_value=_client_answering(text, usage)):
|
||||
out = ''.join(stream_answer('۲+۳؟', None, 'هشتم', result=result))
|
||||
self.assertEqual(out, text)
|
||||
# «بیتغییر» جز پاک شدنِ اعراب/«ی»ِ اضافه (content.plain_math.plain_persian)، که روی هر پاسخ اعمال میشود.
|
||||
self.assertEqual(out, plain_persian(text))
|
||||
self.assertFalse(result['refused'])
|
||||
self.assertEqual(result['usage'], {'promptTokens': 1200, 'completionTokens': 80})
|
||||
|
||||
@@ -141,7 +144,7 @@ class GuardrailThroughTheApiTests(TestCase):
|
||||
done = json.loads(events[-1][1])
|
||||
else:
|
||||
text, done = response.json()['answer'], response.json()
|
||||
self.assertEqual(text, guardrails.REFUSAL)
|
||||
self.assertEqual(text, plain_persian(guardrails.REFUSAL))
|
||||
self.assertEqual(done['usedChunks'], [])
|
||||
self.assertEqual(Message.objects.filter(conversation__user_id=self.user.id, role='user').count(), 2)
|
||||
self.assertEqual(
|
||||
@@ -153,7 +156,7 @@ class GuardrailThroughTheApiTests(TestCase):
|
||||
patch('rag.services.clients.get_openai_client', return_value=_client_answering(MARKER)):
|
||||
response = self.client.post('/api/ai/ask', {'userQuestion': 'حالم خوب نیست، با دوستم دعوام شده'},
|
||||
format='json')
|
||||
self.assertEqual(response.json()['answer'], guardrails.REFUSAL)
|
||||
self.assertEqual(response.json()['answer'], plain_persian(guardrails.REFUSAL))
|
||||
self.assertEqual(response.json()['usedChunks'], [])
|
||||
self.assertEqual(Message.objects.get(role='assistant').text, guardrails.REFUSAL)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user