tutor runs on gpt-4.1-mini, and primary-school "/" is read as decimal or fraction by rule
Two benchmark rounds (14 models on 30 problems, then gpt-4o-mini vs gpt-4.1-mini on 102 problems under three prompt versions, hand-graded): gpt-4.1-mini scored 95-96/102 against 85-90 for gpt-4o-mini, at 0.12 vs 0.04 cents per answer. The default now lives in settings so the deploy needs no .env edit. The context note "in the book / is a decimal point, not a fraction" was written for the upper-grade books but went to every grade, so gpt-4.1-mini computed 1/7 + 1/8 as 3.5. For primary grades the note now says / can be either, and a reading rule sits next to the question itself (the same rule at the end of the system prompt had no effect). Slash problems went from 11 to 14 of 17; three are still wrong. generate_answer now reports the provider's exact per-call cost, reasoning tokens and whether the raw output broke the plain-text rule. The benchmark fixes primary grades being sent with no book context, caches retrieval so runs compare on identical context, and retries Qdrant timeouts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -140,14 +140,15 @@ class GenerateAnswerSanitizesTests(TestCase):
|
||||
return SimpleNamespace(chat=SimpleNamespace(completions=completions))
|
||||
|
||||
def test_latex_from_the_model_never_reaches_the_student(self):
|
||||
from rag.services.generation import generate_answer
|
||||
from rag.services.generation import PROMPT_VERSION, generate_answer
|
||||
|
||||
with patch('rag.services.clients.get_openai_client',
|
||||
return_value=self._client(r'۱) \(x^2 = 4\)' '\n' r'✅ جواب: \(x = \pm 2\)')):
|
||||
answer, _f, meta = generate_answer('x² = 4', None, 'نهم')
|
||||
self.assertEqual(answer, '۱) x² = 4\n✅ جواب: x = ±2')
|
||||
self.assertTrue(meta['sanitized'])
|
||||
self.assertEqual(meta['promptVersion'], 'tutor-v3-stepwise-plain')
|
||||
self.assertTrue(meta['rawMarkup'])
|
||||
self.assertEqual(meta['promptVersion'], PROMPT_VERSION)
|
||||
|
||||
|
||||
class DisputedSolutionTests(SimpleTestCase):
|
||||
@@ -300,3 +301,32 @@ class QuestionFocusTests(TestCase):
|
||||
response = self.client.post('/api/ai/ask', {'userQuestion': 'x' * 2001}, format='json')
|
||||
self.assertEqual(response.status_code, 400)
|
||||
generate.assert_not_called()
|
||||
|
||||
|
||||
class SlashReadingPromptTests(SimpleTestCase):
|
||||
"""
|
||||
«/» در سوالِ دبستانی هم ممیز است هم کسر. سنجشِ ۲۰۲۶-۰۹-۲۶ نشان داد جملهی
|
||||
«در کتاب / ممیز است» که برای دبیرستان نوشته شده بود، gpt-4.1-mini را وادار
|
||||
میکرد ۱/۷ + ۱/۸ را ۳/۵ حساب کند. این تستها فقط متنِ پرامپت را میپایند،
|
||||
نه رفتارِ مدل را.
|
||||
"""
|
||||
|
||||
def _user(self, question, grade, contexts=None):
|
||||
from rag.services.generation import build_messages
|
||||
return build_messages(question, contexts, grade)[-1]['content']
|
||||
|
||||
def test_primary_question_with_slash_carries_the_reading_rule(self):
|
||||
text = self._user('۱/۷ + ۱/۸ را حساب کن', 'ششم', [{'summary': 'بخشی از کتاب'}])
|
||||
self.assertIn('معنای هر «/»', text)
|
||||
self.assertNotIn('نه کسر', text)
|
||||
|
||||
def test_rule_also_sent_without_book_context(self):
|
||||
self.assertIn('معنای هر «/»', self._user('۳/۴ ÷ ۲/۵', 'ششم'))
|
||||
|
||||
def test_upper_grades_keep_the_decimal_note_and_skip_the_rule(self):
|
||||
text = self._user('حد (x+1)/(x-2)', 'دهم', [{'summary': 'بخشی از کتاب'}])
|
||||
self.assertIn('۳/۱۰۴ یعنی 3.104', text)
|
||||
self.assertNotIn('معنای هر «/»', text)
|
||||
|
||||
def test_primary_question_without_slash_is_unchanged(self):
|
||||
self.assertEqual(self._user('۲۰ × ۲۰ = ؟', 'سوم'), '۲۰ × ۲۰ = ؟')
|
||||
|
||||
Reference in New Issue
Block a user