tutor runs on gpt-4.1-mini, and primary-school "/" is read as decimal or fraction by rule
Two benchmark rounds (14 models on 30 problems, then gpt-4o-mini vs gpt-4.1-mini on 102 problems under three prompt versions, hand-graded): gpt-4.1-mini scored 95-96/102 against 85-90 for gpt-4o-mini, at 0.12 vs 0.04 cents per answer. The default now lives in settings so the deploy needs no .env edit. The context note "in the book / is a decimal point, not a fraction" was written for the upper-grade books but went to every grade, so gpt-4.1-mini computed 1/7 + 1/8 as 3.5. For primary grades the note now says / can be either, and a reading rule sits next to the question itself (the same rule at the end of the system prompt had no effect). Slash problems went from 11 to 14 of 17; three are still wrong. generate_answer now reports the provider's exact per-call cost, reasoning tokens and whether the raw output broke the plain-text rule. The benchmark fixes primary grades being sent with no book context, caches retrieval so runs compare on identical context, and retries Qdrant timeouts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -69,6 +69,9 @@ JUDGE_PROMPT = """یک مسئله‌ی ریاضیِ مدرسه، جوابِ در
|
||||
|
||||
# نشانه‌ی LaTeX/markdown در خروجیِ *خامِ* مدل (پیش از تورِ ایمنی)
|
||||
_STEP_LINE = re.compile(r'^\s*([0-9۰-۹]+[)\.)]|مرحله)', re.M)
|
||||
# جمله‌ی انگلیسی/چینی وسطِ پاسخِ فارسی (نامِ تابع‌ها مثلِ sin/lim شمرده نمی‌شوند)
|
||||
_FOREIGN_WORD = re.compile(r'\b(?!(?:sin|cos|tan|cot|log|lim|ln|cm|mm|km|kg|erg|Erg)\b)[A-Za-z]{4,}\b')
|
||||
_CJK = re.compile(r'[\u3040-\u30ff\u4e00-\u9fff]')
|
||||
|
||||
|
||||
_KEY_NOT_AN_ANSWER = re.compile(r'کتاب|اثبات|درست است|صحیح است|مطابق|توضیح')
|
||||
@@ -93,7 +96,9 @@ def _stable_order(chunk_id: int, seed: str) -> str:
|
||||
|
||||
|
||||
def _grade_of(item) -> str:
|
||||
return (item['grade'] or item['collection'].split()[-1]).strip()
|
||||
# از نامِ collection، مثلِ خودِ اپ. `grade_title`ِ کتاب‌های دبستان «سوم دبستان»
|
||||
# است؛ با آن نه collection پیدا می‌شد (retrieval خالی) نه لحنِ دبستان.
|
||||
return item['collection'].split()[-1].strip()
|
||||
|
||||
|
||||
def _usage_cents() -> float | None:
|
||||
@@ -133,6 +138,9 @@ class Command(BaseCommand):
|
||||
# فقط از ایران جواب می‌دهد: نمونه را اینجا بنویس، روی سرور بخوان.
|
||||
parser.add_argument('--export-sample', default=None, metavar='PATH')
|
||||
parser.add_argument('--sample-file', default=None, metavar='PATH')
|
||||
# context یک بار ساخته و ذخیره می‌شود تا چند اجرا (مثلاً دو نسخه‌ی پرامپت)
|
||||
# دقیقاً همان بخش‌های کتاب را ببینند و Qdrant هم فقط یک بار صدا زده شود.
|
||||
parser.add_argument('--context-cache', default=None, metavar='PATH')
|
||||
|
||||
# ------------------------------------------------------------------ sample
|
||||
def _sample(self, per_grade, collections, seed):
|
||||
@@ -160,7 +168,16 @@ class Command(BaseCommand):
|
||||
# ----------------------------------------------------------------- context
|
||||
def _context(self, item):
|
||||
subject = item['collection'].rsplit(' ', 1)[0]
|
||||
hits, _meta = retrieve_context(item['problem'], subject=subject, grade_title=_grade_of(item))
|
||||
for attempt in range(3):
|
||||
try:
|
||||
hits, _meta = retrieve_context(item['problem'], subject=subject,
|
||||
grade_title=_grade_of(item))
|
||||
break
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# timeoutِ Qdrant زیرِ بارِ هم‌زمان یک بار کلِ اجرا را کشت.
|
||||
if is_account_limit(exc) or attempt == 2:
|
||||
raise
|
||||
time.sleep(3 * (attempt + 1))
|
||||
needle = ' '.join(item['problem'].split())[:60]
|
||||
# همان تکه (و هر تکه‌ای که عینِ این مسئله را دارد) بیرون: وگرنه مدل
|
||||
# فقط راه‌حلِ کتاب را رونویسی می‌کند و سنجه بی‌معنا می‌شود.
|
||||
@@ -179,13 +196,14 @@ class Command(BaseCommand):
|
||||
)
|
||||
data = json.loads(completion.choices[0].message.content or '{}')
|
||||
verdict = data.get('verdict')
|
||||
cost = getattr(getattr(completion, 'usage', None), 'cost', None) or 0
|
||||
if verdict in ('correct', 'wrong', 'unclear'):
|
||||
return verdict, str(data.get('why') or '')[:120]
|
||||
return verdict, str(data.get('why') or '')[:120], cost
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if is_account_limit(exc):
|
||||
raise AccountLimitReached(str(exc)[:160]) from exc
|
||||
time.sleep(2 * (attempt + 1))
|
||||
return 'unclear', 'داوری شکست خورد'
|
||||
return 'unclear', 'داوری شکست خورد', 0
|
||||
|
||||
# ------------------------------------------------------------------ handle
|
||||
def handle(self, *args, **opts):
|
||||
@@ -219,16 +237,28 @@ class Command(BaseCommand):
|
||||
|
||||
# context یک بار برای همه‌ی مدل‌ها: مقایسه منصفانه و embedding ارزان‌تر.
|
||||
contexts = {}
|
||||
if not opts['no_context']:
|
||||
cache = opts['context_cache']
|
||||
if cache and not opts['no_context']:
|
||||
try:
|
||||
with open(cache, encoding='utf-8') as fh:
|
||||
contexts = {int(k): v for k, v in json.load(fh).items()}
|
||||
self.stdout.write(f'context از {cache} ({len(contexts)} مسئله)')
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
if not opts['no_context'] and any(item['id'] not in contexts for item in items):
|
||||
self.stdout.write('retrieval …')
|
||||
try:
|
||||
with ThreadPoolExecutor(opts['workers']) as pool:
|
||||
for item, ctx in zip(items, pool.map(self._context, items)):
|
||||
todo = [item for item in items if item['id'] not in contexts]
|
||||
for item, ctx in zip(todo, pool.map(self._context, todo)):
|
||||
contexts[item['id']] = ctx
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if is_account_limit(exc):
|
||||
raise CommandError(f'⛔ اعتبارِ حساب تمام شده است: {str(exc)[:160]}') from exc
|
||||
raise
|
||||
if cache:
|
||||
with open(cache, 'w', encoding='utf-8') as fh:
|
||||
json.dump(contexts, fh, ensure_ascii=False)
|
||||
|
||||
# نتیجه‌ی هر مدل همان لحظه روی دیسک می‌رود. اجرای اول وسطِ کار با تمام
|
||||
# شدنِ اعتبار مرد و چون فقط آخرِ کار ذخیره می‌کرد، پاسخ‌های چهار مدلِ
|
||||
@@ -286,6 +316,11 @@ class Command(BaseCommand):
|
||||
after = _usage_cents()
|
||||
wall = time.monotonic() - started
|
||||
cost = (after - before) if before is not None and after is not None else None
|
||||
# هزینه‌ی دقیقِ هر پاسخ از usage.cost، اگر provider بفرستد. دقیق‌تر از
|
||||
# تفاضلِ billing است و اجرای هم‌زمانِ چند مدل را هم خراب نمی‌کند.
|
||||
per_call = [a[2].get('costUsd') for a in answered if not a[4]]
|
||||
if per_call and all(c is not None for c in per_call):
|
||||
cost = 100 * sum(per_call)
|
||||
|
||||
rows = []
|
||||
for item, answer, meta, seconds, error in answered:
|
||||
@@ -294,8 +329,16 @@ class Command(BaseCommand):
|
||||
'tier': grade_tier(_grade_of(item)), 'problem': item['problem'],
|
||||
'key': item['key'], 'answer': answer, 'seconds': round(seconds, 1),
|
||||
'sanitized': bool(meta.get('sanitized')),
|
||||
'rawMarkup': bool(meta.get('rawMarkup')),
|
||||
'promptTokens': meta.get('promptTokens', 0),
|
||||
'completionTokens': meta.get('completionTokens', 0),
|
||||
'reasoningTokens': meta.get('reasoningTokens', 0),
|
||||
'costUsd': meta.get('costUsd'),
|
||||
'hasFinal': '✅' in answer, 'steps': len(_STEP_LINE.findall(answer)),
|
||||
'chars': len(answer), 'hasCheck': 'وارسی' in answer,
|
||||
'hasPractice': 'امتحان کن' in answer,
|
||||
'foreignWords': len(_FOREIGN_WORD.findall(answer)),
|
||||
'cjk': len(_CJK.findall(answer)),
|
||||
'error': error, 'verdict': 'error', 'why': '',
|
||||
})
|
||||
errors = sum(1 for r in rows if r['error'])
|
||||
@@ -308,10 +351,11 @@ class Command(BaseCommand):
|
||||
with ThreadPoolExecutor(opts['workers']) as pool:
|
||||
verdicts = list(pool.map(
|
||||
lambda r: self._judge(opts['judge'], chunks_by_id[r['chunkId']], r['answer']), todo))
|
||||
for r, (verdict, why) in zip(todo, verdicts):
|
||||
r['verdict'], r['why'] = verdict, why
|
||||
for r, (verdict, why, judge_cost) in zip(todo, verdicts):
|
||||
r['verdict'], r['why'], r['judgeCostUsd'] = verdict, why, judge_cost
|
||||
results.extend(rows)
|
||||
summary[model] = {'costCents': cost, 'wallSeconds': round(wall, 1)}
|
||||
summary[model] = {'costCents': cost, 'wallSeconds': round(wall, 1),
|
||||
'judgeCostCents': round(100 * sum(v[2] for v in verdicts), 3)}
|
||||
|
||||
def _report(self, models, results, summary):
|
||||
tiers = ['primary', 'middle', 'upper']
|
||||
|
||||
Reference in New Issue
Block a user