Zareian
ac93e17305
Self-contained app; wired into settings/urls in the next commit. Data
stays in the parallel team's live Qdrant — nothing is copied into our DB.
apps.py
- RagConfig.
models.py
- LessonChunk: DB-of-record for one summarized lesson chunk
(original_text + summary + embedding_vector) mirrored as a single
Qdrant point; loose lesson_id/grade_id ints, no cross-app FK.
LessonChunkImage for future multimodal lessons.
migrations/0001_initial.py
- initial tables for the two models.
services/clients.py
- lazy, lru_cache'd OpenAI + Qdrant clients (Rag_backend builds them at
import time and crashes without keys). RagNotConfigured raised when a
key/lib is missing; Qdrant client uses check_compatibility=False for
the 1.19-client / 1.17-server minor version gap.
services/config.py
- rag_is_configured(), collection-name template, grade-title resolution
(RAG_GRADE_TITLE_OVERRIDES -> content.Grade.title), retrieval top-k.
services/known_collections.py
- exact live Qdrant catalog: 6 collection names, math chapter-number ->
exact "فصل N: ..." string tables per (subject, grade), and the
علوم/اجتماعی lesson-number -> title maps. filter_key_for_subject()
picks `chapter` for math, `lesson_number` otherwise;
resolve_math_chapter() maps a plain chapter number (Persian or Latin
digits) to the stored string.
services/embedding.py
- embed_text() pinned to OPENAI_EMBEDDING_MODEL (text-embedding-3-small,
1536d) so vectors match the other team's existing collections.
services/chunking.py
- split_text_into_chunks(), summarize_chunk() (gpt-4o-mini, JSON out);
plain exceptions instead of ninja HttpError.
services/qdrant_store.py
- ensure_collection(), upsert_points(), search(): the lesson filter
matches the value both as str and native, since Qdrant stores
lesson_number as int and chapter as string.
services/retrieval.py
- resolve_target() -> (collection, filter_key, filter_value), reused by
the cache key and debug block.
- retrieve_context() -> (chunks, meta): embed -> Qdrant search scoped to
one lesson/chapter -> top-k. Math payloads (no `summary`) fall back to
generalSolution/problem/solution; a 404 collection returns empty.
services/generation.py
- generate_answer() and stream_answer() (token iterator for SSE);
context-grounded Persian prompt that explains from retrieved text
rather than refusing.
services/pdf.py
- textbook PDF -> page images -> gpt-4o vision problem-type extraction
(adapted from Rag_backend mathapp); returns dicts, no on-disk JSON.
pymupdf/pdf2image optional — missing libs raise RagNotConfigured.
services/ingest.py
- ingest_text() / ingest_pdf(): chunk -> summarize -> embed -> persist
LessonChunk + upsert Qdrant, in one transaction.
services/answer_cache.py
- short-TTL cache of the final answer keyed by
sha1(collection | filter_value | normalized-question).
services/tutor.py
- thin answer_question() (retrieve -> generate) for direct callers.
serializers.py, urls.py, views.py
- POST /api/rag/ingest/text, POST /api/rag/ingest/pdf (HasInternalServiceKey,
throttle scope rag_ingest), GET /api/rag/chunks review list.
RagProviderUnavailable -> clean 503.
admin.py
- LessonChunk admin with an inline for its images.
management/commands/rag_catalog.py
- read-only live Qdrant listing (--values, --json); never writes.
tests.py
- mocked-client tests: ask-with-retrieval, ingest, SSE stream, answer
cache, and rag_is_configured() gating.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
67 lines
2.7 KiB
Python
67 lines
2.7 KiB
Python
"""
|
|
لایه‌ی نازک روی Qdrant: ساختِ collection، upsert نقاط، و جستجوی برداری.
|
|
|
|
برداشتِ پاک‌سازی‌شده از subjects/services/qdrant.py مخزنِ Rag_backend.
|
|
هیچ کلاینتِ سطحِ ماژولی ساخته نمی‌شه؛ همه از get_qdrant_client() lazy.
|
|
"""
|
|
from django.conf import settings
|
|
|
|
from . import clients
|
|
|
|
|
|
def ensure_collection(name: str, vector_size: int | None = None) -> None:
|
|
client = clients.get_qdrant_client()
|
|
vector_size = vector_size or settings.OPENAI_EMBEDDING_DIM
|
|
existing = {c.name for c in client.get_collections().collections}
|
|
if name in existing:
|
|
return
|
|
from qdrant_client.models import Distance, VectorParams
|
|
|
|
client.create_collection(
|
|
collection_name=name,
|
|
vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE),
|
|
)
|
|
|
|
|
|
def upsert_points(collection: str, points: list[dict]) -> int:
|
|
"""points: [{id, vector, payload}, ...]"""
|
|
if not points:
|
|
return 0
|
|
client = clients.get_qdrant_client()
|
|
from qdrant_client.models import PointStruct
|
|
|
|
structs = [
|
|
PointStruct(id=str(p['id']), vector=p['vector'], payload=p['payload'])
|
|
for p in points
|
|
]
|
|
client.upsert(collection_name=collection, points=structs, wait=True)
|
|
return len(structs)
|
|
|
|
|
|
def search(collection: str, query_vector: list[float], *, lesson_filter_value=None,
|
|
lesson_filter_key: str | None = None, limit: int = 4) -> list[dict]:
|
|
"""
|
|
نزدیک‌ترین نقاط به بردارِ پرسش، در صورتِ نیاز محدود به یک درس.
|
|
خروجی: [{score, payload}, ...] — payload همون چیزیه که تیمِ دیگه ذخیره
|
|
کرده (معمولاً شاملِ summary / lesson_title / lesson_number).
|
|
"""
|
|
client = clients.get_qdrant_client()
|
|
query_filter = None
|
|
if lesson_filter_value is not None and lesson_filter_key:
|
|
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
|
|
|
# دادهٔ تیمِ دیگه lesson_number/chapter را همیشه به‌صورتِ رشته ذخیره
|
|
# کرده ("5")، پس مقدار را به رشته تبدیل می‌کنیم تا فیلتر بخورد.
|
|
query_filter = Filter(should=[
|
|
FieldCondition(key=lesson_filter_key, match=MatchValue(value=str(lesson_filter_value))),
|
|
FieldCondition(key=lesson_filter_key, match=MatchValue(value=lesson_filter_value)),
|
|
])
|
|
|
|
response = client.query_points(
|
|
collection_name=collection,
|
|
query=query_vector,
|
|
limit=limit,
|
|
query_filter=query_filter,
|
|
)
|
|
return [{'score': p.score, 'payload': p.payload or {}} for p in response.points]
|