Zareian
277927046d
Since 2026-10-07 the cache is the shared Redis on the box. Django's RedisCache raises on any network error and DRF reads the cache for every throttled request, so a Redis outage meant 500 on almost every route, /admin/login/ and OTP request included (all seven drill tests fail on the stock backend). config.cache.ResilientRedisCache turns Redis errors into logged (rate-limited ERROR) misses: get->default, set/delete->ignored, add->True, incr->ValueError. Cache-based caps (DRF throttles, hourly OTP window, admin-login limiter, payment reconcile lock) are NOT enforced during an outage (fail-open); DB-backed caps stay. /api/health returns 503 meanwhile. KEY_PREFIX namespaces our keys on the shared Redis; 1s connect/read timeouts stop a hung Redis from holding requests for gunicorn's 120s. Tested: 11 tests, incl. a real local redis-server killed mid-test and restarted without restarting the app. Not tested yet on the server at the time of this commit. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
181 lines
8.0 KiB
Python
181 lines
8.0 KiB
Python
"""
|
||
قطعیِ Redisِ مشترک نباید API را ۵۰۰ کند (config/cache.py). دو دسته:
|
||
۱. Redis در دسترس نیست (پورتِ بسته، و آدرسِ بی‌جواب برای timeout): هر نوع کارکرد را می‌سنجد.
|
||
۲. یک redis-serverِ واقعیِ محلی (اگر نصب باشد): کارکردِ عادی، پیشوندِ کلید، و «وسطِ کار خاموش و دوباره روشن».
|
||
"""
|
||
import shutil
|
||
import socket
|
||
import subprocess
|
||
import tempfile
|
||
import time
|
||
import unittest
|
||
from unittest import mock
|
||
|
||
from django.core.cache import caches
|
||
from django.test import TestCase, override_settings
|
||
from rest_framework.test import APIClient
|
||
|
||
from accounts.authentication import issue_access_token
|
||
from accounts.models import User
|
||
from content.models import Grade, Lesson, Node, Unit
|
||
|
||
from . import cache as cache_module
|
||
|
||
BACKEND = 'config.cache.ResilientRedisCache'
|
||
|
||
|
||
def _caches(location, prefix=None):
|
||
conf = {'BACKEND': BACKEND, 'LOCATION': location,
|
||
'OPTIONS': {'socket_connect_timeout': 1, 'socket_timeout': 1}}
|
||
if prefix:
|
||
conf['KEY_PREFIX'] = prefix
|
||
return {'default': conf}
|
||
|
||
|
||
class _Quiet:
|
||
def setUp(self):
|
||
super().setUp()
|
||
cache_module._last_logged = 0.0
|
||
|
||
|
||
@override_settings(CACHES=_caches('redis://127.0.0.1:1/0'))
|
||
class RedisDownTests(_Quiet, TestCase):
|
||
"""پورتِ ۱ بسته است: هر اتصال فوراً رد می‌شود."""
|
||
|
||
def test_every_operation_degrades_the_documented_way(self):
|
||
c = caches['default']
|
||
with self.assertLogs('config.cache', 'ERROR'):
|
||
self.assertEqual(c.get('k', 'dflt'), 'dflt')
|
||
self.assertIsNone(c.set('k', 1, 10))
|
||
self.assertIs(c.add('k', 1, 10), True) # «کلید نبود» → فراخواننده ادامه می‌دهد
|
||
self.assertIs(c.delete('k'), False)
|
||
self.assertEqual(c.get_many(['a', 'b']), {})
|
||
self.assertEqual(c.set_many({'a': 1}), ['a'])
|
||
self.assertIs(c.has_key('k'), False)
|
||
with self.assertRaises(ValueError): # همان استثنایی که فراخواننده‌ها می‌شناسند
|
||
c.incr('k')
|
||
self.assertEqual(c.get_or_set('k', lambda: 'built', 10), 'built')
|
||
|
||
def test_errors_are_logged_but_not_flooded(self):
|
||
c = caches['default']
|
||
with self.assertLogs('config.cache', 'ERROR') as logs:
|
||
c.get('a'); c.get('b'); c.set('c', 1)
|
||
self.assertEqual(len(logs.output), 1)
|
||
|
||
def test_throttled_public_endpoint_still_answers(self):
|
||
grade = Grade.objects.create(title='G', icon='', color='', number=88)
|
||
lesson = Lesson.objects.create(node=Node.objects.create(
|
||
unit=Unit.objects.create(grade=grade, title='U', order=1), title='N', order=1), title='L')
|
||
client = APIClient()
|
||
for _ in range(5):
|
||
self.assertEqual(client.get(f'/api/lessons/{lesson.id}/questions').status_code, 200)
|
||
self.assertEqual(client.get('/api/grades').status_code, 200)
|
||
|
||
def test_authenticated_endpoint_still_answers(self):
|
||
user = User.objects.create(phone='09120000888')
|
||
client = APIClient()
|
||
client.credentials(HTTP_AUTHORIZATION=f'Bearer {issue_access_token(user.id)}')
|
||
self.assertEqual(client.get('/api/user/hearts').status_code, 200)
|
||
self.assertEqual(client.get('/api/home').status_code, 200)
|
||
|
||
def test_otp_request_still_works_because_its_caps_live_in_the_database(self):
|
||
with mock.patch('accounts.views.send_otp_sms'):
|
||
response = APIClient().post('/api/auth/otp/request', {'phone': '09120000889'}, format='json')
|
||
self.assertEqual(response.status_code, 200, response.content)
|
||
|
||
def test_admin_login_page_is_not_blocked(self):
|
||
response = self.client.post('/admin/login/', {'username': 'x', 'password': 'y'})
|
||
self.assertNotEqual(response.status_code, 429)
|
||
self.assertNotEqual(response.status_code, 500)
|
||
|
||
def test_health_says_so(self):
|
||
response = APIClient().get('/api/health')
|
||
self.assertEqual(response.status_code, 503)
|
||
self.assertIs(response.json()['checks']['cache']['ok'], False)
|
||
|
||
|
||
class RedisHangsTests(_Quiet, TestCase):
|
||
def test_an_unanswering_redis_costs_about_a_second_not_a_hang(self):
|
||
# آدرسی که جواب نمی‌دهد (بلاک‌هول)؛ اگر شبکه فوراً خطا بدهد هم تست درست است.
|
||
with override_settings(CACHES=_caches('redis://10.255.255.1:6379/0')):
|
||
started = time.monotonic()
|
||
self.assertEqual(caches['default'].get('k', 'dflt'), 'dflt')
|
||
self.assertLess(time.monotonic() - started, 4)
|
||
|
||
|
||
def _free_port():
|
||
with socket.socket() as s:
|
||
s.bind(('127.0.0.1', 0))
|
||
return s.getsockname()[1]
|
||
|
||
|
||
@unittest.skipUnless(shutil.which('redis-server'), 'redis-server نصب نیست')
|
||
class RealRedisTests(_Quiet, TestCase):
|
||
def setUp(self):
|
||
super().setUp()
|
||
self.port = _free_port()
|
||
self.dir = tempfile.mkdtemp()
|
||
self.proc = None
|
||
self.start()
|
||
|
||
def tearDown(self):
|
||
self.stop()
|
||
shutil.rmtree(self.dir, ignore_errors=True)
|
||
super().tearDown()
|
||
|
||
def start(self):
|
||
self.proc = subprocess.Popen(
|
||
['redis-server', '--port', str(self.port), '--bind', '127.0.0.1', '--save', '',
|
||
'--appendonly', 'no', '--dir', self.dir],
|
||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||
deadline = time.monotonic() + 5
|
||
while time.monotonic() < deadline:
|
||
with socket.socket() as s:
|
||
if s.connect_ex(('127.0.0.1', self.port)) == 0:
|
||
return
|
||
time.sleep(0.05)
|
||
self.fail('redis-server بالا نیامد')
|
||
|
||
def stop(self):
|
||
if self.proc and self.proc.poll() is None:
|
||
self.proc.terminate()
|
||
self.proc.wait(timeout=5)
|
||
|
||
def test_normal_behaviour_and_key_prefix(self):
|
||
import redis
|
||
with override_settings(CACHES=_caches(f'redis://127.0.0.1:{self.port}/1', prefix='tadrisino-test')):
|
||
c = caches['default']
|
||
c.set('k', {'a': 1}, 30)
|
||
self.assertEqual(c.get('k'), {'a': 1})
|
||
self.assertTrue(c.add('n', 0, 30))
|
||
self.assertFalse(c.add('n', 0, 30))
|
||
self.assertEqual(c.incr('n'), 1)
|
||
with self.assertRaises(ValueError):
|
||
c.incr('missing')
|
||
raw = redis.Redis(host='127.0.0.1', port=self.port, db=1)
|
||
self.assertTrue(all(b'tadrisino-test' in k for k in raw.keys('*')))
|
||
self.assertEqual(redis.Redis(host='127.0.0.1', port=self.port, db=0).dbsize(), 0) # دیتابیسِ دیگران دست نخورد
|
||
self.assertGreater(raw.ttl(raw.keys('*k')[0]), 0) # TTL گذاشته شده؛ کلیدِ بی‌انقضا نمی‌ماند
|
||
|
||
def test_throttle_is_shared_and_health_is_ok(self):
|
||
with override_settings(CACHES=_caches(f'redis://127.0.0.1:{self.port}/1', prefix='t')):
|
||
response = APIClient().get('/api/health')
|
||
self.assertIs(response.json()['checks']['cache']['ok'], True)
|
||
|
||
def test_redis_dying_mid_flight_and_coming_back_without_restarting_the_app(self):
|
||
with override_settings(CACHES=_caches(f'redis://127.0.0.1:{self.port}/1', prefix='t')):
|
||
c = caches['default']
|
||
c.set('k', 'before', 60)
|
||
self.assertEqual(c.get('k'), 'before')
|
||
|
||
self.stop() # Redis می‌میرد
|
||
with self.assertLogs('config.cache', 'ERROR'):
|
||
self.assertEqual(c.get('k', 'miss'), 'miss')
|
||
self.assertEqual(APIClient().get('/api/grades').status_code, 200)
|
||
self.assertEqual(APIClient().get('/api/health').status_code, 503)
|
||
|
||
self.start() # برمی‌گردد، بدونِ ری‌استارتِ برنامه
|
||
c.set('k2', 'after', 60)
|
||
self.assertEqual(c.get('k2'), 'after')
|
||
self.assertEqual(APIClient().get('/api/health').status_code, 200)
|