Batch feed ingestion, add a readiness check, fail open without Redis (C2, C8)

Ingestion did a get_or_create per entry. It now prefetches existing
articles for the poll in one query, bulk-inserts new ones and
bulk-updates changed ones (falling back to row-by-row only if a
concurrent fetch wins a race), and dispatches rule evaluation in
bounded chunks so a huge first fetch is several small tasks.

/health/ stays a DB-only liveness check (upgrade.sh gates on it right
after restart). New /health/ready/ also checks Redis and that
poll_due_feeds ticked within five minutes — proving beat and a worker
are both alive — and returns 503 with per-check detail otherwise.

The sidebar unread-count cache and login throttle now degrade instead
of erroring when Redis is unreachable: every authenticated page and
every login would otherwise 500 during a cache outage. Found while
verifying in a sandbox with no Redis.

Also includes the dashboard's oldest-article metric (core/views.py).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5.5 committed 2026-11-01 08:51:20 -07:00
1 parent 8d2900424a
commit e5f1277014
14 files changed
+354 -48

No files matched your search

+1
View File
@@ -45,4 +45,5 @@ def _setup_exempt_paths():
reverse("pwa:service_worker"),
reverse("pwa:offline"),
reverse("core:health"),
reverse("core:health_ready"),
}
+18
View File
@@ -72,3 +72,21 @@ def test_too_many_failed_attempts_against_one_username_blocks_that_username(clie
response = client.post(reverse("accounts:login"), {"username": "bob", "password": "pw-bob-123"})
assert response.status_code == 302
assert response.wsgi_request.user == bob
@pytest.mark.django_db
def test_login_still_works_when_the_throttle_cache_is_down(client, monkeypatch):
from unittest.mock import Mock
from django.contrib.auth import get_user_model
get_user_model().objects.create_user("alice", password="pw-alice-123")
broken = Mock(side_effect=ConnectionError("redis down"))
monkeypatch.setattr("apps.accounts.views.cache.get", broken)
monkeypatch.setattr("apps.accounts.views.cache.incr", broken)
monkeypatch.setattr("apps.accounts.views.cache.set", broken)
bad = client.post(reverse("accounts:login"), {"username": "alice", "password": "wrong"})
assert bad.status_code == 200
good = client.post(reverse("accounts:login"), {"username": "alice", "password": "pw-alice-123"})
assert good.status_code == 302
+18 -4
View File
@@ -1,3 +1,5 @@
import logging
from django.contrib import messages
from django.contrib.auth import login
from django.contrib.auth.decorators import user_passes_test
@@ -10,6 +12,8 @@ from .forms import CreateUserForm, SetupForm
from .middleware import SetupWizardMiddleware
from .models import SetupLock, User
logger = logging.getLogger(__name__)
is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff)
# Two independent throttles, either of which can block an attempt: by
@@ -41,14 +45,24 @@ def _attempts_key(kind: str, value: str) -> str:
def _too_many_attempts(key: str, limit: int) -> bool:
return cache.get(key, 0) >= limit
# Fails open if the cache (Redis) is unreachable: logging in must keep
# working through a cache outage, at the cost of no throttling until
# it's back — an outage shouldn't lock every user out of the app.
try:
return cache.get(key, 0) >= limit
except Exception:
logger.warning("Login throttle cache unavailable; not rate limiting", exc_info=True)
return False
def _record_failed_attempt(key: str) -> None:
try:
cache.incr(key)
except ValueError:
cache.set(key, 1, LOGIN_RATE_LIMIT_WINDOW_SECONDS)
try:
cache.incr(key)
except ValueError:
cache.set(key, 1, LOGIN_RATE_LIMIT_WINDOW_SECONDS)
except Exception:
logger.warning("Login throttle cache unavailable; attempt not recorded", exc_info=True)
class LoginView(DjangoLoginView):