Private
Public Access
Batch feed ingestion, add a readiness check, fail open without Redis (C2, C8)
Ingestion did a get_or_create per entry. It now prefetches existing articles for the poll in one query, bulk-inserts new ones and bulk-updates changed ones (falling back to row-by-row only if a concurrent fetch wins a race), and dispatches rule evaluation in bounded chunks so a huge first fetch is several small tasks. /health/ stays a DB-only liveness check (upgrade.sh gates on it right after restart). New /health/ready/ also checks Redis and that poll_due_feeds ticked within five minutes — proving beat and a worker are both alive — and returns 503 with per-check detail otherwise. The sidebar unread-count cache and login throttle now degrade instead of erroring when Redis is unreachable: every authenticated page and every login would otherwise 500 during a cache outage. Found while verifying in a sandbox with no Redis. Also includes the dashboard's oldest-article metric (core/views.py). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8d2900424a
commit
e5f1277014
14 files changed
+354
-48
No files matched your search
@@ -1,5 +1,9 @@
|
||||
import logging
|
||||
|
||||
from django.core.cache import cache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Short enough that a missed invalidation point self-heals within one
|
||||
# page load's worth of staleness; long enough to actually absorb repeat
|
||||
# authenticated-page renders (apps.core.context_processors.sidebar runs
|
||||
@@ -19,4 +23,7 @@ def invalidate_sidebar_unread_cache(user_id: int) -> None:
|
||||
out the TTL above. Rule-driven state changes during background feed
|
||||
ingestion aren't invalidated this way — those self-heal within the
|
||||
TTL, same as a missed call site here would."""
|
||||
cache.delete(sidebar_unread_cache_key(user_id))
|
||||
try:
|
||||
cache.delete(sidebar_unread_cache_key(user_id))
|
||||
except Exception:
|
||||
logger.warning("Sidebar cache unavailable; nothing to invalidate")
|
||||
@@ -22,6 +22,48 @@ def test_health_check_works_before_first_run_setup(client):
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_health_ready_is_degraded_until_a_poll_tick_has_happened(client):
|
||||
response = client.get(reverse("core:health_ready"))
|
||||
assert response.status_code == 503
|
||||
body = response.json()
|
||||
assert body["status"] == "degraded"
|
||||
assert body["checks"]["database"] is True
|
||||
assert body["checks"]["redis"] is True
|
||||
assert body["checks"]["poll_heartbeat"] is False
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_health_ready_is_ok_after_poll_due_feeds_runs(client):
|
||||
from apps.feeds.tasks import poll_due_feeds
|
||||
|
||||
poll_due_feeds()
|
||||
|
||||
response = client.get(reverse("core:health_ready"))
|
||||
assert response.status_code == 200
|
||||
assert response.json()["checks"] == {
|
||||
"database": True,
|
||||
"redis": True,
|
||||
"poll_heartbeat": True,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_health_ready_flags_a_stale_poll_heartbeat(client):
|
||||
from datetime import timedelta
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
|
||||
|
||||
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now() - timedelta(minutes=30))
|
||||
|
||||
response = client.get(reverse("core:health_ready"))
|
||||
assert response.status_code == 503
|
||||
assert response.json()["checks"]["poll_heartbeat"] is False
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_dashboard_requires_staff(client, django_user_model):
|
||||
django_user_model.objects.create_user("regular", password="pw-regular-123")
|
||||
|
||||
@@ -6,6 +6,7 @@ app_name = "core"
|
||||
|
||||
urlpatterns = [
|
||||
path("health/", views.health, name="health"),
|
||||
path("health/ready/", views.health_ready, name="health_ready"),
|
||||
path("dashboard/", views.dashboard, name="dashboard"),
|
||||
path("dashboard/feeds/<int:feed_id>/retry/", views.retry_feed, name="retry_feed"),
|
||||
]
|
||||
+50
-3
@@ -1,20 +1,27 @@
|
||||
from django.contrib import messages
|
||||
from django.contrib.auth.decorators import user_passes_test
|
||||
from django.core.cache import cache
|
||||
from django.db import connection
|
||||
from django.db.models import Count, Q
|
||||
from django.db.models import Count, Min, Q
|
||||
from django.http import JsonResponse
|
||||
from django.shortcuts import get_object_or_404, redirect, render
|
||||
from django.utils import timezone
|
||||
from django.views.decorators.http import require_POST
|
||||
|
||||
from apps.articles.models import Article
|
||||
from apps.feeds.models import Feed, Subscription
|
||||
from apps.feeds.tasks import fetch_feed, reset_feed_backoff
|
||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, fetch_feed, reset_feed_backoff
|
||||
|
||||
is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff)
|
||||
|
||||
|
||||
# Beat ticks poll_due_feeds every minute; this many minutes without a tick
|
||||
# means beat or the worker (or the broker between them) isn't working.
|
||||
POLL_HEARTBEAT_MAX_AGE_SECONDS = 5 * 60
|
||||
|
||||
|
||||
def health(request):
|
||||
"""Unauthenticated liveness/readiness check — reachable even before
|
||||
"""Unauthenticated liveness check — reachable even before
|
||||
first-run setup (see SetupWizardMiddleware). Used by scripts/upgrade.sh
|
||||
to confirm a freshly restarted deploy actually came up, and is a
|
||||
reasonable target for any external uptime monitor too."""
|
||||
@@ -26,6 +33,45 @@ def health(request):
|
||||
return JsonResponse({"status": "ok"})
|
||||
|
||||
|
||||
def health_ready(request):
|
||||
"""Readiness: liveness plus everything background fetching depends on
|
||||
(C8) — the database, the Redis cache/broker, and a recent
|
||||
poll_due_feeds tick, which proves beat is scheduling AND a worker is
|
||||
consuming. /health/ above stays DB-only on purpose: scripts/upgrade.sh
|
||||
gates a deploy on it right after restart, before beat has had a
|
||||
chance to tick. Point an uptime monitor at this one instead."""
|
||||
checks = {"database": False, "redis": False, "poll_heartbeat": False}
|
||||
detail = {}
|
||||
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SELECT 1")
|
||||
checks["database"] = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
cache.set("health-ready-probe", "1", timeout=10)
|
||||
checks["redis"] = cache.get("health-ready-probe") == "1"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
last_tick = cache.get(POLL_HEARTBEAT_CACHE_KEY)
|
||||
except Exception:
|
||||
last_tick = None
|
||||
if last_tick is not None:
|
||||
age = (timezone.now() - last_tick).total_seconds()
|
||||
detail["poll_heartbeat_age_seconds"] = int(age)
|
||||
checks["poll_heartbeat"] = age <= POLL_HEARTBEAT_MAX_AGE_SECONDS
|
||||
|
||||
ok = all(checks.values())
|
||||
return JsonResponse(
|
||||
{"status": "ok" if ok else "degraded", "checks": checks, **detail},
|
||||
status=200 if ok else 503,
|
||||
)
|
||||
|
||||
|
||||
@is_admin
|
||||
def dashboard(request):
|
||||
feeds = Feed.objects.annotate(subscriber_count=Count("subscriptions"))
|
||||
@@ -39,6 +85,7 @@ def dashboard(request):
|
||||
"error_feed_count": erroring_feeds.count(),
|
||||
"subscription_count": Subscription.objects.count(),
|
||||
"article_count": Article.objects.count(),
|
||||
"oldest_article_at": Article.objects.aggregate(oldest=Min("fetched_at"))["oldest"],
|
||||
}
|
||||
|
||||
return render(
|
||||
|
||||
Reference in new issue
Block a user