From fd494a9c0860bf22b6800f8b06712d3da214ef5c Mon Sep 17 00:00:00 2001 From: Keith Smith Date: Sun, 1 Nov 2026 09:30:00 -0700 Subject: [PATCH] Make the readiness check catch fetching that is scheduled but failing (#27) /health/ready/ proved beat and a worker were alive but not that fetching worked: every feed could be failing, or the queue could be hours behind, and it would still report ok. apps.core.health adds three checks: Celery queue depth (real Redis list length, over 1000 fails), a successful fetch within three poll intervals (30 min minimum, with a grace period for a brand-new install), and the share of active feeds failing (over half, once there are at least four). The payload also reports queue depth, active/failing feed counts, and the age of the last successful fetch. Verified against real Redis with queued Celery tasks, and the suite on PostgreSQL. docs/install.md lists each check. Co-Authored-By: Claude Sonnet 5.5 --- apps/core/health.py | 83 ++++++++++++++++++++++++++ apps/core/tests.py | 141 ++++++++++++++++++++++++++++++++++++++++++-- apps/core/views.py | 11 +++- docs/install.md | 28 +++++++-- 4 files changed, 252 insertions(+), 11 deletions(-) create mode 100644 apps/core/health.py diff --git a/apps/core/health.py b/apps/core/health.py new file mode 100644 index 0000000..31110ff --- /dev/null +++ b/apps/core/health.py @@ -0,0 +1,83 @@ +from datetime import timedelta + +import redis +from django.conf import settings +from django.db.models import Max, Min +from django.utils import timezone + +from apps.feeds.models import Feed + +from .models import SiteSettings + +# More than this many tasks waiting means workers aren't keeping up (or +# aren't running) — fetches are piling up faster than they complete. +MAX_QUEUE_DEPTH = 1000 + +# Past this share of active feeds failing, it's far more likely something +# on our side (DNS, outbound network, a bad deploy) than every publisher +# breaking at once. Needs a handful of feeds first so one broken feed out +# of two doesn't read as an outage. +MAX_FAILING_RATIO = 0.5 +MIN_FEEDS_FOR_RATIO = 4 + +MIN_SUCCESS_WINDOW_MINUTES = 30 + + +def queue_depth() -> int | None: + """Tasks waiting in Celery's default queue, or None if the broker + can't be reached. Celery's Redis transport keeps each queue as a Redis + list named after the queue.""" + try: + client = redis.Redis.from_url( + settings.CELERY_BROKER_URL, socket_connect_timeout=2, socket_timeout=2 + ) + return int(client.llen("celery")) + except Exception: + return None + + +def _success_window() -> timedelta: + floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15) + base = max(SiteSettings.load().fetch_interval_minutes, floor) + return timedelta(minutes=max(3 * base, MIN_SUCCESS_WINDOW_MINUTES)) + + +def fetch_pipeline_status(now=None) -> tuple[dict[str, bool], dict]: + """Checks that fetching is actually *succeeding*, not just scheduled + (the poll heartbeat only proves beat and a worker are alive): returns + ({check: passed}, detail) covering queue depth, whether any feed has + been fetched successfully recently, and the share of feeds failing. + """ + now = now or timezone.now() + detail: dict = {} + + depth = queue_depth() + detail["queue_depth"] = depth + checks = {"queue_depth": depth is not None and depth <= MAX_QUEUE_DEPTH} + + active = Feed.objects.filter(is_active=True) + active_count = active.count() + failing = active.filter(fetch_error_count__gt=0).count() + detail["active_feeds"] = active_count + detail["failing_feeds"] = failing + + window = _success_window() + last_success = active.filter( + last_fetch_status__in=[Feed.STATUS_OK, Feed.STATUS_NOT_MODIFIED] + ).aggregate(latest=Max("last_fetched_at"))["latest"] + if last_success is not None: + detail["last_successful_fetch_age_seconds"] = int((now - last_success).total_seconds()) + recent_success = now - last_success <= window + elif active_count == 0: + recent_success = True + else: + # Nothing has ever fetched successfully — only a problem once + # the oldest feed has had a full window to be fetched. + oldest = active.aggregate(oldest=Min("created_at"))["oldest"] + recent_success = now - oldest <= window + checks["recent_fetch_success"] = recent_success + + checks["feed_failures"] = not ( + active_count >= MIN_FEEDS_FOR_RATIO and failing / active_count > MAX_FAILING_RATIO + ) + return checks, detail diff --git a/apps/core/tests.py b/apps/core/tests.py index 327a649..8591059 100644 --- a/apps/core/tests.py +++ b/apps/core/tests.py @@ -22,8 +22,17 @@ def test_health_check_works_before_first_run_setup(client): assert response.status_code == 200 +@pytest.fixture +def broker_queue(monkeypatch): + """Stands in for the Redis broker's queue length (tests have no Redis): + set `broker_queue["depth"]` to a number, or None for "unreachable".""" + state = {"depth": 0} + monkeypatch.setattr("apps.core.health.queue_depth", lambda: state["depth"]) + return state + + @pytest.mark.django_db -def test_health_ready_is_degraded_until_a_poll_tick_has_happened(client): +def test_health_ready_is_degraded_until_a_poll_tick_has_happened(client, broker_queue): response = client.get(reverse("core:health_ready")) assert response.status_code == 503 body = response.json() @@ -34,22 +43,26 @@ def test_health_ready_is_degraded_until_a_poll_tick_has_happened(client): @pytest.mark.django_db -def test_health_ready_is_ok_after_poll_due_feeds_runs(client): +def test_health_ready_is_ok_after_poll_due_feeds_runs(client, broker_queue): from apps.feeds.tasks import poll_due_feeds poll_due_feeds() response = client.get(reverse("core:health_ready")) assert response.status_code == 200 - assert response.json()["checks"] == { + checks = response.json()["checks"] + assert checks == { "database": True, "redis": True, "poll_heartbeat": True, + "queue_depth": True, + "recent_fetch_success": True, + "feed_failures": True, } @pytest.mark.django_db -def test_health_ready_flags_a_stale_poll_heartbeat(client): +def test_health_ready_flags_a_stale_poll_heartbeat(client, broker_queue): from datetime import timedelta from django.core.cache import cache @@ -119,3 +132,123 @@ def test_retry_feed_resets_backoff_and_requeues(client, django_user_model): assert response.status_code == 302 mock_reset.assert_called_once_with(feed.id) mock_fetch.assert_called_once_with(feed.id) + + +def _ready(client): + from django.core.cache import cache + from django.utils import timezone + + from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY + + # Record the heartbeat directly: running poll_due_feeds here would + # execute fetches eagerly (CELERY_TASK_ALWAYS_EAGER) and rewrite the + # feed states each test sets up. + cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now()) + response = client.get(reverse("core:health_ready")) + return response.status_code, response.json() + + +@pytest.mark.django_db +def test_health_ready_flags_a_backed_up_queue(client, broker_queue): + from .health import MAX_QUEUE_DEPTH + + broker_queue["depth"] = MAX_QUEUE_DEPTH + 1 + + status, body = _ready(client) + + assert status == 503 + assert body["checks"]["queue_depth"] is False + assert body["queue_depth"] == MAX_QUEUE_DEPTH + 1 + + +@pytest.mark.django_db +def test_health_ready_flags_an_unreachable_broker_queue(client, broker_queue): + broker_queue["depth"] = None + + status, body = _ready(client) + + assert status == 503 + assert body["checks"]["queue_depth"] is False + + +@pytest.mark.django_db +def test_health_ready_flags_no_recent_successful_fetch(client, broker_queue): + from datetime import timedelta + + from django.utils import timezone + + feed = Feed.objects.create( + feed_url="https://a.example.com/feed.xml", + last_fetch_status=Feed.STATUS_OK, + last_fetched_at=timezone.now() - timedelta(days=2), + ) + Feed.objects.filter(pk=feed.pk).update(created_at=timezone.now() - timedelta(days=30)) + + status, body = _ready(client) + + assert status == 503 + assert body["checks"]["recent_fetch_success"] is False + assert body["last_successful_fetch_age_seconds"] > 86400 + + +@pytest.mark.django_db +def test_health_ready_is_ok_with_a_recent_successful_fetch(client, broker_queue): + from django.utils import timezone + + Feed.objects.create( + feed_url="https://a.example.com/feed.xml", + last_fetch_status=Feed.STATUS_NOT_MODIFIED, + last_fetched_at=timezone.now(), + ) + + status, body = _ready(client) + + assert status == 200 + assert body["active_feeds"] == 1 + assert body["failing_feeds"] == 0 + + +@pytest.mark.django_db +def test_health_ready_gives_a_brand_new_feed_time_before_it_counts_as_stale(client, broker_queue): + Feed.objects.create(feed_url="https://new.example.com/feed.xml") + + status, body = _ready(client) + + assert status == 200 + assert body["checks"]["recent_fetch_success"] is True + + +@pytest.mark.django_db +def test_health_ready_flags_most_feeds_failing_but_not_a_single_broken_one(client, broker_queue): + from django.utils import timezone + + now = timezone.now() + healthy = Feed.objects.create( + feed_url="https://ok.example.com/feed.xml", + last_fetch_status=Feed.STATUS_OK, + last_fetched_at=now, + ) + broken = [ + Feed.objects.create( + feed_url=f"https://bad{i}.example.com/feed.xml", + last_fetch_status=Feed.STATUS_ERROR, + fetch_error_count=3, + last_fetched_at=now, + ) + for i in range(2) + ] + + status, body = _ready(client) + assert status == 200, "2 of 3 failing is below the minimum feed count for the ratio" + + Feed.objects.create( + feed_url="https://bad9.example.com/feed.xml", + last_fetch_status=Feed.STATUS_ERROR, + fetch_error_count=1, + last_fetched_at=now, + ) + status, body = _ready(client) + assert status == 503 + assert body["checks"]["feed_failures"] is False + assert body["failing_feeds"] == 3 + assert healthy.pk and broken diff --git a/apps/core/views.py b/apps/core/views.py index d811834..e76df8f 100644 --- a/apps/core/views.py +++ b/apps/core/views.py @@ -12,6 +12,8 @@ from apps.articles.models import Article from apps.feeds.models import Feed, Subscription from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, fetch_feed, reset_feed_backoff +from .health import fetch_pipeline_status + is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff) @@ -37,7 +39,9 @@ def health_ready(request): """Readiness: liveness plus everything background fetching depends on (C8) — the database, the Redis cache/broker, and a recent poll_due_feeds tick, which proves beat is scheduling AND a worker is - consuming. /health/ above stays DB-only on purpose: scripts/upgrade.sh + consuming. Beyond that tick, apps.core.health checks fetching is + actually succeeding: queue depth, a recent successful fetch, and the + share of feeds failing. /health/ above stays DB-only on purpose: scripts/upgrade.sh gates a deploy on it right after restart, before beat has had a chance to tick. Point an uptime monitor at this one instead.""" checks = {"database": False, "redis": False, "poll_heartbeat": False} @@ -65,6 +69,11 @@ def health_ready(request): detail["poll_heartbeat_age_seconds"] = int(age) checks["poll_heartbeat"] = age <= POLL_HEARTBEAT_MAX_AGE_SECONDS + if checks["database"]: + pipeline_checks, pipeline_detail = fetch_pipeline_status() + checks.update(pipeline_checks) + detail.update(pipeline_detail) + ok = all(checks.values()) return JsonResponse( {"status": "ok" if ok else "degraded", "checks": checks, **detail}, diff --git a/docs/install.md b/docs/install.md index 8af8a1e..a9fe374 100644 --- a/docs/install.md +++ b/docs/install.md @@ -314,17 +314,33 @@ misconfigured feed or an outage at the source. `/health/` is an unauthenticated liveness check (DB connectivity only) — `scripts/upgrade.sh` uses it to confirm a deploy actually came back up. -Point an external uptime monitor at `/health/ready/` instead: it also checks -Redis and that the background poller has ticked in the last five minutes -(proving Celery beat is scheduling *and* a worker is consuming), and returns -503 with a per-check breakdown when any of them fails — so "the site loads -but nothing is being fetched" actually alerts. +Point an external uptime monitor at `/health/ready/` instead. It returns 503 +with a per-check breakdown when any of these fail: + +- `database` and `redis` — reachable. +- `poll_heartbeat` — the background poller ticked in the last five minutes + (proves Celery beat is scheduling *and* a worker is consuming). +- `queue_depth` — no more than 1000 tasks waiting in the Celery queue + (workers aren't keeping up, or aren't running). +- `recent_fetch_success` — at least one feed has been fetched successfully + within three poll intervals (minimum 30 minutes), so "scheduled but every + fetch is failing" is caught. A brand-new install gets that long before it + counts as stale. +- `feed_failures` — no more than half of your active feeds are failing, once + you have at least four (a handful of dead feeds is normal; most of them + failing at once points at your server, DNS, or outbound network). + +The payload also reports `queue_depth`, `active_feeds`, `failing_feeds`, and +the age of the last successful fetch, so you can graph them. **Retention.** By default every fetched article is kept forever. In the Django admin, "Site settings" has a *Retention days* field: set it to delete articles fetched longer ago than that, nightly (03:30 UTC, via Celery beat). Articles anyone has starred or saved for later are always kept; read state and tags for -deleted articles go with them. The dashboard shows the article count and the +deleted articles go with them. A small record (feed and entry id) is kept for +each deleted article so a feed that still lists it can't bring it back as new +and unread, and entries whose own publish date is already older than the window +aren't fetched in the first place. The dashboard shows the article count and the age of the oldest one, so you can see what retention is doing. **SQLite.** SQLite installs run in WAL mode with a 20-second busy timeout (set