From 91a4c360a4a5263b3be56f4deaf47e517e184401 Mon Sep 17 00:00:00 2001 From: Keith Smith Date: Sun, 1 Nov 2026 12:04:01 -0700 Subject: [PATCH] Retry transient fetch failures promptly instead of waiting a poll interval (R2-T5, #41) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An unexpected exception while fetching was recorded and re-raised, then the feed waited for its next scheduled poll — which under error backoff can be hours — even for a locked SQLite file or a dropped database connection that clears in seconds. fetch_feed now retries OperationalError/InterfaceError up to three times with doubling delays (30s, 60s, 120s), holding the feed's claim so the poller can't also fetch it and without counting it against the feed. Anything else, and a transient error after the retries are spent, keep the existing path: record the error, back off, release the claim, re-raise. Time limits are unchanged. Verified with a real Celery worker against a real Redis broker, with the fetch rigged to fail twice: retries fired after 1s and 2s (delays shortened for the probe), the third attempt succeeded, and the feed ended healthy, unclaimed and scheduled. (Celery's eager test mode mishandles retries, so the unit tests mock retry() and assert the countdowns, claim and error-count behavior instead.) Co-Authored-By: Claude Sonnet 5.5 --- apps/feeds/tasks.py | 49 +++++++++++++++++++---- apps/feeds/test_tasks.py | 85 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 8 deletions(-) diff --git a/apps/feeds/tasks.py b/apps/feeds/tasks.py index 5d25fa4..fba0fda 100644 --- a/apps/feeds/tasks.py +++ b/apps/feeds/tasks.py @@ -5,6 +5,7 @@ from io import BytesIO from celery import shared_task from django.contrib.auth import get_user_model from django.core.cache import cache +from django.db import InterfaceError, OperationalError from django.db.models import Q from django.utils import timezone @@ -119,9 +120,32 @@ def _schedule_next_fetch(feed: Feed) -> None: FETCH_FEED_SOFT_TIME_LIMIT = 60 FETCH_FEED_TIME_LIMIT = 90 +# Failures that say nothing about the feed and usually clear up in +# seconds — a locked SQLite file, a dropped database connection — are +# retried promptly (R2-T5) instead of waiting out the next poll interval, +# which under error backoff can be hours. Bounded: three attempts, 30s / +# 60s / 120s apart, then the normal "record the error and back off" path. +# Anything else (a parser bug, the soft time limit) is not retried: that +# would just repeat the same failure. The claim is held throughout so the +# poller can't also fetch the feed; it's far shorter than CLAIM_STALE_MINUTES. +TRANSIENT_FETCH_ERRORS = (OperationalError, InterfaceError) +FETCH_MAX_RETRIES = 3 +FETCH_RETRY_BASE_DELAY_SECONDS = 30 -@shared_task(time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT) -def fetch_feed(feed_id: int): + +def _record_unexpected_failure(feed: Feed) -> None: + logger.exception("Feed %s: unexpected error during fetch", feed.feed_url) + record_fetch_error(feed, "Unexpected error — see worker logs") + feed.last_fetched_at = timezone.now() + feed.claimed_at = None + _schedule_next_fetch(feed) + feed.save() + + +@shared_task( + bind=True, time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT +) +def fetch_feed(self, feed_id: int): try: feed = Feed.objects.get(pk=feed_id, is_active=True) except Feed.DoesNotExist: @@ -129,6 +153,20 @@ def fetch_feed(feed_id: int): try: result = fetch_and_ingest(feed) + except TRANSIENT_FETCH_ERRORS as exc: + if self.request.retries < FETCH_MAX_RETRIES: + delay = FETCH_RETRY_BASE_DELAY_SECONDS * 2**self.request.retries + logger.warning( + "Feed %s: transient error (%s); retry %d/%d in %ds", + feed.feed_url, + type(exc).__name__, + self.request.retries + 1, + FETCH_MAX_RETRIES, + delay, + ) + raise self.retry(exc=exc, countdown=delay, max_retries=FETCH_MAX_RETRIES) + _record_unexpected_failure(feed) + raise except Exception: # fetch_and_ingest already handles the expected failure modes # (network errors, SSRF-blocked targets, unparseable feeds) by @@ -140,12 +178,7 @@ def fetch_feed(feed_id: int): # CLAIM_STALE_MINUTES, and no record of what happened. Re-raised # afterward so Celery still marks the task FAILED (visible in # worker logs/monitoring) instead of swallowing it. - logger.exception("Feed %s: unexpected error during fetch", feed.feed_url) - record_fetch_error(feed, "Unexpected error — see worker logs") - feed.last_fetched_at = timezone.now() - feed.claimed_at = None - _schedule_next_fetch(feed) - feed.save() + _record_unexpected_failure(feed) raise feed.claimed_at = None diff --git a/apps/feeds/test_tasks.py b/apps/feeds/test_tasks.py index 02507f4..5b7a530 100644 --- a/apps/feeds/test_tasks.py +++ b/apps/feeds/test_tasks.py @@ -318,3 +318,88 @@ def test_add_feed_still_subscribes_when_the_broker_is_down(client): assert Subscription.objects.filter( user=user, feed__feed_url="https://example.com/new.xml" ).exists() + + +class TestTransientFailureRetries: + """R2-T5: a locked database or dropped connection is retried promptly + with bounded backoff; everything else keeps the record-and-back-off + behavior.""" + + @pytest.mark.django_db + def test_a_transient_database_error_is_retried_without_penalizing_the_feed(self): + from celery.exceptions import Retry + from django.db import OperationalError + + feed = Feed.objects.create( + feed_url="https://example.com/feed.xml", claimed_at=timezone.now() + ) + + with ( + patch("apps.feeds.tasks.fetch_and_ingest", side_effect=OperationalError("locked")), + patch.object(fetch_feed, "retry", side_effect=Retry()) as mock_retry, + ): + with pytest.raises(Retry): + fetch_feed(feed.id) + + assert mock_retry.call_args.kwargs["countdown"] == 30 + feed.refresh_from_db() + assert feed.fetch_error_count == 0, "the feed wasn't at fault" + assert feed.claimed_at is not None, "claim is held so the poller can't double-fetch" + + @pytest.mark.django_db + def test_backoff_doubles_each_attempt(self): + from celery.exceptions import Retry + from django.db import InterfaceError + + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + delays = [] + for attempt in range(3): + fetch_feed.push_request(retries=attempt) + try: + with ( + patch("apps.feeds.tasks.fetch_and_ingest", side_effect=InterfaceError("gone")), + patch.object(fetch_feed, "retry", side_effect=Retry()) as mock_retry, + ): + with pytest.raises(Retry): + fetch_feed(feed.id) + delays.append(mock_retry.call_args.kwargs["countdown"]) + finally: + fetch_feed.pop_request() + + assert delays == [30, 60, 120] + + @pytest.mark.django_db + def test_after_the_retries_are_spent_it_is_recorded_as_a_normal_failure(self): + from django.db import OperationalError + + from apps.feeds.tasks import FETCH_MAX_RETRIES + + feed = Feed.objects.create( + feed_url="https://example.com/feed.xml", claimed_at=timezone.now() + ) + + fetch_feed.push_request(retries=FETCH_MAX_RETRIES) + try: + with patch("apps.feeds.tasks.fetch_and_ingest", side_effect=OperationalError("locked")): + with pytest.raises(OperationalError): + fetch_feed(feed.id) + finally: + fetch_feed.pop_request() + + feed.refresh_from_db() + assert feed.fetch_error_count == 1 + assert feed.claimed_at is None + assert feed.next_fetch_at is not None + + @pytest.mark.django_db + def test_other_exceptions_are_not_retried(self): + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + + with ( + patch("apps.feeds.tasks.fetch_and_ingest", side_effect=RuntimeError("parser bug")), + patch.object(fetch_feed, "retry") as mock_retry, + ): + with pytest.raises(RuntimeError): + fetch_feed(feed.id) + + mock_retry.assert_not_called()