diff --git a/apps/feeds/tasks.py b/apps/feeds/tasks.py index 5d25fa4..fba0fda 100644 --- a/apps/feeds/tasks.py +++ b/apps/feeds/tasks.py @@ -5,6 +5,7 @@ from io import BytesIO from celery import shared_task from django.contrib.auth import get_user_model from django.core.cache import cache +from django.db import InterfaceError, OperationalError from django.db.models import Q from django.utils import timezone @@ -119,9 +120,32 @@ def _schedule_next_fetch(feed: Feed) -> None: FETCH_FEED_SOFT_TIME_LIMIT = 60 FETCH_FEED_TIME_LIMIT = 90 +# Failures that say nothing about the feed and usually clear up in +# seconds — a locked SQLite file, a dropped database connection — are +# retried promptly (R2-T5) instead of waiting out the next poll interval, +# which under error backoff can be hours. Bounded: three attempts, 30s / +# 60s / 120s apart, then the normal "record the error and back off" path. +# Anything else (a parser bug, the soft time limit) is not retried: that +# would just repeat the same failure. The claim is held throughout so the +# poller can't also fetch the feed; it's far shorter than CLAIM_STALE_MINUTES. +TRANSIENT_FETCH_ERRORS = (OperationalError, InterfaceError) +FETCH_MAX_RETRIES = 3 +FETCH_RETRY_BASE_DELAY_SECONDS = 30 -@shared_task(time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT) -def fetch_feed(feed_id: int): + +def _record_unexpected_failure(feed: Feed) -> None: + logger.exception("Feed %s: unexpected error during fetch", feed.feed_url) + record_fetch_error(feed, "Unexpected error — see worker logs") + feed.last_fetched_at = timezone.now() + feed.claimed_at = None + _schedule_next_fetch(feed) + feed.save() + + +@shared_task( + bind=True, time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT +) +def fetch_feed(self, feed_id: int): try: feed = Feed.objects.get(pk=feed_id, is_active=True) except Feed.DoesNotExist: @@ -129,6 +153,20 @@ def fetch_feed(feed_id: int): try: result = fetch_and_ingest(feed) + except TRANSIENT_FETCH_ERRORS as exc: + if self.request.retries < FETCH_MAX_RETRIES: + delay = FETCH_RETRY_BASE_DELAY_SECONDS * 2**self.request.retries + logger.warning( + "Feed %s: transient error (%s); retry %d/%d in %ds", + feed.feed_url, + type(exc).__name__, + self.request.retries + 1, + FETCH_MAX_RETRIES, + delay, + ) + raise self.retry(exc=exc, countdown=delay, max_retries=FETCH_MAX_RETRIES) + _record_unexpected_failure(feed) + raise except Exception: # fetch_and_ingest already handles the expected failure modes # (network errors, SSRF-blocked targets, unparseable feeds) by @@ -140,12 +178,7 @@ def fetch_feed(feed_id: int): # CLAIM_STALE_MINUTES, and no record of what happened. Re-raised # afterward so Celery still marks the task FAILED (visible in # worker logs/monitoring) instead of swallowing it. - logger.exception("Feed %s: unexpected error during fetch", feed.feed_url) - record_fetch_error(feed, "Unexpected error — see worker logs") - feed.last_fetched_at = timezone.now() - feed.claimed_at = None - _schedule_next_fetch(feed) - feed.save() + _record_unexpected_failure(feed) raise feed.claimed_at = None diff --git a/apps/feeds/test_tasks.py b/apps/feeds/test_tasks.py index 02507f4..5b7a530 100644 --- a/apps/feeds/test_tasks.py +++ b/apps/feeds/test_tasks.py @@ -318,3 +318,88 @@ def test_add_feed_still_subscribes_when_the_broker_is_down(client): assert Subscription.objects.filter( user=user, feed__feed_url="https://example.com/new.xml" ).exists() + + +class TestTransientFailureRetries: + """R2-T5: a locked database or dropped connection is retried promptly + with bounded backoff; everything else keeps the record-and-back-off + behavior.""" + + @pytest.mark.django_db + def test_a_transient_database_error_is_retried_without_penalizing_the_feed(self): + from celery.exceptions import Retry + from django.db import OperationalError + + feed = Feed.objects.create( + feed_url="https://example.com/feed.xml", claimed_at=timezone.now() + ) + + with ( + patch("apps.feeds.tasks.fetch_and_ingest", side_effect=OperationalError("locked")), + patch.object(fetch_feed, "retry", side_effect=Retry()) as mock_retry, + ): + with pytest.raises(Retry): + fetch_feed(feed.id) + + assert mock_retry.call_args.kwargs["countdown"] == 30 + feed.refresh_from_db() + assert feed.fetch_error_count == 0, "the feed wasn't at fault" + assert feed.claimed_at is not None, "claim is held so the poller can't double-fetch" + + @pytest.mark.django_db + def test_backoff_doubles_each_attempt(self): + from celery.exceptions import Retry + from django.db import InterfaceError + + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + delays = [] + for attempt in range(3): + fetch_feed.push_request(retries=attempt) + try: + with ( + patch("apps.feeds.tasks.fetch_and_ingest", side_effect=InterfaceError("gone")), + patch.object(fetch_feed, "retry", side_effect=Retry()) as mock_retry, + ): + with pytest.raises(Retry): + fetch_feed(feed.id) + delays.append(mock_retry.call_args.kwargs["countdown"]) + finally: + fetch_feed.pop_request() + + assert delays == [30, 60, 120] + + @pytest.mark.django_db + def test_after_the_retries_are_spent_it_is_recorded_as_a_normal_failure(self): + from django.db import OperationalError + + from apps.feeds.tasks import FETCH_MAX_RETRIES + + feed = Feed.objects.create( + feed_url="https://example.com/feed.xml", claimed_at=timezone.now() + ) + + fetch_feed.push_request(retries=FETCH_MAX_RETRIES) + try: + with patch("apps.feeds.tasks.fetch_and_ingest", side_effect=OperationalError("locked")): + with pytest.raises(OperationalError): + fetch_feed(feed.id) + finally: + fetch_feed.pop_request() + + feed.refresh_from_db() + assert feed.fetch_error_count == 1 + assert feed.claimed_at is None + assert feed.next_fetch_at is not None + + @pytest.mark.django_db + def test_other_exceptions_are_not_retried(self): + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + + with ( + patch("apps.feeds.tasks.fetch_and_ingest", side_effect=RuntimeError("parser bug")), + patch.object(fetch_feed, "retry") as mock_retry, + ): + with pytest.raises(RuntimeError): + fetch_feed(feed.id) + + mock_retry.assert_not_called()