Retry transient fetch failures promptly instead of waiting a poll interval (R2-T5, #41)

An unexpected exception while fetching was recorded and re-raised, then
the feed waited for its next scheduled poll — which under error backoff
can be hours — even for a locked SQLite file or a dropped database
connection that clears in seconds.

fetch_feed now retries OperationalError/InterfaceError up to three times
with doubling delays (30s, 60s, 120s), holding the feed's claim so the
poller can't also fetch it and without counting it against the feed.
Anything else, and a transient error after the retries are spent, keep
the existing path: record the error, back off, release the claim,
re-raise. Time limits are unchanged.

Verified with a real Celery worker against a real Redis broker, with
the fetch rigged to fail twice: retries fired after 1s and 2s (delays
shortened for the probe), the third attempt succeeded, and the feed
ended healthy, unclaimed and scheduled. (Celery's eager test mode
mishandles retries, so the unit tests mock retry() and assert the
countdowns, claim and error-count behavior instead.)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5.5 committed 2026-11-01 12:04:01 -07:00
1 parent d6e310b127
commit 91a4c360a4
2 files changed
+126 -8

No files matched your search

+41 -8
View File
@@ -5,6 +5,7 @@ from io import BytesIO
from celery import shared_task
from django.contrib.auth import get_user_model
from django.core.cache import cache
from django.db import InterfaceError, OperationalError
from django.db.models import Q
from django.utils import timezone
@@ -119,9 +120,32 @@ def _schedule_next_fetch(feed: Feed) -> None:
FETCH_FEED_SOFT_TIME_LIMIT = 60
FETCH_FEED_TIME_LIMIT = 90
# Failures that say nothing about the feed and usually clear up in
# seconds — a locked SQLite file, a dropped database connection — are
# retried promptly (R2-T5) instead of waiting out the next poll interval,
# which under error backoff can be hours. Bounded: three attempts, 30s /
# 60s / 120s apart, then the normal "record the error and back off" path.
# Anything else (a parser bug, the soft time limit) is not retried: that
# would just repeat the same failure. The claim is held throughout so the
# poller can't also fetch the feed; it's far shorter than CLAIM_STALE_MINUTES.
TRANSIENT_FETCH_ERRORS = (OperationalError, InterfaceError)
FETCH_MAX_RETRIES = 3
FETCH_RETRY_BASE_DELAY_SECONDS = 30
@shared_task(time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT)
def fetch_feed(feed_id: int):
def _record_unexpected_failure(feed: Feed) -> None:
logger.exception("Feed %s: unexpected error during fetch", feed.feed_url)
record_fetch_error(feed, "Unexpected error — see worker logs")
feed.last_fetched_at = timezone.now()
feed.claimed_at = None
_schedule_next_fetch(feed)
feed.save()
@shared_task(
bind=True, time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT
)
def fetch_feed(self, feed_id: int):
try:
feed = Feed.objects.get(pk=feed_id, is_active=True)
except Feed.DoesNotExist:
@@ -129,6 +153,20 @@ def fetch_feed(feed_id: int):
try:
result = fetch_and_ingest(feed)
except TRANSIENT_FETCH_ERRORS as exc:
if self.request.retries < FETCH_MAX_RETRIES:
delay = FETCH_RETRY_BASE_DELAY_SECONDS * 2**self.request.retries
logger.warning(
"Feed %s: transient error (%s); retry %d/%d in %ds",
feed.feed_url,
type(exc).__name__,
self.request.retries + 1,
FETCH_MAX_RETRIES,
delay,
)
raise self.retry(exc=exc, countdown=delay, max_retries=FETCH_MAX_RETRIES)
_record_unexpected_failure(feed)
raise
except Exception:
# fetch_and_ingest already handles the expected failure modes
# (network errors, SSRF-blocked targets, unparseable feeds) by
@@ -140,12 +178,7 @@ def fetch_feed(feed_id: int):
# CLAIM_STALE_MINUTES, and no record of what happened. Re-raised
# afterward so Celery still marks the task FAILED (visible in
# worker logs/monitoring) instead of swallowing it.
logger.exception("Feed %s: unexpected error during fetch", feed.feed_url)
record_fetch_error(feed, "Unexpected error — see worker logs")
feed.last_fetched_at = timezone.now()
feed.claimed_at = None
_schedule_next_fetch(feed)
feed.save()
_record_unexpected_failure(feed)
raise
feed.claimed_at = None
+85
View File
@@ -318,3 +318,88 @@ def test_add_feed_still_subscribes_when_the_broker_is_down(client):
assert Subscription.objects.filter(
user=user, feed__feed_url="https://example.com/new.xml"
).exists()
class TestTransientFailureRetries:
"""R2-T5: a locked database or dropped connection is retried promptly
with bounded backoff; everything else keeps the record-and-back-off
behavior."""
@pytest.mark.django_db
def test_a_transient_database_error_is_retried_without_penalizing_the_feed(self):
from celery.exceptions import Retry
from django.db import OperationalError
feed = Feed.objects.create(
feed_url="https://example.com/feed.xml", claimed_at=timezone.now()
)
with (
patch("apps.feeds.tasks.fetch_and_ingest", side_effect=OperationalError("locked")),
patch.object(fetch_feed, "retry", side_effect=Retry()) as mock_retry,
):
with pytest.raises(Retry):
fetch_feed(feed.id)
assert mock_retry.call_args.kwargs["countdown"] == 30
feed.refresh_from_db()
assert feed.fetch_error_count == 0, "the feed wasn't at fault"
assert feed.claimed_at is not None, "claim is held so the poller can't double-fetch"
@pytest.mark.django_db
def test_backoff_doubles_each_attempt(self):
from celery.exceptions import Retry
from django.db import InterfaceError
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
delays = []
for attempt in range(3):
fetch_feed.push_request(retries=attempt)
try:
with (
patch("apps.feeds.tasks.fetch_and_ingest", side_effect=InterfaceError("gone")),
patch.object(fetch_feed, "retry", side_effect=Retry()) as mock_retry,
):
with pytest.raises(Retry):
fetch_feed(feed.id)
delays.append(mock_retry.call_args.kwargs["countdown"])
finally:
fetch_feed.pop_request()
assert delays == [30, 60, 120]
@pytest.mark.django_db
def test_after_the_retries_are_spent_it_is_recorded_as_a_normal_failure(self):
from django.db import OperationalError
from apps.feeds.tasks import FETCH_MAX_RETRIES
feed = Feed.objects.create(
feed_url="https://example.com/feed.xml", claimed_at=timezone.now()
)
fetch_feed.push_request(retries=FETCH_MAX_RETRIES)
try:
with patch("apps.feeds.tasks.fetch_and_ingest", side_effect=OperationalError("locked")):
with pytest.raises(OperationalError):
fetch_feed(feed.id)
finally:
fetch_feed.pop_request()
feed.refresh_from_db()
assert feed.fetch_error_count == 1
assert feed.claimed_at is None
assert feed.next_fetch_at is not None
@pytest.mark.django_db
def test_other_exceptions_are_not_retried(self):
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with (
patch("apps.feeds.tasks.fetch_and_ingest", side_effect=RuntimeError("parser bug")),
patch.object(fetch_feed, "retry") as mock_retry,
):
with pytest.raises(RuntimeError):
fetch_feed(feed.id)
mock_retry.assert_not_called()