Private
Public Access
Retry transient fetch failures promptly instead of waiting a poll interval (R2-T5, #41)
An unexpected exception while fetching was recorded and re-raised, then the feed waited for its next scheduled poll — which under error backoff can be hours — even for a locked SQLite file or a dropped database connection that clears in seconds. fetch_feed now retries OperationalError/InterfaceError up to three times with doubling delays (30s, 60s, 120s), holding the feed's claim so the poller can't also fetch it and without counting it against the feed. Anything else, and a transient error after the retries are spent, keep the existing path: record the error, back off, release the claim, re-raise. Time limits are unchanged. Verified with a real Celery worker against a real Redis broker, with the fetch rigged to fail twice: retries fired after 1s and 2s (delays shortened for the probe), the third attempt succeeded, and the feed ended healthy, unclaimed and scheduled. (Celery's eager test mode mishandles retries, so the unit tests mock retry() and assert the countdowns, claim and error-count behavior instead.) Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d6e310b127
commit
91a4c360a4
2 files changed
+126
-8
No files matched your search
@@ -318,3 +318,88 @@ def test_add_feed_still_subscribes_when_the_broker_is_down(client):
|
||||
assert Subscription.objects.filter(
|
||||
user=user, feed__feed_url="https://example.com/new.xml"
|
||||
).exists()
|
||||
|
||||
|
||||
class TestTransientFailureRetries:
|
||||
"""R2-T5: a locked database or dropped connection is retried promptly
|
||||
with bounded backoff; everything else keeps the record-and-back-off
|
||||
behavior."""
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_a_transient_database_error_is_retried_without_penalizing_the_feed(self):
|
||||
from celery.exceptions import Retry
|
||||
from django.db import OperationalError
|
||||
|
||||
feed = Feed.objects.create(
|
||||
feed_url="https://example.com/feed.xml", claimed_at=timezone.now()
|
||||
)
|
||||
|
||||
with (
|
||||
patch("apps.feeds.tasks.fetch_and_ingest", side_effect=OperationalError("locked")),
|
||||
patch.object(fetch_feed, "retry", side_effect=Retry()) as mock_retry,
|
||||
):
|
||||
with pytest.raises(Retry):
|
||||
fetch_feed(feed.id)
|
||||
|
||||
assert mock_retry.call_args.kwargs["countdown"] == 30
|
||||
feed.refresh_from_db()
|
||||
assert feed.fetch_error_count == 0, "the feed wasn't at fault"
|
||||
assert feed.claimed_at is not None, "claim is held so the poller can't double-fetch"
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_backoff_doubles_each_attempt(self):
|
||||
from celery.exceptions import Retry
|
||||
from django.db import InterfaceError
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
delays = []
|
||||
for attempt in range(3):
|
||||
fetch_feed.push_request(retries=attempt)
|
||||
try:
|
||||
with (
|
||||
patch("apps.feeds.tasks.fetch_and_ingest", side_effect=InterfaceError("gone")),
|
||||
patch.object(fetch_feed, "retry", side_effect=Retry()) as mock_retry,
|
||||
):
|
||||
with pytest.raises(Retry):
|
||||
fetch_feed(feed.id)
|
||||
delays.append(mock_retry.call_args.kwargs["countdown"])
|
||||
finally:
|
||||
fetch_feed.pop_request()
|
||||
|
||||
assert delays == [30, 60, 120]
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_after_the_retries_are_spent_it_is_recorded_as_a_normal_failure(self):
|
||||
from django.db import OperationalError
|
||||
|
||||
from apps.feeds.tasks import FETCH_MAX_RETRIES
|
||||
|
||||
feed = Feed.objects.create(
|
||||
feed_url="https://example.com/feed.xml", claimed_at=timezone.now()
|
||||
)
|
||||
|
||||
fetch_feed.push_request(retries=FETCH_MAX_RETRIES)
|
||||
try:
|
||||
with patch("apps.feeds.tasks.fetch_and_ingest", side_effect=OperationalError("locked")):
|
||||
with pytest.raises(OperationalError):
|
||||
fetch_feed(feed.id)
|
||||
finally:
|
||||
fetch_feed.pop_request()
|
||||
|
||||
feed.refresh_from_db()
|
||||
assert feed.fetch_error_count == 1
|
||||
assert feed.claimed_at is None
|
||||
assert feed.next_fetch_at is not None
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_other_exceptions_are_not_retried(self):
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with (
|
||||
patch("apps.feeds.tasks.fetch_and_ingest", side_effect=RuntimeError("parser bug")),
|
||||
patch.object(fetch_feed, "retry") as mock_retry,
|
||||
):
|
||||
with pytest.raises(RuntimeError):
|
||||
fetch_feed(feed.id)
|
||||
|
||||
mock_retry.assert_not_called()
|
||||
Reference in new issue
Block a user