Route every feed fetch through one atomic claim (R2-T1, #32)

Adding a feed and the admin retry queued fetch_feed directly without
claiming the feed, so the periodic poller (which sees a never-fetched
or just-reset feed as due) could claim and fetch it too: duplicate
fetches, and one task overwriting another's claim/ETag/next-fetch state.
The retry path also queued the backoff reset and the fetch as two
independent tasks that could run in either order.

claim_feed/enqueue_fetch in apps.feeds.tasks are now the only way a
fetch gets dispatched — the poller, add-feed and retry all use them, so
a feed can only have one fetch in flight. If queueing fails the claim is
released (the poller recovers it), and add-feed still saves the
subscription during a broker outage. The retry resets backoff
synchronously before claiming, and says so when a fetch is already
running.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5.5 committed 2026-11-01 11:56:23 -07:00
1 parent 0975a7634b
commit 85914f9107
5 files changed
+184 -33

No files matched your search

+19 -6
View File
@@ -113,7 +113,7 @@ def test_dashboard_lists_erroring_feeds(client, django_user_model):
@pytest.mark.django_db @pytest.mark.django_db
def test_retry_feed_resets_backoff_and_requeues(client, django_user_model): def test_retry_feed_resets_backoff_then_claims_and_queues_one_fetch(client, django_user_model):
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True) django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
client.login(username="admin", password="pw-admin-123") client.login(username="admin", password="pw-admin-123")
@@ -123,15 +123,28 @@ def test_retry_feed_resets_backoff_and_requeues(client, django_user_model):
last_fetch_status=Feed.STATUS_ERROR, last_fetch_status=Feed.STATUS_ERROR,
) )
with ( with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch:
patch("apps.core.views.reset_feed_backoff.delay") as mock_reset,
patch("apps.core.views.fetch_feed.delay") as mock_fetch,
):
response = client.post(reverse("core:retry_feed", args=[feed.id])) response = client.post(reverse("core:retry_feed", args=[feed.id]))
assert response.status_code == 302 assert response.status_code == 302
mock_reset.assert_called_once_with(feed.id)
mock_fetch.assert_called_once_with(feed.id) mock_fetch.assert_called_once_with(feed.id)
feed.refresh_from_db()
assert feed.fetch_error_count == 0
assert feed.claimed_at is not None, "the fetch is claimed so the poller can't also take it"
@pytest.mark.django_db
def test_retry_feed_does_not_queue_a_second_fetch_while_one_is_in_flight(client, django_user_model):
from django.utils import timezone
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
client.login(username="admin", password="pw-admin-123")
feed = Feed.objects.create(feed_url="https://busy.example.com/feed", claimed_at=timezone.now())
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch:
client.post(reverse("core:retry_feed", args=[feed.id]))
mock_fetch.assert_not_called()
def _ready(client): def _ready(client):
+9 -4
View File
@@ -10,7 +10,7 @@ from django.views.decorators.http import require_POST
from apps.articles.models import Article from apps.articles.models import Article
from apps.feeds.models import Feed, Subscription from apps.feeds.models import Feed, Subscription
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, fetch_feed, reset_feed_backoff from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, enqueue_fetch, reset_backoff
from .health import fetch_pipeline_status from .health import fetch_pipeline_status
@@ -108,7 +108,12 @@ def dashboard(request):
@require_POST @require_POST
def retry_feed(request, feed_id): def retry_feed(request, feed_id):
feed = get_object_or_404(Feed, pk=feed_id) feed = get_object_or_404(Feed, pk=feed_id)
reset_feed_backoff.delay(feed.id) # Reset first, synchronously: as separate queued tasks the reset could
fetch_feed.delay(feed.id) # land after the fetch and undo the schedule it just set.
messages.success(request, f"Queued an immediate retry for '{feed.title or feed.feed_url}'.") reset_backoff(feed.id)
name = feed.title or feed.feed_url
if enqueue_fetch(feed.id):
messages.success(request, f"Queued an immediate retry for '{name}'.")
else:
messages.info(request, f"'{name}' is already being fetched right now.")
return redirect("core:dashboard") return redirect("core:dashboard")
+55 -21
View File
@@ -47,30 +47,57 @@ def poll_due_feeds():
""" """
now = timezone.now() now = timezone.now()
cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS) cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS)
claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
due_feed_ids = list( due_feed_ids = list(
Feed.objects.filter(is_active=True) Feed.objects.filter(is_active=True)
.filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now)) .filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now))
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before)) .filter(
Q(claimed_at__isnull=True)
| Q(claimed_at__lt=now - timedelta(minutes=CLAIM_STALE_MINUTES))
)
.order_by("next_fetch_at") .order_by("next_fetch_at")
.values_list("id", flat=True)[:POLL_BATCH_SIZE] .values_list("id", flat=True)[:POLL_BATCH_SIZE]
) )
claimed_ids = [] dispatched = 0
for feed_id in due_feed_ids: for feed_id in due_feed_ids:
# The UPDATE's WHERE clause re-checks staleness at the database try:
# level: it only claims rows nobody else has already claimed more if enqueue_fetch(feed_id):
# recently than CLAIM_STALE_MINUTES ago, so two workers racing this dispatched += 1
# same loop can't both get rowcount=1 for the same feed. except Exception:
claimed = ( # Broker trouble: enqueue_fetch already released this feed's
Feed.objects.filter(id=feed_id, is_active=True) # claim, so it's simply picked up again next tick.
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before)) logger.exception("Could not queue a fetch for feed %s", feed_id)
.update(claimed_at=now) return dispatched
)
if claimed:
claimed_ids.append(feed_id) def claim_feed(feed_id: int) -> bool:
for feed_id in claimed_ids: """Atomically marks a feed as having a fetch in flight (an UPDATE whose
WHERE clause re-checks staleness, so two callers racing for the same
feed can't both get rowcount=1). Every path that dispatches a fetch —
the periodic poller, a newly added feed, an admin retry — goes through
this, so they can never run the same feed concurrently."""
now = timezone.now()
stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
return bool(
Feed.objects.filter(id=feed_id, is_active=True)
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=stale_before))
.update(claimed_at=now)
)
def enqueue_fetch(feed_id: int) -> bool:
"""Claim a feed and queue its fetch. False means another fetch of it is
already in flight (nothing was queued — that one will do the work).
If queueing itself fails the claim is released and the error re-raised,
so a broker outage doesn't leave the feed claimed until it goes stale.
"""
if not claim_feed(feed_id):
return False
try:
fetch_feed.delay(feed_id) fetch_feed.delay(feed_id)
return len(claimed_ids) except Exception:
Feed.objects.filter(pk=feed_id).update(claimed_at=None)
raise
return True
def _schedule_next_fetch(feed: Feed) -> None: def _schedule_next_fetch(feed: Feed) -> None:
@@ -148,13 +175,20 @@ def fetch_feed(feed_id: int):
} }
def reset_backoff(feed_id: int) -> None:
"""Clear a feed's error count and due-date backoff so it's retried at
its normal configured interval instead of the backed-off one
(next_fetch_at=None also makes it immediately due, same as a feed
that's never been fetched). A plain synchronous function so a caller
can finish the reset *before* dispatching the fetch — as an async task
it could land after the fetch and undo its freshly scheduled
next_fetch_at."""
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
@shared_task @shared_task
def reset_feed_backoff(feed_id: int): def reset_feed_backoff(feed_id: int):
"""Admin-triggered: clear a feed's error count and due-date backoff so reset_backoff(feed_id)
it's retried at its normal configured interval instead of the backed-
off one (next_fetch_at=None also makes it immediately due for the
next poll_due_feeds tick, same as a feed that's never been fetched)."""
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
@shared_task @shared_task
+90
View File
@@ -3,6 +3,7 @@ from unittest.mock import patch
import pytest import pytest
import responses import responses
from django.urls import reverse
from django.utils import timezone from django.utils import timezone
from apps.articles.models import Article from apps.articles.models import Article
@@ -228,3 +229,92 @@ def test_fetch_feed_dispatches_rule_evaluation_in_bounded_chunks(monkeypatch):
chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list] chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list]
assert chunk_sizes == [2, 2, 1] assert chunk_sizes == [2, 2, 1]
@pytest.mark.django_db
def test_enqueue_fetch_claims_the_feed_and_queues_once():
from apps.feeds.tasks import enqueue_fetch
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
first = enqueue_fetch(feed.id)
second = enqueue_fetch(feed.id)
assert (first, second) == (True, False)
mock_delay.assert_called_once_with(feed.id)
@pytest.mark.django_db
def test_the_poller_does_not_dispatch_a_feed_a_direct_enqueue_already_claimed():
"""T1: a just-added feed (next_fetch_at NULL, i.e. due) is claimed by
the add-feed path, so the periodic poller must not fetch it too."""
from apps.feeds.tasks import enqueue_fetch
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
enqueue_fetch(feed.id)
dispatched = poll_due_feeds()
assert dispatched == 0
assert mock_delay.call_count == 1
@pytest.mark.django_db
def test_enqueue_fetch_releases_the_claim_if_queueing_fails():
from apps.feeds.tasks import enqueue_fetch
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
with pytest.raises(ConnectionError):
enqueue_fetch(feed.id)
feed.refresh_from_db()
assert feed.claimed_at is None
@pytest.mark.django_db
def test_poller_survives_a_broker_failure_and_leaves_feeds_unclaimed():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
assert poll_due_feeds() == 0
feed.refresh_from_db()
assert feed.claimed_at is None
@pytest.mark.django_db
def test_add_feed_claims_its_initial_fetch(client):
from django.contrib.auth import get_user_model
get_user_model().objects.create_user("alice", password="pw-alice-123")
client.login(username="alice", password="pw-alice-123")
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"})
poll_due_feeds()
assert mock_delay.call_count == 1
@pytest.mark.django_db
def test_add_feed_still_subscribes_when_the_broker_is_down(client):
from django.contrib.auth import get_user_model
from apps.feeds.models import Subscription
user = get_user_model().objects.create_user("alice", password="pw-alice-123")
client.login(username="alice", password="pw-alice-123")
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
response = client.post(
reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"}
)
assert response.status_code == 302
assert Subscription.objects.filter(
user=user, feed__feed_url="https://example.com/new.xml"
).exists()
+11 -2
View File
@@ -1,3 +1,4 @@
import logging
from io import BytesIO from io import BytesIO
from django.contrib import messages from django.contrib import messages
@@ -12,7 +13,9 @@ from apps.articles.models import SubscriptionTag, Tag
from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm
from .models import Feed, Subscription from .models import Feed, Subscription
from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml
from .tasks import fetch_feed, import_opml_task from .tasks import enqueue_fetch, import_opml_task
logger = logging.getLogger(__name__)
OPML_ASYNC_THRESHOLD = 50 OPML_ASYNC_THRESHOLD = 50
@@ -42,7 +45,13 @@ def add_feed(request):
# slow or large feed would otherwise tie up a Gunicorn worker for # slow or large feed would otherwise tie up a Gunicorn worker for
# the duration of the request; fetch_feed handles the save/error # the duration of the request; fetch_feed handles the save/error
# bookkeeping the same way the periodic poll's dispatch does. # bookkeeping the same way the periodic poll's dispatch does.
fetch_feed.delay(feed.pk) try:
enqueue_fetch(feed.pk)
except Exception:
# The subscription itself is saved; with the broker down the
# periodic poller picks the (never-fetched) feed up once it's
# back, so this isn't worth failing the request over.
logger.exception("Could not queue the initial fetch for feed %s", feed.pk)
messages.success( messages.success(
request, request,
f"Subscribed to {feed_url}. Fetching its articles now — " f"Subscribed to {feed_url}. Fetching its articles now — "