Route every feed fetch through one atomic claim (R2-T1, #32)

Adding a feed and the admin retry queued fetch_feed directly without
claiming the feed, so the periodic poller (which sees a never-fetched
or just-reset feed as due) could claim and fetch it too: duplicate
fetches, and one task overwriting another's claim/ETag/next-fetch state.
The retry path also queued the backoff reset and the fetch as two
independent tasks that could run in either order.

claim_feed/enqueue_fetch in apps.feeds.tasks are now the only way a
fetch gets dispatched — the poller, add-feed and retry all use them, so
a feed can only have one fetch in flight. If queueing fails the claim is
released (the poller recovers it), and add-feed still saves the
subscription during a broker outage. The retry resets backoff
synchronously before claiming, and says so when a fetch is already
running.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5.5 committed 2026-11-01 11:56:23 -07:00
1 parent 0975a7634b
commit 85914f9107
5 files changed
+184 -33

No files matched your search

+19 -6
View File
@@ -113,7 +113,7 @@ def test_dashboard_lists_erroring_feeds(client, django_user_model):
@pytest.mark.django_db
def test_retry_feed_resets_backoff_and_requeues(client, django_user_model):
def test_retry_feed_resets_backoff_then_claims_and_queues_one_fetch(client, django_user_model):
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
client.login(username="admin", password="pw-admin-123")
@@ -123,15 +123,28 @@ def test_retry_feed_resets_backoff_and_requeues(client, django_user_model):
last_fetch_status=Feed.STATUS_ERROR,
)
with (
patch("apps.core.views.reset_feed_backoff.delay") as mock_reset,
patch("apps.core.views.fetch_feed.delay") as mock_fetch,
):
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch:
response = client.post(reverse("core:retry_feed", args=[feed.id]))
assert response.status_code == 302
mock_reset.assert_called_once_with(feed.id)
mock_fetch.assert_called_once_with(feed.id)
feed.refresh_from_db()
assert feed.fetch_error_count == 0
assert feed.claimed_at is not None, "the fetch is claimed so the poller can't also take it"
@pytest.mark.django_db
def test_retry_feed_does_not_queue_a_second_fetch_while_one_is_in_flight(client, django_user_model):
from django.utils import timezone
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
client.login(username="admin", password="pw-admin-123")
feed = Feed.objects.create(feed_url="https://busy.example.com/feed", claimed_at=timezone.now())
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch:
client.post(reverse("core:retry_feed", args=[feed.id]))
mock_fetch.assert_not_called()
def _ready(client):
+9 -4
View File
@@ -10,7 +10,7 @@ from django.views.decorators.http import require_POST
from apps.articles.models import Article
from apps.feeds.models import Feed, Subscription
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, fetch_feed, reset_feed_backoff
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, enqueue_fetch, reset_backoff
from .health import fetch_pipeline_status
@@ -108,7 +108,12 @@ def dashboard(request):
@require_POST
def retry_feed(request, feed_id):
feed = get_object_or_404(Feed, pk=feed_id)
reset_feed_backoff.delay(feed.id)
fetch_feed.delay(feed.id)
messages.success(request, f"Queued an immediate retry for '{feed.title or feed.feed_url}'.")
# Reset first, synchronously: as separate queued tasks the reset could
# land after the fetch and undo the schedule it just set.
reset_backoff(feed.id)
name = feed.title or feed.feed_url
if enqueue_fetch(feed.id):
messages.success(request, f"Queued an immediate retry for '{name}'.")
else:
messages.info(request, f"'{name}' is already being fetched right now.")
return redirect("core:dashboard")
+55 -21
View File
@@ -47,30 +47,57 @@ def poll_due_feeds():
"""
now = timezone.now()
cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS)
claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
due_feed_ids = list(
Feed.objects.filter(is_active=True)
.filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now))
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
.filter(
Q(claimed_at__isnull=True)
| Q(claimed_at__lt=now - timedelta(minutes=CLAIM_STALE_MINUTES))
)
.order_by("next_fetch_at")
.values_list("id", flat=True)[:POLL_BATCH_SIZE]
)
claimed_ids = []
dispatched = 0
for feed_id in due_feed_ids:
# The UPDATE's WHERE clause re-checks staleness at the database
# level: it only claims rows nobody else has already claimed more
# recently than CLAIM_STALE_MINUTES ago, so two workers racing this
# same loop can't both get rowcount=1 for the same feed.
claimed = (
Feed.objects.filter(id=feed_id, is_active=True)
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
.update(claimed_at=now)
)
if claimed:
claimed_ids.append(feed_id)
for feed_id in claimed_ids:
try:
if enqueue_fetch(feed_id):
dispatched += 1
except Exception:
# Broker trouble: enqueue_fetch already released this feed's
# claim, so it's simply picked up again next tick.
logger.exception("Could not queue a fetch for feed %s", feed_id)
return dispatched
def claim_feed(feed_id: int) -> bool:
"""Atomically marks a feed as having a fetch in flight (an UPDATE whose
WHERE clause re-checks staleness, so two callers racing for the same
feed can't both get rowcount=1). Every path that dispatches a fetch —
the periodic poller, a newly added feed, an admin retry — goes through
this, so they can never run the same feed concurrently."""
now = timezone.now()
stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
return bool(
Feed.objects.filter(id=feed_id, is_active=True)
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=stale_before))
.update(claimed_at=now)
)
def enqueue_fetch(feed_id: int) -> bool:
"""Claim a feed and queue its fetch. False means another fetch of it is
already in flight (nothing was queued — that one will do the work).
If queueing itself fails the claim is released and the error re-raised,
so a broker outage doesn't leave the feed claimed until it goes stale.
"""
if not claim_feed(feed_id):
return False
try:
fetch_feed.delay(feed_id)
return len(claimed_ids)
except Exception:
Feed.objects.filter(pk=feed_id).update(claimed_at=None)
raise
return True
def _schedule_next_fetch(feed: Feed) -> None:
@@ -148,13 +175,20 @@ def fetch_feed(feed_id: int):
}
def reset_backoff(feed_id: int) -> None:
"""Clear a feed's error count and due-date backoff so it's retried at
its normal configured interval instead of the backed-off one
(next_fetch_at=None also makes it immediately due, same as a feed
that's never been fetched). A plain synchronous function so a caller
can finish the reset *before* dispatching the fetch — as an async task
it could land after the fetch and undo its freshly scheduled
next_fetch_at."""
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
@shared_task
def reset_feed_backoff(feed_id: int):
"""Admin-triggered: clear a feed's error count and due-date backoff so
it's retried at its normal configured interval instead of the backed-
off one (next_fetch_at=None also makes it immediately due for the
next poll_due_feeds tick, same as a feed that's never been fetched)."""
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
reset_backoff(feed_id)
@shared_task
+90
View File
@@ -3,6 +3,7 @@ from unittest.mock import patch
import pytest
import responses
from django.urls import reverse
from django.utils import timezone
from apps.articles.models import Article
@@ -228,3 +229,92 @@ def test_fetch_feed_dispatches_rule_evaluation_in_bounded_chunks(monkeypatch):
chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list]
assert chunk_sizes == [2, 2, 1]
@pytest.mark.django_db
def test_enqueue_fetch_claims_the_feed_and_queues_once():
from apps.feeds.tasks import enqueue_fetch
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
first = enqueue_fetch(feed.id)
second = enqueue_fetch(feed.id)
assert (first, second) == (True, False)
mock_delay.assert_called_once_with(feed.id)
@pytest.mark.django_db
def test_the_poller_does_not_dispatch_a_feed_a_direct_enqueue_already_claimed():
"""T1: a just-added feed (next_fetch_at NULL, i.e. due) is claimed by
the add-feed path, so the periodic poller must not fetch it too."""
from apps.feeds.tasks import enqueue_fetch
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
enqueue_fetch(feed.id)
dispatched = poll_due_feeds()
assert dispatched == 0
assert mock_delay.call_count == 1
@pytest.mark.django_db
def test_enqueue_fetch_releases_the_claim_if_queueing_fails():
from apps.feeds.tasks import enqueue_fetch
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
with pytest.raises(ConnectionError):
enqueue_fetch(feed.id)
feed.refresh_from_db()
assert feed.claimed_at is None
@pytest.mark.django_db
def test_poller_survives_a_broker_failure_and_leaves_feeds_unclaimed():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
assert poll_due_feeds() == 0
feed.refresh_from_db()
assert feed.claimed_at is None
@pytest.mark.django_db
def test_add_feed_claims_its_initial_fetch(client):
from django.contrib.auth import get_user_model
get_user_model().objects.create_user("alice", password="pw-alice-123")
client.login(username="alice", password="pw-alice-123")
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"})
poll_due_feeds()
assert mock_delay.call_count == 1
@pytest.mark.django_db
def test_add_feed_still_subscribes_when_the_broker_is_down(client):
from django.contrib.auth import get_user_model
from apps.feeds.models import Subscription
user = get_user_model().objects.create_user("alice", password="pw-alice-123")
client.login(username="alice", password="pw-alice-123")
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
response = client.post(
reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"}
)
assert response.status_code == 302
assert Subscription.objects.filter(
user=user, feed__feed_url="https://example.com/new.xml"
).exists()
+11 -2
View File
@@ -1,3 +1,4 @@
import logging
from io import BytesIO
from django.contrib import messages
@@ -12,7 +13,9 @@ from apps.articles.models import SubscriptionTag, Tag
from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm
from .models import Feed, Subscription
from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml
from .tasks import fetch_feed, import_opml_task
from .tasks import enqueue_fetch, import_opml_task
logger = logging.getLogger(__name__)
OPML_ASYNC_THRESHOLD = 50
@@ -42,7 +45,13 @@ def add_feed(request):
# slow or large feed would otherwise tie up a Gunicorn worker for
# the duration of the request; fetch_feed handles the save/error
# bookkeeping the same way the periodic poll's dispatch does.
fetch_feed.delay(feed.pk)
try:
enqueue_fetch(feed.pk)
except Exception:
# The subscription itself is saved; with the broker down the
# periodic poller picks the (never-fetched) feed up once it's
# back, so this isn't worth failing the request over.
logger.exception("Could not queue the initial fetch for feed %s", feed.pk)
messages.success(
request,
f"Subscribed to {feed_url}. Fetching its articles now — "