From 85914f910749de90ee2a372ce9388396fcf7fc9b Mon Sep 17 00:00:00 2001 From: Keith Smith Date: Sun, 1 Nov 2026 11:56:23 -0700 Subject: [PATCH] Route every feed fetch through one atomic claim (R2-T1, #32) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding a feed and the admin retry queued fetch_feed directly without claiming the feed, so the periodic poller (which sees a never-fetched or just-reset feed as due) could claim and fetch it too: duplicate fetches, and one task overwriting another's claim/ETag/next-fetch state. The retry path also queued the backoff reset and the fetch as two independent tasks that could run in either order. claim_feed/enqueue_fetch in apps.feeds.tasks are now the only way a fetch gets dispatched — the poller, add-feed and retry all use them, so a feed can only have one fetch in flight. If queueing fails the claim is released (the poller recovers it), and add-feed still saves the subscription during a broker outage. The retry resets backoff synchronously before claiming, and says so when a fetch is already running. Co-Authored-By: Claude Sonnet 5.5 --- apps/core/tests.py | 25 ++++++++--- apps/core/views.py | 13 ++++-- apps/feeds/tasks.py | 76 +++++++++++++++++++++++---------- apps/feeds/test_tasks.py | 90 ++++++++++++++++++++++++++++++++++++++++ apps/feeds/views.py | 13 +++++- 5 files changed, 184 insertions(+), 33 deletions(-) diff --git a/apps/core/tests.py b/apps/core/tests.py index 8591059..81dfaa1 100644 --- a/apps/core/tests.py +++ b/apps/core/tests.py @@ -113,7 +113,7 @@ def test_dashboard_lists_erroring_feeds(client, django_user_model): @pytest.mark.django_db -def test_retry_feed_resets_backoff_and_requeues(client, django_user_model): +def test_retry_feed_resets_backoff_then_claims_and_queues_one_fetch(client, django_user_model): django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True) client.login(username="admin", password="pw-admin-123") @@ -123,15 +123,28 @@ def test_retry_feed_resets_backoff_and_requeues(client, django_user_model): last_fetch_status=Feed.STATUS_ERROR, ) - with ( - patch("apps.core.views.reset_feed_backoff.delay") as mock_reset, - patch("apps.core.views.fetch_feed.delay") as mock_fetch, - ): + with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch: response = client.post(reverse("core:retry_feed", args=[feed.id])) assert response.status_code == 302 - mock_reset.assert_called_once_with(feed.id) mock_fetch.assert_called_once_with(feed.id) + feed.refresh_from_db() + assert feed.fetch_error_count == 0 + assert feed.claimed_at is not None, "the fetch is claimed so the poller can't also take it" + + +@pytest.mark.django_db +def test_retry_feed_does_not_queue_a_second_fetch_while_one_is_in_flight(client, django_user_model): + from django.utils import timezone + + django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True) + client.login(username="admin", password="pw-admin-123") + feed = Feed.objects.create(feed_url="https://busy.example.com/feed", claimed_at=timezone.now()) + + with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch: + client.post(reverse("core:retry_feed", args=[feed.id])) + + mock_fetch.assert_not_called() def _ready(client): diff --git a/apps/core/views.py b/apps/core/views.py index e76df8f..988f1c5 100644 --- a/apps/core/views.py +++ b/apps/core/views.py @@ -10,7 +10,7 @@ from django.views.decorators.http import require_POST from apps.articles.models import Article from apps.feeds.models import Feed, Subscription -from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, fetch_feed, reset_feed_backoff +from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, enqueue_fetch, reset_backoff from .health import fetch_pipeline_status @@ -108,7 +108,12 @@ def dashboard(request): @require_POST def retry_feed(request, feed_id): feed = get_object_or_404(Feed, pk=feed_id) - reset_feed_backoff.delay(feed.id) - fetch_feed.delay(feed.id) - messages.success(request, f"Queued an immediate retry for '{feed.title or feed.feed_url}'.") + # Reset first, synchronously: as separate queued tasks the reset could + # land after the fetch and undo the schedule it just set. + reset_backoff(feed.id) + name = feed.title or feed.feed_url + if enqueue_fetch(feed.id): + messages.success(request, f"Queued an immediate retry for '{name}'.") + else: + messages.info(request, f"'{name}' is already being fetched right now.") return redirect("core:dashboard") diff --git a/apps/feeds/tasks.py b/apps/feeds/tasks.py index 5a330a3..e8f6bc1 100644 --- a/apps/feeds/tasks.py +++ b/apps/feeds/tasks.py @@ -47,30 +47,57 @@ def poll_due_feeds(): """ now = timezone.now() cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS) - claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES) due_feed_ids = list( Feed.objects.filter(is_active=True) .filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now)) - .filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before)) + .filter( + Q(claimed_at__isnull=True) + | Q(claimed_at__lt=now - timedelta(minutes=CLAIM_STALE_MINUTES)) + ) .order_by("next_fetch_at") .values_list("id", flat=True)[:POLL_BATCH_SIZE] ) - claimed_ids = [] + dispatched = 0 for feed_id in due_feed_ids: - # The UPDATE's WHERE clause re-checks staleness at the database - # level: it only claims rows nobody else has already claimed more - # recently than CLAIM_STALE_MINUTES ago, so two workers racing this - # same loop can't both get rowcount=1 for the same feed. - claimed = ( - Feed.objects.filter(id=feed_id, is_active=True) - .filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before)) - .update(claimed_at=now) - ) - if claimed: - claimed_ids.append(feed_id) - for feed_id in claimed_ids: + try: + if enqueue_fetch(feed_id): + dispatched += 1 + except Exception: + # Broker trouble: enqueue_fetch already released this feed's + # claim, so it's simply picked up again next tick. + logger.exception("Could not queue a fetch for feed %s", feed_id) + return dispatched + + +def claim_feed(feed_id: int) -> bool: + """Atomically marks a feed as having a fetch in flight (an UPDATE whose + WHERE clause re-checks staleness, so two callers racing for the same + feed can't both get rowcount=1). Every path that dispatches a fetch — + the periodic poller, a newly added feed, an admin retry — goes through + this, so they can never run the same feed concurrently.""" + now = timezone.now() + stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES) + return bool( + Feed.objects.filter(id=feed_id, is_active=True) + .filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=stale_before)) + .update(claimed_at=now) + ) + + +def enqueue_fetch(feed_id: int) -> bool: + """Claim a feed and queue its fetch. False means another fetch of it is + already in flight (nothing was queued — that one will do the work). + If queueing itself fails the claim is released and the error re-raised, + so a broker outage doesn't leave the feed claimed until it goes stale. + """ + if not claim_feed(feed_id): + return False + try: fetch_feed.delay(feed_id) - return len(claimed_ids) + except Exception: + Feed.objects.filter(pk=feed_id).update(claimed_at=None) + raise + return True def _schedule_next_fetch(feed: Feed) -> None: @@ -148,13 +175,20 @@ def fetch_feed(feed_id: int): } +def reset_backoff(feed_id: int) -> None: + """Clear a feed's error count and due-date backoff so it's retried at + its normal configured interval instead of the backed-off one + (next_fetch_at=None also makes it immediately due, same as a feed + that's never been fetched). A plain synchronous function so a caller + can finish the reset *before* dispatching the fetch — as an async task + it could land after the fetch and undo its freshly scheduled + next_fetch_at.""" + Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None) + + @shared_task def reset_feed_backoff(feed_id: int): - """Admin-triggered: clear a feed's error count and due-date backoff so - it's retried at its normal configured interval instead of the backed- - off one (next_fetch_at=None also makes it immediately due for the - next poll_due_feeds tick, same as a feed that's never been fetched).""" - Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None) + reset_backoff(feed_id) @shared_task diff --git a/apps/feeds/test_tasks.py b/apps/feeds/test_tasks.py index a06fa58..02507f4 100644 --- a/apps/feeds/test_tasks.py +++ b/apps/feeds/test_tasks.py @@ -3,6 +3,7 @@ from unittest.mock import patch import pytest import responses +from django.urls import reverse from django.utils import timezone from apps.articles.models import Article @@ -228,3 +229,92 @@ def test_fetch_feed_dispatches_rule_evaluation_in_bounded_chunks(monkeypatch): chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list] assert chunk_sizes == [2, 2, 1] + + +@pytest.mark.django_db +def test_enqueue_fetch_claims_the_feed_and_queues_once(): + from apps.feeds.tasks import enqueue_fetch + + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + + with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay: + first = enqueue_fetch(feed.id) + second = enqueue_fetch(feed.id) + + assert (first, second) == (True, False) + mock_delay.assert_called_once_with(feed.id) + + +@pytest.mark.django_db +def test_the_poller_does_not_dispatch_a_feed_a_direct_enqueue_already_claimed(): + """T1: a just-added feed (next_fetch_at NULL, i.e. due) is claimed by + the add-feed path, so the periodic poller must not fetch it too.""" + from apps.feeds.tasks import enqueue_fetch + + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + + with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay: + enqueue_fetch(feed.id) + dispatched = poll_due_feeds() + + assert dispatched == 0 + assert mock_delay.call_count == 1 + + +@pytest.mark.django_db +def test_enqueue_fetch_releases_the_claim_if_queueing_fails(): + from apps.feeds.tasks import enqueue_fetch + + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + + with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")): + with pytest.raises(ConnectionError): + enqueue_fetch(feed.id) + + feed.refresh_from_db() + assert feed.claimed_at is None + + +@pytest.mark.django_db +def test_poller_survives_a_broker_failure_and_leaves_feeds_unclaimed(): + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + + with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")): + assert poll_due_feeds() == 0 + + feed.refresh_from_db() + assert feed.claimed_at is None + + +@pytest.mark.django_db +def test_add_feed_claims_its_initial_fetch(client): + from django.contrib.auth import get_user_model + + get_user_model().objects.create_user("alice", password="pw-alice-123") + client.login(username="alice", password="pw-alice-123") + + with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay: + client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"}) + poll_due_feeds() + + assert mock_delay.call_count == 1 + + +@pytest.mark.django_db +def test_add_feed_still_subscribes_when_the_broker_is_down(client): + from django.contrib.auth import get_user_model + + from apps.feeds.models import Subscription + + user = get_user_model().objects.create_user("alice", password="pw-alice-123") + client.login(username="alice", password="pw-alice-123") + + with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")): + response = client.post( + reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"} + ) + + assert response.status_code == 302 + assert Subscription.objects.filter( + user=user, feed__feed_url="https://example.com/new.xml" + ).exists() diff --git a/apps/feeds/views.py b/apps/feeds/views.py index 9f53f95..f8567ca 100644 --- a/apps/feeds/views.py +++ b/apps/feeds/views.py @@ -1,3 +1,4 @@ +import logging from io import BytesIO from django.contrib import messages @@ -12,7 +13,9 @@ from apps.articles.models import SubscriptionTag, Tag from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm from .models import Feed, Subscription from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml -from .tasks import fetch_feed, import_opml_task +from .tasks import enqueue_fetch, import_opml_task + +logger = logging.getLogger(__name__) OPML_ASYNC_THRESHOLD = 50 @@ -42,7 +45,13 @@ def add_feed(request): # slow or large feed would otherwise tie up a Gunicorn worker for # the duration of the request; fetch_feed handles the save/error # bookkeeping the same way the periodic poll's dispatch does. - fetch_feed.delay(feed.pk) + try: + enqueue_fetch(feed.pk) + except Exception: + # The subscription itself is saved; with the broker down the + # periodic poller picks the (never-fetched) feed up once it's + # back, so this isn't worth failing the request over. + logger.exception("Could not queue the initial fetch for feed %s", feed.pk) messages.success( request, f"Subscribed to {feed_url}. Fetching its articles now — "