Private
Public Access
Route every feed fetch through one atomic claim (R2-T1, #32)
Adding a feed and the admin retry queued fetch_feed directly without claiming the feed, so the periodic poller (which sees a never-fetched or just-reset feed as due) could claim and fetch it too: duplicate fetches, and one task overwriting another's claim/ETag/next-fetch state. The retry path also queued the backoff reset and the fetch as two independent tasks that could run in either order. claim_feed/enqueue_fetch in apps.feeds.tasks are now the only way a fetch gets dispatched — the poller, add-feed and retry all use them, so a feed can only have one fetch in flight. If queueing fails the claim is released (the poller recovers it), and add-feed still saves the subscription during a broker outage. The retry resets backoff synchronously before claiming, and says so when a fetch is already running. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0975a7634b
commit
85914f9107
5 files changed
+184
-33
No files matched your search
+19
-6
@@ -113,7 +113,7 @@ def test_dashboard_lists_erroring_feeds(client, django_user_model):
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_retry_feed_resets_backoff_and_requeues(client, django_user_model):
|
||||
def test_retry_feed_resets_backoff_then_claims_and_queues_one_fetch(client, django_user_model):
|
||||
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
|
||||
client.login(username="admin", password="pw-admin-123")
|
||||
|
||||
@@ -123,15 +123,28 @@ def test_retry_feed_resets_backoff_and_requeues(client, django_user_model):
|
||||
last_fetch_status=Feed.STATUS_ERROR,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("apps.core.views.reset_feed_backoff.delay") as mock_reset,
|
||||
patch("apps.core.views.fetch_feed.delay") as mock_fetch,
|
||||
):
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch:
|
||||
response = client.post(reverse("core:retry_feed", args=[feed.id]))
|
||||
|
||||
assert response.status_code == 302
|
||||
mock_reset.assert_called_once_with(feed.id)
|
||||
mock_fetch.assert_called_once_with(feed.id)
|
||||
feed.refresh_from_db()
|
||||
assert feed.fetch_error_count == 0
|
||||
assert feed.claimed_at is not None, "the fetch is claimed so the poller can't also take it"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_retry_feed_does_not_queue_a_second_fetch_while_one_is_in_flight(client, django_user_model):
|
||||
from django.utils import timezone
|
||||
|
||||
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
|
||||
client.login(username="admin", password="pw-admin-123")
|
||||
feed = Feed.objects.create(feed_url="https://busy.example.com/feed", claimed_at=timezone.now())
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch:
|
||||
client.post(reverse("core:retry_feed", args=[feed.id]))
|
||||
|
||||
mock_fetch.assert_not_called()
|
||||
|
||||
|
||||
def _ready(client):
|
||||
|
||||
+9
-4
@@ -10,7 +10,7 @@ from django.views.decorators.http import require_POST
|
||||
|
||||
from apps.articles.models import Article
|
||||
from apps.feeds.models import Feed, Subscription
|
||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, fetch_feed, reset_feed_backoff
|
||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, enqueue_fetch, reset_backoff
|
||||
|
||||
from .health import fetch_pipeline_status
|
||||
|
||||
@@ -108,7 +108,12 @@ def dashboard(request):
|
||||
@require_POST
|
||||
def retry_feed(request, feed_id):
|
||||
feed = get_object_or_404(Feed, pk=feed_id)
|
||||
reset_feed_backoff.delay(feed.id)
|
||||
fetch_feed.delay(feed.id)
|
||||
messages.success(request, f"Queued an immediate retry for '{feed.title or feed.feed_url}'.")
|
||||
# Reset first, synchronously: as separate queued tasks the reset could
|
||||
# land after the fetch and undo the schedule it just set.
|
||||
reset_backoff(feed.id)
|
||||
name = feed.title or feed.feed_url
|
||||
if enqueue_fetch(feed.id):
|
||||
messages.success(request, f"Queued an immediate retry for '{name}'.")
|
||||
else:
|
||||
messages.info(request, f"'{name}' is already being fetched right now.")
|
||||
return redirect("core:dashboard")
|
||||
+55
-21
@@ -47,30 +47,57 @@ def poll_due_feeds():
|
||||
"""
|
||||
now = timezone.now()
|
||||
cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS)
|
||||
claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
|
||||
due_feed_ids = list(
|
||||
Feed.objects.filter(is_active=True)
|
||||
.filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now))
|
||||
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
|
||||
.filter(
|
||||
Q(claimed_at__isnull=True)
|
||||
| Q(claimed_at__lt=now - timedelta(minutes=CLAIM_STALE_MINUTES))
|
||||
)
|
||||
.order_by("next_fetch_at")
|
||||
.values_list("id", flat=True)[:POLL_BATCH_SIZE]
|
||||
)
|
||||
claimed_ids = []
|
||||
dispatched = 0
|
||||
for feed_id in due_feed_ids:
|
||||
# The UPDATE's WHERE clause re-checks staleness at the database
|
||||
# level: it only claims rows nobody else has already claimed more
|
||||
# recently than CLAIM_STALE_MINUTES ago, so two workers racing this
|
||||
# same loop can't both get rowcount=1 for the same feed.
|
||||
claimed = (
|
||||
Feed.objects.filter(id=feed_id, is_active=True)
|
||||
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
|
||||
.update(claimed_at=now)
|
||||
)
|
||||
if claimed:
|
||||
claimed_ids.append(feed_id)
|
||||
for feed_id in claimed_ids:
|
||||
try:
|
||||
if enqueue_fetch(feed_id):
|
||||
dispatched += 1
|
||||
except Exception:
|
||||
# Broker trouble: enqueue_fetch already released this feed's
|
||||
# claim, so it's simply picked up again next tick.
|
||||
logger.exception("Could not queue a fetch for feed %s", feed_id)
|
||||
return dispatched
|
||||
|
||||
|
||||
def claim_feed(feed_id: int) -> bool:
|
||||
"""Atomically marks a feed as having a fetch in flight (an UPDATE whose
|
||||
WHERE clause re-checks staleness, so two callers racing for the same
|
||||
feed can't both get rowcount=1). Every path that dispatches a fetch —
|
||||
the periodic poller, a newly added feed, an admin retry — goes through
|
||||
this, so they can never run the same feed concurrently."""
|
||||
now = timezone.now()
|
||||
stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
|
||||
return bool(
|
||||
Feed.objects.filter(id=feed_id, is_active=True)
|
||||
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=stale_before))
|
||||
.update(claimed_at=now)
|
||||
)
|
||||
|
||||
|
||||
def enqueue_fetch(feed_id: int) -> bool:
|
||||
"""Claim a feed and queue its fetch. False means another fetch of it is
|
||||
already in flight (nothing was queued — that one will do the work).
|
||||
If queueing itself fails the claim is released and the error re-raised,
|
||||
so a broker outage doesn't leave the feed claimed until it goes stale.
|
||||
"""
|
||||
if not claim_feed(feed_id):
|
||||
return False
|
||||
try:
|
||||
fetch_feed.delay(feed_id)
|
||||
return len(claimed_ids)
|
||||
except Exception:
|
||||
Feed.objects.filter(pk=feed_id).update(claimed_at=None)
|
||||
raise
|
||||
return True
|
||||
|
||||
|
||||
def _schedule_next_fetch(feed: Feed) -> None:
|
||||
@@ -148,13 +175,20 @@ def fetch_feed(feed_id: int):
|
||||
}
|
||||
|
||||
|
||||
def reset_backoff(feed_id: int) -> None:
|
||||
"""Clear a feed's error count and due-date backoff so it's retried at
|
||||
its normal configured interval instead of the backed-off one
|
||||
(next_fetch_at=None also makes it immediately due, same as a feed
|
||||
that's never been fetched). A plain synchronous function so a caller
|
||||
can finish the reset *before* dispatching the fetch — as an async task
|
||||
it could land after the fetch and undo its freshly scheduled
|
||||
next_fetch_at."""
|
||||
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
|
||||
|
||||
|
||||
@shared_task
|
||||
def reset_feed_backoff(feed_id: int):
|
||||
"""Admin-triggered: clear a feed's error count and due-date backoff so
|
||||
it's retried at its normal configured interval instead of the backed-
|
||||
off one (next_fetch_at=None also makes it immediately due for the
|
||||
next poll_due_feeds tick, same as a feed that's never been fetched)."""
|
||||
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
|
||||
reset_backoff(feed_id)
|
||||
|
||||
|
||||
@shared_task
|
||||
|
||||
@@ -3,6 +3,7 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import responses
|
||||
from django.urls import reverse
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.articles.models import Article
|
||||
@@ -228,3 +229,92 @@ def test_fetch_feed_dispatches_rule_evaluation_in_bounded_chunks(monkeypatch):
|
||||
|
||||
chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list]
|
||||
assert chunk_sizes == [2, 2, 1]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_enqueue_fetch_claims_the_feed_and_queues_once():
|
||||
from apps.feeds.tasks import enqueue_fetch
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||
first = enqueue_fetch(feed.id)
|
||||
second = enqueue_fetch(feed.id)
|
||||
|
||||
assert (first, second) == (True, False)
|
||||
mock_delay.assert_called_once_with(feed.id)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_the_poller_does_not_dispatch_a_feed_a_direct_enqueue_already_claimed():
|
||||
"""T1: a just-added feed (next_fetch_at NULL, i.e. due) is claimed by
|
||||
the add-feed path, so the periodic poller must not fetch it too."""
|
||||
from apps.feeds.tasks import enqueue_fetch
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||
enqueue_fetch(feed.id)
|
||||
dispatched = poll_due_feeds()
|
||||
|
||||
assert dispatched == 0
|
||||
assert mock_delay.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_enqueue_fetch_releases_the_claim_if_queueing_fails():
|
||||
from apps.feeds.tasks import enqueue_fetch
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
||||
with pytest.raises(ConnectionError):
|
||||
enqueue_fetch(feed.id)
|
||||
|
||||
feed.refresh_from_db()
|
||||
assert feed.claimed_at is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_poller_survives_a_broker_failure_and_leaves_feeds_unclaimed():
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
||||
assert poll_due_feeds() == 0
|
||||
|
||||
feed.refresh_from_db()
|
||||
assert feed.claimed_at is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_add_feed_claims_its_initial_fetch(client):
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
get_user_model().objects.create_user("alice", password="pw-alice-123")
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||
client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"})
|
||||
poll_due_feeds()
|
||||
|
||||
assert mock_delay.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_add_feed_still_subscribes_when_the_broker_is_down(client):
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
from apps.feeds.models import Subscription
|
||||
|
||||
user = get_user_model().objects.create_user("alice", password="pw-alice-123")
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
||||
response = client.post(
|
||||
reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"}
|
||||
)
|
||||
|
||||
assert response.status_code == 302
|
||||
assert Subscription.objects.filter(
|
||||
user=user, feed__feed_url="https://example.com/new.xml"
|
||||
).exists()
|
||||
+11
-2
@@ -1,3 +1,4 @@
|
||||
import logging
|
||||
from io import BytesIO
|
||||
|
||||
from django.contrib import messages
|
||||
@@ -12,7 +13,9 @@ from apps.articles.models import SubscriptionTag, Tag
|
||||
from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm
|
||||
from .models import Feed, Subscription
|
||||
from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml
|
||||
from .tasks import fetch_feed, import_opml_task
|
||||
from .tasks import enqueue_fetch, import_opml_task
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPML_ASYNC_THRESHOLD = 50
|
||||
|
||||
@@ -42,7 +45,13 @@ def add_feed(request):
|
||||
# slow or large feed would otherwise tie up a Gunicorn worker for
|
||||
# the duration of the request; fetch_feed handles the save/error
|
||||
# bookkeeping the same way the periodic poll's dispatch does.
|
||||
fetch_feed.delay(feed.pk)
|
||||
try:
|
||||
enqueue_fetch(feed.pk)
|
||||
except Exception:
|
||||
# The subscription itself is saved; with the broker down the
|
||||
# periodic poller picks the (never-fetched) feed up once it's
|
||||
# back, so this isn't worth failing the request over.
|
||||
logger.exception("Could not queue the initial fetch for feed %s", feed.pk)
|
||||
messages.success(
|
||||
request,
|
||||
f"Subscribed to {feed_url}. Fetching its articles now — "
|
||||
|
||||
Reference in new issue
Block a user