Private
Public Access
Route every feed fetch through one atomic claim (R2-T1, #32)
Adding a feed and the admin retry queued fetch_feed directly without claiming the feed, so the periodic poller (which sees a never-fetched or just-reset feed as due) could claim and fetch it too: duplicate fetches, and one task overwriting another's claim/ETag/next-fetch state. The retry path also queued the backoff reset and the fetch as two independent tasks that could run in either order. claim_feed/enqueue_fetch in apps.feeds.tasks are now the only way a fetch gets dispatched — the poller, add-feed and retry all use them, so a feed can only have one fetch in flight. If queueing fails the claim is released (the poller recovers it), and add-feed still saves the subscription during a broker outage. The retry resets backoff synchronously before claiming, and says so when a fetch is already running. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0975a7634b
commit
85914f9107
5 files changed
+181
-30
No files matched your search
+19
-6
@@ -113,7 +113,7 @@ def test_dashboard_lists_erroring_feeds(client, django_user_model):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
def test_retry_feed_resets_backoff_and_requeues(client, django_user_model):
|
def test_retry_feed_resets_backoff_then_claims_and_queues_one_fetch(client, django_user_model):
|
||||||
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
|
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
|
||||||
client.login(username="admin", password="pw-admin-123")
|
client.login(username="admin", password="pw-admin-123")
|
||||||
|
|
||||||
@@ -123,15 +123,28 @@ def test_retry_feed_resets_backoff_and_requeues(client, django_user_model):
|
|||||||
last_fetch_status=Feed.STATUS_ERROR,
|
last_fetch_status=Feed.STATUS_ERROR,
|
||||||
)
|
)
|
||||||
|
|
||||||
with (
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch:
|
||||||
patch("apps.core.views.reset_feed_backoff.delay") as mock_reset,
|
|
||||||
patch("apps.core.views.fetch_feed.delay") as mock_fetch,
|
|
||||||
):
|
|
||||||
response = client.post(reverse("core:retry_feed", args=[feed.id]))
|
response = client.post(reverse("core:retry_feed", args=[feed.id]))
|
||||||
|
|
||||||
assert response.status_code == 302
|
assert response.status_code == 302
|
||||||
mock_reset.assert_called_once_with(feed.id)
|
|
||||||
mock_fetch.assert_called_once_with(feed.id)
|
mock_fetch.assert_called_once_with(feed.id)
|
||||||
|
feed.refresh_from_db()
|
||||||
|
assert feed.fetch_error_count == 0
|
||||||
|
assert feed.claimed_at is not None, "the fetch is claimed so the poller can't also take it"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_retry_feed_does_not_queue_a_second_fetch_while_one_is_in_flight(client, django_user_model):
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
|
||||||
|
client.login(username="admin", password="pw-admin-123")
|
||||||
|
feed = Feed.objects.create(feed_url="https://busy.example.com/feed", claimed_at=timezone.now())
|
||||||
|
|
||||||
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch:
|
||||||
|
client.post(reverse("core:retry_feed", args=[feed.id]))
|
||||||
|
|
||||||
|
mock_fetch.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
def _ready(client):
|
def _ready(client):
|
||||||
|
|||||||
+9
-4
@@ -10,7 +10,7 @@ from django.views.decorators.http import require_POST
|
|||||||
|
|
||||||
from apps.articles.models import Article
|
from apps.articles.models import Article
|
||||||
from apps.feeds.models import Feed, Subscription
|
from apps.feeds.models import Feed, Subscription
|
||||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, fetch_feed, reset_feed_backoff
|
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, enqueue_fetch, reset_backoff
|
||||||
|
|
||||||
from .health import fetch_pipeline_status
|
from .health import fetch_pipeline_status
|
||||||
|
|
||||||
@@ -108,7 +108,12 @@ def dashboard(request):
|
|||||||
@require_POST
|
@require_POST
|
||||||
def retry_feed(request, feed_id):
|
def retry_feed(request, feed_id):
|
||||||
feed = get_object_or_404(Feed, pk=feed_id)
|
feed = get_object_or_404(Feed, pk=feed_id)
|
||||||
reset_feed_backoff.delay(feed.id)
|
# Reset first, synchronously: as separate queued tasks the reset could
|
||||||
fetch_feed.delay(feed.id)
|
# land after the fetch and undo the schedule it just set.
|
||||||
messages.success(request, f"Queued an immediate retry for '{feed.title or feed.feed_url}'.")
|
reset_backoff(feed.id)
|
||||||
|
name = feed.title or feed.feed_url
|
||||||
|
if enqueue_fetch(feed.id):
|
||||||
|
messages.success(request, f"Queued an immediate retry for '{name}'.")
|
||||||
|
else:
|
||||||
|
messages.info(request, f"'{name}' is already being fetched right now.")
|
||||||
return redirect("core:dashboard")
|
return redirect("core:dashboard")
|
||||||
+52
-18
@@ -47,30 +47,57 @@ def poll_due_feeds():
|
|||||||
"""
|
"""
|
||||||
now = timezone.now()
|
now = timezone.now()
|
||||||
cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS)
|
cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS)
|
||||||
claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
|
|
||||||
due_feed_ids = list(
|
due_feed_ids = list(
|
||||||
Feed.objects.filter(is_active=True)
|
Feed.objects.filter(is_active=True)
|
||||||
.filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now))
|
.filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now))
|
||||||
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
|
.filter(
|
||||||
|
Q(claimed_at__isnull=True)
|
||||||
|
| Q(claimed_at__lt=now - timedelta(minutes=CLAIM_STALE_MINUTES))
|
||||||
|
)
|
||||||
.order_by("next_fetch_at")
|
.order_by("next_fetch_at")
|
||||||
.values_list("id", flat=True)[:POLL_BATCH_SIZE]
|
.values_list("id", flat=True)[:POLL_BATCH_SIZE]
|
||||||
)
|
)
|
||||||
claimed_ids = []
|
dispatched = 0
|
||||||
for feed_id in due_feed_ids:
|
for feed_id in due_feed_ids:
|
||||||
# The UPDATE's WHERE clause re-checks staleness at the database
|
try:
|
||||||
# level: it only claims rows nobody else has already claimed more
|
if enqueue_fetch(feed_id):
|
||||||
# recently than CLAIM_STALE_MINUTES ago, so two workers racing this
|
dispatched += 1
|
||||||
# same loop can't both get rowcount=1 for the same feed.
|
except Exception:
|
||||||
claimed = (
|
# Broker trouble: enqueue_fetch already released this feed's
|
||||||
|
# claim, so it's simply picked up again next tick.
|
||||||
|
logger.exception("Could not queue a fetch for feed %s", feed_id)
|
||||||
|
return dispatched
|
||||||
|
|
||||||
|
|
||||||
|
def claim_feed(feed_id: int) -> bool:
|
||||||
|
"""Atomically marks a feed as having a fetch in flight (an UPDATE whose
|
||||||
|
WHERE clause re-checks staleness, so two callers racing for the same
|
||||||
|
feed can't both get rowcount=1). Every path that dispatches a fetch —
|
||||||
|
the periodic poller, a newly added feed, an admin retry — goes through
|
||||||
|
this, so they can never run the same feed concurrently."""
|
||||||
|
now = timezone.now()
|
||||||
|
stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
|
||||||
|
return bool(
|
||||||
Feed.objects.filter(id=feed_id, is_active=True)
|
Feed.objects.filter(id=feed_id, is_active=True)
|
||||||
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
|
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=stale_before))
|
||||||
.update(claimed_at=now)
|
.update(claimed_at=now)
|
||||||
)
|
)
|
||||||
if claimed:
|
|
||||||
claimed_ids.append(feed_id)
|
|
||||||
for feed_id in claimed_ids:
|
def enqueue_fetch(feed_id: int) -> bool:
|
||||||
|
"""Claim a feed and queue its fetch. False means another fetch of it is
|
||||||
|
already in flight (nothing was queued — that one will do the work).
|
||||||
|
If queueing itself fails the claim is released and the error re-raised,
|
||||||
|
so a broker outage doesn't leave the feed claimed until it goes stale.
|
||||||
|
"""
|
||||||
|
if not claim_feed(feed_id):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
fetch_feed.delay(feed_id)
|
fetch_feed.delay(feed_id)
|
||||||
return len(claimed_ids)
|
except Exception:
|
||||||
|
Feed.objects.filter(pk=feed_id).update(claimed_at=None)
|
||||||
|
raise
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def _schedule_next_fetch(feed: Feed) -> None:
|
def _schedule_next_fetch(feed: Feed) -> None:
|
||||||
@@ -148,13 +175,20 @@ def fetch_feed(feed_id: int):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def reset_backoff(feed_id: int) -> None:
|
||||||
|
"""Clear a feed's error count and due-date backoff so it's retried at
|
||||||
|
its normal configured interval instead of the backed-off one
|
||||||
|
(next_fetch_at=None also makes it immediately due, same as a feed
|
||||||
|
that's never been fetched). A plain synchronous function so a caller
|
||||||
|
can finish the reset *before* dispatching the fetch — as an async task
|
||||||
|
it could land after the fetch and undo its freshly scheduled
|
||||||
|
next_fetch_at."""
|
||||||
|
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def reset_feed_backoff(feed_id: int):
|
def reset_feed_backoff(feed_id: int):
|
||||||
"""Admin-triggered: clear a feed's error count and due-date backoff so
|
reset_backoff(feed_id)
|
||||||
it's retried at its normal configured interval instead of the backed-
|
|
||||||
off one (next_fetch_at=None also makes it immediately due for the
|
|
||||||
next poll_due_feeds tick, same as a feed that's never been fetched)."""
|
|
||||||
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
|
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ from unittest.mock import patch
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import responses
|
import responses
|
||||||
|
from django.urls import reverse
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
from apps.articles.models import Article
|
from apps.articles.models import Article
|
||||||
@@ -228,3 +229,92 @@ def test_fetch_feed_dispatches_rule_evaluation_in_bounded_chunks(monkeypatch):
|
|||||||
|
|
||||||
chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list]
|
chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list]
|
||||||
assert chunk_sizes == [2, 2, 1]
|
assert chunk_sizes == [2, 2, 1]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_enqueue_fetch_claims_the_feed_and_queues_once():
|
||||||
|
from apps.feeds.tasks import enqueue_fetch
|
||||||
|
|
||||||
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||||
|
|
||||||
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||||
|
first = enqueue_fetch(feed.id)
|
||||||
|
second = enqueue_fetch(feed.id)
|
||||||
|
|
||||||
|
assert (first, second) == (True, False)
|
||||||
|
mock_delay.assert_called_once_with(feed.id)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_the_poller_does_not_dispatch_a_feed_a_direct_enqueue_already_claimed():
|
||||||
|
"""T1: a just-added feed (next_fetch_at NULL, i.e. due) is claimed by
|
||||||
|
the add-feed path, so the periodic poller must not fetch it too."""
|
||||||
|
from apps.feeds.tasks import enqueue_fetch
|
||||||
|
|
||||||
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||||
|
|
||||||
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||||
|
enqueue_fetch(feed.id)
|
||||||
|
dispatched = poll_due_feeds()
|
||||||
|
|
||||||
|
assert dispatched == 0
|
||||||
|
assert mock_delay.call_count == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_enqueue_fetch_releases_the_claim_if_queueing_fails():
|
||||||
|
from apps.feeds.tasks import enqueue_fetch
|
||||||
|
|
||||||
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||||
|
|
||||||
|
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
enqueue_fetch(feed.id)
|
||||||
|
|
||||||
|
feed.refresh_from_db()
|
||||||
|
assert feed.claimed_at is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_poller_survives_a_broker_failure_and_leaves_feeds_unclaimed():
|
||||||
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||||
|
|
||||||
|
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
||||||
|
assert poll_due_feeds() == 0
|
||||||
|
|
||||||
|
feed.refresh_from_db()
|
||||||
|
assert feed.claimed_at is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_add_feed_claims_its_initial_fetch(client):
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
|
||||||
|
get_user_model().objects.create_user("alice", password="pw-alice-123")
|
||||||
|
client.login(username="alice", password="pw-alice-123")
|
||||||
|
|
||||||
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||||
|
client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"})
|
||||||
|
poll_due_feeds()
|
||||||
|
|
||||||
|
assert mock_delay.call_count == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_add_feed_still_subscribes_when_the_broker_is_down(client):
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
|
||||||
|
from apps.feeds.models import Subscription
|
||||||
|
|
||||||
|
user = get_user_model().objects.create_user("alice", password="pw-alice-123")
|
||||||
|
client.login(username="alice", password="pw-alice-123")
|
||||||
|
|
||||||
|
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
||||||
|
response = client.post(
|
||||||
|
reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 302
|
||||||
|
assert Subscription.objects.filter(
|
||||||
|
user=user, feed__feed_url="https://example.com/new.xml"
|
||||||
|
).exists()
|
||||||
+11
-2
@@ -1,3 +1,4 @@
|
|||||||
|
import logging
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
|
||||||
from django.contrib import messages
|
from django.contrib import messages
|
||||||
@@ -12,7 +13,9 @@ from apps.articles.models import SubscriptionTag, Tag
|
|||||||
from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm
|
from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm
|
||||||
from .models import Feed, Subscription
|
from .models import Feed, Subscription
|
||||||
from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml
|
from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml
|
||||||
from .tasks import fetch_feed, import_opml_task
|
from .tasks import enqueue_fetch, import_opml_task
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
OPML_ASYNC_THRESHOLD = 50
|
OPML_ASYNC_THRESHOLD = 50
|
||||||
|
|
||||||
@@ -42,7 +45,13 @@ def add_feed(request):
|
|||||||
# slow or large feed would otherwise tie up a Gunicorn worker for
|
# slow or large feed would otherwise tie up a Gunicorn worker for
|
||||||
# the duration of the request; fetch_feed handles the save/error
|
# the duration of the request; fetch_feed handles the save/error
|
||||||
# bookkeeping the same way the periodic poll's dispatch does.
|
# bookkeeping the same way the periodic poll's dispatch does.
|
||||||
fetch_feed.delay(feed.pk)
|
try:
|
||||||
|
enqueue_fetch(feed.pk)
|
||||||
|
except Exception:
|
||||||
|
# The subscription itself is saved; with the broker down the
|
||||||
|
# periodic poller picks the (never-fetched) feed up once it's
|
||||||
|
# back, so this isn't worth failing the request over.
|
||||||
|
logger.exception("Could not queue the initial fetch for feed %s", feed.pk)
|
||||||
messages.success(
|
messages.success(
|
||||||
request,
|
request,
|
||||||
f"Subscribed to {feed_url}. Fetching its articles now — "
|
f"Subscribed to {feed_url}. Fetching its articles now — "
|
||||||
|
|||||||
Reference in new issue
Block a user