Private
Public Access
Route every feed fetch through one atomic claim (R2-T1, #32)
Adding a feed and the admin retry queued fetch_feed directly without claiming the feed, so the periodic poller (which sees a never-fetched or just-reset feed as due) could claim and fetch it too: duplicate fetches, and one task overwriting another's claim/ETag/next-fetch state. The retry path also queued the backoff reset and the fetch as two independent tasks that could run in either order. claim_feed/enqueue_fetch in apps.feeds.tasks are now the only way a fetch gets dispatched — the poller, add-feed and retry all use them, so a feed can only have one fetch in flight. If queueing fails the claim is released (the poller recovers it), and add-feed still saves the subscription during a broker outage. The retry resets backoff synchronously before claiming, and says so when a fetch is already running. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0975a7634b
commit
85914f9107
5 files changed
+184
-33
No files matched your search
@@ -3,6 +3,7 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import responses
|
||||
from django.urls import reverse
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.articles.models import Article
|
||||
@@ -228,3 +229,92 @@ def test_fetch_feed_dispatches_rule_evaluation_in_bounded_chunks(monkeypatch):
|
||||
|
||||
chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list]
|
||||
assert chunk_sizes == [2, 2, 1]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_enqueue_fetch_claims_the_feed_and_queues_once():
|
||||
from apps.feeds.tasks import enqueue_fetch
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||
first = enqueue_fetch(feed.id)
|
||||
second = enqueue_fetch(feed.id)
|
||||
|
||||
assert (first, second) == (True, False)
|
||||
mock_delay.assert_called_once_with(feed.id)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_the_poller_does_not_dispatch_a_feed_a_direct_enqueue_already_claimed():
|
||||
"""T1: a just-added feed (next_fetch_at NULL, i.e. due) is claimed by
|
||||
the add-feed path, so the periodic poller must not fetch it too."""
|
||||
from apps.feeds.tasks import enqueue_fetch
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||
enqueue_fetch(feed.id)
|
||||
dispatched = poll_due_feeds()
|
||||
|
||||
assert dispatched == 0
|
||||
assert mock_delay.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_enqueue_fetch_releases_the_claim_if_queueing_fails():
|
||||
from apps.feeds.tasks import enqueue_fetch
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
||||
with pytest.raises(ConnectionError):
|
||||
enqueue_fetch(feed.id)
|
||||
|
||||
feed.refresh_from_db()
|
||||
assert feed.claimed_at is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_poller_survives_a_broker_failure_and_leaves_feeds_unclaimed():
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
||||
assert poll_due_feeds() == 0
|
||||
|
||||
feed.refresh_from_db()
|
||||
assert feed.claimed_at is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_add_feed_claims_its_initial_fetch(client):
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
get_user_model().objects.create_user("alice", password="pw-alice-123")
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||
client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"})
|
||||
poll_due_feeds()
|
||||
|
||||
assert mock_delay.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_add_feed_still_subscribes_when_the_broker_is_down(client):
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
from apps.feeds.models import Subscription
|
||||
|
||||
user = get_user_model().objects.create_user("alice", password="pw-alice-123")
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
||||
response = client.post(
|
||||
reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"}
|
||||
)
|
||||
|
||||
assert response.status_code == 302
|
||||
assert Subscription.objects.filter(
|
||||
user=user, feed__feed_url="https://example.com/new.xml"
|
||||
).exists()
|
||||
Reference in new issue
Block a user