Private
Public Access
Add an atomic per-feed claim to poll_due_feeds (T1)
poll_due_feeds scanned for due feeds and dispatched fetch_feed without marking them claimed, so overlapping beat ticks or a slow-running fetch could dispatch the same feed twice, racing on ETag/status/error- count writes. Feed.claimed_at is now set via an UPDATE ... WHERE claimed_at IS NULL OR claimed_at < stale-cutoff, so only one caller's UPDATE can ever match a given row; fetch_feed clears it when done. A claim older than CLAIM_STALE_MINUTES is treated as abandoned so a worker dying mid-fetch doesn't wedge the feed forever. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d7b433e580
commit
b7d9ee672c
4 files changed
+108
-4
No files matched your search
@@ -40,6 +40,58 @@ def test_poll_due_feeds_dispatches_only_due_feeds():
|
||||
assert inactive_but_overdue.id not in dispatched_ids
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_poll_due_feeds_does_not_reclaim_a_feed_it_just_claimed():
|
||||
# Models a second beat tick (or an overlapping worker) running
|
||||
# poll_due_feeds before the first claimed fetch_feed has finished —
|
||||
# the feed must only be dispatched once.
|
||||
overdue = Feed.objects.create(
|
||||
feed_url="https://c.example.com/feed.xml",
|
||||
last_fetched_at=timezone.now() - timedelta(hours=2),
|
||||
fetch_interval_minutes=60,
|
||||
)
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||
first_count = poll_due_feeds()
|
||||
second_count = poll_due_feeds()
|
||||
|
||||
assert first_count == 1
|
||||
assert second_count == 0
|
||||
assert mock_delay.call_count == 1
|
||||
overdue.refresh_from_db()
|
||||
assert overdue.claimed_at is not None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_poll_due_feeds_reclaims_a_stale_claim():
|
||||
from apps.feeds.tasks import CLAIM_STALE_MINUTES
|
||||
|
||||
overdue = Feed.objects.create(
|
||||
feed_url="https://c.example.com/feed.xml",
|
||||
last_fetched_at=timezone.now() - timedelta(hours=2),
|
||||
fetch_interval_minutes=60,
|
||||
claimed_at=timezone.now() - timedelta(minutes=CLAIM_STALE_MINUTES + 1),
|
||||
)
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||
dispatched_count = poll_due_feeds()
|
||||
|
||||
assert dispatched_count == 1
|
||||
assert mock_delay.call_args.args[0] == overdue.id
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_fetch_feed_task_clears_its_claim():
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", claimed_at=timezone.now())
|
||||
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
||||
|
||||
fetch_feed(feed.id)
|
||||
|
||||
feed.refresh_from_db()
|
||||
assert feed.claimed_at is None
|
||||
|
||||
|
||||
GOOD_FEED_XML = """<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel>
|
||||
<title>Example Blog</title>
|
||||
|
||||
Reference in new issue
Block a user