Private
Public Access
poll_due_feeds scanned for due feeds and dispatched fetch_feed without marking them claimed, so overlapping beat ticks or a slow-running fetch could dispatch the same feed twice, racing on ETag/status/error- count writes. Feed.claimed_at is now set via an UPDATE ... WHERE claimed_at IS NULL OR claimed_at < stale-cutoff, so only one caller's UPDATE can ever match a given row; fetch_feed clears it when done. A claim older than CLAIM_STALE_MINUTES is treated as abandoned so a worker dying mid-fetch doesn't wedge the feed forever. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
130 lines
4.1 KiB
Python
130 lines
4.1 KiB
Python
from datetime import timedelta
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
import responses
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.models import Article
|
|
from apps.feeds.models import Feed
|
|
from apps.feeds.tasks import fetch_feed, poll_due_feeds
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_dispatches_only_due_feeds():
|
|
never_fetched = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
|
recently_fetched = Feed.objects.create(
|
|
feed_url="https://b.example.com/feed.xml",
|
|
last_fetched_at=timezone.now(),
|
|
fetch_interval_minutes=60,
|
|
)
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
fetch_interval_minutes=60,
|
|
)
|
|
inactive_but_overdue = Feed.objects.create(
|
|
feed_url="https://d.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
fetch_interval_minutes=60,
|
|
is_active=False,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
dispatched_count = poll_due_feeds()
|
|
|
|
dispatched_ids = {call.args[0] for call in mock_delay.call_args_list}
|
|
assert dispatched_ids == {never_fetched.id, overdue.id}
|
|
assert dispatched_count == 2
|
|
assert recently_fetched.id not in dispatched_ids
|
|
assert inactive_but_overdue.id not in dispatched_ids
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_does_not_reclaim_a_feed_it_just_claimed():
|
|
# Models a second beat tick (or an overlapping worker) running
|
|
# poll_due_feeds before the first claimed fetch_feed has finished —
|
|
# the feed must only be dispatched once.
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
fetch_interval_minutes=60,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
first_count = poll_due_feeds()
|
|
second_count = poll_due_feeds()
|
|
|
|
assert first_count == 1
|
|
assert second_count == 0
|
|
assert mock_delay.call_count == 1
|
|
overdue.refresh_from_db()
|
|
assert overdue.claimed_at is not None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_reclaims_a_stale_claim():
|
|
from apps.feeds.tasks import CLAIM_STALE_MINUTES
|
|
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
fetch_interval_minutes=60,
|
|
claimed_at=timezone.now() - timedelta(minutes=CLAIM_STALE_MINUTES + 1),
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
dispatched_count = poll_due_feeds()
|
|
|
|
assert dispatched_count == 1
|
|
assert mock_delay.call_args.args[0] == overdue.id
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_task_clears_its_claim():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", claimed_at=timezone.now())
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
fetch_feed(feed.id)
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.claimed_at is None
|
|
|
|
|
|
GOOD_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Example Blog</title>
|
|
<item>
|
|
<title>First post</title>
|
|
<guid>guid-1</guid>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_task_ingests_and_saves_feed_state():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
result = fetch_feed(feed.id)
|
|
|
|
assert result["status"] == Feed.STATUS_OK
|
|
assert result["new"] == 1
|
|
feed.refresh_from_db()
|
|
assert feed.last_fetch_status == Feed.STATUS_OK
|
|
assert Article.objects.filter(feed=feed, guid="guid-1").exists()
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_skips_inactive_feed():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", is_active=False)
|
|
assert fetch_feed(feed.id) is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_skips_missing_feed():
|
|
assert fetch_feed(999999) is None
|