Files
Evection/apps/feeds/test_tasks.py
T
ksmithandClaude Sonnet 5.5 85914f9107 Route every feed fetch through one atomic claim (R2-T1, #32)
Adding a feed and the admin retry queued fetch_feed directly without
claiming the feed, so the periodic poller (which sees a never-fetched
or just-reset feed as due) could claim and fetch it too: duplicate
fetches, and one task overwriting another's claim/ETag/next-fetch state.
The retry path also queued the backoff reset and the fetch as two
independent tasks that could run in either order.

claim_feed/enqueue_fetch in apps.feeds.tasks are now the only way a
fetch gets dispatched — the poller, add-feed and retry all use them, so
a feed can only have one fetch in flight. If queueing fails the claim is
released (the poller recovers it), and add-feed still saves the
subscription during a broker outage. The retry resets backoff
synchronously before claiming, and says so when a fetch is already
running.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-01 11:56:23 -07:00

321 lines
11 KiB
Python

from datetime import timedelta
from unittest.mock import patch
import pytest
import responses
from django.urls import reverse
from django.utils import timezone
from apps.articles.models import Article
from apps.feeds.models import Feed
from apps.feeds.tasks import fetch_feed, poll_due_feeds
@pytest.mark.django_db
def test_poll_due_feeds_dispatches_only_due_feeds():
# next_fetch_at (set by fetch_feed itself, C1), not last_fetched_at,
# is what poll_due_feeds' query filters on now.
never_fetched = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
recently_fetched = Feed.objects.create(
feed_url="https://b.example.com/feed.xml",
last_fetched_at=timezone.now(),
next_fetch_at=timezone.now() + timedelta(hours=1),
)
overdue = Feed.objects.create(
feed_url="https://c.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
next_fetch_at=timezone.now() - timedelta(hours=1),
)
inactive_but_overdue = Feed.objects.create(
feed_url="https://d.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
next_fetch_at=timezone.now() - timedelta(hours=1),
is_active=False,
)
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
dispatched_count = poll_due_feeds()
dispatched_ids = {call.args[0] for call in mock_delay.call_args_list}
assert dispatched_ids == {never_fetched.id, overdue.id}
assert dispatched_count == 2
assert recently_fetched.id not in dispatched_ids
assert inactive_but_overdue.id not in dispatched_ids
@pytest.mark.django_db
def test_poll_due_feeds_does_not_reclaim_a_feed_it_just_claimed():
# Models a second beat tick (or an overlapping worker) running
# poll_due_feeds before the first claimed fetch_feed has finished —
# the feed must only be dispatched once.
overdue = Feed.objects.create(
feed_url="https://c.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
next_fetch_at=timezone.now() - timedelta(hours=1),
)
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
first_count = poll_due_feeds()
second_count = poll_due_feeds()
assert first_count == 1
assert second_count == 0
assert mock_delay.call_count == 1
overdue.refresh_from_db()
assert overdue.claimed_at is not None
@pytest.mark.django_db
def test_poll_due_feeds_reclaims_a_stale_claim():
from apps.feeds.tasks import CLAIM_STALE_MINUTES
overdue = Feed.objects.create(
feed_url="https://c.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
next_fetch_at=timezone.now() - timedelta(hours=1),
claimed_at=timezone.now() - timedelta(minutes=CLAIM_STALE_MINUTES + 1),
)
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
dispatched_count = poll_due_feeds()
assert dispatched_count == 1
assert mock_delay.call_args.args[0] == overdue.id
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_task_clears_its_claim():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", claimed_at=timezone.now())
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
fetch_feed(feed.id)
feed.refresh_from_db()
assert feed.claimed_at is None
GOOD_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Example Blog</title>
<item>
<title>First post</title>
<guid>guid-1</guid>
</item>
</channel></rss>
"""
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_task_ingests_and_saves_feed_state():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
result = fetch_feed(feed.id)
assert result["status"] == Feed.STATUS_OK
assert result["new"] == 1
feed.refresh_from_db()
assert feed.last_fetch_status == Feed.STATUS_OK
assert Article.objects.filter(feed=feed, guid="guid-1").exists()
@pytest.mark.django_db
def test_fetch_feed_task_skips_inactive_feed():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", is_active=False)
assert fetch_feed(feed.id) is None
@pytest.mark.django_db
def test_fetch_feed_task_skips_missing_feed():
assert fetch_feed(999999) is None
@pytest.mark.django_db
def test_fetch_feed_task_records_error_and_clears_claim_on_unexpected_exception():
feed = Feed.objects.create(
feed_url="https://example.com/feed.xml",
claimed_at=timezone.now(),
fetch_error_count=0,
)
with patch("apps.feeds.tasks.fetch_and_ingest", side_effect=RuntimeError("boom")):
with pytest.raises(RuntimeError, match="boom"):
fetch_feed(feed.id)
feed.refresh_from_db()
assert feed.last_fetch_status == Feed.STATUS_ERROR
assert feed.fetch_error_count == 1
assert feed.claimed_at is None
assert feed.last_fetched_at is not None
assert feed.next_fetch_at is not None
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_task_sets_next_fetch_at_so_poll_due_feeds_can_find_it_later():
"""next_fetch_at (C1) is what poll_due_feeds' query filters on — a
feed fetch_feed just succeeded on must come back due again only after
the configured interval, not immediately."""
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
fetch_feed(feed.id)
feed.refresh_from_db()
assert feed.next_fetch_at is not None
assert feed.next_fetch_at > timezone.now()
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
poll_due_feeds()
mock_delay.assert_not_called()
@pytest.mark.django_db
def test_reset_feed_backoff_makes_the_feed_immediately_due():
from apps.feeds.tasks import reset_feed_backoff
feed = Feed.objects.create(
feed_url="https://example.com/feed.xml",
fetch_error_count=5,
last_fetched_at=timezone.now(),
next_fetch_at=timezone.now() + timedelta(hours=12),
)
reset_feed_backoff(feed.id)
feed.refresh_from_db()
assert feed.fetch_error_count == 0
assert feed.next_fetch_at is None
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_schedules_the_next_fetch_using_the_site_wide_interval():
"""fetch_feed (not poll_due_feeds — see apps.feeds.tasks.
_schedule_next_fetch, C1) is what reads SiteSettings' site-wide
interval and sets Feed.next_fetch_at from it after every fetch."""
from apps.core.models import SiteSettings
SiteSettings.objects.create(fetch_interval_minutes=120)
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
before = timezone.now()
fetch_feed(feed.id)
feed.refresh_from_db()
assert feed.next_fetch_at is not None
expected = before + timedelta(minutes=120)
assert abs((feed.next_fetch_at - expected).total_seconds()) < 5
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_dispatches_rule_evaluation_in_bounded_chunks(monkeypatch):
monkeypatch.setattr("apps.feeds.tasks.RULE_DISPATCH_CHUNK_SIZE", 2)
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
items = "".join(f"<item><title>T{i}</title><guid>g{i}</guid></item>" for i in range(5))
responses.add(
responses.GET,
feed.feed_url,
body=f'<?xml version="1.0"?><rss version="2.0"><channel>{items}</channel></rss>',
status=200,
)
with patch("apps.feeds.tasks.apply_filter_rules_for_feed.delay") as mock_delay:
fetch_feed(feed.id)
chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list]
assert chunk_sizes == [2, 2, 1]
@pytest.mark.django_db
def test_enqueue_fetch_claims_the_feed_and_queues_once():
from apps.feeds.tasks import enqueue_fetch
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
first = enqueue_fetch(feed.id)
second = enqueue_fetch(feed.id)
assert (first, second) == (True, False)
mock_delay.assert_called_once_with(feed.id)
@pytest.mark.django_db
def test_the_poller_does_not_dispatch_a_feed_a_direct_enqueue_already_claimed():
"""T1: a just-added feed (next_fetch_at NULL, i.e. due) is claimed by
the add-feed path, so the periodic poller must not fetch it too."""
from apps.feeds.tasks import enqueue_fetch
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
enqueue_fetch(feed.id)
dispatched = poll_due_feeds()
assert dispatched == 0
assert mock_delay.call_count == 1
@pytest.mark.django_db
def test_enqueue_fetch_releases_the_claim_if_queueing_fails():
from apps.feeds.tasks import enqueue_fetch
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
with pytest.raises(ConnectionError):
enqueue_fetch(feed.id)
feed.refresh_from_db()
assert feed.claimed_at is None
@pytest.mark.django_db
def test_poller_survives_a_broker_failure_and_leaves_feeds_unclaimed():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
assert poll_due_feeds() == 0
feed.refresh_from_db()
assert feed.claimed_at is None
@pytest.mark.django_db
def test_add_feed_claims_its_initial_fetch(client):
from django.contrib.auth import get_user_model
get_user_model().objects.create_user("alice", password="pw-alice-123")
client.login(username="alice", password="pw-alice-123")
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"})
poll_due_feeds()
assert mock_delay.call_count == 1
@pytest.mark.django_db
def test_add_feed_still_subscribes_when_the_broker_is_down(client):
from django.contrib.auth import get_user_model
from apps.feeds.models import Subscription
user = get_user_model().objects.create_user("alice", password="pw-alice-123")
client.login(username="alice", password="pw-alice-123")
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
response = client.post(
reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"}
)
assert response.status_code == 302
assert Subscription.objects.filter(
user=user, feed__feed_url="https://example.com/new.xml"
).exists()