Private
Public Access
One 5-minute poll (/new-since/) now drives the new-articles banner, the tab dot and the sidebar counts, across all subscriptions rather than the selected feed/tag. The banner lives in the base layout with a server-rendered baseline id and reloads the current view on click; new articles are never inserted automatically. Sidebar counts are patched in place by the poll, and immediately after reading or starring (HX-Trigger: countsChanged). Ingestion, filter rules and subscription changes now drop the cached counts instead of waiting out the TTL. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
425 lines
15 KiB
Python
425 lines
15 KiB
Python
from datetime import timedelta
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
import responses
|
|
from django.urls import reverse
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.models import Article
|
|
from apps.feeds.models import Feed
|
|
from apps.feeds.tasks import fetch_feed, poll_due_feeds
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_dispatches_only_due_feeds():
|
|
# next_fetch_at (set by fetch_feed itself, C1), not last_fetched_at,
|
|
# is what poll_due_feeds' query filters on now.
|
|
never_fetched = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
|
recently_fetched = Feed.objects.create(
|
|
feed_url="https://b.example.com/feed.xml",
|
|
last_fetched_at=timezone.now(),
|
|
next_fetch_at=timezone.now() + timedelta(hours=1),
|
|
)
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
next_fetch_at=timezone.now() - timedelta(hours=1),
|
|
)
|
|
inactive_but_overdue = Feed.objects.create(
|
|
feed_url="https://d.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
next_fetch_at=timezone.now() - timedelta(hours=1),
|
|
is_active=False,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
dispatched_count = poll_due_feeds()
|
|
|
|
dispatched_ids = {call.args[0] for call in mock_delay.call_args_list}
|
|
assert dispatched_ids == {never_fetched.id, overdue.id}
|
|
assert dispatched_count == 2
|
|
assert recently_fetched.id not in dispatched_ids
|
|
assert inactive_but_overdue.id not in dispatched_ids
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_does_not_reclaim_a_feed_it_just_claimed():
|
|
# Models a second beat tick (or an overlapping worker) running
|
|
# poll_due_feeds before the first claimed fetch_feed has finished —
|
|
# the feed must only be dispatched once.
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
next_fetch_at=timezone.now() - timedelta(hours=1),
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
first_count = poll_due_feeds()
|
|
second_count = poll_due_feeds()
|
|
|
|
assert first_count == 1
|
|
assert second_count == 0
|
|
assert mock_delay.call_count == 1
|
|
overdue.refresh_from_db()
|
|
assert overdue.claimed_at is not None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_reclaims_a_stale_claim():
|
|
from apps.feeds.tasks import CLAIM_STALE_MINUTES
|
|
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
next_fetch_at=timezone.now() - timedelta(hours=1),
|
|
claimed_at=timezone.now() - timedelta(minutes=CLAIM_STALE_MINUTES + 1),
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
dispatched_count = poll_due_feeds()
|
|
|
|
assert dispatched_count == 1
|
|
assert mock_delay.call_args.args[0] == overdue.id
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_task_clears_its_claim():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", claimed_at=timezone.now())
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
fetch_feed(feed.id)
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.claimed_at is None
|
|
|
|
|
|
GOOD_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Example Blog</title>
|
|
<item>
|
|
<title>First post</title>
|
|
<guid>guid-1</guid>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_task_ingests_and_saves_feed_state():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
result = fetch_feed(feed.id)
|
|
|
|
assert result["status"] == Feed.STATUS_OK
|
|
assert result["new"] == 1
|
|
feed.refresh_from_db()
|
|
assert feed.last_fetch_status == Feed.STATUS_OK
|
|
assert Article.objects.filter(feed=feed, guid="guid-1").exists()
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_drops_subscribers_cached_sidebar_counts(django_user_model):
|
|
"""New articles must show in the sidebar at the next refresh, not after
|
|
the counts cache's TTL."""
|
|
from apps.articles.queries import unread_counts_by_feed
|
|
from apps.feeds.models import Subscription
|
|
|
|
user = django_user_model.objects.create_user("reader", password="pw-reader-123")
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
Subscription.objects.create(user=user, feed=feed)
|
|
assert unread_counts_by_feed(user) == {} # now cached
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
fetch_feed(feed.id)
|
|
|
|
assert unread_counts_by_feed(user) == {feed.id: 1}
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_skips_inactive_feed():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", is_active=False)
|
|
assert fetch_feed(feed.id) is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_skips_missing_feed():
|
|
assert fetch_feed(999999) is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_records_error_and_clears_claim_on_unexpected_exception():
|
|
feed = Feed.objects.create(
|
|
feed_url="https://example.com/feed.xml",
|
|
claimed_at=timezone.now(),
|
|
fetch_error_count=0,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_and_ingest", side_effect=RuntimeError("boom")):
|
|
with pytest.raises(RuntimeError, match="boom"):
|
|
fetch_feed(feed.id)
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.last_fetch_status == Feed.STATUS_ERROR
|
|
assert feed.fetch_error_count == 1
|
|
assert feed.claimed_at is None
|
|
assert feed.last_fetched_at is not None
|
|
assert feed.next_fetch_at is not None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_task_sets_next_fetch_at_so_poll_due_feeds_can_find_it_later():
|
|
"""next_fetch_at (C1) is what poll_due_feeds' query filters on — a
|
|
feed fetch_feed just succeeded on must come back due again only after
|
|
the configured interval, not immediately."""
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
fetch_feed(feed.id)
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.next_fetch_at is not None
|
|
assert feed.next_fetch_at > timezone.now()
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
poll_due_feeds()
|
|
mock_delay.assert_not_called()
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_reset_feed_backoff_makes_the_feed_immediately_due():
|
|
from apps.feeds.tasks import reset_feed_backoff
|
|
|
|
feed = Feed.objects.create(
|
|
feed_url="https://example.com/feed.xml",
|
|
fetch_error_count=5,
|
|
last_fetched_at=timezone.now(),
|
|
next_fetch_at=timezone.now() + timedelta(hours=12),
|
|
)
|
|
|
|
reset_feed_backoff(feed.id)
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.fetch_error_count == 0
|
|
assert feed.next_fetch_at is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_schedules_the_next_fetch_using_the_site_wide_interval():
|
|
"""fetch_feed (not poll_due_feeds — see apps.feeds.tasks.
|
|
_schedule_next_fetch, C1) is what reads SiteSettings' site-wide
|
|
interval and sets Feed.next_fetch_at from it after every fetch."""
|
|
from apps.core.models import SiteSettings
|
|
|
|
SiteSettings.objects.create(fetch_interval_minutes=120)
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
before = timezone.now()
|
|
fetch_feed(feed.id)
|
|
feed.refresh_from_db()
|
|
|
|
assert feed.next_fetch_at is not None
|
|
expected = before + timedelta(minutes=120)
|
|
assert abs((feed.next_fetch_at - expected).total_seconds()) < 5
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_dispatches_rule_evaluation_in_bounded_chunks(monkeypatch):
|
|
monkeypatch.setattr("apps.feeds.tasks.RULE_DISPATCH_CHUNK_SIZE", 2)
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
items = "".join(f"<item><title>T{i}</title><guid>g{i}</guid></item>" for i in range(5))
|
|
responses.add(
|
|
responses.GET,
|
|
feed.feed_url,
|
|
body=f'<?xml version="1.0"?><rss version="2.0"><channel>{items}</channel></rss>',
|
|
status=200,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.apply_filter_rules_for_feed.delay") as mock_delay:
|
|
fetch_feed(feed.id)
|
|
|
|
chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list]
|
|
assert chunk_sizes == [2, 2, 1]
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_enqueue_fetch_claims_the_feed_and_queues_once():
|
|
from apps.feeds.tasks import enqueue_fetch
|
|
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
first = enqueue_fetch(feed.id)
|
|
second = enqueue_fetch(feed.id)
|
|
|
|
assert (first, second) == (True, False)
|
|
mock_delay.assert_called_once_with(feed.id)
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_the_poller_does_not_dispatch_a_feed_a_direct_enqueue_already_claimed():
|
|
"""T1: a just-added feed (next_fetch_at NULL, i.e. due) is claimed by
|
|
the add-feed path, so the periodic poller must not fetch it too."""
|
|
from apps.feeds.tasks import enqueue_fetch
|
|
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
enqueue_fetch(feed.id)
|
|
dispatched = poll_due_feeds()
|
|
|
|
assert dispatched == 0
|
|
assert mock_delay.call_count == 1
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_enqueue_fetch_releases_the_claim_if_queueing_fails():
|
|
from apps.feeds.tasks import enqueue_fetch
|
|
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
|
with pytest.raises(ConnectionError):
|
|
enqueue_fetch(feed.id)
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.claimed_at is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poller_survives_a_broker_failure_and_leaves_feeds_unclaimed():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
|
assert poll_due_feeds() == 0
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.claimed_at is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_add_feed_claims_its_initial_fetch(client):
|
|
from django.contrib.auth import get_user_model
|
|
|
|
get_user_model().objects.create_user("alice", password="pw-alice-123")
|
|
client.login(username="alice", password="pw-alice-123")
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"})
|
|
poll_due_feeds()
|
|
|
|
assert mock_delay.call_count == 1
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_add_feed_still_subscribes_when_the_broker_is_down(client):
|
|
from django.contrib.auth import get_user_model
|
|
|
|
from apps.feeds.models import Subscription
|
|
|
|
user = get_user_model().objects.create_user("alice", password="pw-alice-123")
|
|
client.login(username="alice", password="pw-alice-123")
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
|
response = client.post(
|
|
reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"}
|
|
)
|
|
|
|
assert response.status_code == 302
|
|
assert Subscription.objects.filter(
|
|
user=user, feed__feed_url="https://example.com/new.xml"
|
|
).exists()
|
|
|
|
|
|
class TestTransientFailureRetries:
|
|
"""R2-T5: a locked database or dropped connection is retried promptly
|
|
with bounded backoff; everything else keeps the record-and-back-off
|
|
behavior."""
|
|
|
|
@pytest.mark.django_db
|
|
def test_a_transient_database_error_is_retried_without_penalizing_the_feed(self):
|
|
from celery.exceptions import Retry
|
|
from django.db import OperationalError
|
|
|
|
feed = Feed.objects.create(
|
|
feed_url="https://example.com/feed.xml", claimed_at=timezone.now()
|
|
)
|
|
|
|
with (
|
|
patch("apps.feeds.tasks.fetch_and_ingest", side_effect=OperationalError("locked")),
|
|
patch.object(fetch_feed, "retry", side_effect=Retry()) as mock_retry,
|
|
):
|
|
with pytest.raises(Retry):
|
|
fetch_feed(feed.id)
|
|
|
|
assert mock_retry.call_args.kwargs["countdown"] == 30
|
|
feed.refresh_from_db()
|
|
assert feed.fetch_error_count == 0, "the feed wasn't at fault"
|
|
assert feed.claimed_at is not None, "claim is held so the poller can't double-fetch"
|
|
|
|
@pytest.mark.django_db
|
|
def test_backoff_doubles_each_attempt(self):
|
|
from celery.exceptions import Retry
|
|
from django.db import InterfaceError
|
|
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
delays = []
|
|
for attempt in range(3):
|
|
fetch_feed.push_request(retries=attempt)
|
|
try:
|
|
with (
|
|
patch("apps.feeds.tasks.fetch_and_ingest", side_effect=InterfaceError("gone")),
|
|
patch.object(fetch_feed, "retry", side_effect=Retry()) as mock_retry,
|
|
):
|
|
with pytest.raises(Retry):
|
|
fetch_feed(feed.id)
|
|
delays.append(mock_retry.call_args.kwargs["countdown"])
|
|
finally:
|
|
fetch_feed.pop_request()
|
|
|
|
assert delays == [30, 60, 120]
|
|
|
|
@pytest.mark.django_db
|
|
def test_after_the_retries_are_spent_it_is_recorded_as_a_normal_failure(self):
|
|
from django.db import OperationalError
|
|
|
|
from apps.feeds.tasks import FETCH_MAX_RETRIES
|
|
|
|
feed = Feed.objects.create(
|
|
feed_url="https://example.com/feed.xml", claimed_at=timezone.now()
|
|
)
|
|
|
|
fetch_feed.push_request(retries=FETCH_MAX_RETRIES)
|
|
try:
|
|
with patch("apps.feeds.tasks.fetch_and_ingest", side_effect=OperationalError("locked")):
|
|
with pytest.raises(OperationalError):
|
|
fetch_feed(feed.id)
|
|
finally:
|
|
fetch_feed.pop_request()
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.fetch_error_count == 1
|
|
assert feed.claimed_at is None
|
|
assert feed.next_fetch_at is not None
|
|
|
|
@pytest.mark.django_db
|
|
def test_other_exceptions_are_not_retried(self):
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
|
|
with (
|
|
patch("apps.feeds.tasks.fetch_and_ingest", side_effect=RuntimeError("parser bug")),
|
|
patch.object(fetch_feed, "retry") as mock_retry,
|
|
):
|
|
with pytest.raises(RuntimeError):
|
|
fetch_feed(feed.id)
|
|
|
|
mock_retry.assert_not_called()
|