Files
Evection/apps/feeds/test_tasks.py
T
ksmithandClaude Sonnet 5.5 e5f1277014 Batch feed ingestion, add a readiness check, fail open without Redis (C2, C8)
Ingestion did a get_or_create per entry. It now prefetches existing
articles for the poll in one query, bulk-inserts new ones and
bulk-updates changed ones (falling back to row-by-row only if a
concurrent fetch wins a race), and dispatches rule evaluation in
bounded chunks so a huge first fetch is several small tasks.

/health/ stays a DB-only liveness check (upgrade.sh gates on it right
after restart). New /health/ready/ also checks Redis and that
poll_due_feeds ticked within five minutes — proving beat and a worker
are both alive — and returns 503 with per-check detail otherwise.

The sidebar unread-count cache and login throttle now degrade instead
of erroring when Redis is unreachable: every authenticated page and
every login would otherwise 500 during a cache outage. Found while
verifying in a sandbox with no Redis.

Also includes the dashboard's oldest-article metric (core/views.py).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-01 08:51:20 -07:00

231 lines
7.7 KiB
Python

from datetime import timedelta
from unittest.mock import patch
import pytest
import responses
from django.utils import timezone
from apps.articles.models import Article
from apps.feeds.models import Feed
from apps.feeds.tasks import fetch_feed, poll_due_feeds
@pytest.mark.django_db
def test_poll_due_feeds_dispatches_only_due_feeds():
# next_fetch_at (set by fetch_feed itself, C1), not last_fetched_at,
# is what poll_due_feeds' query filters on now.
never_fetched = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
recently_fetched = Feed.objects.create(
feed_url="https://b.example.com/feed.xml",
last_fetched_at=timezone.now(),
next_fetch_at=timezone.now() + timedelta(hours=1),
)
overdue = Feed.objects.create(
feed_url="https://c.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
next_fetch_at=timezone.now() - timedelta(hours=1),
)
inactive_but_overdue = Feed.objects.create(
feed_url="https://d.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
next_fetch_at=timezone.now() - timedelta(hours=1),
is_active=False,
)
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
dispatched_count = poll_due_feeds()
dispatched_ids = {call.args[0] for call in mock_delay.call_args_list}
assert dispatched_ids == {never_fetched.id, overdue.id}
assert dispatched_count == 2
assert recently_fetched.id not in dispatched_ids
assert inactive_but_overdue.id not in dispatched_ids
@pytest.mark.django_db
def test_poll_due_feeds_does_not_reclaim_a_feed_it_just_claimed():
# Models a second beat tick (or an overlapping worker) running
# poll_due_feeds before the first claimed fetch_feed has finished —
# the feed must only be dispatched once.
overdue = Feed.objects.create(
feed_url="https://c.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
next_fetch_at=timezone.now() - timedelta(hours=1),
)
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
first_count = poll_due_feeds()
second_count = poll_due_feeds()
assert first_count == 1
assert second_count == 0
assert mock_delay.call_count == 1
overdue.refresh_from_db()
assert overdue.claimed_at is not None
@pytest.mark.django_db
def test_poll_due_feeds_reclaims_a_stale_claim():
from apps.feeds.tasks import CLAIM_STALE_MINUTES
overdue = Feed.objects.create(
feed_url="https://c.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
next_fetch_at=timezone.now() - timedelta(hours=1),
claimed_at=timezone.now() - timedelta(minutes=CLAIM_STALE_MINUTES + 1),
)
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
dispatched_count = poll_due_feeds()
assert dispatched_count == 1
assert mock_delay.call_args.args[0] == overdue.id
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_task_clears_its_claim():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", claimed_at=timezone.now())
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
fetch_feed(feed.id)
feed.refresh_from_db()
assert feed.claimed_at is None
GOOD_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Example Blog</title>
<item>
<title>First post</title>
<guid>guid-1</guid>
</item>
</channel></rss>
"""
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_task_ingests_and_saves_feed_state():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
result = fetch_feed(feed.id)
assert result["status"] == Feed.STATUS_OK
assert result["new"] == 1
feed.refresh_from_db()
assert feed.last_fetch_status == Feed.STATUS_OK
assert Article.objects.filter(feed=feed, guid="guid-1").exists()
@pytest.mark.django_db
def test_fetch_feed_task_skips_inactive_feed():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", is_active=False)
assert fetch_feed(feed.id) is None
@pytest.mark.django_db
def test_fetch_feed_task_skips_missing_feed():
assert fetch_feed(999999) is None
@pytest.mark.django_db
def test_fetch_feed_task_records_error_and_clears_claim_on_unexpected_exception():
feed = Feed.objects.create(
feed_url="https://example.com/feed.xml",
claimed_at=timezone.now(),
fetch_error_count=0,
)
with patch("apps.feeds.tasks.fetch_and_ingest", side_effect=RuntimeError("boom")):
with pytest.raises(RuntimeError, match="boom"):
fetch_feed(feed.id)
feed.refresh_from_db()
assert feed.last_fetch_status == Feed.STATUS_ERROR
assert feed.fetch_error_count == 1
assert feed.claimed_at is None
assert feed.last_fetched_at is not None
assert feed.next_fetch_at is not None
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_task_sets_next_fetch_at_so_poll_due_feeds_can_find_it_later():
"""next_fetch_at (C1) is what poll_due_feeds' query filters on — a
feed fetch_feed just succeeded on must come back due again only after
the configured interval, not immediately."""
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
fetch_feed(feed.id)
feed.refresh_from_db()
assert feed.next_fetch_at is not None
assert feed.next_fetch_at > timezone.now()
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
poll_due_feeds()
mock_delay.assert_not_called()
@pytest.mark.django_db
def test_reset_feed_backoff_makes_the_feed_immediately_due():
from apps.feeds.tasks import reset_feed_backoff
feed = Feed.objects.create(
feed_url="https://example.com/feed.xml",
fetch_error_count=5,
last_fetched_at=timezone.now(),
next_fetch_at=timezone.now() + timedelta(hours=12),
)
reset_feed_backoff(feed.id)
feed.refresh_from_db()
assert feed.fetch_error_count == 0
assert feed.next_fetch_at is None
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_schedules_the_next_fetch_using_the_site_wide_interval():
"""fetch_feed (not poll_due_feeds — see apps.feeds.tasks.
_schedule_next_fetch, C1) is what reads SiteSettings' site-wide
interval and sets Feed.next_fetch_at from it after every fetch."""
from apps.core.models import SiteSettings
SiteSettings.objects.create(fetch_interval_minutes=120)
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
before = timezone.now()
fetch_feed(feed.id)
feed.refresh_from_db()
assert feed.next_fetch_at is not None
expected = before + timedelta(minutes=120)
assert abs((feed.next_fetch_at - expected).total_seconds()) < 5
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_dispatches_rule_evaluation_in_bounded_chunks(monkeypatch):
monkeypatch.setattr("apps.feeds.tasks.RULE_DISPATCH_CHUNK_SIZE", 2)
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
items = "".join(f"<item><title>T{i}</title><guid>g{i}</guid></item>" for i in range(5))
responses.add(
responses.GET,
feed.feed_url,
body=f'<?xml version="1.0"?><rss version="2.0"><channel>{items}</channel></rss>',
status=200,
)
with patch("apps.feeds.tasks.apply_filter_rules_for_feed.delay") as mock_delay:
fetch_feed(feed.id)
chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list]
assert chunk_sizes == [2, 2, 1]