Private
Public Access
Ingestion did a get_or_create per entry. It now prefetches existing articles for the poll in one query, bulk-inserts new ones and bulk-updates changed ones (falling back to row-by-row only if a concurrent fetch wins a race), and dispatches rule evaluation in bounded chunks so a huge first fetch is several small tasks. /health/ stays a DB-only liveness check (upgrade.sh gates on it right after restart). New /health/ready/ also checks Redis and that poll_due_feeds ticked within five minutes — proving beat and a worker are both alive — and returns 503 with per-check detail otherwise. The sidebar unread-count cache and login throttle now degrade instead of erroring when Redis is unreachable: every authenticated page and every login would otherwise 500 during a cache outage. Found while verifying in a sandbox with no Redis. Also includes the dashboard's oldest-article metric (core/views.py). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
231 lines
7.7 KiB
Python
231 lines
7.7 KiB
Python
from datetime import timedelta
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
import responses
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.models import Article
|
|
from apps.feeds.models import Feed
|
|
from apps.feeds.tasks import fetch_feed, poll_due_feeds
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_dispatches_only_due_feeds():
|
|
# next_fetch_at (set by fetch_feed itself, C1), not last_fetched_at,
|
|
# is what poll_due_feeds' query filters on now.
|
|
never_fetched = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
|
recently_fetched = Feed.objects.create(
|
|
feed_url="https://b.example.com/feed.xml",
|
|
last_fetched_at=timezone.now(),
|
|
next_fetch_at=timezone.now() + timedelta(hours=1),
|
|
)
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
next_fetch_at=timezone.now() - timedelta(hours=1),
|
|
)
|
|
inactive_but_overdue = Feed.objects.create(
|
|
feed_url="https://d.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
next_fetch_at=timezone.now() - timedelta(hours=1),
|
|
is_active=False,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
dispatched_count = poll_due_feeds()
|
|
|
|
dispatched_ids = {call.args[0] for call in mock_delay.call_args_list}
|
|
assert dispatched_ids == {never_fetched.id, overdue.id}
|
|
assert dispatched_count == 2
|
|
assert recently_fetched.id not in dispatched_ids
|
|
assert inactive_but_overdue.id not in dispatched_ids
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_does_not_reclaim_a_feed_it_just_claimed():
|
|
# Models a second beat tick (or an overlapping worker) running
|
|
# poll_due_feeds before the first claimed fetch_feed has finished —
|
|
# the feed must only be dispatched once.
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
next_fetch_at=timezone.now() - timedelta(hours=1),
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
first_count = poll_due_feeds()
|
|
second_count = poll_due_feeds()
|
|
|
|
assert first_count == 1
|
|
assert second_count == 0
|
|
assert mock_delay.call_count == 1
|
|
overdue.refresh_from_db()
|
|
assert overdue.claimed_at is not None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_reclaims_a_stale_claim():
|
|
from apps.feeds.tasks import CLAIM_STALE_MINUTES
|
|
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
next_fetch_at=timezone.now() - timedelta(hours=1),
|
|
claimed_at=timezone.now() - timedelta(minutes=CLAIM_STALE_MINUTES + 1),
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
dispatched_count = poll_due_feeds()
|
|
|
|
assert dispatched_count == 1
|
|
assert mock_delay.call_args.args[0] == overdue.id
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_task_clears_its_claim():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", claimed_at=timezone.now())
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
fetch_feed(feed.id)
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.claimed_at is None
|
|
|
|
|
|
GOOD_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Example Blog</title>
|
|
<item>
|
|
<title>First post</title>
|
|
<guid>guid-1</guid>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_task_ingests_and_saves_feed_state():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
result = fetch_feed(feed.id)
|
|
|
|
assert result["status"] == Feed.STATUS_OK
|
|
assert result["new"] == 1
|
|
feed.refresh_from_db()
|
|
assert feed.last_fetch_status == Feed.STATUS_OK
|
|
assert Article.objects.filter(feed=feed, guid="guid-1").exists()
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_skips_inactive_feed():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", is_active=False)
|
|
assert fetch_feed(feed.id) is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_skips_missing_feed():
|
|
assert fetch_feed(999999) is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_records_error_and_clears_claim_on_unexpected_exception():
|
|
feed = Feed.objects.create(
|
|
feed_url="https://example.com/feed.xml",
|
|
claimed_at=timezone.now(),
|
|
fetch_error_count=0,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_and_ingest", side_effect=RuntimeError("boom")):
|
|
with pytest.raises(RuntimeError, match="boom"):
|
|
fetch_feed(feed.id)
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.last_fetch_status == Feed.STATUS_ERROR
|
|
assert feed.fetch_error_count == 1
|
|
assert feed.claimed_at is None
|
|
assert feed.last_fetched_at is not None
|
|
assert feed.next_fetch_at is not None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_task_sets_next_fetch_at_so_poll_due_feeds_can_find_it_later():
|
|
"""next_fetch_at (C1) is what poll_due_feeds' query filters on — a
|
|
feed fetch_feed just succeeded on must come back due again only after
|
|
the configured interval, not immediately."""
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
fetch_feed(feed.id)
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.next_fetch_at is not None
|
|
assert feed.next_fetch_at > timezone.now()
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
poll_due_feeds()
|
|
mock_delay.assert_not_called()
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_reset_feed_backoff_makes_the_feed_immediately_due():
|
|
from apps.feeds.tasks import reset_feed_backoff
|
|
|
|
feed = Feed.objects.create(
|
|
feed_url="https://example.com/feed.xml",
|
|
fetch_error_count=5,
|
|
last_fetched_at=timezone.now(),
|
|
next_fetch_at=timezone.now() + timedelta(hours=12),
|
|
)
|
|
|
|
reset_feed_backoff(feed.id)
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.fetch_error_count == 0
|
|
assert feed.next_fetch_at is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_schedules_the_next_fetch_using_the_site_wide_interval():
|
|
"""fetch_feed (not poll_due_feeds — see apps.feeds.tasks.
|
|
_schedule_next_fetch, C1) is what reads SiteSettings' site-wide
|
|
interval and sets Feed.next_fetch_at from it after every fetch."""
|
|
from apps.core.models import SiteSettings
|
|
|
|
SiteSettings.objects.create(fetch_interval_minutes=120)
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
before = timezone.now()
|
|
fetch_feed(feed.id)
|
|
feed.refresh_from_db()
|
|
|
|
assert feed.next_fetch_at is not None
|
|
expected = before + timedelta(minutes=120)
|
|
assert abs((feed.next_fetch_at - expected).total_seconds()) < 5
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_dispatches_rule_evaluation_in_bounded_chunks(monkeypatch):
|
|
monkeypatch.setattr("apps.feeds.tasks.RULE_DISPATCH_CHUNK_SIZE", 2)
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
items = "".join(f"<item><title>T{i}</title><guid>g{i}</guid></item>" for i in range(5))
|
|
responses.add(
|
|
responses.GET,
|
|
feed.feed_url,
|
|
body=f'<?xml version="1.0"?><rss version="2.0"><channel>{items}</channel></rss>',
|
|
status=200,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.apply_filter_rules_for_feed.delay") as mock_delay:
|
|
fetch_feed(feed.id)
|
|
|
|
chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list]
|
|
assert chunk_sizes == [2, 2, 1]
|