Private
Public Access
fetch_feed had no time limit and no handling for anything beyond the network/parse errors fetch_and_ingest already catches internally — an unexpected exception (a parser/sanitizer bug, a DB error) crashed the task with no error recorded on the Feed, the claim left held for the full CLAIM_STALE_MINUTES, and no visibility beyond a bare traceback. fetch_feed now catches that, records it the same way a handled failure would be (reusing fetcher.record_fetch_error, promoted from a private helper), clears the claim, and re-raises so Celery still marks the task FAILED. soft_time_limit/time_limit bound a single fetch that somehow outruns FETCH_TIMEOUT_SECONDS + ssrf_guard's response cap during parsing/ingestion. Also: a worker process killed mid-task (OOM, deploy restart, crash) previously lost that task silently — Celery's default is to ack a message the moment a worker *starts* it. CELERY_TASK_ACKS_LATE + CELERY_TASK_REJECT_ON_WORKER_LOST (config/settings/base.py) make the message redeliverable instead, and CELERY_WORKER_PREFETCH_MULTIPLIER=1 keeps a worker from grabbing a batch of tasks it could lose all at once. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
149 lines
4.7 KiB
Python
149 lines
4.7 KiB
Python
from datetime import timedelta
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
import responses
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.models import Article
|
|
from apps.feeds.models import Feed
|
|
from apps.feeds.tasks import fetch_feed, poll_due_feeds
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_dispatches_only_due_feeds():
|
|
never_fetched = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
|
recently_fetched = Feed.objects.create(
|
|
feed_url="https://b.example.com/feed.xml",
|
|
last_fetched_at=timezone.now(),
|
|
fetch_interval_minutes=60,
|
|
)
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
fetch_interval_minutes=60,
|
|
)
|
|
inactive_but_overdue = Feed.objects.create(
|
|
feed_url="https://d.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
fetch_interval_minutes=60,
|
|
is_active=False,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
dispatched_count = poll_due_feeds()
|
|
|
|
dispatched_ids = {call.args[0] for call in mock_delay.call_args_list}
|
|
assert dispatched_ids == {never_fetched.id, overdue.id}
|
|
assert dispatched_count == 2
|
|
assert recently_fetched.id not in dispatched_ids
|
|
assert inactive_but_overdue.id not in dispatched_ids
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_does_not_reclaim_a_feed_it_just_claimed():
|
|
# Models a second beat tick (or an overlapping worker) running
|
|
# poll_due_feeds before the first claimed fetch_feed has finished —
|
|
# the feed must only be dispatched once.
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
fetch_interval_minutes=60,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
first_count = poll_due_feeds()
|
|
second_count = poll_due_feeds()
|
|
|
|
assert first_count == 1
|
|
assert second_count == 0
|
|
assert mock_delay.call_count == 1
|
|
overdue.refresh_from_db()
|
|
assert overdue.claimed_at is not None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_reclaims_a_stale_claim():
|
|
from apps.feeds.tasks import CLAIM_STALE_MINUTES
|
|
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
fetch_interval_minutes=60,
|
|
claimed_at=timezone.now() - timedelta(minutes=CLAIM_STALE_MINUTES + 1),
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
dispatched_count = poll_due_feeds()
|
|
|
|
assert dispatched_count == 1
|
|
assert mock_delay.call_args.args[0] == overdue.id
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_task_clears_its_claim():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", claimed_at=timezone.now())
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
fetch_feed(feed.id)
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.claimed_at is None
|
|
|
|
|
|
GOOD_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Example Blog</title>
|
|
<item>
|
|
<title>First post</title>
|
|
<guid>guid-1</guid>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_task_ingests_and_saves_feed_state():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
result = fetch_feed(feed.id)
|
|
|
|
assert result["status"] == Feed.STATUS_OK
|
|
assert result["new"] == 1
|
|
feed.refresh_from_db()
|
|
assert feed.last_fetch_status == Feed.STATUS_OK
|
|
assert Article.objects.filter(feed=feed, guid="guid-1").exists()
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_skips_inactive_feed():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", is_active=False)
|
|
assert fetch_feed(feed.id) is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_skips_missing_feed():
|
|
assert fetch_feed(999999) is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_records_error_and_clears_claim_on_unexpected_exception():
|
|
feed = Feed.objects.create(
|
|
feed_url="https://example.com/feed.xml",
|
|
claimed_at=timezone.now(),
|
|
fetch_error_count=0,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_and_ingest", side_effect=RuntimeError("boom")):
|
|
with pytest.raises(RuntimeError, match="boom"):
|
|
fetch_feed(feed.id)
|
|
|
|
feed.refresh_from_db()
|
|
assert feed.last_fetch_status == Feed.STATUS_ERROR
|
|
assert feed.fetch_error_count == 1
|
|
assert feed.claimed_at is None
|
|
assert feed.last_fetched_at is not None
|