diff --git a/apps/core/test_site_settings.py b/apps/core/test_site_settings.py index 5141ac4..a57be87 100644 --- a/apps/core/test_site_settings.py +++ b/apps/core/test_site_settings.py @@ -1,13 +1,6 @@ -from datetime import timedelta -from unittest.mock import patch - import pytest from django.contrib.auth import get_user_model from django.urls import reverse -from django.utils import timezone - -from apps.feeds.models import Feed -from apps.feeds.tasks import poll_due_feeds from .models import SiteSettings @@ -30,26 +23,6 @@ def test_load_returns_the_existing_row_without_creating_a_second_one(): assert SiteSettings.objects.count() == 1 -@pytest.mark.django_db -def test_poll_due_feeds_uses_the_site_wide_interval_for_every_feed(): - SiteSettings.objects.create(fetch_interval_minutes=120) - due = Feed.objects.create( - feed_url="https://a.example.com/feed.xml", - last_fetched_at=timezone.now() - timedelta(minutes=150), - ) - not_due = Feed.objects.create( - feed_url="https://b.example.com/feed.xml", - last_fetched_at=timezone.now() - timedelta(minutes=90), - ) - - with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay: - poll_due_feeds() - - dispatched_ids = {call.args[0] for call in mock_delay.call_args_list} - assert dispatched_ids == {due.id} - assert not_due.id not in dispatched_ids - - @pytest.mark.django_db def test_admin_changelist_redirects_straight_to_the_change_form(client): assert not SiteSettings.objects.exists() diff --git a/apps/feeds/migrations/0005_feed_next_fetch_at_and_more.py b/apps/feeds/migrations/0005_feed_next_fetch_at_and_more.py new file mode 100644 index 0000000..74da9e3 --- /dev/null +++ b/apps/feeds/migrations/0005_feed_next_fetch_at_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.2.18 on 2026-10-31 22:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("feeds", "0004_remove_feed_fetch_interval_minutes"), + ] + + operations = [ + migrations.AddField( + model_name="feed", + name="next_fetch_at", + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddIndex( + model_name="feed", + index=models.Index( + fields=["is_active", "next_fetch_at"], name="feeds_feed_is_acti_5c232e_idx" + ), + ), + ] diff --git a/apps/feeds/migrations/0006_backfill_feed_next_fetch_at.py b/apps/feeds/migrations/0006_backfill_feed_next_fetch_at.py new file mode 100644 index 0000000..c4cee12 --- /dev/null +++ b/apps/feeds/migrations/0006_backfill_feed_next_fetch_at.py @@ -0,0 +1,49 @@ +from datetime import timedelta + +from django.conf import settings +from django.db import migrations + + +def backfill_next_fetch_at(apps, schema_editor): + """One-time backfill for feeds that existed before next_fetch_at (C1) — + everything created after this migration gets it set by fetch_feed + itself. Mirrors the exact formula apps.feeds.services.fetcher. + effective_interval_minutes used to compute on every poll tick, just + run once here instead of forever after. + """ + Feed = apps.get_model("feeds", "Feed") + SiteSettings = apps.get_model("core", "SiteSettings") + + configured_interval = 60 + settings_row = SiteSettings.objects.filter(pk=1).first() + if settings_row is not None: + configured_interval = settings_row.fetch_interval_minutes + + floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15) + base = max(configured_interval, floor) + max_backoff = 24 * 60 + + for feed in Feed.objects.filter(last_fetched_at__isnull=False).only( + "id", "last_fetched_at", "fetch_error_count" + ): + if feed.fetch_error_count <= 0: + interval = base + else: + interval = min(base * (2**feed.fetch_error_count), max_backoff) + feed.next_fetch_at = feed.last_fetched_at + timedelta(minutes=interval) + feed.save(update_fields=["next_fetch_at"]) + + +def noop_reverse(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + dependencies = [ + ("feeds", "0005_feed_next_fetch_at_and_more"), + ("core", "0001_initial"), + ] + + operations = [ + migrations.RunPython(backfill_next_fetch_at, noop_reverse), + ] diff --git a/apps/feeds/models.py b/apps/feeds/models.py index 6a69cdf..0a90767 100644 --- a/apps/feeds/models.py +++ b/apps/feeds/models.py @@ -42,6 +42,15 @@ class Feed(models.Model): fetch_error_count = models.PositiveIntegerField(default=0) is_active = models.BooleanField(default=True) + # Set by fetch_feed right after every fetch attempt (success or error) + # to last_fetched_at + effective_interval_minutes — precomputed once, + # at write time, instead of poll_due_feeds re-deriving due-ness (which + # depends on fetch_error_count's exponential backoff) for every active + # feed on every tick (C1). NULL means "due now", same as a feed that's + # never been fetched. Indexed with is_active below so the scheduler's + # query is a direct index range scan, not a full table scan. + next_fetch_at = models.DateTimeField(null=True, blank=True) + # Set by poll_due_feeds' atomic claim (an UPDATE ... WHERE clause, not # select_for_update — Celery Beat and fetch_feed run in separate # transactions) and cleared once fetch_feed finishes, so overlapping @@ -53,6 +62,9 @@ class Feed(models.Model): created_at = models.DateTimeField(auto_now_add=True) + class Meta: + indexes = [models.Index(fields=["is_active", "next_fetch_at"])] + def __str__(self): return self.title or self.feed_url diff --git a/apps/feeds/tasks.py b/apps/feeds/tasks.py index 82c7751..f200b77 100644 --- a/apps/feeds/tasks.py +++ b/apps/feeds/tasks.py @@ -22,27 +22,31 @@ logger = logging.getLogger(__name__) # while a worker that died mid-fetch doesn't wedge the feed forever. CLAIM_STALE_MINUTES = 10 +# Caps how many feeds one poll_due_feeds tick will claim/dispatch, so a +# large backlog (first run after a long outage, a huge import) spreads +# across several beat ticks instead of enqueuing everything — and every +# bounded Feed query below stays index-only work, not a full scan (C1). +POLL_BATCH_SIZE = 500 + @shared_task def poll_due_feeds(): - """Find every active Feed whose polling interval has elapsed, atomically - claim it (so overlapping beat ticks or a slow-running previous fetch - can't dispatch the same feed twice), and dispatch a fetch_feed task for - each claimed feed — fan-out happens here, not via one Celery Beat entry - per feed. + """Find a bounded batch of active Feeds whose next_fetch_at has + elapsed (set by fetch_feed itself after every attempt — see Feed. + next_fetch_at), atomically claim them (so overlapping beat ticks or a + slow-running previous fetch can't dispatch the same feed twice), and + dispatch a fetch_feed task for each claimed feed — fan-out happens + here, not via one Celery Beat entry per feed. """ now = timezone.now() claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES) - # Loaded once per call, not per feed — the interval is site-wide - # (apps.core.models.SiteSettings), so every feed below shares it. - configured_interval = SiteSettings.load().fetch_interval_minutes - due_feed_ids = [ - feed.id - for feed in Feed.objects.filter(is_active=True).only( - "id", "last_fetched_at", "fetch_error_count" - ) - if _is_due(feed, configured_interval) - ] + due_feed_ids = list( + Feed.objects.filter(is_active=True) + .filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now)) + .filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before)) + .order_by("next_fetch_at") + .values_list("id", flat=True)[:POLL_BATCH_SIZE] + ) claimed_ids = [] for feed_id in due_feed_ids: # The UPDATE's WHERE clause re-checks staleness at the database @@ -61,11 +65,14 @@ def poll_due_feeds(): return len(claimed_ids) -def _is_due(feed: Feed, configured_interval_minutes: int) -> bool: - if feed.last_fetched_at is None: - return True - interval = effective_interval_minutes(feed, configured_interval_minutes) - return feed.last_fetched_at <= timezone.now() - timedelta(minutes=interval) +def _schedule_next_fetch(feed: Feed) -> None: + """Sets feed.next_fetch_at from its (just-updated) fetch_error_count — + called right before every feed.save() in fetch_feed, so the next + poll_due_feeds tick can find it with a plain indexed range query + instead of recomputing every active feed's backoff in Python (C1).""" + configured_interval = SiteSettings.load().fetch_interval_minutes + interval = effective_interval_minutes(feed, configured_interval) + feed.next_fetch_at = timezone.now() + timedelta(minutes=interval) # Bounds a single feed's fetch+parse+ingest, independent of @@ -102,10 +109,12 @@ def fetch_feed(feed_id: int): record_fetch_error(feed, "Unexpected error — see worker logs") feed.last_fetched_at = timezone.now() feed.claimed_at = None + _schedule_next_fetch(feed) feed.save() raise feed.claimed_at = None + _schedule_next_fetch(feed) feed.save() changed_article_ids = result.new_article_ids + result.updated_article_ids @@ -127,9 +136,11 @@ def fetch_feed(feed_id: int): @shared_task def reset_feed_backoff(feed_id: int): - """Admin-triggered: clear a feed's error count so it's retried at its - normal configured interval instead of the backed-off one.""" - Feed.objects.filter(pk=feed_id).update(fetch_error_count=0) + """Admin-triggered: clear a feed's error count and due-date backoff so + it's retried at its normal configured interval instead of the backed- + off one (next_fetch_at=None also makes it immediately due for the + next poll_due_feeds tick, same as a feed that's never been fetched).""" + Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None) @shared_task diff --git a/apps/feeds/test_tasks.py b/apps/feeds/test_tasks.py index ab82ba6..37c8cf8 100644 --- a/apps/feeds/test_tasks.py +++ b/apps/feeds/test_tasks.py @@ -12,18 +12,23 @@ from apps.feeds.tasks import fetch_feed, poll_due_feeds @pytest.mark.django_db def test_poll_due_feeds_dispatches_only_due_feeds(): + # next_fetch_at (set by fetch_feed itself, C1), not last_fetched_at, + # is what poll_due_feeds' query filters on now. never_fetched = Feed.objects.create(feed_url="https://a.example.com/feed.xml") recently_fetched = Feed.objects.create( feed_url="https://b.example.com/feed.xml", last_fetched_at=timezone.now(), + next_fetch_at=timezone.now() + timedelta(hours=1), ) overdue = Feed.objects.create( feed_url="https://c.example.com/feed.xml", last_fetched_at=timezone.now() - timedelta(hours=2), + next_fetch_at=timezone.now() - timedelta(hours=1), ) inactive_but_overdue = Feed.objects.create( feed_url="https://d.example.com/feed.xml", last_fetched_at=timezone.now() - timedelta(hours=2), + next_fetch_at=timezone.now() - timedelta(hours=1), is_active=False, ) @@ -45,6 +50,7 @@ def test_poll_due_feeds_does_not_reclaim_a_feed_it_just_claimed(): overdue = Feed.objects.create( feed_url="https://c.example.com/feed.xml", last_fetched_at=timezone.now() - timedelta(hours=2), + next_fetch_at=timezone.now() - timedelta(hours=1), ) with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay: @@ -65,6 +71,7 @@ def test_poll_due_feeds_reclaims_a_stale_claim(): overdue = Feed.objects.create( feed_url="https://c.example.com/feed.xml", last_fetched_at=timezone.now() - timedelta(hours=2), + next_fetch_at=timezone.now() - timedelta(hours=1), claimed_at=timezone.now() - timedelta(minutes=CLAIM_STALE_MINUTES + 1), ) @@ -141,3 +148,63 @@ def test_fetch_feed_task_records_error_and_clears_claim_on_unexpected_exception( assert feed.fetch_error_count == 1 assert feed.claimed_at is None assert feed.last_fetched_at is not None + assert feed.next_fetch_at is not None + + +@pytest.mark.django_db +@responses.activate +def test_fetch_feed_task_sets_next_fetch_at_so_poll_due_feeds_can_find_it_later(): + """next_fetch_at (C1) is what poll_due_feeds' query filters on — a + feed fetch_feed just succeeded on must come back due again only after + the configured interval, not immediately.""" + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200) + + fetch_feed(feed.id) + + feed.refresh_from_db() + assert feed.next_fetch_at is not None + assert feed.next_fetch_at > timezone.now() + + with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay: + poll_due_feeds() + mock_delay.assert_not_called() + + +@pytest.mark.django_db +def test_reset_feed_backoff_makes_the_feed_immediately_due(): + from apps.feeds.tasks import reset_feed_backoff + + feed = Feed.objects.create( + feed_url="https://example.com/feed.xml", + fetch_error_count=5, + last_fetched_at=timezone.now(), + next_fetch_at=timezone.now() + timedelta(hours=12), + ) + + reset_feed_backoff(feed.id) + + feed.refresh_from_db() + assert feed.fetch_error_count == 0 + assert feed.next_fetch_at is None + + +@pytest.mark.django_db +@responses.activate +def test_fetch_feed_schedules_the_next_fetch_using_the_site_wide_interval(): + """fetch_feed (not poll_due_feeds — see apps.feeds.tasks. + _schedule_next_fetch, C1) is what reads SiteSettings' site-wide + interval and sets Feed.next_fetch_at from it after every fetch.""" + from apps.core.models import SiteSettings + + SiteSettings.objects.create(fetch_interval_minutes=120) + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200) + + before = timezone.now() + fetch_feed(feed.id) + feed.refresh_from_db() + + assert feed.next_fetch_at is not None + expected = before + timedelta(minutes=120) + assert abs((feed.next_fetch_at - expected).total_seconds()) < 5