From dc8b9ff83b27d687f6a2652b1f4fda0a9b045c23 Mon Sep 17 00:00:00 2001 From: Keith Smith Date: Sat, 31 Oct 2026 12:59:47 -0600 Subject: [PATCH] Make the feed-poll interval site-wide instead of per-feed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces Feed.fetch_interval_minutes with a single admin-editable SiteSettings.fetch_interval_minutes (apps.core), following the same singleton-row pattern as accounts.SetupLock. Per-feed error backoff is unchanged — only the configured baseline interval is now shared across every feed, loaded once per poll_due_feeds run rather than queried per feed. EVECTION_MIN_FETCH_INTERVAL_MINUTES keeps working as the floor underneath it. SiteSettings is registered in Django admin as a singleton: "Add" is only offered when no row exists yet, and delete is always refused. Co-Authored-By: Claude Sonnet 5 --- apps/core/admin.py | 19 ++++- apps/core/migrations/0001_initial.py | 33 +++++++++ apps/core/models.py | 36 ++++++++- apps/core/test_site_settings.py | 74 +++++++++++++++++++ ...0004_remove_feed_fetch_interval_minutes.py | 17 +++++ apps/feeds/models.py | 6 +- apps/feeds/services/fetcher.py | 15 ++-- apps/feeds/tasks.py | 12 ++- apps/feeds/test_fetcher.py | 2 +- apps/feeds/test_tasks.py | 5 -- docs/install.md | 4 + 11 files changed, 203 insertions(+), 20 deletions(-) create mode 100644 apps/core/migrations/0001_initial.py create mode 100644 apps/core/test_site_settings.py create mode 100644 apps/feeds/migrations/0004_remove_feed_fetch_interval_minutes.py diff --git a/apps/core/admin.py b/apps/core/admin.py index 846f6b4..78fdcab 100644 --- a/apps/core/admin.py +++ b/apps/core/admin.py @@ -1 +1,18 @@ -# Register your models here. +from django.contrib import admin + +from .models import SiteSettings + + +@admin.register(SiteSettings) +class SiteSettingsAdmin(admin.ModelAdmin): + list_display = ["fetch_interval_minutes"] + fields = ["fetch_interval_minutes"] + + def has_add_permission(self, request): + # Singleton — only offer "Add" when the row doesn't exist yet + # (a fresh install before the first poll_due_feeds run or admin + # visit has created it via SiteSettings.load()). + return not SiteSettings.objects.exists() + + def has_delete_permission(self, request, obj=None): + return False diff --git a/apps/core/migrations/0001_initial.py b/apps/core/migrations/0001_initial.py new file mode 100644 index 0000000..43d619e --- /dev/null +++ b/apps/core/migrations/0001_initial.py @@ -0,0 +1,33 @@ +# Generated by Django 5.2.18 on 2026-10-31 18:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="SiteSettings", + fields=[ + ( + "id", + models.PositiveSmallIntegerField(default=1, primary_key=True, serialize=False), + ), + ( + "fetch_interval_minutes", + models.PositiveIntegerField( + default=60, + help_text="How often, in minutes, every feed is polled for new articles. Can't be set lower than EVECTION_MIN_FETCH_INTERVAL_MINUTES (see .env).", + ), + ), + ], + options={ + "verbose_name": "Site settings", + "verbose_name_plural": "Site settings", + }, + ), + ] diff --git a/apps/core/models.py b/apps/core/models.py index 6b20219..2b6186c 100644 --- a/apps/core/models.py +++ b/apps/core/models.py @@ -1 +1,35 @@ -# Create your models here. +from django.db import models + + +class SiteSettings(models.Model): + """Singleton row (pk=1, same pattern as accounts.SetupLock) holding + instance-wide configuration an operator reasonably wants to change + without touching the server or .env — editable via Django admin. + Currently just the feed-poll interval, which used to be a per-feed + field; it's site-wide now (see apps.feeds.tasks.poll_due_feeds). + """ + + id = models.PositiveSmallIntegerField(primary_key=True, default=1) + fetch_interval_minutes = models.PositiveIntegerField( + default=60, + help_text=( + "How often, in minutes, every feed is polled for new articles. " + "Can't be set lower than EVECTION_MIN_FETCH_INTERVAL_MINUTES (see .env)." + ), + ) + + class Meta: + verbose_name = "Site settings" + verbose_name_plural = "Site settings" + + def __str__(self): + return "Site settings" + + @classmethod + def load(cls): + """Returns the singleton row, creating it with defaults on first + use so callers never need to special-case "no row exists yet" + (e.g. a fresh install whose first poll_due_feeds run hasn't + created it, and whose admin hasn't visited the settings page).""" + obj, _ = cls.objects.get_or_create(pk=1) + return obj diff --git a/apps/core/test_site_settings.py b/apps/core/test_site_settings.py new file mode 100644 index 0000000..e40adee --- /dev/null +++ b/apps/core/test_site_settings.py @@ -0,0 +1,74 @@ +from datetime import timedelta +from unittest.mock import patch + +import pytest +from django.contrib.auth import get_user_model +from django.urls import reverse +from django.utils import timezone + +from apps.feeds.models import Feed +from apps.feeds.tasks import poll_due_feeds + +from .models import SiteSettings + +User = get_user_model() + + +@pytest.mark.django_db +def test_load_creates_the_singleton_row_with_defaults(): + assert not SiteSettings.objects.exists() + settings_row = SiteSettings.load() + assert settings_row.fetch_interval_minutes == 60 + assert SiteSettings.objects.count() == 1 + + +@pytest.mark.django_db +def test_load_returns_the_existing_row_without_creating_a_second_one(): + SiteSettings.objects.create(fetch_interval_minutes=30) + settings_row = SiteSettings.load() + assert settings_row.fetch_interval_minutes == 30 + assert SiteSettings.objects.count() == 1 + + +@pytest.mark.django_db +def test_poll_due_feeds_uses_the_site_wide_interval_for_every_feed(): + SiteSettings.objects.create(fetch_interval_minutes=120) + due = Feed.objects.create( + feed_url="https://a.example.com/feed.xml", + last_fetched_at=timezone.now() - timedelta(minutes=150), + ) + not_due = Feed.objects.create( + feed_url="https://b.example.com/feed.xml", + last_fetched_at=timezone.now() - timedelta(minutes=90), + ) + + with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay: + poll_due_feeds() + + dispatched_ids = {call.args[0] for call in mock_delay.call_args_list} + assert dispatched_ids == {due.id} + assert not_due.id not in dispatched_ids + + +@pytest.mark.django_db +def test_admin_offers_add_only_once_and_never_offers_delete(client): + # ModelAdmin.has_add_permission/has_delete_permission returning False + # makes Django raise PermissionDenied rather than build a 403 response + # directly — the test client's default behavior re-raises that for + # debugging convenience instead of returning it as a response. + client.raise_request_exception = False + + User.objects.create_superuser("admin", password="pw-admin-123") + client.login(username="admin", password="pw-admin-123") + + add_url = reverse("admin:core_sitesettings_add") + response = client.get(add_url) + assert response.status_code == 200 # no row yet — "Add" is offered + + settings_row = SiteSettings.load() + response = client.get(add_url) + assert response.status_code == 403 # a row exists now — add is refused + + delete_url = reverse("admin:core_sitesettings_delete", args=[settings_row.pk]) + response = client.get(delete_url) + assert response.status_code == 403 diff --git a/apps/feeds/migrations/0004_remove_feed_fetch_interval_minutes.py b/apps/feeds/migrations/0004_remove_feed_fetch_interval_minutes.py new file mode 100644 index 0000000..53409c0 --- /dev/null +++ b/apps/feeds/migrations/0004_remove_feed_fetch_interval_minutes.py @@ -0,0 +1,17 @@ +# Generated by Django 5.2.18 on 2026-10-31 18:56 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("feeds", "0003_feed_claimed_at"), + ] + + operations = [ + migrations.RemoveField( + model_name="feed", + name="fetch_interval_minutes", + ), + ] diff --git a/apps/feeds/models.py b/apps/feeds/models.py index ef89b27..6a69cdf 100644 --- a/apps/feeds/models.py +++ b/apps/feeds/models.py @@ -35,8 +35,10 @@ class Feed(models.Model): ) last_error_message = models.TextField(blank=True) - # Instance-wide, not a per-user preference — see class docstring. - fetch_interval_minutes = models.PositiveIntegerField(default=60) + # The poll interval itself is site-wide now (apps.core.models. + # SiteSettings), not per-feed — only the error-driven backoff on top + # of it still varies per feed (see apps.feeds.services.fetcher. + # effective_interval_minutes). fetch_error_count = models.PositiveIntegerField(default=0) is_active = models.BooleanField(default=True) diff --git a/apps/feeds/services/fetcher.py b/apps/feeds/services/fetcher.py index 1743a56..1080843 100644 --- a/apps/feeds/services/fetcher.py +++ b/apps/feeds/services/fetcher.py @@ -124,14 +124,17 @@ def _entry_content_html(entry) -> str: return getattr(entry, "summary", "") or "" -def effective_interval_minutes(feed: Feed) -> int: - """The actual polling interval to use, applying exponential backoff on - top of the admin-configured ``fetch_interval_minutes`` for feeds that - are currently erroring. ``fetch_interval_minutes`` itself is never - mutated by the fetcher — it's the admin's chosen baseline. +def effective_interval_minutes(feed: Feed, configured_interval_minutes: int) -> int: + """The actual polling interval to use for `feed`, applying exponential + backoff on top of the site-wide `configured_interval_minutes` (see + apps.core.models.SiteSettings) for a feed that's currently erroring. + Takes the configured value as a parameter rather than querying + SiteSettings itself, since this runs once per feed (apps.feeds.tasks + calls it from a list comprehension over every active feed) and the + site-wide value doesn't vary by feed. """ floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15) - base = max(feed.fetch_interval_minutes, floor) + base = max(configured_interval_minutes, floor) if feed.fetch_error_count <= 0: return base backoff = base * (2**feed.fetch_error_count) diff --git a/apps/feeds/tasks.py b/apps/feeds/tasks.py index 199ba9d..82c7751 100644 --- a/apps/feeds/tasks.py +++ b/apps/feeds/tasks.py @@ -8,6 +8,7 @@ from django.db.models import Q from django.utils import timezone from apps.articles.tasks import apply_filter_rules_for_feed +from apps.core.models import SiteSettings from .models import Feed from .opml import import_opml @@ -32,12 +33,15 @@ def poll_due_feeds(): """ now = timezone.now() claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES) + # Loaded once per call, not per feed — the interval is site-wide + # (apps.core.models.SiteSettings), so every feed below shares it. + configured_interval = SiteSettings.load().fetch_interval_minutes due_feed_ids = [ feed.id for feed in Feed.objects.filter(is_active=True).only( - "id", "last_fetched_at", "fetch_interval_minutes", "fetch_error_count" + "id", "last_fetched_at", "fetch_error_count" ) - if _is_due(feed) + if _is_due(feed, configured_interval) ] claimed_ids = [] for feed_id in due_feed_ids: @@ -57,10 +61,10 @@ def poll_due_feeds(): return len(claimed_ids) -def _is_due(feed: Feed) -> bool: +def _is_due(feed: Feed, configured_interval_minutes: int) -> bool: if feed.last_fetched_at is None: return True - interval = effective_interval_minutes(feed) + interval = effective_interval_minutes(feed, configured_interval_minutes) return feed.last_fetched_at <= timezone.now() - timedelta(minutes=interval) diff --git a/apps/feeds/test_fetcher.py b/apps/feeds/test_fetcher.py index 1d3e23a..d653cbc 100644 --- a/apps/feeds/test_fetcher.py +++ b/apps/feeds/test_fetcher.py @@ -136,7 +136,7 @@ def test_unparseable_feed_records_error_and_backs_off(feed): assert result.status == Feed.STATUS_ERROR assert feed.fetch_error_count == 1 - assert effective_interval_minutes(feed) > feed.fetch_interval_minutes + assert effective_interval_minutes(feed, 60) > 60 @pytest.mark.django_db diff --git a/apps/feeds/test_tasks.py b/apps/feeds/test_tasks.py index e6ce875..ab82ba6 100644 --- a/apps/feeds/test_tasks.py +++ b/apps/feeds/test_tasks.py @@ -16,17 +16,14 @@ def test_poll_due_feeds_dispatches_only_due_feeds(): recently_fetched = Feed.objects.create( feed_url="https://b.example.com/feed.xml", last_fetched_at=timezone.now(), - fetch_interval_minutes=60, ) overdue = Feed.objects.create( feed_url="https://c.example.com/feed.xml", last_fetched_at=timezone.now() - timedelta(hours=2), - fetch_interval_minutes=60, ) inactive_but_overdue = Feed.objects.create( feed_url="https://d.example.com/feed.xml", last_fetched_at=timezone.now() - timedelta(hours=2), - fetch_interval_minutes=60, is_active=False, ) @@ -48,7 +45,6 @@ def test_poll_due_feeds_does_not_reclaim_a_feed_it_just_claimed(): overdue = Feed.objects.create( feed_url="https://c.example.com/feed.xml", last_fetched_at=timezone.now() - timedelta(hours=2), - fetch_interval_minutes=60, ) with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay: @@ -69,7 +65,6 @@ def test_poll_due_feeds_reclaims_a_stale_claim(): overdue = Feed.objects.create( feed_url="https://c.example.com/feed.xml", last_fetched_at=timezone.now() - timedelta(hours=2), - fetch_interval_minutes=60, claimed_at=timezone.now() - timedelta(minutes=CLAIM_STALE_MINUTES + 1), ) diff --git a/docs/install.md b/docs/install.md index 8eaf12b..7cbc6bb 100644 --- a/docs/install.md +++ b/docs/install.md @@ -143,6 +143,10 @@ At minimum, set the following in `.env`: - `EVECTION_GUNICORN_BIND` — the `address:port` gunicorn listens on; see "Reverse proxy" below for how to pick this +How often every feed is polled is a site-wide setting (Django admin → Site +settings), not a `.env` value — `EVECTION_MIN_FETCH_INTERVAL_MINUTES` only sets +a floor under it, so it can't be configured lower than that without editing `.env`. + Evection reads `config.settings.prod` when run via the commands below (see each systemd unit's `DJANGO_SETTINGS_MODULE`). That module turns on HTTPS-only cookies, HSTS, and a Content-Security-Policy header on top of the base settings, and