From 37e658c49a135a5e1ab1d5afabba6f01a6c43532 Mon Sep 17 00:00:00 2001 From: Keith Smith Date: Sun, 1 Nov 2026 08:51:20 -0700 Subject: [PATCH] Add configurable article retention (C7) Articles and full HTML bodies accumulated forever, growing the database, search index, and backups without bound. Site settings gets a "Retention days" field (0, the default, keeps everything); a nightly beat task deletes articles fetched longer ago than that in bounded chunks, always keeping anything a user starred or saved for later. Read state and tags cascade with the article and the FTS triggers drop search entries. The dashboard shows the article count and oldest article's age, and docs/install.md covers retention, the readiness check, and SQLite's WAL tuning. Co-Authored-By: Claude Sonnet 5.5 --- apps/articles/tasks.py | 40 +++++++++- apps/articles/test_retention.py | 74 +++++++++++++++++++ apps/core/admin.py | 2 +- .../0002_sitesettings_retention_days.py | 18 +++++ apps/core/models.py | 10 +++ config/celery.py | 4 + docs/install.md | 22 +++++- templates/core/dashboard.html | 2 +- 8 files changed, 166 insertions(+), 6 deletions(-) create mode 100644 apps/articles/test_retention.py create mode 100644 apps/core/migrations/0002_sitesettings_retention_days.py diff --git a/apps/articles/tasks.py b/apps/articles/tasks.py index 6022287..8e01ea7 100644 --- a/apps/articles/tasks.py +++ b/apps/articles/tasks.py @@ -1,10 +1,14 @@ -from celery import shared_task +from datetime import timedelta +from celery import shared_task +from django.utils import timezone + +from apps.core.models import SiteSettings from apps.feeds.models import Subscription from apps.rules.engine import evaluate_rules_for_article from apps.rules.models import FilterRule -from .models import Article, SubscriptionTag +from .models import Article, ArticleTag, SubscriptionTag @shared_task @@ -57,3 +61,35 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]): evaluated += 1 return evaluated + + +PURGE_CHUNK_SIZE = 500 + + +@shared_task +def purge_old_articles(): + """Deletes articles older than SiteSettings.retention_days (C7), in + chunks so one run never holds a huge delete in a single transaction. + Anything a user starred or saved for later (a system ArticleTag) is + kept — those are explicit "keep this" signals. UserArticleState and + ArticleTag rows go with their article via ON DELETE CASCADE, and the + SQLite FTS triggers drop the search entries. 0 days means keep + everything forever (the default). + """ + retention_days = SiteSettings.load().retention_days + if retention_days <= 0: + return 0 + + cutoff = timezone.now() - timedelta(days=retention_days) + keep = ArticleTag.objects.filter(tag__is_system=True).values("article_id") + deleted_total = 0 + while True: + chunk = list( + Article.objects.filter(fetched_at__lt=cutoff) + .exclude(pk__in=keep) + .values_list("pk", flat=True)[:PURGE_CHUNK_SIZE] + ) + if not chunk: + return deleted_total + Article.objects.filter(pk__in=chunk).delete() + deleted_total += len(chunk) diff --git a/apps/articles/test_retention.py b/apps/articles/test_retention.py new file mode 100644 index 0000000..18fb8c9 --- /dev/null +++ b/apps/articles/test_retention.py @@ -0,0 +1,74 @@ +from datetime import timedelta + +import pytest +from django.contrib.auth import get_user_model +from django.utils import timezone + +from apps.core.models import SiteSettings +from apps.feeds.models import Feed + +from .models import STARRED_TAG_NAME, Article, ArticleTag, UserArticleState, get_system_tag +from .tasks import purge_old_articles + +User = get_user_model() + + +def _article(feed, guid, age_days): + article = Article.objects.create(feed=feed, guid=guid, title=guid) + Article.objects.filter(pk=article.pk).update( + fetched_at=timezone.now() - timedelta(days=age_days) + ) + return article + + +@pytest.fixture +def feed(db): + return Feed.objects.create(feed_url="https://example.com/feed.xml") + + +@pytest.mark.django_db +def test_purge_is_a_noop_with_the_default_keep_forever_setting(feed): + _article(feed, "ancient", age_days=5000) + + assert purge_old_articles() == 0 + assert Article.objects.count() == 1 + + +@pytest.mark.django_db +def test_purge_deletes_only_articles_older_than_the_retention_window(feed): + SiteSettings.objects.create(retention_days=30) + old = _article(feed, "old", age_days=45) + recent = _article(feed, "recent", age_days=5) + + assert purge_old_articles() == 1 + assert not Article.objects.filter(pk=old.pk).exists() + assert Article.objects.filter(pk=recent.pk).exists() + + +@pytest.mark.django_db +def test_purge_keeps_starred_articles_and_cascades_user_state(feed): + SiteSettings.objects.create(retention_days=30) + user = User.objects.create_user("alice", password="pw-alice-123") + starred = _article(feed, "starred-old", age_days=90) + ArticleTag.objects.create( + user=user, article=starred, tag=get_system_tag(user, STARRED_TAG_NAME) + ) + doomed = _article(feed, "plain-old", age_days=90) + state = UserArticleState.objects.create(user=user, article=doomed, is_read=True) + + purge_old_articles() + + assert Article.objects.filter(pk=starred.pk).exists() + assert not Article.objects.filter(pk=doomed.pk).exists() + assert not UserArticleState.objects.filter(pk=state.pk).exists() + + +@pytest.mark.django_db +def test_purge_works_through_more_than_one_chunk(feed, monkeypatch): + monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2) + SiteSettings.objects.create(retention_days=30) + for i in range(5): + _article(feed, f"old-{i}", age_days=60) + + assert purge_old_articles() == 5 + assert Article.objects.count() == 0 diff --git a/apps/core/admin.py b/apps/core/admin.py index bfa42c6..29b6cf1 100644 --- a/apps/core/admin.py +++ b/apps/core/admin.py @@ -14,7 +14,7 @@ class SiteSettingsAdmin(admin.ModelAdmin): install. No changelist/add/delete views exist at all here. """ - fields = ["fetch_interval_minutes"] + fields = ["fetch_interval_minutes", "retention_days"] def has_add_permission(self, request): return False diff --git a/apps/core/migrations/0002_sitesettings_retention_days.py b/apps/core/migrations/0002_sitesettings_retention_days.py new file mode 100644 index 0000000..49a9800 --- /dev/null +++ b/apps/core/migrations/0002_sitesettings_retention_days.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.18 on 2026-11-01 15:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='sitesettings', + name='retention_days', + field=models.PositiveIntegerField(default=0, help_text="Articles older than this many days (by when they were fetched) are deleted daily, along with everyone's read state and tags for them — except articles anyone has starred or saved for later. 0 keeps everything forever."), + ), + ] diff --git a/apps/core/models.py b/apps/core/models.py index 2b6186c..aedd86b 100644 --- a/apps/core/models.py +++ b/apps/core/models.py @@ -18,6 +18,16 @@ class SiteSettings(models.Model): ), ) + retention_days = models.PositiveIntegerField( + default=0, + help_text=( + "Articles older than this many days (by when they were fetched) are " + "deleted daily, along with everyone's read state and tags for them — " + "except articles anyone has starred or saved for later. 0 keeps " + "everything forever." + ), + ) + class Meta: verbose_name = "Site settings" verbose_name_plural = "Site settings" diff --git a/config/celery.py b/config/celery.py index 487f488..198db1d 100644 --- a/config/celery.py +++ b/config/celery.py @@ -17,4 +17,8 @@ app.conf.beat_schedule = { "task": "apps.feeds.tasks.poll_due_feeds", "schedule": crontab(minute="*/1"), }, + "purge-old-articles": { + "task": "apps.articles.tasks.purge_old_articles", + "schedule": crontab(hour=3, minute=30), + }, } diff --git a/docs/install.md b/docs/install.md index a837518..8af8a1e 100644 --- a/docs/install.md +++ b/docs/install.md @@ -313,8 +313,26 @@ surfaces any feeds that are failing to fetch, which is the fastest way to catch misconfigured feed or an outage at the source. `/health/` is an unauthenticated liveness check (DB connectivity only) — -`scripts/upgrade.sh` uses it to confirm a deploy actually came back up, and -it's a reasonable target for an external uptime monitor too. +`scripts/upgrade.sh` uses it to confirm a deploy actually came back up. +Point an external uptime monitor at `/health/ready/` instead: it also checks +Redis and that the background poller has ticked in the last five minutes +(proving Celery beat is scheduling *and* a worker is consuming), and returns +503 with a per-check breakdown when any of them fails — so "the site loads +but nothing is being fetched" actually alerts. + +**Retention.** By default every fetched article is kept forever. In the Django +admin, "Site settings" has a *Retention days* field: set it to delete articles +fetched longer ago than that, nightly (03:30 UTC, via Celery beat). Articles +anyone has starred or saved for later are always kept; read state and tags for +deleted articles go with them. The dashboard shows the article count and the +age of the oldest one, so you can see what retention is doing. + +**SQLite.** SQLite installs run in WAL mode with a 20-second busy timeout (set +automatically in `config/settings/base.py`) so gunicorn, the Celery worker, and +the full-text-search triggers can share the file without "database is locked" +errors. That's plenty for a personal or small-team instance; for many users or +a very large archive, use PostgreSQL (its search uses a stored, GIN-indexed +vector kept current by a trigger, added by migration automatically). ## Backups diff --git a/templates/core/dashboard.html b/templates/core/dashboard.html index 340bff7..bc4c8b6 100644 --- a/templates/core/dashboard.html +++ b/templates/core/dashboard.html @@ -6,7 +6,7 @@