diff --git a/apps/articles/tasks.py b/apps/articles/tasks.py index 6022287..8e01ea7 100644 --- a/apps/articles/tasks.py +++ b/apps/articles/tasks.py @@ -1,10 +1,14 @@ -from celery import shared_task +from datetime import timedelta +from celery import shared_task +from django.utils import timezone + +from apps.core.models import SiteSettings from apps.feeds.models import Subscription from apps.rules.engine import evaluate_rules_for_article from apps.rules.models import FilterRule -from .models import Article, SubscriptionTag +from .models import Article, ArticleTag, SubscriptionTag @shared_task @@ -57,3 +61,35 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]): evaluated += 1 return evaluated + + +PURGE_CHUNK_SIZE = 500 + + +@shared_task +def purge_old_articles(): + """Deletes articles older than SiteSettings.retention_days (C7), in + chunks so one run never holds a huge delete in a single transaction. + Anything a user starred or saved for later (a system ArticleTag) is + kept — those are explicit "keep this" signals. UserArticleState and + ArticleTag rows go with their article via ON DELETE CASCADE, and the + SQLite FTS triggers drop the search entries. 0 days means keep + everything forever (the default). + """ + retention_days = SiteSettings.load().retention_days + if retention_days <= 0: + return 0 + + cutoff = timezone.now() - timedelta(days=retention_days) + keep = ArticleTag.objects.filter(tag__is_system=True).values("article_id") + deleted_total = 0 + while True: + chunk = list( + Article.objects.filter(fetched_at__lt=cutoff) + .exclude(pk__in=keep) + .values_list("pk", flat=True)[:PURGE_CHUNK_SIZE] + ) + if not chunk: + return deleted_total + Article.objects.filter(pk__in=chunk).delete() + deleted_total += len(chunk) diff --git a/apps/articles/test_retention.py b/apps/articles/test_retention.py new file mode 100644 index 0000000..18fb8c9 --- /dev/null +++ b/apps/articles/test_retention.py @@ -0,0 +1,74 @@ +from datetime import timedelta + +import pytest +from django.contrib.auth import get_user_model +from django.utils import timezone + +from apps.core.models import SiteSettings +from apps.feeds.models import Feed + +from .models import STARRED_TAG_NAME, Article, ArticleTag, UserArticleState, get_system_tag +from .tasks import purge_old_articles + +User = get_user_model() + + +def _article(feed, guid, age_days): + article = Article.objects.create(feed=feed, guid=guid, title=guid) + Article.objects.filter(pk=article.pk).update( + fetched_at=timezone.now() - timedelta(days=age_days) + ) + return article + + +@pytest.fixture +def feed(db): + return Feed.objects.create(feed_url="https://example.com/feed.xml") + + +@pytest.mark.django_db +def test_purge_is_a_noop_with_the_default_keep_forever_setting(feed): + _article(feed, "ancient", age_days=5000) + + assert purge_old_articles() == 0 + assert Article.objects.count() == 1 + + +@pytest.mark.django_db +def test_purge_deletes_only_articles_older_than_the_retention_window(feed): + SiteSettings.objects.create(retention_days=30) + old = _article(feed, "old", age_days=45) + recent = _article(feed, "recent", age_days=5) + + assert purge_old_articles() == 1 + assert not Article.objects.filter(pk=old.pk).exists() + assert Article.objects.filter(pk=recent.pk).exists() + + +@pytest.mark.django_db +def test_purge_keeps_starred_articles_and_cascades_user_state(feed): + SiteSettings.objects.create(retention_days=30) + user = User.objects.create_user("alice", password="pw-alice-123") + starred = _article(feed, "starred-old", age_days=90) + ArticleTag.objects.create( + user=user, article=starred, tag=get_system_tag(user, STARRED_TAG_NAME) + ) + doomed = _article(feed, "plain-old", age_days=90) + state = UserArticleState.objects.create(user=user, article=doomed, is_read=True) + + purge_old_articles() + + assert Article.objects.filter(pk=starred.pk).exists() + assert not Article.objects.filter(pk=doomed.pk).exists() + assert not UserArticleState.objects.filter(pk=state.pk).exists() + + +@pytest.mark.django_db +def test_purge_works_through_more_than_one_chunk(feed, monkeypatch): + monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2) + SiteSettings.objects.create(retention_days=30) + for i in range(5): + _article(feed, f"old-{i}", age_days=60) + + assert purge_old_articles() == 5 + assert Article.objects.count() == 0 diff --git a/apps/core/admin.py b/apps/core/admin.py index bfa42c6..29b6cf1 100644 --- a/apps/core/admin.py +++ b/apps/core/admin.py @@ -14,7 +14,7 @@ class SiteSettingsAdmin(admin.ModelAdmin): install. No changelist/add/delete views exist at all here. """ - fields = ["fetch_interval_minutes"] + fields = ["fetch_interval_minutes", "retention_days"] def has_add_permission(self, request): return False diff --git a/apps/core/migrations/0002_sitesettings_retention_days.py b/apps/core/migrations/0002_sitesettings_retention_days.py new file mode 100644 index 0000000..49a9800 --- /dev/null +++ b/apps/core/migrations/0002_sitesettings_retention_days.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.18 on 2026-11-01 15:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='sitesettings', + name='retention_days', + field=models.PositiveIntegerField(default=0, help_text="Articles older than this many days (by when they were fetched) are deleted daily, along with everyone's read state and tags for them — except articles anyone has starred or saved for later. 0 keeps everything forever."), + ), + ] diff --git a/apps/core/models.py b/apps/core/models.py index 2b6186c..aedd86b 100644 --- a/apps/core/models.py +++ b/apps/core/models.py @@ -18,6 +18,16 @@ class SiteSettings(models.Model): ), ) + retention_days = models.PositiveIntegerField( + default=0, + help_text=( + "Articles older than this many days (by when they were fetched) are " + "deleted daily, along with everyone's read state and tags for them — " + "except articles anyone has starred or saved for later. 0 keeps " + "everything forever." + ), + ) + class Meta: verbose_name = "Site settings" verbose_name_plural = "Site settings" diff --git a/config/celery.py b/config/celery.py index 487f488..198db1d 100644 --- a/config/celery.py +++ b/config/celery.py @@ -17,4 +17,8 @@ app.conf.beat_schedule = { "task": "apps.feeds.tasks.poll_due_feeds", "schedule": crontab(minute="*/1"), }, + "purge-old-articles": { + "task": "apps.articles.tasks.purge_old_articles", + "schedule": crontab(hour=3, minute=30), + }, } diff --git a/docs/install.md b/docs/install.md index a837518..8af8a1e 100644 --- a/docs/install.md +++ b/docs/install.md @@ -313,8 +313,26 @@ surfaces any feeds that are failing to fetch, which is the fastest way to catch misconfigured feed or an outage at the source. `/health/` is an unauthenticated liveness check (DB connectivity only) — -`scripts/upgrade.sh` uses it to confirm a deploy actually came back up, and -it's a reasonable target for an external uptime monitor too. +`scripts/upgrade.sh` uses it to confirm a deploy actually came back up. +Point an external uptime monitor at `/health/ready/` instead: it also checks +Redis and that the background poller has ticked in the last five minutes +(proving Celery beat is scheduling *and* a worker is consuming), and returns +503 with a per-check breakdown when any of them fails — so "the site loads +but nothing is being fetched" actually alerts. + +**Retention.** By default every fetched article is kept forever. In the Django +admin, "Site settings" has a *Retention days* field: set it to delete articles +fetched longer ago than that, nightly (03:30 UTC, via Celery beat). Articles +anyone has starred or saved for later are always kept; read state and tags for +deleted articles go with them. The dashboard shows the article count and the +age of the oldest one, so you can see what retention is doing. + +**SQLite.** SQLite installs run in WAL mode with a 20-second busy timeout (set +automatically in `config/settings/base.py`) so gunicorn, the Celery worker, and +the full-text-search triggers can share the file without "database is locked" +errors. That's plenty for a personal or small-team instance; for many users or +a very large archive, use PostgreSQL (its search uses a stored, GIN-indexed +vector kept current by a trigger, added by migration automatically). ## Backups diff --git a/templates/core/dashboard.html b/templates/core/dashboard.html index 340bff7..bc4c8b6 100644 --- a/templates/core/dashboard.html +++ b/templates/core/dashboard.html @@ -6,7 +6,7 @@