Add configurable article retention (C7)

Articles and full HTML bodies accumulated forever, growing the
database, search index, and backups without bound. Site settings gets
a "Retention days" field (0, the default, keeps everything); a nightly
beat task deletes articles fetched longer ago than that in bounded
chunks, always keeping anything a user starred or saved for later.
Read state and tags cascade with the article and the FTS triggers drop
search entries. The dashboard shows the article count and oldest
article's age, and docs/install.md covers retention, the readiness
check, and SQLite's WAL tuning.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5.5 committed 2026-11-01 08:51:20 -07:00
1 parent e5f1277014
commit 37e658c49a
8 files changed
+166 -6

No files matched your search

+38 -2
View File
@@ -1,10 +1,14 @@
from celery import shared_task
from datetime import timedelta
from celery import shared_task
from django.utils import timezone
from apps.core.models import SiteSettings
from apps.feeds.models import Subscription
from apps.rules.engine import evaluate_rules_for_article
from apps.rules.models import FilterRule
from .models import Article, SubscriptionTag
from .models import Article, ArticleTag, SubscriptionTag
@shared_task
@@ -57,3 +61,35 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
evaluated += 1
return evaluated
PURGE_CHUNK_SIZE = 500
@shared_task
def purge_old_articles():
"""Deletes articles older than SiteSettings.retention_days (C7), in
chunks so one run never holds a huge delete in a single transaction.
Anything a user starred or saved for later (a system ArticleTag) is
kept — those are explicit "keep this" signals. UserArticleState and
ArticleTag rows go with their article via ON DELETE CASCADE, and the
SQLite FTS triggers drop the search entries. 0 days means keep
everything forever (the default).
"""
retention_days = SiteSettings.load().retention_days
if retention_days <= 0:
return 0
cutoff = timezone.now() - timedelta(days=retention_days)
keep = ArticleTag.objects.filter(tag__is_system=True).values("article_id")
deleted_total = 0
while True:
chunk = list(
Article.objects.filter(fetched_at__lt=cutoff)
.exclude(pk__in=keep)
.values_list("pk", flat=True)[:PURGE_CHUNK_SIZE]
)
if not chunk:
return deleted_total
Article.objects.filter(pk__in=chunk).delete()
deleted_total += len(chunk)
+74
View File
@@ -0,0 +1,74 @@
from datetime import timedelta
import pytest
from django.contrib.auth import get_user_model
from django.utils import timezone
from apps.core.models import SiteSettings
from apps.feeds.models import Feed
from .models import STARRED_TAG_NAME, Article, ArticleTag, UserArticleState, get_system_tag
from .tasks import purge_old_articles
User = get_user_model()
def _article(feed, guid, age_days):
article = Article.objects.create(feed=feed, guid=guid, title=guid)
Article.objects.filter(pk=article.pk).update(
fetched_at=timezone.now() - timedelta(days=age_days)
)
return article
@pytest.fixture
def feed(db):
return Feed.objects.create(feed_url="https://example.com/feed.xml")
@pytest.mark.django_db
def test_purge_is_a_noop_with_the_default_keep_forever_setting(feed):
_article(feed, "ancient", age_days=5000)
assert purge_old_articles() == 0
assert Article.objects.count() == 1
@pytest.mark.django_db
def test_purge_deletes_only_articles_older_than_the_retention_window(feed):
SiteSettings.objects.create(retention_days=30)
old = _article(feed, "old", age_days=45)
recent = _article(feed, "recent", age_days=5)
assert purge_old_articles() == 1
assert not Article.objects.filter(pk=old.pk).exists()
assert Article.objects.filter(pk=recent.pk).exists()
@pytest.mark.django_db
def test_purge_keeps_starred_articles_and_cascades_user_state(feed):
SiteSettings.objects.create(retention_days=30)
user = User.objects.create_user("alice", password="pw-alice-123")
starred = _article(feed, "starred-old", age_days=90)
ArticleTag.objects.create(
user=user, article=starred, tag=get_system_tag(user, STARRED_TAG_NAME)
)
doomed = _article(feed, "plain-old", age_days=90)
state = UserArticleState.objects.create(user=user, article=doomed, is_read=True)
purge_old_articles()
assert Article.objects.filter(pk=starred.pk).exists()
assert not Article.objects.filter(pk=doomed.pk).exists()
assert not UserArticleState.objects.filter(pk=state.pk).exists()
@pytest.mark.django_db
def test_purge_works_through_more_than_one_chunk(feed, monkeypatch):
monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2)
SiteSettings.objects.create(retention_days=30)
for i in range(5):
_article(feed, f"old-{i}", age_days=60)
assert purge_old_articles() == 5
assert Article.objects.count() == 0
+1 -1
View File
@@ -14,7 +14,7 @@ class SiteSettingsAdmin(admin.ModelAdmin):
install. No changelist/add/delete views exist at all here.
"""
fields = ["fetch_interval_minutes"]
fields = ["fetch_interval_minutes", "retention_days"]
def has_add_permission(self, request):
return False
@@ -0,0 +1,18 @@
# Generated by Django 5.2.18 on 2026-11-01 15:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='sitesettings',
name='retention_days',
field=models.PositiveIntegerField(default=0, help_text="Articles older than this many days (by when they were fetched) are deleted daily, along with everyone's read state and tags for them — except articles anyone has starred or saved for later. 0 keeps everything forever."),
),
]
+10
View File
@@ -18,6 +18,16 @@ class SiteSettings(models.Model):
),
)
retention_days = models.PositiveIntegerField(
default=0,
help_text=(
"Articles older than this many days (by when they were fetched) are "
"deleted daily, along with everyone's read state and tags for them — "
"except articles anyone has starred or saved for later. 0 keeps "
"everything forever."
),
)
class Meta:
verbose_name = "Site settings"
verbose_name_plural = "Site settings"
+4
View File
@@ -17,4 +17,8 @@ app.conf.beat_schedule = {
"task": "apps.feeds.tasks.poll_due_feeds",
"schedule": crontab(minute="*/1"),
},
"purge-old-articles": {
"task": "apps.articles.tasks.purge_old_articles",
"schedule": crontab(hour=3, minute=30),
},
}
+20 -2
View File
@@ -313,8 +313,26 @@ surfaces any feeds that are failing to fetch, which is the fastest way to catch
misconfigured feed or an outage at the source.
`/health/` is an unauthenticated liveness check (DB connectivity only) —
`scripts/upgrade.sh` uses it to confirm a deploy actually came back up, and
it's a reasonable target for an external uptime monitor too.
`scripts/upgrade.sh` uses it to confirm a deploy actually came back up.
Point an external uptime monitor at `/health/ready/` instead: it also checks
Redis and that the background poller has ticked in the last five minutes
(proving Celery beat is scheduling *and* a worker is consuming), and returns
503 with a per-check breakdown when any of them fails — so "the site loads
but nothing is being fetched" actually alerts.
**Retention.** By default every fetched article is kept forever. In the Django
admin, "Site settings" has a *Retention days* field: set it to delete articles
fetched longer ago than that, nightly (03:30 UTC, via Celery beat). Articles
anyone has starred or saved for later are always kept; read state and tags for
deleted articles go with them. The dashboard shows the article count and the
age of the oldest one, so you can see what retention is doing.
**SQLite.** SQLite installs run in WAL mode with a 20-second busy timeout (set
automatically in `config/settings/base.py`) so gunicorn, the Celery worker, and
the full-text-search triggers can share the file without "database is locked"
errors. That's plenty for a personal or small-team instance; for many users or
a very large archive, use PostgreSQL (its search uses a stored, GIN-indexed
vector kept current by a trigger, added by migration automatically).
## Backups
+1 -1
View File
@@ -6,7 +6,7 @@
<ul class="dashboard-stats">
<li>{{ stats.feed_count }} feed(s) — {{ stats.active_feed_count }} active</li>
<li>{{ stats.subscription_count }} subscription(s)</li>
<li>{{ stats.article_count }} article(s)</li>
<li>{{ stats.article_count }} article(s){% if stats.oldest_article_at %}, oldest fetched <time datetime="{{ stats.oldest_article_at|date:"c" }}" data-local-format="long">{{ stats.oldest_article_at|date:"M j, Y H:i" }}</time>{% endif %}</li>
<li>{{ stats.error_feed_count }} feed(s) currently erroring</li>
</ul>