Private
Public Access
Add configurable article retention (C7)
Articles and full HTML bodies accumulated forever, growing the database, search index, and backups without bound. Site settings gets a "Retention days" field (0, the default, keeps everything); a nightly beat task deletes articles fetched longer ago than that in bounded chunks, always keeping anything a user starred or saved for later. Read state and tags cascade with the article and the FTS triggers drop search entries. The dashboard shows the article count and oldest article's age, and docs/install.md covers retention, the readiness check, and SQLite's WAL tuning. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e5f1277014
commit
37e658c49a
8 files changed
+166
-6
No files matched your search
+38
-2
@@ -1,10 +1,14 @@
|
||||
from celery import shared_task
|
||||
from datetime import timedelta
|
||||
|
||||
from celery import shared_task
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.core.models import SiteSettings
|
||||
from apps.feeds.models import Subscription
|
||||
from apps.rules.engine import evaluate_rules_for_article
|
||||
from apps.rules.models import FilterRule
|
||||
|
||||
from .models import Article, SubscriptionTag
|
||||
from .models import Article, ArticleTag, SubscriptionTag
|
||||
|
||||
|
||||
@shared_task
|
||||
@@ -57,3 +61,35 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
||||
evaluated += 1
|
||||
|
||||
return evaluated
|
||||
|
||||
|
||||
PURGE_CHUNK_SIZE = 500
|
||||
|
||||
|
||||
@shared_task
|
||||
def purge_old_articles():
|
||||
"""Deletes articles older than SiteSettings.retention_days (C7), in
|
||||
chunks so one run never holds a huge delete in a single transaction.
|
||||
Anything a user starred or saved for later (a system ArticleTag) is
|
||||
kept — those are explicit "keep this" signals. UserArticleState and
|
||||
ArticleTag rows go with their article via ON DELETE CASCADE, and the
|
||||
SQLite FTS triggers drop the search entries. 0 days means keep
|
||||
everything forever (the default).
|
||||
"""
|
||||
retention_days = SiteSettings.load().retention_days
|
||||
if retention_days <= 0:
|
||||
return 0
|
||||
|
||||
cutoff = timezone.now() - timedelta(days=retention_days)
|
||||
keep = ArticleTag.objects.filter(tag__is_system=True).values("article_id")
|
||||
deleted_total = 0
|
||||
while True:
|
||||
chunk = list(
|
||||
Article.objects.filter(fetched_at__lt=cutoff)
|
||||
.exclude(pk__in=keep)
|
||||
.values_list("pk", flat=True)[:PURGE_CHUNK_SIZE]
|
||||
)
|
||||
if not chunk:
|
||||
return deleted_total
|
||||
Article.objects.filter(pk__in=chunk).delete()
|
||||
deleted_total += len(chunk)
|
||||
@@ -0,0 +1,74 @@
|
||||
from datetime import timedelta
|
||||
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.core.models import SiteSettings
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
from .models import STARRED_TAG_NAME, Article, ArticleTag, UserArticleState, get_system_tag
|
||||
from .tasks import purge_old_articles
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
def _article(feed, guid, age_days):
|
||||
article = Article.objects.create(feed=feed, guid=guid, title=guid)
|
||||
Article.objects.filter(pk=article.pk).update(
|
||||
fetched_at=timezone.now() - timedelta(days=age_days)
|
||||
)
|
||||
return article
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def feed(db):
|
||||
return Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_purge_is_a_noop_with_the_default_keep_forever_setting(feed):
|
||||
_article(feed, "ancient", age_days=5000)
|
||||
|
||||
assert purge_old_articles() == 0
|
||||
assert Article.objects.count() == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_purge_deletes_only_articles_older_than_the_retention_window(feed):
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
old = _article(feed, "old", age_days=45)
|
||||
recent = _article(feed, "recent", age_days=5)
|
||||
|
||||
assert purge_old_articles() == 1
|
||||
assert not Article.objects.filter(pk=old.pk).exists()
|
||||
assert Article.objects.filter(pk=recent.pk).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_purge_keeps_starred_articles_and_cascades_user_state(feed):
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
user = User.objects.create_user("alice", password="pw-alice-123")
|
||||
starred = _article(feed, "starred-old", age_days=90)
|
||||
ArticleTag.objects.create(
|
||||
user=user, article=starred, tag=get_system_tag(user, STARRED_TAG_NAME)
|
||||
)
|
||||
doomed = _article(feed, "plain-old", age_days=90)
|
||||
state = UserArticleState.objects.create(user=user, article=doomed, is_read=True)
|
||||
|
||||
purge_old_articles()
|
||||
|
||||
assert Article.objects.filter(pk=starred.pk).exists()
|
||||
assert not Article.objects.filter(pk=doomed.pk).exists()
|
||||
assert not UserArticleState.objects.filter(pk=state.pk).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_purge_works_through_more_than_one_chunk(feed, monkeypatch):
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2)
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
for i in range(5):
|
||||
_article(feed, f"old-{i}", age_days=60)
|
||||
|
||||
assert purge_old_articles() == 5
|
||||
assert Article.objects.count() == 0
|
||||
Reference in new issue
Block a user