From 72bffbe97f36e152ab3844172ee77a894053aeb0 Mon Sep 17 00:00:00 2001 From: Keith Smith Date: Sun, 1 Nov 2026 12:00:52 -0700 Subject: [PATCH] Expire purge tombstones and time-box the retention purge (R2-T2, R2-T4; #38, #40) Tombstones were kept forever, so enabling retention left a permanent row per deleted article. They now expire after 365 days (cleaned up by the same nightly task, even when retention is off); the documented price is that an entry a feed still lists after a year, with no or a recent publish date, can reappear. The purge looped until every eligible row was gone with no run budget. A run now works for at most four minutes, then queues a continuation, so a huge first purge can't monopolise a worker or starve polling; hard time limits back that up, and each chunk is its own transaction so an interrupted run just resumes. Adds indexes on Article.fetched_at and PurgedArticle.purged_at for the scans these do. Co-Authored-By: Claude Sonnet 5.5 --- ...articles_ar_fetched_eb5501_idx_and_more.py | 22 ++++++ apps/articles/models.py | 7 +- apps/articles/tasks.py | 72 ++++++++++++++++--- apps/articles/test_retention.py | 61 ++++++++++++++++ 4 files changed, 153 insertions(+), 9 deletions(-) create mode 100644 apps/articles/migrations/0006_article_articles_ar_fetched_eb5501_idx_and_more.py diff --git a/apps/articles/migrations/0006_article_articles_ar_fetched_eb5501_idx_and_more.py b/apps/articles/migrations/0006_article_articles_ar_fetched_eb5501_idx_and_more.py new file mode 100644 index 0000000..ffa4582 --- /dev/null +++ b/apps/articles/migrations/0006_article_articles_ar_fetched_eb5501_idx_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.18 on 2026-11-01 19:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('articles', '0005_purgedarticle'), + ('feeds', '0006_backfill_feed_next_fetch_at'), + ] + + operations = [ + migrations.AddIndex( + model_name='article', + index=models.Index(fields=['fetched_at'], name='articles_ar_fetched_eb5501_idx'), + ), + migrations.AddIndex( + model_name='purgedarticle', + index=models.Index(fields=['purged_at'], name='articles_pu_purged__27d417_idx'), + ), + ] diff --git a/apps/articles/models.py b/apps/articles/models.py index 18f1190..462a1db 100644 --- a/apps/articles/models.py +++ b/apps/articles/models.py @@ -25,7 +25,11 @@ class Article(models.Model): class Meta: unique_together = [("feed", "guid")] - indexes = [models.Index(fields=["feed", "published_at"])] + indexes = [ + models.Index(fields=["feed", "published_at"]), + # The retention purge scans for the oldest articles by this. + models.Index(fields=["fetched_at"]), + ] ordering = ["-published_at", "-fetched_at"] def __str__(self): @@ -47,6 +51,7 @@ class PurgedArticle(models.Model): class Meta: unique_together = [("feed", "guid")] + indexes = [models.Index(fields=["purged_at"])] class UserArticleState(models.Model): diff --git a/apps/articles/tasks.py b/apps/articles/tasks.py index 84bbd8f..e2adc3c 100644 --- a/apps/articles/tasks.py +++ b/apps/articles/tasks.py @@ -1,3 +1,5 @@ +import logging +import time from datetime import timedelta from celery import shared_task @@ -11,6 +13,8 @@ from apps.rules.models import FilterRule from .models import Article, ArticleTag, PurgedArticle, SubscriptionTag +logger = logging.getLogger(__name__) + @shared_task def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]): @@ -66,21 +70,65 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]): PURGE_CHUNK_SIZE = 500 +# One run works for at most this long, then queues itself again (R2-T4) so +# a huge first purge never holds a worker for hours or starves feed +# polling; the hard limits below are the backstop for a single stuck chunk. +PURGE_RUN_BUDGET_SECONDS = 240 +PURGE_CONTINUATION_DELAY_SECONDS = 30 +PURGE_SOFT_TIME_LIMIT = 540 +PURGE_TIME_LIMIT = 600 -@shared_task +# How long a purged article's tombstone is kept (R2-T2). It only needs to +# outlive the time a feed keeps *listing* the entry; after this, an entry a +# feed still publishes (and that has no publish date, or a recent one) can +# be ingested as new again — the price of the tombstone table not growing +# forever. Documented in docs/install.md. +TOMBSTONE_TTL_DAYS = 365 + + +def _expire_tombstones(out_of_time) -> int: + cutoff = timezone.now() - timedelta(days=TOMBSTONE_TTL_DAYS) + removed = 0 + while not out_of_time(): + chunk = list( + PurgedArticle.objects.filter(purged_at__lt=cutoff).values_list("pk", flat=True)[ + :PURGE_CHUNK_SIZE + ] + ) + if not chunk: + break + PurgedArticle.objects.filter(pk__in=chunk).delete() + removed += len(chunk) + return removed + + +@shared_task(time_limit=PURGE_TIME_LIMIT, soft_time_limit=PURGE_SOFT_TIME_LIMIT) def purge_old_articles(): """Deletes articles older than SiteSettings.retention_days (C7), in - chunks so one run never holds a huge delete in a single transaction. - Anything a user starred or saved for later (a system ArticleTag) is - kept — those are explicit "keep this" signals. UserArticleState and - ArticleTag rows go with their article via ON DELETE CASCADE, and the - SQLite FTS triggers drop the search entries. 0 days means keep - everything forever (the default). + chunks so one run never holds a huge delete in a single transaction, + and within a time budget — if it runs out it queues a continuation + (R2-T4). Anything a user starred or saved for later (a system + ArticleTag) is kept — those are explicit "keep this" signals. + UserArticleState and ArticleTag rows go with their article via ON + DELETE CASCADE, and the SQLite FTS triggers drop the search entries. + 0 days means keep everything forever (the default). Each deleted article leaves a PurgedArticle tombstone first, so a feed that still lists the entry can't bring it back as a new, unread - article on its next poll (see apps.feeds.services.fetcher). + article on its next poll (see apps.feeds.services.fetcher). Tombstones + older than TOMBSTONE_TTL_DAYS are removed here too, whether or not + retention is currently on (R2-T2). Returns the number of articles + deleted by this run. """ + started = time.monotonic() + + def out_of_time() -> bool: + return time.monotonic() - started > PURGE_RUN_BUDGET_SECONDS + + expired = _expire_tombstones(out_of_time) + if expired: + logger.info("Removed %d expired purge tombstone(s)", expired) + retention_days = SiteSettings.load().retention_days if retention_days <= 0: return 0 @@ -103,3 +151,11 @@ def purge_old_articles(): ) Article.objects.filter(pk__in=[pk for pk, _feed_id, _guid in chunk]).delete() deleted_total += len(chunk) + if out_of_time(): + logger.info( + "Retention purge hit its %ds budget after %d article(s); continuing shortly", + PURGE_RUN_BUDGET_SECONDS, + deleted_total, + ) + purge_old_articles.apply_async(countdown=PURGE_CONTINUATION_DELAY_SECONDS) + return deleted_total diff --git a/apps/articles/test_retention.py b/apps/articles/test_retention.py index 7e079c3..b92dbd0 100644 --- a/apps/articles/test_retention.py +++ b/apps/articles/test_retention.py @@ -89,3 +89,64 @@ def test_purge_leaves_a_tombstone_per_deleted_article(feed): "gone-1", "gone-2", } + + +@pytest.mark.django_db +def test_expired_tombstones_are_removed_even_when_retention_is_off(feed): + from .models import PurgedArticle + from .tasks import TOMBSTONE_TTL_DAYS + + old = PurgedArticle.objects.create(feed=feed, guid="old") + fresh = PurgedArticle.objects.create(feed=feed, guid="fresh") + PurgedArticle.objects.filter(pk=old.pk).update( + purged_at=timezone.now() - timedelta(days=TOMBSTONE_TTL_DAYS + 1) + ) + assert SiteSettings.load().retention_days == 0 + + purge_old_articles() + + assert not PurgedArticle.objects.filter(pk=old.pk).exists() + assert PurgedArticle.objects.filter(pk=fresh.pk).exists() + + +@pytest.mark.django_db +def test_a_run_that_exhausts_its_budget_queues_a_continuation(feed, monkeypatch): + """R2-T4: a huge purge doesn't hold a worker for its whole length — + after the budget it does what it can and queues another run.""" + from unittest.mock import patch + + monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2) + monkeypatch.setattr("apps.articles.tasks.PURGE_RUN_BUDGET_SECONDS", -1) # already over + SiteSettings.objects.create(retention_days=30) + for i in range(5): + _article(feed, f"old-{i}", age_days=60) + + with patch("apps.articles.tasks.purge_old_articles.apply_async") as mock_continue: + deleted = purge_old_articles() + + assert deleted == 2, "one chunk, then it yields" + assert Article.objects.count() == 3 + mock_continue.assert_called_once() + assert mock_continue.call_args.kwargs["countdown"] > 0 + + +@pytest.mark.django_db +def test_continuation_runs_finish_the_job(feed, monkeypatch): + from unittest.mock import patch + + monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2) + SiteSettings.objects.create(retention_days=30) + for i in range(5): + _article(feed, f"old-{i}", age_days=60) + + total = 0 + for run in range(5): + # Budget runs out after every chunk, as it would on a huge archive. + monkeypatch.setattr("apps.articles.tasks.PURGE_RUN_BUDGET_SECONDS", -1) + with patch("apps.articles.tasks.purge_old_articles.apply_async") as mock_continue: + total += purge_old_articles() + if not mock_continue.called: + break + + assert total == 5 + assert Article.objects.count() == 0