diff --git a/apps/articles/migrations/0006_article_articles_ar_fetched_eb5501_idx_and_more.py b/apps/articles/migrations/0006_article_articles_ar_fetched_eb5501_idx_and_more.py new file mode 100644 index 0000000..ffa4582 --- /dev/null +++ b/apps/articles/migrations/0006_article_articles_ar_fetched_eb5501_idx_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.18 on 2026-11-01 19:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('articles', '0005_purgedarticle'), + ('feeds', '0006_backfill_feed_next_fetch_at'), + ] + + operations = [ + migrations.AddIndex( + model_name='article', + index=models.Index(fields=['fetched_at'], name='articles_ar_fetched_eb5501_idx'), + ), + migrations.AddIndex( + model_name='purgedarticle', + index=models.Index(fields=['purged_at'], name='articles_pu_purged__27d417_idx'), + ), + ] diff --git a/apps/articles/models.py b/apps/articles/models.py index 18f1190..462a1db 100644 --- a/apps/articles/models.py +++ b/apps/articles/models.py @@ -25,7 +25,11 @@ class Article(models.Model): class Meta: unique_together = [("feed", "guid")] - indexes = [models.Index(fields=["feed", "published_at"])] + indexes = [ + models.Index(fields=["feed", "published_at"]), + # The retention purge scans for the oldest articles by this. + models.Index(fields=["fetched_at"]), + ] ordering = ["-published_at", "-fetched_at"] def __str__(self): @@ -47,6 +51,7 @@ class PurgedArticle(models.Model): class Meta: unique_together = [("feed", "guid")] + indexes = [models.Index(fields=["purged_at"])] class UserArticleState(models.Model): diff --git a/apps/articles/tasks.py b/apps/articles/tasks.py index 84bbd8f..e2adc3c 100644 --- a/apps/articles/tasks.py +++ b/apps/articles/tasks.py @@ -1,3 +1,5 @@ +import logging +import time from datetime import timedelta from celery import shared_task @@ -11,6 +13,8 @@ from apps.rules.models import FilterRule from .models import Article, ArticleTag, PurgedArticle, SubscriptionTag +logger = logging.getLogger(__name__) + @shared_task def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]): @@ -66,21 +70,65 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]): PURGE_CHUNK_SIZE = 500 +# One run works for at most this long, then queues itself again (R2-T4) so +# a huge first purge never holds a worker for hours or starves feed +# polling; the hard limits below are the backstop for a single stuck chunk. +PURGE_RUN_BUDGET_SECONDS = 240 +PURGE_CONTINUATION_DELAY_SECONDS = 30 +PURGE_SOFT_TIME_LIMIT = 540 +PURGE_TIME_LIMIT = 600 -@shared_task +# How long a purged article's tombstone is kept (R2-T2). It only needs to +# outlive the time a feed keeps *listing* the entry; after this, an entry a +# feed still publishes (and that has no publish date, or a recent one) can +# be ingested as new again — the price of the tombstone table not growing +# forever. Documented in docs/install.md. +TOMBSTONE_TTL_DAYS = 365 + + +def _expire_tombstones(out_of_time) -> int: + cutoff = timezone.now() - timedelta(days=TOMBSTONE_TTL_DAYS) + removed = 0 + while not out_of_time(): + chunk = list( + PurgedArticle.objects.filter(purged_at__lt=cutoff).values_list("pk", flat=True)[ + :PURGE_CHUNK_SIZE + ] + ) + if not chunk: + break + PurgedArticle.objects.filter(pk__in=chunk).delete() + removed += len(chunk) + return removed + + +@shared_task(time_limit=PURGE_TIME_LIMIT, soft_time_limit=PURGE_SOFT_TIME_LIMIT) def purge_old_articles(): """Deletes articles older than SiteSettings.retention_days (C7), in - chunks so one run never holds a huge delete in a single transaction. - Anything a user starred or saved for later (a system ArticleTag) is - kept — those are explicit "keep this" signals. UserArticleState and - ArticleTag rows go with their article via ON DELETE CASCADE, and the - SQLite FTS triggers drop the search entries. 0 days means keep - everything forever (the default). + chunks so one run never holds a huge delete in a single transaction, + and within a time budget — if it runs out it queues a continuation + (R2-T4). Anything a user starred or saved for later (a system + ArticleTag) is kept — those are explicit "keep this" signals. + UserArticleState and ArticleTag rows go with their article via ON + DELETE CASCADE, and the SQLite FTS triggers drop the search entries. + 0 days means keep everything forever (the default). Each deleted article leaves a PurgedArticle tombstone first, so a feed that still lists the entry can't bring it back as a new, unread - article on its next poll (see apps.feeds.services.fetcher). + article on its next poll (see apps.feeds.services.fetcher). Tombstones + older than TOMBSTONE_TTL_DAYS are removed here too, whether or not + retention is currently on (R2-T2). Returns the number of articles + deleted by this run. """ + started = time.monotonic() + + def out_of_time() -> bool: + return time.monotonic() - started > PURGE_RUN_BUDGET_SECONDS + + expired = _expire_tombstones(out_of_time) + if expired: + logger.info("Removed %d expired purge tombstone(s)", expired) + retention_days = SiteSettings.load().retention_days if retention_days <= 0: return 0 @@ -103,3 +151,11 @@ def purge_old_articles(): ) Article.objects.filter(pk__in=[pk for pk, _feed_id, _guid in chunk]).delete() deleted_total += len(chunk) + if out_of_time(): + logger.info( + "Retention purge hit its %ds budget after %d article(s); continuing shortly", + PURGE_RUN_BUDGET_SECONDS, + deleted_total, + ) + purge_old_articles.apply_async(countdown=PURGE_CONTINUATION_DELAY_SECONDS) + return deleted_total diff --git a/apps/articles/test_retention.py b/apps/articles/test_retention.py index 7e079c3..b92dbd0 100644 --- a/apps/articles/test_retention.py +++ b/apps/articles/test_retention.py @@ -89,3 +89,64 @@ def test_purge_leaves_a_tombstone_per_deleted_article(feed): "gone-1", "gone-2", } + + +@pytest.mark.django_db +def test_expired_tombstones_are_removed_even_when_retention_is_off(feed): + from .models import PurgedArticle + from .tasks import TOMBSTONE_TTL_DAYS + + old = PurgedArticle.objects.create(feed=feed, guid="old") + fresh = PurgedArticle.objects.create(feed=feed, guid="fresh") + PurgedArticle.objects.filter(pk=old.pk).update( + purged_at=timezone.now() - timedelta(days=TOMBSTONE_TTL_DAYS + 1) + ) + assert SiteSettings.load().retention_days == 0 + + purge_old_articles() + + assert not PurgedArticle.objects.filter(pk=old.pk).exists() + assert PurgedArticle.objects.filter(pk=fresh.pk).exists() + + +@pytest.mark.django_db +def test_a_run_that_exhausts_its_budget_queues_a_continuation(feed, monkeypatch): + """R2-T4: a huge purge doesn't hold a worker for its whole length — + after the budget it does what it can and queues another run.""" + from unittest.mock import patch + + monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2) + monkeypatch.setattr("apps.articles.tasks.PURGE_RUN_BUDGET_SECONDS", -1) # already over + SiteSettings.objects.create(retention_days=30) + for i in range(5): + _article(feed, f"old-{i}", age_days=60) + + with patch("apps.articles.tasks.purge_old_articles.apply_async") as mock_continue: + deleted = purge_old_articles() + + assert deleted == 2, "one chunk, then it yields" + assert Article.objects.count() == 3 + mock_continue.assert_called_once() + assert mock_continue.call_args.kwargs["countdown"] > 0 + + +@pytest.mark.django_db +def test_continuation_runs_finish_the_job(feed, monkeypatch): + from unittest.mock import patch + + monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2) + SiteSettings.objects.create(retention_days=30) + for i in range(5): + _article(feed, f"old-{i}", age_days=60) + + total = 0 + for run in range(5): + # Budget runs out after every chunk, as it would on a huge archive. + monkeypatch.setattr("apps.articles.tasks.PURGE_RUN_BUDGET_SECONDS", -1) + with patch("apps.articles.tasks.purge_old_articles.apply_async") as mock_continue: + total += purge_old_articles() + if not mock_continue.called: + break + + assert total == 5 + assert Article.objects.count() == 0