From f8cd1eb33e4a5bc5ee6dae651c048e98fff232b6 Mon Sep 17 00:00:00 2001 From: Keith Smith Date: Sun, 1 Nov 2026 09:30:00 -0700 Subject: [PATCH] Stop retention from bringing deleted articles back (#29) The nightly purge deleted articles, but a feed that still listed an entry would re-insert it on its next poll as a brand-new, unread article. For feeds with long archives, old items could reappear after every retention run. The purge now leaves a (feed, guid) tombstone per deleted article, and ingestion skips tombstoned guids. With retention on, entries whose own publish date is already outside the window are not ingested at all, so a long archive isn't fetched only to be purged and tombstoned the same night. Undated entries rely on the tombstone. Co-Authored-By: Claude Sonnet 5.5 --- .../articles/migrations/0005_purgedarticle.py | 27 ++++++++ apps/articles/models.py | 17 ++++++ apps/articles/tasks.py | 16 ++++- apps/articles/test_retention.py | 17 ++++++ apps/feeds/services/fetcher.py | 23 ++++++- apps/feeds/test_fetcher.py | 61 +++++++++++++++++++ 6 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 apps/articles/migrations/0005_purgedarticle.py diff --git a/apps/articles/migrations/0005_purgedarticle.py b/apps/articles/migrations/0005_purgedarticle.py new file mode 100644 index 0000000..c23c45c --- /dev/null +++ b/apps/articles/migrations/0005_purgedarticle.py @@ -0,0 +1,27 @@ +# Generated by Django 5.2.18 on 2026-11-01 16:28 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('articles', '0004_starred_saved_as_system_tags'), + ('feeds', '0006_backfill_feed_next_fetch_at'), + ] + + operations = [ + migrations.CreateModel( + name='PurgedArticle', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('guid', models.CharField(max_length=1000)), + ('purged_at', models.DateTimeField(auto_now_add=True)), + ('feed', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='purged_articles', to='feeds.feed')), + ], + options={ + 'unique_together': {('feed', 'guid')}, + }, + ), + ] diff --git a/apps/articles/models.py b/apps/articles/models.py index 9bb3b27..18f1190 100644 --- a/apps/articles/models.py +++ b/apps/articles/models.py @@ -32,6 +32,23 @@ class Article(models.Model): return self.title or self.guid +class PurgedArticle(models.Model): + """Tombstone left by the retention purge (apps.articles.tasks. + purge_old_articles) for every article it deletes: a feed that still + lists an entry after we purged it would otherwise be re-inserted as a + brand-new, unread article on its next poll. Ingestion skips any guid + that has a tombstone here. Only the (feed, guid) pair is kept — a + small fraction of the article it replaces. + """ + + feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="purged_articles") + guid = models.CharField(max_length=1000) + purged_at = models.DateTimeField(auto_now_add=True) + + class Meta: + unique_together = [("feed", "guid")] + + class UserArticleState(models.Model): """Per-user read/hide state for an Article. diff --git a/apps/articles/tasks.py b/apps/articles/tasks.py index 8e01ea7..84bbd8f 100644 --- a/apps/articles/tasks.py +++ b/apps/articles/tasks.py @@ -1,6 +1,7 @@ from datetime import timedelta from celery import shared_task +from django.db import transaction from django.utils import timezone from apps.core.models import SiteSettings @@ -8,7 +9,7 @@ from apps.feeds.models import Subscription from apps.rules.engine import evaluate_rules_for_article from apps.rules.models import FilterRule -from .models import Article, ArticleTag, SubscriptionTag +from .models import Article, ArticleTag, PurgedArticle, SubscriptionTag @shared_task @@ -75,6 +76,10 @@ def purge_old_articles(): ArticleTag rows go with their article via ON DELETE CASCADE, and the SQLite FTS triggers drop the search entries. 0 days means keep everything forever (the default). + + Each deleted article leaves a PurgedArticle tombstone first, so a feed + that still lists the entry can't bring it back as a new, unread + article on its next poll (see apps.feeds.services.fetcher). """ retention_days = SiteSettings.load().retention_days if retention_days <= 0: @@ -87,9 +92,14 @@ def purge_old_articles(): chunk = list( Article.objects.filter(fetched_at__lt=cutoff) .exclude(pk__in=keep) - .values_list("pk", flat=True)[:PURGE_CHUNK_SIZE] + .values_list("pk", "feed_id", "guid")[:PURGE_CHUNK_SIZE] ) if not chunk: return deleted_total - Article.objects.filter(pk__in=chunk).delete() + with transaction.atomic(): + PurgedArticle.objects.bulk_create( + [PurgedArticle(feed_id=feed_id, guid=guid) for _pk, feed_id, guid in chunk], + ignore_conflicts=True, + ) + Article.objects.filter(pk__in=[pk for pk, _feed_id, _guid in chunk]).delete() deleted_total += len(chunk) diff --git a/apps/articles/test_retention.py b/apps/articles/test_retention.py index 18fb8c9..7e079c3 100644 --- a/apps/articles/test_retention.py +++ b/apps/articles/test_retention.py @@ -72,3 +72,20 @@ def test_purge_works_through_more_than_one_chunk(feed, monkeypatch): assert purge_old_articles() == 5 assert Article.objects.count() == 0 + + +@pytest.mark.django_db +def test_purge_leaves_a_tombstone_per_deleted_article(feed): + from .models import PurgedArticle + + SiteSettings.objects.create(retention_days=30) + _article(feed, "gone-1", age_days=60) + _article(feed, "gone-2", age_days=60) + _article(feed, "kept", age_days=1) + + purge_old_articles() + + assert set(PurgedArticle.objects.filter(feed=feed).values_list("guid", flat=True)) == { + "gone-1", + "gone-2", + } diff --git a/apps/feeds/services/fetcher.py b/apps/feeds/services/fetcher.py index e8a0b34..9247fb1 100644 --- a/apps/feeds/services/fetcher.py +++ b/apps/feeds/services/fetcher.py @@ -11,7 +11,7 @@ import hashlib import logging from calendar import timegm from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta from datetime import timezone as dt_timezone import feedparser @@ -21,7 +21,8 @@ from django.conf import settings from django.db import IntegrityError, transaction from django.utils import timezone -from apps.articles.models import Article +from apps.articles.models import Article, PurgedArticle +from apps.core.models import SiteSettings from apps.feeds.models import Feed from .ssrf_guard import SSRFBlocked, safe_get @@ -263,6 +264,24 @@ def fetch_and_ingest(feed: Feed) -> FetchResult: "content_hash": _content_hash(title, link, author, summary, content_html, published_at), } + # Entries the retention purge already deleted (tombstoned) and, when + # retention is on, entries so old by their own publish date that the + # next purge would delete them again, are not (re-)ingested — see + # apps.articles.tasks.purge_old_articles. + retention_days = SiteSettings.load().retention_days + if retention_days > 0: + too_old_before = timezone.now() - timedelta(days=retention_days) + for guid in [ + g + for g, f in parsed_entries.items() + if f["published_at"] is not None and f["published_at"] < too_old_before + ]: + del parsed_entries[guid] + for guid in PurgedArticle.objects.filter(feed=feed, guid__in=parsed_entries).values_list( + "guid", flat=True + ): + del parsed_entries[guid] + existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)} to_create = [ diff --git a/apps/feeds/test_fetcher.py b/apps/feeds/test_fetcher.py index 7a5806c..ba59ce0 100644 --- a/apps/feeds/test_fetcher.py +++ b/apps/feeds/test_fetcher.py @@ -338,3 +338,64 @@ def test_ingest_falls_back_to_row_by_row_when_a_concurrent_insert_wins(feed): assert len(result.new_article_ids) == 3 assert Article.objects.filter(feed=feed).count() == 3 + + +@pytest.mark.django_db +@responses.activate +def test_purged_article_is_not_resurrected_by_a_feed_that_still_lists_it(feed): + """#29: a feed still listing an entry the retention purge deleted must + not bring it back as a brand-new, unread article.""" + from apps.articles.tasks import purge_old_articles + from apps.core.models import SiteSettings + + SiteSettings.objects.create(retention_days=30) + responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200) + fetch_and_ingest(feed) + Article.objects.filter(feed=feed, guid="guid-0").update( + fetched_at=timezone.now() - dt.timedelta(days=90) + ) + assert purge_old_articles() == 1 + assert not Article.objects.filter(feed=feed, guid="guid-0").exists() + + result = fetch_and_ingest(feed) + + assert result.new_article_ids == [] + assert not Article.objects.filter(feed=feed, guid="guid-0").exists() + assert Article.objects.filter(feed=feed).count() == 2 + + +@pytest.mark.django_db +@responses.activate +def test_entries_older_than_the_retention_window_are_not_ingested(feed): + """With retention on, a feed's long archive isn't ingested only to be + purged (and tombstoned) the same night.""" + from apps.core.models import SiteSettings + + SiteSettings.objects.create(retention_days=30) + xml = ( + 'T' + "Ancientold" + "Mon, 01 Jan 2024 00:00:00 GMT" + "Undatedundated" + "" + ) + responses.add(responses.GET, FEED_URL, body=xml, status=200) + + result = fetch_and_ingest(feed) + + assert len(result.new_article_ids) == 1 + assert Article.objects.filter(feed=feed).get().guid == "undated" + + +@pytest.mark.django_db +@responses.activate +def test_old_entries_are_ingested_normally_when_retention_is_off(feed): + xml = ( + 'T' + "Ancientold" + "Mon, 01 Jan 2024 00:00:00 GMT" + "" + ) + responses.add(responses.GET, FEED_URL, body=xml, status=200) + + assert len(fetch_and_ingest(feed).new_article_ids) == 1