Stop retention from bringing deleted articles back (#29)

The nightly purge deleted articles, but a feed that still listed an
entry would re-insert it on its next poll as a brand-new, unread
article. For feeds with long archives, old items could reappear after
every retention run.

The purge now leaves a (feed, guid) tombstone per deleted article, and
ingestion skips tombstoned guids. With retention on, entries whose own
publish date is already outside the window are not ingested at all, so
a long archive isn't fetched only to be purged and tombstoned the same
night. Undated entries rely on the tombstone.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5.5 committed 2026-11-01 09:30:00 -07:00
1 parent 73dbee9400
commit f8cd1eb33e
6 files changed
+156 -5

No files matched your search

@@ -0,0 +1,27 @@
# Generated by Django 5.2.18 on 2026-11-01 16:28
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('articles', '0004_starred_saved_as_system_tags'),
('feeds', '0006_backfill_feed_next_fetch_at'),
]
operations = [
migrations.CreateModel(
name='PurgedArticle',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('guid', models.CharField(max_length=1000)),
('purged_at', models.DateTimeField(auto_now_add=True)),
('feed', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='purged_articles', to='feeds.feed')),
],
options={
'unique_together': {('feed', 'guid')},
},
),
]
+17
View File
@@ -32,6 +32,23 @@ class Article(models.Model):
return self.title or self.guid
class PurgedArticle(models.Model):
"""Tombstone left by the retention purge (apps.articles.tasks.
purge_old_articles) for every article it deletes: a feed that still
lists an entry after we purged it would otherwise be re-inserted as a
brand-new, unread article on its next poll. Ingestion skips any guid
that has a tombstone here. Only the (feed, guid) pair is kept — a
small fraction of the article it replaces.
"""
feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="purged_articles")
guid = models.CharField(max_length=1000)
purged_at = models.DateTimeField(auto_now_add=True)
class Meta:
unique_together = [("feed", "guid")]
class UserArticleState(models.Model):
"""Per-user read/hide state for an Article.
+13 -3
View File
@@ -1,6 +1,7 @@
from datetime import timedelta
from celery import shared_task
from django.db import transaction
from django.utils import timezone
from apps.core.models import SiteSettings
@@ -8,7 +9,7 @@ from apps.feeds.models import Subscription
from apps.rules.engine import evaluate_rules_for_article
from apps.rules.models import FilterRule
from .models import Article, ArticleTag, SubscriptionTag
from .models import Article, ArticleTag, PurgedArticle, SubscriptionTag
@shared_task
@@ -75,6 +76,10 @@ def purge_old_articles():
ArticleTag rows go with their article via ON DELETE CASCADE, and the
SQLite FTS triggers drop the search entries. 0 days means keep
everything forever (the default).
Each deleted article leaves a PurgedArticle tombstone first, so a feed
that still lists the entry can't bring it back as a new, unread
article on its next poll (see apps.feeds.services.fetcher).
"""
retention_days = SiteSettings.load().retention_days
if retention_days <= 0:
@@ -87,9 +92,14 @@ def purge_old_articles():
chunk = list(
Article.objects.filter(fetched_at__lt=cutoff)
.exclude(pk__in=keep)
.values_list("pk", flat=True)[:PURGE_CHUNK_SIZE]
.values_list("pk", "feed_id", "guid")[:PURGE_CHUNK_SIZE]
)
if not chunk:
return deleted_total
Article.objects.filter(pk__in=chunk).delete()
with transaction.atomic():
PurgedArticle.objects.bulk_create(
[PurgedArticle(feed_id=feed_id, guid=guid) for _pk, feed_id, guid in chunk],
ignore_conflicts=True,
)
Article.objects.filter(pk__in=[pk for pk, _feed_id, _guid in chunk]).delete()
deleted_total += len(chunk)
+17
View File
@@ -72,3 +72,20 @@ def test_purge_works_through_more_than_one_chunk(feed, monkeypatch):
assert purge_old_articles() == 5
assert Article.objects.count() == 0
@pytest.mark.django_db
def test_purge_leaves_a_tombstone_per_deleted_article(feed):
from .models import PurgedArticle
SiteSettings.objects.create(retention_days=30)
_article(feed, "gone-1", age_days=60)
_article(feed, "gone-2", age_days=60)
_article(feed, "kept", age_days=1)
purge_old_articles()
assert set(PurgedArticle.objects.filter(feed=feed).values_list("guid", flat=True)) == {
"gone-1",
"gone-2",
}
+21 -2
View File
@@ -11,7 +11,7 @@ import hashlib
import logging
from calendar import timegm
from dataclasses import dataclass
from datetime import datetime
from datetime import datetime, timedelta
from datetime import timezone as dt_timezone
import feedparser
@@ -21,7 +21,8 @@ from django.conf import settings
from django.db import IntegrityError, transaction
from django.utils import timezone
from apps.articles.models import Article
from apps.articles.models import Article, PurgedArticle
from apps.core.models import SiteSettings
from apps.feeds.models import Feed
from .ssrf_guard import SSRFBlocked, safe_get
@@ -263,6 +264,24 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
"content_hash": _content_hash(title, link, author, summary, content_html, published_at),
}
# Entries the retention purge already deleted (tombstoned) and, when
# retention is on, entries so old by their own publish date that the
# next purge would delete them again, are not (re-)ingested — see
# apps.articles.tasks.purge_old_articles.
retention_days = SiteSettings.load().retention_days
if retention_days > 0:
too_old_before = timezone.now() - timedelta(days=retention_days)
for guid in [
g
for g, f in parsed_entries.items()
if f["published_at"] is not None and f["published_at"] < too_old_before
]:
del parsed_entries[guid]
for guid in PurgedArticle.objects.filter(feed=feed, guid__in=parsed_entries).values_list(
"guid", flat=True
):
del parsed_entries[guid]
existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)}
to_create = [
+61
View File
@@ -338,3 +338,64 @@ def test_ingest_falls_back_to_row_by_row_when_a_concurrent_insert_wins(feed):
assert len(result.new_article_ids) == 3
assert Article.objects.filter(feed=feed).count() == 3
@pytest.mark.django_db
@responses.activate
def test_purged_article_is_not_resurrected_by_a_feed_that_still_lists_it(feed):
"""#29: a feed still listing an entry the retention purge deleted must
not bring it back as a brand-new, unread article."""
from apps.articles.tasks import purge_old_articles
from apps.core.models import SiteSettings
SiteSettings.objects.create(retention_days=30)
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
fetch_and_ingest(feed)
Article.objects.filter(feed=feed, guid="guid-0").update(
fetched_at=timezone.now() - dt.timedelta(days=90)
)
assert purge_old_articles() == 1
assert not Article.objects.filter(feed=feed, guid="guid-0").exists()
result = fetch_and_ingest(feed)
assert result.new_article_ids == []
assert not Article.objects.filter(feed=feed, guid="guid-0").exists()
assert Article.objects.filter(feed=feed).count() == 2
@pytest.mark.django_db
@responses.activate
def test_entries_older_than_the_retention_window_are_not_ingested(feed):
"""With retention on, a feed's long archive isn't ingested only to be
purged (and tombstoned) the same night."""
from apps.core.models import SiteSettings
SiteSettings.objects.create(retention_days=30)
xml = (
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
"<item><title>Ancient</title><guid>old</guid>"
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate></item>"
"<item><title>Undated</title><guid>undated</guid></item>"
"</channel></rss>"
)
responses.add(responses.GET, FEED_URL, body=xml, status=200)
result = fetch_and_ingest(feed)
assert len(result.new_article_ids) == 1
assert Article.objects.filter(feed=feed).get().guid == "undated"
@pytest.mark.django_db
@responses.activate
def test_old_entries_are_ingested_normally_when_retention_is_off(feed):
xml = (
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
"<item><title>Ancient</title><guid>old</guid>"
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate></item>"
"</channel></rss>"
)
responses.add(responses.GET, FEED_URL, body=xml, status=200)
assert len(fetch_and_ingest(feed).new_article_ids) == 1