Private
Public Access
Stop retention from bringing deleted articles back (#29)
The nightly purge deleted articles, but a feed that still listed an entry would re-insert it on its next poll as a brand-new, unread article. For feeds with long archives, old items could reappear after every retention run. The purge now leaves a (feed, guid) tombstone per deleted article, and ingestion skips tombstoned guids. With retention on, entries whose own publish date is already outside the window are not ingested at all, so a long archive isn't fetched only to be purged and tombstoned the same night. Undated entries rely on the tombstone. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
73dbee9400
commit
f8cd1eb33e
6 files changed
+156
-5
No files matched your search
@@ -0,0 +1,27 @@
|
|||||||
|
# Generated by Django 5.2.18 on 2026-11-01 16:28
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('articles', '0004_starred_saved_as_system_tags'),
|
||||||
|
('feeds', '0006_backfill_feed_next_fetch_at'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name='PurgedArticle',
|
||||||
|
fields=[
|
||||||
|
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||||
|
('guid', models.CharField(max_length=1000)),
|
||||||
|
('purged_at', models.DateTimeField(auto_now_add=True)),
|
||||||
|
('feed', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='purged_articles', to='feeds.feed')),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
'unique_together': {('feed', 'guid')},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
]
|
||||||
@@ -32,6 +32,23 @@ class Article(models.Model):
|
|||||||
return self.title or self.guid
|
return self.title or self.guid
|
||||||
|
|
||||||
|
|
||||||
|
class PurgedArticle(models.Model):
|
||||||
|
"""Tombstone left by the retention purge (apps.articles.tasks.
|
||||||
|
purge_old_articles) for every article it deletes: a feed that still
|
||||||
|
lists an entry after we purged it would otherwise be re-inserted as a
|
||||||
|
brand-new, unread article on its next poll. Ingestion skips any guid
|
||||||
|
that has a tombstone here. Only the (feed, guid) pair is kept — a
|
||||||
|
small fraction of the article it replaces.
|
||||||
|
"""
|
||||||
|
|
||||||
|
feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="purged_articles")
|
||||||
|
guid = models.CharField(max_length=1000)
|
||||||
|
purged_at = models.DateTimeField(auto_now_add=True)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
unique_together = [("feed", "guid")]
|
||||||
|
|
||||||
|
|
||||||
class UserArticleState(models.Model):
|
class UserArticleState(models.Model):
|
||||||
"""Per-user read/hide state for an Article.
|
"""Per-user read/hide state for an Article.
|
||||||
|
|
||||||
|
|||||||
+13
-3
@@ -1,6 +1,7 @@
|
|||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
|
||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
|
from django.db import transaction
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
from apps.core.models import SiteSettings
|
from apps.core.models import SiteSettings
|
||||||
@@ -8,7 +9,7 @@ from apps.feeds.models import Subscription
|
|||||||
from apps.rules.engine import evaluate_rules_for_article
|
from apps.rules.engine import evaluate_rules_for_article
|
||||||
from apps.rules.models import FilterRule
|
from apps.rules.models import FilterRule
|
||||||
|
|
||||||
from .models import Article, ArticleTag, SubscriptionTag
|
from .models import Article, ArticleTag, PurgedArticle, SubscriptionTag
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
@@ -75,6 +76,10 @@ def purge_old_articles():
|
|||||||
ArticleTag rows go with their article via ON DELETE CASCADE, and the
|
ArticleTag rows go with their article via ON DELETE CASCADE, and the
|
||||||
SQLite FTS triggers drop the search entries. 0 days means keep
|
SQLite FTS triggers drop the search entries. 0 days means keep
|
||||||
everything forever (the default).
|
everything forever (the default).
|
||||||
|
|
||||||
|
Each deleted article leaves a PurgedArticle tombstone first, so a feed
|
||||||
|
that still lists the entry can't bring it back as a new, unread
|
||||||
|
article on its next poll (see apps.feeds.services.fetcher).
|
||||||
"""
|
"""
|
||||||
retention_days = SiteSettings.load().retention_days
|
retention_days = SiteSettings.load().retention_days
|
||||||
if retention_days <= 0:
|
if retention_days <= 0:
|
||||||
@@ -87,9 +92,14 @@ def purge_old_articles():
|
|||||||
chunk = list(
|
chunk = list(
|
||||||
Article.objects.filter(fetched_at__lt=cutoff)
|
Article.objects.filter(fetched_at__lt=cutoff)
|
||||||
.exclude(pk__in=keep)
|
.exclude(pk__in=keep)
|
||||||
.values_list("pk", flat=True)[:PURGE_CHUNK_SIZE]
|
.values_list("pk", "feed_id", "guid")[:PURGE_CHUNK_SIZE]
|
||||||
)
|
)
|
||||||
if not chunk:
|
if not chunk:
|
||||||
return deleted_total
|
return deleted_total
|
||||||
Article.objects.filter(pk__in=chunk).delete()
|
with transaction.atomic():
|
||||||
|
PurgedArticle.objects.bulk_create(
|
||||||
|
[PurgedArticle(feed_id=feed_id, guid=guid) for _pk, feed_id, guid in chunk],
|
||||||
|
ignore_conflicts=True,
|
||||||
|
)
|
||||||
|
Article.objects.filter(pk__in=[pk for pk, _feed_id, _guid in chunk]).delete()
|
||||||
deleted_total += len(chunk)
|
deleted_total += len(chunk)
|
||||||
@@ -72,3 +72,20 @@ def test_purge_works_through_more_than_one_chunk(feed, monkeypatch):
|
|||||||
|
|
||||||
assert purge_old_articles() == 5
|
assert purge_old_articles() == 5
|
||||||
assert Article.objects.count() == 0
|
assert Article.objects.count() == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_purge_leaves_a_tombstone_per_deleted_article(feed):
|
||||||
|
from .models import PurgedArticle
|
||||||
|
|
||||||
|
SiteSettings.objects.create(retention_days=30)
|
||||||
|
_article(feed, "gone-1", age_days=60)
|
||||||
|
_article(feed, "gone-2", age_days=60)
|
||||||
|
_article(feed, "kept", age_days=1)
|
||||||
|
|
||||||
|
purge_old_articles()
|
||||||
|
|
||||||
|
assert set(PurgedArticle.objects.filter(feed=feed).values_list("guid", flat=True)) == {
|
||||||
|
"gone-1",
|
||||||
|
"gone-2",
|
||||||
|
}
|
||||||
@@ -11,7 +11,7 @@ import hashlib
|
|||||||
import logging
|
import logging
|
||||||
from calendar import timegm
|
from calendar import timegm
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime, timedelta
|
||||||
from datetime import timezone as dt_timezone
|
from datetime import timezone as dt_timezone
|
||||||
|
|
||||||
import feedparser
|
import feedparser
|
||||||
@@ -21,7 +21,8 @@ from django.conf import settings
|
|||||||
from django.db import IntegrityError, transaction
|
from django.db import IntegrityError, transaction
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
from apps.articles.models import Article
|
from apps.articles.models import Article, PurgedArticle
|
||||||
|
from apps.core.models import SiteSettings
|
||||||
from apps.feeds.models import Feed
|
from apps.feeds.models import Feed
|
||||||
|
|
||||||
from .ssrf_guard import SSRFBlocked, safe_get
|
from .ssrf_guard import SSRFBlocked, safe_get
|
||||||
@@ -263,6 +264,24 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
|||||||
"content_hash": _content_hash(title, link, author, summary, content_html, published_at),
|
"content_hash": _content_hash(title, link, author, summary, content_html, published_at),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Entries the retention purge already deleted (tombstoned) and, when
|
||||||
|
# retention is on, entries so old by their own publish date that the
|
||||||
|
# next purge would delete them again, are not (re-)ingested — see
|
||||||
|
# apps.articles.tasks.purge_old_articles.
|
||||||
|
retention_days = SiteSettings.load().retention_days
|
||||||
|
if retention_days > 0:
|
||||||
|
too_old_before = timezone.now() - timedelta(days=retention_days)
|
||||||
|
for guid in [
|
||||||
|
g
|
||||||
|
for g, f in parsed_entries.items()
|
||||||
|
if f["published_at"] is not None and f["published_at"] < too_old_before
|
||||||
|
]:
|
||||||
|
del parsed_entries[guid]
|
||||||
|
for guid in PurgedArticle.objects.filter(feed=feed, guid__in=parsed_entries).values_list(
|
||||||
|
"guid", flat=True
|
||||||
|
):
|
||||||
|
del parsed_entries[guid]
|
||||||
|
|
||||||
existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)}
|
existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)}
|
||||||
|
|
||||||
to_create = [
|
to_create = [
|
||||||
|
|||||||
@@ -338,3 +338,64 @@ def test_ingest_falls_back_to_row_by_row_when_a_concurrent_insert_wins(feed):
|
|||||||
|
|
||||||
assert len(result.new_article_ids) == 3
|
assert len(result.new_article_ids) == 3
|
||||||
assert Article.objects.filter(feed=feed).count() == 3
|
assert Article.objects.filter(feed=feed).count() == 3
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
@responses.activate
|
||||||
|
def test_purged_article_is_not_resurrected_by_a_feed_that_still_lists_it(feed):
|
||||||
|
"""#29: a feed still listing an entry the retention purge deleted must
|
||||||
|
not bring it back as a brand-new, unread article."""
|
||||||
|
from apps.articles.tasks import purge_old_articles
|
||||||
|
from apps.core.models import SiteSettings
|
||||||
|
|
||||||
|
SiteSettings.objects.create(retention_days=30)
|
||||||
|
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
|
||||||
|
fetch_and_ingest(feed)
|
||||||
|
Article.objects.filter(feed=feed, guid="guid-0").update(
|
||||||
|
fetched_at=timezone.now() - dt.timedelta(days=90)
|
||||||
|
)
|
||||||
|
assert purge_old_articles() == 1
|
||||||
|
assert not Article.objects.filter(feed=feed, guid="guid-0").exists()
|
||||||
|
|
||||||
|
result = fetch_and_ingest(feed)
|
||||||
|
|
||||||
|
assert result.new_article_ids == []
|
||||||
|
assert not Article.objects.filter(feed=feed, guid="guid-0").exists()
|
||||||
|
assert Article.objects.filter(feed=feed).count() == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
@responses.activate
|
||||||
|
def test_entries_older_than_the_retention_window_are_not_ingested(feed):
|
||||||
|
"""With retention on, a feed's long archive isn't ingested only to be
|
||||||
|
purged (and tombstoned) the same night."""
|
||||||
|
from apps.core.models import SiteSettings
|
||||||
|
|
||||||
|
SiteSettings.objects.create(retention_days=30)
|
||||||
|
xml = (
|
||||||
|
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
|
||||||
|
"<item><title>Ancient</title><guid>old</guid>"
|
||||||
|
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate></item>"
|
||||||
|
"<item><title>Undated</title><guid>undated</guid></item>"
|
||||||
|
"</channel></rss>"
|
||||||
|
)
|
||||||
|
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
||||||
|
|
||||||
|
result = fetch_and_ingest(feed)
|
||||||
|
|
||||||
|
assert len(result.new_article_ids) == 1
|
||||||
|
assert Article.objects.filter(feed=feed).get().guid == "undated"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
@responses.activate
|
||||||
|
def test_old_entries_are_ingested_normally_when_retention_is_off(feed):
|
||||||
|
xml = (
|
||||||
|
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
|
||||||
|
"<item><title>Ancient</title><guid>old</guid>"
|
||||||
|
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate></item>"
|
||||||
|
"</channel></rss>"
|
||||||
|
)
|
||||||
|
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
||||||
|
|
||||||
|
assert len(fetch_and_ingest(feed).new_article_ids) == 1
|
||||||
Reference in new issue
Block a user