Private
Public Access
Stop retention from bringing deleted articles back (#29)
The nightly purge deleted articles, but a feed that still listed an entry would re-insert it on its next poll as a brand-new, unread article. For feeds with long archives, old items could reappear after every retention run. The purge now leaves a (feed, guid) tombstone per deleted article, and ingestion skips tombstoned guids. With retention on, entries whose own publish date is already outside the window are not ingested at all, so a long archive isn't fetched only to be purged and tombstoned the same night. Undated entries rely on the tombstone. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
73dbee9400
commit
f8cd1eb33e
6 files changed
+156
-5
No files matched your search
@@ -11,7 +11,7 @@ import hashlib
|
||||
import logging
|
||||
from calendar import timegm
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import timezone as dt_timezone
|
||||
|
||||
import feedparser
|
||||
@@ -21,7 +21,8 @@ from django.conf import settings
|
||||
from django.db import IntegrityError, transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.articles.models import Article
|
||||
from apps.articles.models import Article, PurgedArticle
|
||||
from apps.core.models import SiteSettings
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
from .ssrf_guard import SSRFBlocked, safe_get
|
||||
@@ -263,6 +264,24 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
||||
"content_hash": _content_hash(title, link, author, summary, content_html, published_at),
|
||||
}
|
||||
|
||||
# Entries the retention purge already deleted (tombstoned) and, when
|
||||
# retention is on, entries so old by their own publish date that the
|
||||
# next purge would delete them again, are not (re-)ingested — see
|
||||
# apps.articles.tasks.purge_old_articles.
|
||||
retention_days = SiteSettings.load().retention_days
|
||||
if retention_days > 0:
|
||||
too_old_before = timezone.now() - timedelta(days=retention_days)
|
||||
for guid in [
|
||||
g
|
||||
for g, f in parsed_entries.items()
|
||||
if f["published_at"] is not None and f["published_at"] < too_old_before
|
||||
]:
|
||||
del parsed_entries[guid]
|
||||
for guid in PurgedArticle.objects.filter(feed=feed, guid__in=parsed_entries).values_list(
|
||||
"guid", flat=True
|
||||
):
|
||||
del parsed_entries[guid]
|
||||
|
||||
existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)}
|
||||
|
||||
to_create = [
|
||||
|
||||
@@ -338,3 +338,64 @@ def test_ingest_falls_back_to_row_by_row_when_a_concurrent_insert_wins(feed):
|
||||
|
||||
assert len(result.new_article_ids) == 3
|
||||
assert Article.objects.filter(feed=feed).count() == 3
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_purged_article_is_not_resurrected_by_a_feed_that_still_lists_it(feed):
|
||||
"""#29: a feed still listing an entry the retention purge deleted must
|
||||
not bring it back as a brand-new, unread article."""
|
||||
from apps.articles.tasks import purge_old_articles
|
||||
from apps.core.models import SiteSettings
|
||||
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
|
||||
fetch_and_ingest(feed)
|
||||
Article.objects.filter(feed=feed, guid="guid-0").update(
|
||||
fetched_at=timezone.now() - dt.timedelta(days=90)
|
||||
)
|
||||
assert purge_old_articles() == 1
|
||||
assert not Article.objects.filter(feed=feed, guid="guid-0").exists()
|
||||
|
||||
result = fetch_and_ingest(feed)
|
||||
|
||||
assert result.new_article_ids == []
|
||||
assert not Article.objects.filter(feed=feed, guid="guid-0").exists()
|
||||
assert Article.objects.filter(feed=feed).count() == 2
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_entries_older_than_the_retention_window_are_not_ingested(feed):
|
||||
"""With retention on, a feed's long archive isn't ingested only to be
|
||||
purged (and tombstoned) the same night."""
|
||||
from apps.core.models import SiteSettings
|
||||
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
xml = (
|
||||
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
|
||||
"<item><title>Ancient</title><guid>old</guid>"
|
||||
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate></item>"
|
||||
"<item><title>Undated</title><guid>undated</guid></item>"
|
||||
"</channel></rss>"
|
||||
)
|
||||
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
||||
|
||||
result = fetch_and_ingest(feed)
|
||||
|
||||
assert len(result.new_article_ids) == 1
|
||||
assert Article.objects.filter(feed=feed).get().guid == "undated"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_old_entries_are_ingested_normally_when_retention_is_off(feed):
|
||||
xml = (
|
||||
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
|
||||
"<item><title>Ancient</title><guid>old</guid>"
|
||||
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate></item>"
|
||||
"</channel></rss>"
|
||||
)
|
||||
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
||||
|
||||
assert len(fetch_and_ingest(feed).new_article_ids) == 1
|
||||
Reference in new issue
Block a user