Files
ksmithandClaude Sonnet 5.5 72bffbe97f Expire purge tombstones and time-box the retention purge (R2-T2, R2-T4; #38, #40)
Tombstones were kept forever, so enabling retention left a permanent row
per deleted article. They now expire after 365 days (cleaned up by the
same nightly task, even when retention is off); the documented price is
that an entry a feed still lists after a year, with no or a recent
publish date, can reappear.

The purge looped until every eligible row was gone with no run budget.
A run now works for at most four minutes, then queues a continuation, so
a huge first purge can't monopolise a worker or starve polling; hard
time limits back that up, and each chunk is its own transaction so an
interrupted run just resumes. Adds indexes on Article.fetched_at and
PurgedArticle.purged_at for the scans these do.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-01 12:00:52 -07:00

153 lines
5.1 KiB
Python

from datetime import timedelta
import pytest
from django.contrib.auth import get_user_model
from django.utils import timezone
from apps.core.models import SiteSettings
from apps.feeds.models import Feed
from .models import STARRED_TAG_NAME, Article, ArticleTag, UserArticleState, get_system_tag
from .tasks import purge_old_articles
User = get_user_model()
def _article(feed, guid, age_days):
article = Article.objects.create(feed=feed, guid=guid, title=guid)
Article.objects.filter(pk=article.pk).update(
fetched_at=timezone.now() - timedelta(days=age_days)
)
return article
@pytest.fixture
def feed(db):
return Feed.objects.create(feed_url="https://example.com/feed.xml")
@pytest.mark.django_db
def test_purge_is_a_noop_with_the_default_keep_forever_setting(feed):
_article(feed, "ancient", age_days=5000)
assert purge_old_articles() == 0
assert Article.objects.count() == 1
@pytest.mark.django_db
def test_purge_deletes_only_articles_older_than_the_retention_window(feed):
SiteSettings.objects.create(retention_days=30)
old = _article(feed, "old", age_days=45)
recent = _article(feed, "recent", age_days=5)
assert purge_old_articles() == 1
assert not Article.objects.filter(pk=old.pk).exists()
assert Article.objects.filter(pk=recent.pk).exists()
@pytest.mark.django_db
def test_purge_keeps_starred_articles_and_cascades_user_state(feed):
SiteSettings.objects.create(retention_days=30)
user = User.objects.create_user("alice", password="pw-alice-123")
starred = _article(feed, "starred-old", age_days=90)
ArticleTag.objects.create(
user=user, article=starred, tag=get_system_tag(user, STARRED_TAG_NAME)
)
doomed = _article(feed, "plain-old", age_days=90)
state = UserArticleState.objects.create(user=user, article=doomed, is_read=True)
purge_old_articles()
assert Article.objects.filter(pk=starred.pk).exists()
assert not Article.objects.filter(pk=doomed.pk).exists()
assert not UserArticleState.objects.filter(pk=state.pk).exists()
@pytest.mark.django_db
def test_purge_works_through_more_than_one_chunk(feed, monkeypatch):
monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2)
SiteSettings.objects.create(retention_days=30)
for i in range(5):
_article(feed, f"old-{i}", age_days=60)
assert purge_old_articles() == 5
assert Article.objects.count() == 0
@pytest.mark.django_db
def test_purge_leaves_a_tombstone_per_deleted_article(feed):
from .models import PurgedArticle
SiteSettings.objects.create(retention_days=30)
_article(feed, "gone-1", age_days=60)
_article(feed, "gone-2", age_days=60)
_article(feed, "kept", age_days=1)
purge_old_articles()
assert set(PurgedArticle.objects.filter(feed=feed).values_list("guid", flat=True)) == {
"gone-1",
"gone-2",
}
@pytest.mark.django_db
def test_expired_tombstones_are_removed_even_when_retention_is_off(feed):
from .models import PurgedArticle
from .tasks import TOMBSTONE_TTL_DAYS
old = PurgedArticle.objects.create(feed=feed, guid="old")
fresh = PurgedArticle.objects.create(feed=feed, guid="fresh")
PurgedArticle.objects.filter(pk=old.pk).update(
purged_at=timezone.now() - timedelta(days=TOMBSTONE_TTL_DAYS + 1)
)
assert SiteSettings.load().retention_days == 0
purge_old_articles()
assert not PurgedArticle.objects.filter(pk=old.pk).exists()
assert PurgedArticle.objects.filter(pk=fresh.pk).exists()
@pytest.mark.django_db
def test_a_run_that_exhausts_its_budget_queues_a_continuation(feed, monkeypatch):
"""R2-T4: a huge purge doesn't hold a worker for its whole length —
after the budget it does what it can and queues another run."""
from unittest.mock import patch
monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2)
monkeypatch.setattr("apps.articles.tasks.PURGE_RUN_BUDGET_SECONDS", -1) # already over
SiteSettings.objects.create(retention_days=30)
for i in range(5):
_article(feed, f"old-{i}", age_days=60)
with patch("apps.articles.tasks.purge_old_articles.apply_async") as mock_continue:
deleted = purge_old_articles()
assert deleted == 2, "one chunk, then it yields"
assert Article.objects.count() == 3
mock_continue.assert_called_once()
assert mock_continue.call_args.kwargs["countdown"] > 0
@pytest.mark.django_db
def test_continuation_runs_finish_the_job(feed, monkeypatch):
from unittest.mock import patch
monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2)
SiteSettings.objects.create(retention_days=30)
for i in range(5):
_article(feed, f"old-{i}", age_days=60)
total = 0
for run in range(5):
# Budget runs out after every chunk, as it would on a huge archive.
monkeypatch.setattr("apps.articles.tasks.PURGE_RUN_BUDGET_SECONDS", -1)
with patch("apps.articles.tasks.purge_old_articles.apply_async") as mock_continue:
total += purge_old_articles()
if not mock_continue.called:
break
assert total == 5
assert Article.objects.count() == 0