Private
Public Access
Tombstones were kept forever, so enabling retention left a permanent row per deleted article. They now expire after 365 days (cleaned up by the same nightly task, even when retention is off); the documented price is that an entry a feed still lists after a year, with no or a recent publish date, can reappear. The purge looped until every eligible row was gone with no run budget. A run now works for at most four minutes, then queues a continuation, so a huge first purge can't monopolise a worker or starve polling; hard time limits back that up, and each chunk is its own transaction so an interrupted run just resumes. Adds indexes on Article.fetched_at and PurgedArticle.purged_at for the scans these do. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2e1c0e31a8
commit
72bffbe97f
4 files changed
+153
-9
No files matched your search
@@ -89,3 +89,64 @@ def test_purge_leaves_a_tombstone_per_deleted_article(feed):
|
||||
"gone-1",
|
||||
"gone-2",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_expired_tombstones_are_removed_even_when_retention_is_off(feed):
|
||||
from .models import PurgedArticle
|
||||
from .tasks import TOMBSTONE_TTL_DAYS
|
||||
|
||||
old = PurgedArticle.objects.create(feed=feed, guid="old")
|
||||
fresh = PurgedArticle.objects.create(feed=feed, guid="fresh")
|
||||
PurgedArticle.objects.filter(pk=old.pk).update(
|
||||
purged_at=timezone.now() - timedelta(days=TOMBSTONE_TTL_DAYS + 1)
|
||||
)
|
||||
assert SiteSettings.load().retention_days == 0
|
||||
|
||||
purge_old_articles()
|
||||
|
||||
assert not PurgedArticle.objects.filter(pk=old.pk).exists()
|
||||
assert PurgedArticle.objects.filter(pk=fresh.pk).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_a_run_that_exhausts_its_budget_queues_a_continuation(feed, monkeypatch):
|
||||
"""R2-T4: a huge purge doesn't hold a worker for its whole length —
|
||||
after the budget it does what it can and queues another run."""
|
||||
from unittest.mock import patch
|
||||
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2)
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_RUN_BUDGET_SECONDS", -1) # already over
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
for i in range(5):
|
||||
_article(feed, f"old-{i}", age_days=60)
|
||||
|
||||
with patch("apps.articles.tasks.purge_old_articles.apply_async") as mock_continue:
|
||||
deleted = purge_old_articles()
|
||||
|
||||
assert deleted == 2, "one chunk, then it yields"
|
||||
assert Article.objects.count() == 3
|
||||
mock_continue.assert_called_once()
|
||||
assert mock_continue.call_args.kwargs["countdown"] > 0
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_continuation_runs_finish_the_job(feed, monkeypatch):
|
||||
from unittest.mock import patch
|
||||
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2)
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
for i in range(5):
|
||||
_article(feed, f"old-{i}", age_days=60)
|
||||
|
||||
total = 0
|
||||
for run in range(5):
|
||||
# Budget runs out after every chunk, as it would on a huge archive.
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_RUN_BUDGET_SECONDS", -1)
|
||||
with patch("apps.articles.tasks.purge_old_articles.apply_async") as mock_continue:
|
||||
total += purge_old_articles()
|
||||
if not mock_continue.called:
|
||||
break
|
||||
|
||||
assert total == 5
|
||||
assert Article.objects.count() == 0
|
||||
Reference in new issue
Block a user