Private
Public Access
Tombstones were kept forever, so enabling retention left a permanent row per deleted article. They now expire after 365 days (cleaned up by the same nightly task, even when retention is off); the documented price is that an entry a feed still lists after a year, with no or a recent publish date, can reappear. The purge looped until every eligible row was gone with no run budget. A run now works for at most four minutes, then queues a continuation, so a huge first purge can't monopolise a worker or starve polling; hard time limits back that up, and each chunk is its own transaction so an interrupted run just resumes. Adds indexes on Article.fetched_at and PurgedArticle.purged_at for the scans these do. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2e1c0e31a8
commit
72bffbe97f
4 files changed
+153
-9
No files matched your search
@@ -0,0 +1,22 @@
|
||||
# Generated by Django 5.2.18 on 2026-11-01 19:00
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('articles', '0005_purgedarticle'),
|
||||
('feeds', '0006_backfill_feed_next_fetch_at'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddIndex(
|
||||
model_name='article',
|
||||
index=models.Index(fields=['fetched_at'], name='articles_ar_fetched_eb5501_idx'),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name='purgedarticle',
|
||||
index=models.Index(fields=['purged_at'], name='articles_pu_purged__27d417_idx'),
|
||||
),
|
||||
]
|
||||
@@ -25,7 +25,11 @@ class Article(models.Model):
|
||||
|
||||
class Meta:
|
||||
unique_together = [("feed", "guid")]
|
||||
indexes = [models.Index(fields=["feed", "published_at"])]
|
||||
indexes = [
|
||||
models.Index(fields=["feed", "published_at"]),
|
||||
# The retention purge scans for the oldest articles by this.
|
||||
models.Index(fields=["fetched_at"]),
|
||||
]
|
||||
ordering = ["-published_at", "-fetched_at"]
|
||||
|
||||
def __str__(self):
|
||||
@@ -47,6 +51,7 @@ class PurgedArticle(models.Model):
|
||||
|
||||
class Meta:
|
||||
unique_together = [("feed", "guid")]
|
||||
indexes = [models.Index(fields=["purged_at"])]
|
||||
|
||||
|
||||
class UserArticleState(models.Model):
|
||||
|
||||
+64
-8
@@ -1,3 +1,5 @@
|
||||
import logging
|
||||
import time
|
||||
from datetime import timedelta
|
||||
|
||||
from celery import shared_task
|
||||
@@ -11,6 +13,8 @@ from apps.rules.models import FilterRule
|
||||
|
||||
from .models import Article, ArticleTag, PurgedArticle, SubscriptionTag
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@shared_task
|
||||
def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
||||
@@ -66,21 +70,65 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
||||
|
||||
PURGE_CHUNK_SIZE = 500
|
||||
|
||||
# One run works for at most this long, then queues itself again (R2-T4) so
|
||||
# a huge first purge never holds a worker for hours or starves feed
|
||||
# polling; the hard limits below are the backstop for a single stuck chunk.
|
||||
PURGE_RUN_BUDGET_SECONDS = 240
|
||||
PURGE_CONTINUATION_DELAY_SECONDS = 30
|
||||
PURGE_SOFT_TIME_LIMIT = 540
|
||||
PURGE_TIME_LIMIT = 600
|
||||
|
||||
@shared_task
|
||||
# How long a purged article's tombstone is kept (R2-T2). It only needs to
|
||||
# outlive the time a feed keeps *listing* the entry; after this, an entry a
|
||||
# feed still publishes (and that has no publish date, or a recent one) can
|
||||
# be ingested as new again — the price of the tombstone table not growing
|
||||
# forever. Documented in docs/install.md.
|
||||
TOMBSTONE_TTL_DAYS = 365
|
||||
|
||||
|
||||
def _expire_tombstones(out_of_time) -> int:
|
||||
cutoff = timezone.now() - timedelta(days=TOMBSTONE_TTL_DAYS)
|
||||
removed = 0
|
||||
while not out_of_time():
|
||||
chunk = list(
|
||||
PurgedArticle.objects.filter(purged_at__lt=cutoff).values_list("pk", flat=True)[
|
||||
:PURGE_CHUNK_SIZE
|
||||
]
|
||||
)
|
||||
if not chunk:
|
||||
break
|
||||
PurgedArticle.objects.filter(pk__in=chunk).delete()
|
||||
removed += len(chunk)
|
||||
return removed
|
||||
|
||||
|
||||
@shared_task(time_limit=PURGE_TIME_LIMIT, soft_time_limit=PURGE_SOFT_TIME_LIMIT)
|
||||
def purge_old_articles():
|
||||
"""Deletes articles older than SiteSettings.retention_days (C7), in
|
||||
chunks so one run never holds a huge delete in a single transaction.
|
||||
Anything a user starred or saved for later (a system ArticleTag) is
|
||||
kept — those are explicit "keep this" signals. UserArticleState and
|
||||
ArticleTag rows go with their article via ON DELETE CASCADE, and the
|
||||
SQLite FTS triggers drop the search entries. 0 days means keep
|
||||
everything forever (the default).
|
||||
chunks so one run never holds a huge delete in a single transaction,
|
||||
and within a time budget — if it runs out it queues a continuation
|
||||
(R2-T4). Anything a user starred or saved for later (a system
|
||||
ArticleTag) is kept — those are explicit "keep this" signals.
|
||||
UserArticleState and ArticleTag rows go with their article via ON
|
||||
DELETE CASCADE, and the SQLite FTS triggers drop the search entries.
|
||||
0 days means keep everything forever (the default).
|
||||
|
||||
Each deleted article leaves a PurgedArticle tombstone first, so a feed
|
||||
that still lists the entry can't bring it back as a new, unread
|
||||
article on its next poll (see apps.feeds.services.fetcher).
|
||||
article on its next poll (see apps.feeds.services.fetcher). Tombstones
|
||||
older than TOMBSTONE_TTL_DAYS are removed here too, whether or not
|
||||
retention is currently on (R2-T2). Returns the number of articles
|
||||
deleted by this run.
|
||||
"""
|
||||
started = time.monotonic()
|
||||
|
||||
def out_of_time() -> bool:
|
||||
return time.monotonic() - started > PURGE_RUN_BUDGET_SECONDS
|
||||
|
||||
expired = _expire_tombstones(out_of_time)
|
||||
if expired:
|
||||
logger.info("Removed %d expired purge tombstone(s)", expired)
|
||||
|
||||
retention_days = SiteSettings.load().retention_days
|
||||
if retention_days <= 0:
|
||||
return 0
|
||||
@@ -103,3 +151,11 @@ def purge_old_articles():
|
||||
)
|
||||
Article.objects.filter(pk__in=[pk for pk, _feed_id, _guid in chunk]).delete()
|
||||
deleted_total += len(chunk)
|
||||
if out_of_time():
|
||||
logger.info(
|
||||
"Retention purge hit its %ds budget after %d article(s); continuing shortly",
|
||||
PURGE_RUN_BUDGET_SECONDS,
|
||||
deleted_total,
|
||||
)
|
||||
purge_old_articles.apply_async(countdown=PURGE_CONTINUATION_DELAY_SECONDS)
|
||||
return deleted_total
|
||||
@@ -89,3 +89,64 @@ def test_purge_leaves_a_tombstone_per_deleted_article(feed):
|
||||
"gone-1",
|
||||
"gone-2",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_expired_tombstones_are_removed_even_when_retention_is_off(feed):
|
||||
from .models import PurgedArticle
|
||||
from .tasks import TOMBSTONE_TTL_DAYS
|
||||
|
||||
old = PurgedArticle.objects.create(feed=feed, guid="old")
|
||||
fresh = PurgedArticle.objects.create(feed=feed, guid="fresh")
|
||||
PurgedArticle.objects.filter(pk=old.pk).update(
|
||||
purged_at=timezone.now() - timedelta(days=TOMBSTONE_TTL_DAYS + 1)
|
||||
)
|
||||
assert SiteSettings.load().retention_days == 0
|
||||
|
||||
purge_old_articles()
|
||||
|
||||
assert not PurgedArticle.objects.filter(pk=old.pk).exists()
|
||||
assert PurgedArticle.objects.filter(pk=fresh.pk).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_a_run_that_exhausts_its_budget_queues_a_continuation(feed, monkeypatch):
|
||||
"""R2-T4: a huge purge doesn't hold a worker for its whole length —
|
||||
after the budget it does what it can and queues another run."""
|
||||
from unittest.mock import patch
|
||||
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2)
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_RUN_BUDGET_SECONDS", -1) # already over
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
for i in range(5):
|
||||
_article(feed, f"old-{i}", age_days=60)
|
||||
|
||||
with patch("apps.articles.tasks.purge_old_articles.apply_async") as mock_continue:
|
||||
deleted = purge_old_articles()
|
||||
|
||||
assert deleted == 2, "one chunk, then it yields"
|
||||
assert Article.objects.count() == 3
|
||||
mock_continue.assert_called_once()
|
||||
assert mock_continue.call_args.kwargs["countdown"] > 0
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_continuation_runs_finish_the_job(feed, monkeypatch):
|
||||
from unittest.mock import patch
|
||||
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2)
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
for i in range(5):
|
||||
_article(feed, f"old-{i}", age_days=60)
|
||||
|
||||
total = 0
|
||||
for run in range(5):
|
||||
# Budget runs out after every chunk, as it would on a huge archive.
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_RUN_BUDGET_SECONDS", -1)
|
||||
with patch("apps.articles.tasks.purge_old_articles.apply_async") as mock_continue:
|
||||
total += purge_old_articles()
|
||||
if not mock_continue.called:
|
||||
break
|
||||
|
||||
assert total == 5
|
||||
assert Article.objects.count() == 0
|
||||
Reference in new issue
Block a user