Private
Public Access
Tombstones were kept forever, so enabling retention left a permanent row per deleted article. They now expire after 365 days (cleaned up by the same nightly task, even when retention is off); the documented price is that an entry a feed still lists after a year, with no or a recent publish date, can reappear. The purge looped until every eligible row was gone with no run budget. A run now works for at most four minutes, then queues a continuation, so a huge first purge can't monopolise a worker or starve polling; hard time limits back that up, and each chunk is its own transaction so an interrupted run just resumes. Adds indexes on Article.fetched_at and PurgedArticle.purged_at for the scans these do. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
162 lines
6.1 KiB
Python
162 lines
6.1 KiB
Python
import logging
|
|
import time
|
|
from datetime import timedelta
|
|
|
|
from celery import shared_task
|
|
from django.db import transaction
|
|
from django.utils import timezone
|
|
|
|
from apps.core.models import SiteSettings
|
|
from apps.feeds.models import Subscription
|
|
from apps.rules.engine import evaluate_rules_for_article
|
|
from apps.rules.models import FilterRule
|
|
|
|
from .models import Article, ArticleTag, PurgedArticle, SubscriptionTag
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@shared_task
|
|
def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
|
"""For every subscriber of ``feed_id``, evaluate their enabled
|
|
FilterRules (scoped to this feed/tag or unscoped) against the given
|
|
articles. Dispatched by the fetch pipeline after new/updated articles
|
|
are committed for a feed.
|
|
|
|
Per-rule scoping is left to evaluate_rules_for_article/rule_in_scope
|
|
rather than filtered into this query, so this and the backlog
|
|
re-apply task (apps.rules.tasks) can't scope rules differently.
|
|
|
|
Rules and subscription tags are fetched once for every subscriber of
|
|
this feed, not once per subscriber (C9) — a popular feed with many
|
|
subscribers previously issued 2 queries per subscriber here.
|
|
"""
|
|
if not article_ids:
|
|
return 0
|
|
|
|
articles = list(Article.objects.filter(pk__in=article_ids).select_related("feed"))
|
|
if not articles:
|
|
return 0
|
|
|
|
subscriptions = list(Subscription.objects.filter(feed_id=feed_id).select_related("user"))
|
|
if not subscriptions:
|
|
return 0
|
|
|
|
rules_by_user_id: dict[int, list[FilterRule]] = {}
|
|
for rule in FilterRule.objects.filter(
|
|
user_id__in={s.user_id for s in subscriptions}, is_enabled=True
|
|
).order_by("priority", "id"):
|
|
rules_by_user_id.setdefault(rule.user_id, []).append(rule)
|
|
|
|
tag_ids_by_subscription_id: dict[int, set[int]] = {}
|
|
for subscription_id, tag_id in SubscriptionTag.objects.filter(
|
|
subscription_id__in=[s.pk for s in subscriptions]
|
|
).values_list("subscription_id", "tag_id"):
|
|
tag_ids_by_subscription_id.setdefault(subscription_id, set()).add(tag_id)
|
|
|
|
evaluated = 0
|
|
for subscription in subscriptions:
|
|
rules = rules_by_user_id.get(subscription.user_id)
|
|
if not rules:
|
|
continue
|
|
subscription_tag_ids = tag_ids_by_subscription_id.get(subscription.pk, set())
|
|
|
|
for article in articles:
|
|
evaluate_rules_for_article(subscription.user, article, rules, subscription_tag_ids)
|
|
evaluated += 1
|
|
|
|
return evaluated
|
|
|
|
|
|
PURGE_CHUNK_SIZE = 500
|
|
|
|
# One run works for at most this long, then queues itself again (R2-T4) so
|
|
# a huge first purge never holds a worker for hours or starves feed
|
|
# polling; the hard limits below are the backstop for a single stuck chunk.
|
|
PURGE_RUN_BUDGET_SECONDS = 240
|
|
PURGE_CONTINUATION_DELAY_SECONDS = 30
|
|
PURGE_SOFT_TIME_LIMIT = 540
|
|
PURGE_TIME_LIMIT = 600
|
|
|
|
# How long a purged article's tombstone is kept (R2-T2). It only needs to
|
|
# outlive the time a feed keeps *listing* the entry; after this, an entry a
|
|
# feed still publishes (and that has no publish date, or a recent one) can
|
|
# be ingested as new again — the price of the tombstone table not growing
|
|
# forever. Documented in docs/install.md.
|
|
TOMBSTONE_TTL_DAYS = 365
|
|
|
|
|
|
def _expire_tombstones(out_of_time) -> int:
|
|
cutoff = timezone.now() - timedelta(days=TOMBSTONE_TTL_DAYS)
|
|
removed = 0
|
|
while not out_of_time():
|
|
chunk = list(
|
|
PurgedArticle.objects.filter(purged_at__lt=cutoff).values_list("pk", flat=True)[
|
|
:PURGE_CHUNK_SIZE
|
|
]
|
|
)
|
|
if not chunk:
|
|
break
|
|
PurgedArticle.objects.filter(pk__in=chunk).delete()
|
|
removed += len(chunk)
|
|
return removed
|
|
|
|
|
|
@shared_task(time_limit=PURGE_TIME_LIMIT, soft_time_limit=PURGE_SOFT_TIME_LIMIT)
|
|
def purge_old_articles():
|
|
"""Deletes articles older than SiteSettings.retention_days (C7), in
|
|
chunks so one run never holds a huge delete in a single transaction,
|
|
and within a time budget — if it runs out it queues a continuation
|
|
(R2-T4). Anything a user starred or saved for later (a system
|
|
ArticleTag) is kept — those are explicit "keep this" signals.
|
|
UserArticleState and ArticleTag rows go with their article via ON
|
|
DELETE CASCADE, and the SQLite FTS triggers drop the search entries.
|
|
0 days means keep everything forever (the default).
|
|
|
|
Each deleted article leaves a PurgedArticle tombstone first, so a feed
|
|
that still lists the entry can't bring it back as a new, unread
|
|
article on its next poll (see apps.feeds.services.fetcher). Tombstones
|
|
older than TOMBSTONE_TTL_DAYS are removed here too, whether or not
|
|
retention is currently on (R2-T2). Returns the number of articles
|
|
deleted by this run.
|
|
"""
|
|
started = time.monotonic()
|
|
|
|
def out_of_time() -> bool:
|
|
return time.monotonic() - started > PURGE_RUN_BUDGET_SECONDS
|
|
|
|
expired = _expire_tombstones(out_of_time)
|
|
if expired:
|
|
logger.info("Removed %d expired purge tombstone(s)", expired)
|
|
|
|
retention_days = SiteSettings.load().retention_days
|
|
if retention_days <= 0:
|
|
return 0
|
|
|
|
cutoff = timezone.now() - timedelta(days=retention_days)
|
|
keep = ArticleTag.objects.filter(tag__is_system=True).values("article_id")
|
|
deleted_total = 0
|
|
while True:
|
|
chunk = list(
|
|
Article.objects.filter(fetched_at__lt=cutoff)
|
|
.exclude(pk__in=keep)
|
|
.values_list("pk", "feed_id", "guid")[:PURGE_CHUNK_SIZE]
|
|
)
|
|
if not chunk:
|
|
return deleted_total
|
|
with transaction.atomic():
|
|
PurgedArticle.objects.bulk_create(
|
|
[PurgedArticle(feed_id=feed_id, guid=guid) for _pk, feed_id, guid in chunk],
|
|
ignore_conflicts=True,
|
|
)
|
|
Article.objects.filter(pk__in=[pk for pk, _feed_id, _guid in chunk]).delete()
|
|
deleted_total += len(chunk)
|
|
if out_of_time():
|
|
logger.info(
|
|
"Retention purge hit its %ds budget after %d article(s); continuing shortly",
|
|
PURGE_RUN_BUDGET_SECONDS,
|
|
deleted_total,
|
|
)
|
|
purge_old_articles.apply_async(countdown=PURGE_CONTINUATION_DELAY_SECONDS)
|
|
return deleted_total
|