import logging import time from datetime import timedelta from celery import shared_task from django.db import transaction from django.utils import timezone from apps.core.cache_utils import invalidate_feed_subscribers_sidebar_cache from apps.core.models import SiteSettings from apps.feeds.models import Subscription from apps.rules.engine import evaluate_rules_for_article from apps.rules.models import FilterRule from .models import Article, ArticleTag, PurgedArticle, SubscriptionTag logger = logging.getLogger(__name__) @shared_task def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]): """For every subscriber of ``feed_id``, evaluate their enabled FilterRules (scoped to this feed/tag or unscoped) against the given articles. Dispatched by the fetch pipeline after new/updated articles are committed for a feed. Per-rule scoping is left to evaluate_rules_for_article/rule_in_scope rather than filtered into this query, so this and the backlog re-apply task (apps.rules.tasks) can't scope rules differently. Rules and subscription tags are fetched once for every subscriber of this feed, not once per subscriber (C9) — a popular feed with many subscribers previously issued 2 queries per subscriber here. """ if not article_ids: return 0 articles = list(Article.objects.filter(pk__in=article_ids).select_related("feed")) if not articles: return 0 subscriptions = list(Subscription.objects.filter(feed_id=feed_id).select_related("user")) if not subscriptions: return 0 rules_by_user_id: dict[int, list[FilterRule]] = {} for rule in FilterRule.objects.filter( user_id__in={s.user_id for s in subscriptions}, is_enabled=True ).order_by("priority", "id"): rules_by_user_id.setdefault(rule.user_id, []).append(rule) tag_ids_by_subscription_id: dict[int, set[int]] = {} for subscription_id, tag_id in SubscriptionTag.objects.filter( subscription_id__in=[s.pk for s in subscriptions] ).values_list("subscription_id", "tag_id"): tag_ids_by_subscription_id.setdefault(subscription_id, set()).add(tag_id) evaluated = 0 for subscription in subscriptions: rules = rules_by_user_id.get(subscription.user_id) if not rules: continue subscription_tag_ids = tag_ids_by_subscription_id.get(subscription.pk, set()) for article in articles: evaluate_rules_for_article(subscription.user, article, rules, subscription_tag_ids) evaluated += 1 if evaluated: # Rules can mark articles read or hide them, which changes counts. invalidate_feed_subscribers_sidebar_cache(feed_id) return evaluated PURGE_CHUNK_SIZE = 500 # One run works for at most this long, then queues itself again (R2-T4) so # a huge first purge never holds a worker for hours or starves feed # polling; the hard limits below are the backstop for a single stuck chunk. PURGE_RUN_BUDGET_SECONDS = 240 PURGE_CONTINUATION_DELAY_SECONDS = 30 PURGE_SOFT_TIME_LIMIT = 540 PURGE_TIME_LIMIT = 600 # How long a purged article's tombstone is kept (R2-T2). It only needs to # outlive the time a feed keeps *listing* the entry; after this, an entry a # feed still publishes (and that has no publish date, or a recent one) can # be ingested as new again — the price of the tombstone table not growing # forever. Documented in docs/install.md. TOMBSTONE_TTL_DAYS = 365 def _expire_tombstones(out_of_time) -> int: cutoff = timezone.now() - timedelta(days=TOMBSTONE_TTL_DAYS) removed = 0 while not out_of_time(): chunk = list( PurgedArticle.objects.filter(purged_at__lt=cutoff).values_list("pk", flat=True)[ :PURGE_CHUNK_SIZE ] ) if not chunk: break PurgedArticle.objects.filter(pk__in=chunk).delete() removed += len(chunk) return removed @shared_task(time_limit=PURGE_TIME_LIMIT, soft_time_limit=PURGE_SOFT_TIME_LIMIT) def purge_old_articles(): """Deletes articles older than SiteSettings.retention_days (C7), in chunks so one run never holds a huge delete in a single transaction, and within a time budget — if it runs out it queues a continuation (R2-T4). Anything a user starred or saved for later (a system ArticleTag) is kept — those are explicit "keep this" signals. UserArticleState and ArticleTag rows go with their article via ON DELETE CASCADE, and the SQLite FTS triggers drop the search entries. 0 days means keep everything forever (the default). Each deleted article leaves a PurgedArticle tombstone first, so a feed that still lists the entry can't bring it back as a new, unread article on its next poll (see apps.feeds.services.fetcher). Tombstones older than TOMBSTONE_TTL_DAYS are removed here too, whether or not retention is currently on (R2-T2). Returns the number of articles deleted by this run. """ started = time.monotonic() def out_of_time() -> bool: return time.monotonic() - started > PURGE_RUN_BUDGET_SECONDS expired = _expire_tombstones(out_of_time) if expired: logger.info("Removed %d expired purge tombstone(s)", expired) retention_days = SiteSettings.load().retention_days if retention_days <= 0: return 0 cutoff = timezone.now() - timedelta(days=retention_days) keep = ArticleTag.objects.filter(tag__is_system=True).values("article_id") deleted_total = 0 while True: chunk = list( Article.objects.filter(fetched_at__lt=cutoff) .exclude(pk__in=keep) .values_list("pk", "feed_id", "guid")[:PURGE_CHUNK_SIZE] ) if not chunk: return deleted_total with transaction.atomic(): PurgedArticle.objects.bulk_create( [PurgedArticle(feed_id=feed_id, guid=guid) for _pk, feed_id, guid in chunk], ignore_conflicts=True, ) Article.objects.filter(pk__in=[pk for pk, _feed_id, _guid in chunk]).delete() deleted_total += len(chunk) if out_of_time(): logger.info( "Retention purge hit its %ds budget after %d article(s); continuing shortly", PURGE_RUN_BUDGET_SECONDS, deleted_total, ) purge_old_articles.apply_async(countdown=PURGE_CONTINUATION_DELAY_SECONDS) return deleted_total