Files
Evection/apps/articles/tasks.py
T
ksmithandClaude Sonnet 5.5 f8cd1eb33e Stop retention from bringing deleted articles back (#29)
The nightly purge deleted articles, but a feed that still listed an
entry would re-insert it on its next poll as a brand-new, unread
article. For feeds with long archives, old items could reappear after
every retention run.

The purge now leaves a (feed, guid) tombstone per deleted article, and
ingestion skips tombstoned guids. With retention on, entries whose own
publish date is already outside the window are not ingested at all, so
a long archive isn't fetched only to be purged and tombstoned the same
night. Undated entries rely on the tombstone.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-01 09:30:00 -07:00

106 lines
4.0 KiB
Python

from datetime import timedelta
from celery import shared_task
from django.db import transaction
from django.utils import timezone
from apps.core.models import SiteSettings
from apps.feeds.models import Subscription
from apps.rules.engine import evaluate_rules_for_article
from apps.rules.models import FilterRule
from .models import Article, ArticleTag, PurgedArticle, SubscriptionTag
@shared_task
def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
"""For every subscriber of ``feed_id``, evaluate their enabled
FilterRules (scoped to this feed/tag or unscoped) against the given
articles. Dispatched by the fetch pipeline after new/updated articles
are committed for a feed.
Per-rule scoping is left to evaluate_rules_for_article/rule_in_scope
rather than filtered into this query, so this and the backlog
re-apply task (apps.rules.tasks) can't scope rules differently.
Rules and subscription tags are fetched once for every subscriber of
this feed, not once per subscriber (C9) — a popular feed with many
subscribers previously issued 2 queries per subscriber here.
"""
if not article_ids:
return 0
articles = list(Article.objects.filter(pk__in=article_ids).select_related("feed"))
if not articles:
return 0
subscriptions = list(Subscription.objects.filter(feed_id=feed_id).select_related("user"))
if not subscriptions:
return 0
rules_by_user_id: dict[int, list[FilterRule]] = {}
for rule in FilterRule.objects.filter(
user_id__in={s.user_id for s in subscriptions}, is_enabled=True
).order_by("priority", "id"):
rules_by_user_id.setdefault(rule.user_id, []).append(rule)
tag_ids_by_subscription_id: dict[int, set[int]] = {}
for subscription_id, tag_id in SubscriptionTag.objects.filter(
subscription_id__in=[s.pk for s in subscriptions]
).values_list("subscription_id", "tag_id"):
tag_ids_by_subscription_id.setdefault(subscription_id, set()).add(tag_id)
evaluated = 0
for subscription in subscriptions:
rules = rules_by_user_id.get(subscription.user_id)
if not rules:
continue
subscription_tag_ids = tag_ids_by_subscription_id.get(subscription.pk, set())
for article in articles:
evaluate_rules_for_article(subscription.user, article, rules, subscription_tag_ids)
evaluated += 1
return evaluated
PURGE_CHUNK_SIZE = 500
@shared_task
def purge_old_articles():
"""Deletes articles older than SiteSettings.retention_days (C7), in
chunks so one run never holds a huge delete in a single transaction.
Anything a user starred or saved for later (a system ArticleTag) is
kept — those are explicit "keep this" signals. UserArticleState and
ArticleTag rows go with their article via ON DELETE CASCADE, and the
SQLite FTS triggers drop the search entries. 0 days means keep
everything forever (the default).
Each deleted article leaves a PurgedArticle tombstone first, so a feed
that still lists the entry can't bring it back as a new, unread
article on its next poll (see apps.feeds.services.fetcher).
"""
retention_days = SiteSettings.load().retention_days
if retention_days <= 0:
return 0
cutoff = timezone.now() - timedelta(days=retention_days)
keep = ArticleTag.objects.filter(tag__is_system=True).values("article_id")
deleted_total = 0
while True:
chunk = list(
Article.objects.filter(fetched_at__lt=cutoff)
.exclude(pk__in=keep)
.values_list("pk", "feed_id", "guid")[:PURGE_CHUNK_SIZE]
)
if not chunk:
return deleted_total
with transaction.atomic():
PurgedArticle.objects.bulk_create(
[PurgedArticle(feed_id=feed_id, guid=guid) for _pk, feed_id, guid in chunk],
ignore_conflicts=True,
)
Article.objects.filter(pk__in=[pk for pk, _feed_id, _guid in chunk]).delete()
deleted_total += len(chunk)