Private
Public Access
Articles and full HTML bodies accumulated forever, growing the database, search index, and backups without bound. Site settings gets a "Retention days" field (0, the default, keeps everything); a nightly beat task deletes articles fetched longer ago than that in bounded chunks, always keeping anything a user starred or saved for later. Read state and tags cascade with the article and the FTS triggers drop search entries. The dashboard shows the article count and oldest article's age, and docs/install.md covers retention, the readiness check, and SQLite's WAL tuning. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
96 lines
3.5 KiB
Python
96 lines
3.5 KiB
Python
from datetime import timedelta
|
|
|
|
from celery import shared_task
|
|
from django.utils import timezone
|
|
|
|
from apps.core.models import SiteSettings
|
|
from apps.feeds.models import Subscription
|
|
from apps.rules.engine import evaluate_rules_for_article
|
|
from apps.rules.models import FilterRule
|
|
|
|
from .models import Article, ArticleTag, SubscriptionTag
|
|
|
|
|
|
@shared_task
|
|
def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
|
"""For every subscriber of ``feed_id``, evaluate their enabled
|
|
FilterRules (scoped to this feed/tag or unscoped) against the given
|
|
articles. Dispatched by the fetch pipeline after new/updated articles
|
|
are committed for a feed.
|
|
|
|
Per-rule scoping is left to evaluate_rules_for_article/rule_in_scope
|
|
rather than filtered into this query, so this and the backlog
|
|
re-apply task (apps.rules.tasks) can't scope rules differently.
|
|
|
|
Rules and subscription tags are fetched once for every subscriber of
|
|
this feed, not once per subscriber (C9) — a popular feed with many
|
|
subscribers previously issued 2 queries per subscriber here.
|
|
"""
|
|
if not article_ids:
|
|
return 0
|
|
|
|
articles = list(Article.objects.filter(pk__in=article_ids).select_related("feed"))
|
|
if not articles:
|
|
return 0
|
|
|
|
subscriptions = list(Subscription.objects.filter(feed_id=feed_id).select_related("user"))
|
|
if not subscriptions:
|
|
return 0
|
|
|
|
rules_by_user_id: dict[int, list[FilterRule]] = {}
|
|
for rule in FilterRule.objects.filter(
|
|
user_id__in={s.user_id for s in subscriptions}, is_enabled=True
|
|
).order_by("priority", "id"):
|
|
rules_by_user_id.setdefault(rule.user_id, []).append(rule)
|
|
|
|
tag_ids_by_subscription_id: dict[int, set[int]] = {}
|
|
for subscription_id, tag_id in SubscriptionTag.objects.filter(
|
|
subscription_id__in=[s.pk for s in subscriptions]
|
|
).values_list("subscription_id", "tag_id"):
|
|
tag_ids_by_subscription_id.setdefault(subscription_id, set()).add(tag_id)
|
|
|
|
evaluated = 0
|
|
for subscription in subscriptions:
|
|
rules = rules_by_user_id.get(subscription.user_id)
|
|
if not rules:
|
|
continue
|
|
subscription_tag_ids = tag_ids_by_subscription_id.get(subscription.pk, set())
|
|
|
|
for article in articles:
|
|
evaluate_rules_for_article(subscription.user, article, rules, subscription_tag_ids)
|
|
evaluated += 1
|
|
|
|
return evaluated
|
|
|
|
|
|
PURGE_CHUNK_SIZE = 500
|
|
|
|
|
|
@shared_task
|
|
def purge_old_articles():
|
|
"""Deletes articles older than SiteSettings.retention_days (C7), in
|
|
chunks so one run never holds a huge delete in a single transaction.
|
|
Anything a user starred or saved for later (a system ArticleTag) is
|
|
kept — those are explicit "keep this" signals. UserArticleState and
|
|
ArticleTag rows go with their article via ON DELETE CASCADE, and the
|
|
SQLite FTS triggers drop the search entries. 0 days means keep
|
|
everything forever (the default).
|
|
"""
|
|
retention_days = SiteSettings.load().retention_days
|
|
if retention_days <= 0:
|
|
return 0
|
|
|
|
cutoff = timezone.now() - timedelta(days=retention_days)
|
|
keep = ArticleTag.objects.filter(tag__is_system=True).values("article_id")
|
|
deleted_total = 0
|
|
while True:
|
|
chunk = list(
|
|
Article.objects.filter(fetched_at__lt=cutoff)
|
|
.exclude(pk__in=keep)
|
|
.values_list("pk", flat=True)[:PURGE_CHUNK_SIZE]
|
|
)
|
|
if not chunk:
|
|
return deleted_total
|
|
Article.objects.filter(pk__in=chunk).delete()
|
|
deleted_total += len(chunk)
|