Centralize rule feed/tag scoping in the engine and fix backlog re-apply (T5)

reapply_rules_for_user_backlog narrowed its *backlog* to the triggering
rule's feed/tag scope, but then loaded every enabled rule for the user
and ran all of them against that backlog with no per-rule scope check
— so re-applying one feed- or tag-scoped rule could also fire an
unrelated rule scoped to a different feed/tag, if it happened to match.

rule_in_scope() moves the feed/tag scope check into the engine itself
(apply_filter_rules_for_feed already computed this per-subscription via
query filters; it now just passes the tag ids and lets the engine
decide), so backlog re-apply and ingest-time dispatch can't drift into
scoping rules differently again.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-10-31 12:19:56 -06:00
1 parent f2ee793cf8
commit 91c0b1dd26
3 files changed
+117 -17

No files matched your search

+12 -12
View File
@@ -1,5 +1,4 @@
from celery import shared_task from celery import shared_task
from django.db.models import Q
from apps.feeds.models import Subscription from apps.feeds.models import Subscription
from apps.rules.engine import evaluate_rules_for_article from apps.rules.engine import evaluate_rules_for_article
@@ -14,6 +13,10 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
FilterRules (scoped to this feed/tag or unscoped) against the given FilterRules (scoped to this feed/tag or unscoped) against the given
articles. Dispatched by the fetch pipeline after new/updated articles articles. Dispatched by the fetch pipeline after new/updated articles
are committed for a feed. are committed for a feed.
Per-rule scoping is left to evaluate_rules_for_article/rule_in_scope
rather than filtered into this query, so this and the backlog
re-apply task (apps.rules.tasks) can't scope rules differently.
""" """
if not article_ids: if not article_ids:
return 0 return 0
@@ -26,24 +29,21 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
evaluated = 0 evaluated = 0
for subscription in subscriptions: for subscription in subscriptions:
rules = FilterRule.objects.filter(user=subscription.user, is_enabled=True).filter( rules = list(
Q(feed_id=feed_id) | Q(feed__isnull=True) FilterRule.objects.filter(user=subscription.user, is_enabled=True).order_by(
"priority", "id"
) )
subscription_tag_ids = list( )
if not rules:
continue
subscription_tag_ids = set(
SubscriptionTag.objects.filter( SubscriptionTag.objects.filter(
user=subscription.user, subscription=subscription user=subscription.user, subscription=subscription
).values_list("tag_id", flat=True) ).values_list("tag_id", flat=True)
) )
if subscription_tag_ids:
rules = rules.filter(Q(tag_id__in=subscription_tag_ids) | Q(tag__isnull=True))
else:
rules = rules.filter(tag__isnull=True)
rules = list(rules.order_by("priority", "id"))
if not rules:
continue
for article in articles: for article in articles:
evaluate_rules_for_article(subscription.user, article, rules) evaluate_rules_for_article(subscription.user, article, rules, subscription_tag_ids)
evaluated += 1 evaluated += 1
return evaluated return evaluated
+68 -4
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
import logging import logging
import re import re
import signal
from django.utils import timezone from django.utils import timezone
@@ -27,6 +28,42 @@ logger = logging.getLogger(__name__)
_TAG_RE = re.compile(r"<[^>]+>") _TAG_RE = re.compile(r"<[^>]+>")
# Bounds how much CPU/time a single user-authored regex rule can burn
# against one article. re has no built-in match timeout, so this uses
# SIGALRM to interrupt a runaway match (catastrophic backtracking) —
# only effective on the main thread of a process with SIGALRM available,
# which covers gunicorn's sync workers and Celery's (default) prefork
# pool; elsewhere it's skipped and MAX_REGEX_HAYSTACK_CHARS is the only
# bound. reapply_rules_for_user (apps.rules.tasks) also carries a Celery
# task time_limit as a second, coarser backstop.
REGEX_TIMEOUT_SECONDS = 1
MAX_REGEX_HAYSTACK_CHARS = 20_000
class _RegexTimeout(Exception):
pass
def _raise_regex_timeout(signum, frame):
raise _RegexTimeout()
def _regex_search_with_timeout(pattern: str, haystack: str, flags: int):
if not hasattr(signal, "SIGALRM"):
return re.search(pattern, haystack, flags)
try:
previous_handler = signal.signal(signal.SIGALRM, _raise_regex_timeout)
except ValueError:
# Not the main thread (e.g. a threaded dev server) — signal
# timers only work there.
return re.search(pattern, haystack, flags)
signal.setitimer(signal.ITIMER_REAL, REGEX_TIMEOUT_SECONDS)
try:
return re.search(pattern, haystack, flags)
finally:
signal.setitimer(signal.ITIMER_REAL, 0)
signal.signal(signal.SIGALRM, previous_handler)
def _haystack(article: Article, match_field: str) -> str: def _haystack(article: Article, match_field: str) -> str:
if match_field == FilterRule.FIELD_TITLE: if match_field == FilterRule.FIELD_TITLE:
@@ -47,8 +84,12 @@ def rule_matches(rule: FilterRule, article: Article) -> bool:
if rule.match_type == FilterRule.MATCH_REGEX: if rule.match_type == FilterRule.MATCH_REGEX:
flags = 0 if rule.case_sensitive else re.IGNORECASE flags = 0 if rule.case_sensitive else re.IGNORECASE
haystack = raw_haystack[:MAX_REGEX_HAYSTACK_CHARS]
try: try:
return re.search(rule.match_value, raw_haystack, flags) is not None return _regex_search_with_timeout(rule.match_value, haystack, flags) is not None
except _RegexTimeout:
logger.warning("Regex timed out for rule %s", rule.pk)
return False
except re.error as exc: except re.error as exc:
logger.warning("Invalid regex in rule %s: %s", rule.pk, exc) logger.warning("Invalid regex in rule %s: %s", rule.pk, exc)
return False return False
@@ -100,13 +141,36 @@ def apply_rule_action(rule: FilterRule, user, article: Article, state: UserArtic
return False return False
def evaluate_rules_for_article(user, article: Article, rules) -> None: def rule_in_scope(rule: FilterRule, feed_id, subscription_tag_ids) -> bool:
"""Evaluates an ordered iterable of rules (already scoped/enabled) """Whether `rule`'s own feed/tag scope (see FilterRule's docstring)
against one article for one user, applying actions as they match.""" covers an article from `feed_id`, given the user's subscription-tag
ids for that feed.
Centralized here — rather than left for each caller to re-derive via
its own query filters — so ingest-time dispatch and backlog re-apply
can't drift into scoping rules differently (see apps.articles.tasks
and apps.rules.tasks).
"""
if rule.feed_id is not None and rule.feed_id != feed_id:
return False
if rule.tag_id is not None and rule.tag_id not in subscription_tag_ids:
return False
return True
def evaluate_rules_for_article(user, article: Article, rules, subscription_tag_ids=()) -> None:
"""Evaluates an ordered iterable of a user's enabled rules against one
article, applying actions as they match. `rules` doesn't need to be
pre-scoped to this article's feed/tags — out-of-scope rules (per
rule_in_scope) are skipped here, so callers can pass one user's whole
enabled ruleset regardless of which feed(s) the article(s) belong to.
"""
matched_any = False matched_any = False
state = None state = None
for rule in rules: for rule in rules:
if not rule_in_scope(rule, article.feed_id, subscription_tag_ids):
continue
if not rule_matches(rule, article): if not rule_matches(rule, article):
continue continue
if state is None: if state is None:
+37 -1
View File
@@ -1,3 +1,5 @@
import time
import pytest import pytest
from django.contrib.auth import get_user_model from django.contrib.auth import get_user_model
@@ -10,7 +12,7 @@ from apps.articles.models import (
) )
from apps.feeds.models import Feed from apps.feeds.models import Feed
from .engine import evaluate_rules_for_article, rule_matches from .engine import evaluate_rules_for_article, rule_in_scope, rule_matches
from .models import FilterRule from .models import FilterRule
User = get_user_model() User = get_user_model()
@@ -87,6 +89,22 @@ class TestRuleMatches:
rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value="(unclosed") rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value="(unclosed")
assert rule_matches(rule, article) is False assert rule_matches(rule, article) is False
def test_catastrophic_backtracking_regex_times_out_instead_of_hanging(self, user, article):
# (a+)+$ against a long run of a's followed by a non-matching
# character is a textbook ReDoS pattern — left unguarded this
# takes on the order of a minute (confirmed locally); the
# REGEX_TIMEOUT_SECONDS guard in apps.rules.engine must interrupt
# it well before that instead of hanging the caller.
article.title = "a" * 30 + "!"
rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value=r"(a+)+$")
start = time.monotonic()
result = rule_matches(rule, article)
elapsed = time.monotonic() - start
assert result is False
assert elapsed < 5
def test_content_field_strips_html_tags(self, user, article): def test_content_field_strips_html_tags(self, user, article):
rule = make_rule(user, match_field=FilterRule.FIELD_CONTENT, match_value="Lots of details") rule = make_rule(user, match_field=FilterRule.FIELD_CONTENT, match_value="Lots of details")
assert rule_matches(rule, article) is True assert rule_matches(rule, article) is True
@@ -104,6 +122,24 @@ class TestRuleMatches:
assert rule_matches(rule, article) is True assert rule_matches(rule, article) is True
@pytest.mark.django_db
class TestRuleInScope:
def test_unscoped_rule_applies_to_any_feed(self, user):
rule = make_rule(user)
assert rule_in_scope(rule, feed_id=999, subscription_tag_ids=set()) is True
def test_feed_scoped_rule_only_applies_to_its_own_feed(self, user, feed):
rule = make_rule(user, feed=feed)
assert rule_in_scope(rule, feed_id=feed.id, subscription_tag_ids=set()) is True
assert rule_in_scope(rule, feed_id=feed.id + 1, subscription_tag_ids=set()) is False
def test_tag_scoped_rule_only_applies_when_the_feed_carries_that_tag(self, user):
tag = Tag.objects.create(user=user, name="work")
rule = make_rule(user, tag=tag)
assert rule_in_scope(rule, feed_id=1, subscription_tag_ids={tag.id}) is True
assert rule_in_scope(rule, feed_id=1, subscription_tag_ids=set()) is False
@pytest.mark.django_db @pytest.mark.django_db
class TestApplyRuleAction: class TestApplyRuleAction:
def test_star_action(self, user, article): def test_star_action(self, user, article):