Private
Public Access
Centralize rule feed/tag scoping in the engine and fix backlog re-apply (T5)
reapply_rules_for_user_backlog narrowed its *backlog* to the triggering rule's feed/tag scope, but then loaded every enabled rule for the user and ran all of them against that backlog with no per-rule scope check — so re-applying one feed- or tag-scoped rule could also fire an unrelated rule scoped to a different feed/tag, if it happened to match. rule_in_scope() moves the feed/tag scope check into the engine itself (apply_filter_rules_for_feed already computed this per-subscription via query filters; it now just passes the tag ids and lets the engine decide), so backlog re-apply and ingest-time dispatch can't drift into scoping rules differently again. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f2ee793cf8
commit
91c0b1dd26
3 files changed
+117
-17
No files matched your search
+12
-12
@@ -1,5 +1,4 @@
|
||||
from celery import shared_task
|
||||
from django.db.models import Q
|
||||
|
||||
from apps.feeds.models import Subscription
|
||||
from apps.rules.engine import evaluate_rules_for_article
|
||||
@@ -14,6 +13,10 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
||||
FilterRules (scoped to this feed/tag or unscoped) against the given
|
||||
articles. Dispatched by the fetch pipeline after new/updated articles
|
||||
are committed for a feed.
|
||||
|
||||
Per-rule scoping is left to evaluate_rules_for_article/rule_in_scope
|
||||
rather than filtered into this query, so this and the backlog
|
||||
re-apply task (apps.rules.tasks) can't scope rules differently.
|
||||
"""
|
||||
if not article_ids:
|
||||
return 0
|
||||
@@ -26,24 +29,21 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
||||
|
||||
evaluated = 0
|
||||
for subscription in subscriptions:
|
||||
rules = FilterRule.objects.filter(user=subscription.user, is_enabled=True).filter(
|
||||
Q(feed_id=feed_id) | Q(feed__isnull=True)
|
||||
rules = list(
|
||||
FilterRule.objects.filter(user=subscription.user, is_enabled=True).order_by(
|
||||
"priority", "id"
|
||||
)
|
||||
)
|
||||
subscription_tag_ids = list(
|
||||
if not rules:
|
||||
continue
|
||||
subscription_tag_ids = set(
|
||||
SubscriptionTag.objects.filter(
|
||||
user=subscription.user, subscription=subscription
|
||||
).values_list("tag_id", flat=True)
|
||||
)
|
||||
if subscription_tag_ids:
|
||||
rules = rules.filter(Q(tag_id__in=subscription_tag_ids) | Q(tag__isnull=True))
|
||||
else:
|
||||
rules = rules.filter(tag__isnull=True)
|
||||
rules = list(rules.order_by("priority", "id"))
|
||||
if not rules:
|
||||
continue
|
||||
|
||||
for article in articles:
|
||||
evaluate_rules_for_article(subscription.user, article, rules)
|
||||
evaluate_rules_for_article(subscription.user, article, rules, subscription_tag_ids)
|
||||
evaluated += 1
|
||||
|
||||
return evaluated
|
||||
+68
-4
@@ -9,6 +9,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import signal
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
@@ -27,6 +28,42 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_TAG_RE = re.compile(r"<[^>]+>")
|
||||
|
||||
# Bounds how much CPU/time a single user-authored regex rule can burn
|
||||
# against one article. re has no built-in match timeout, so this uses
|
||||
# SIGALRM to interrupt a runaway match (catastrophic backtracking) —
|
||||
# only effective on the main thread of a process with SIGALRM available,
|
||||
# which covers gunicorn's sync workers and Celery's (default) prefork
|
||||
# pool; elsewhere it's skipped and MAX_REGEX_HAYSTACK_CHARS is the only
|
||||
# bound. reapply_rules_for_user (apps.rules.tasks) also carries a Celery
|
||||
# task time_limit as a second, coarser backstop.
|
||||
REGEX_TIMEOUT_SECONDS = 1
|
||||
MAX_REGEX_HAYSTACK_CHARS = 20_000
|
||||
|
||||
|
||||
class _RegexTimeout(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _raise_regex_timeout(signum, frame):
|
||||
raise _RegexTimeout()
|
||||
|
||||
|
||||
def _regex_search_with_timeout(pattern: str, haystack: str, flags: int):
|
||||
if not hasattr(signal, "SIGALRM"):
|
||||
return re.search(pattern, haystack, flags)
|
||||
try:
|
||||
previous_handler = signal.signal(signal.SIGALRM, _raise_regex_timeout)
|
||||
except ValueError:
|
||||
# Not the main thread (e.g. a threaded dev server) — signal
|
||||
# timers only work there.
|
||||
return re.search(pattern, haystack, flags)
|
||||
signal.setitimer(signal.ITIMER_REAL, REGEX_TIMEOUT_SECONDS)
|
||||
try:
|
||||
return re.search(pattern, haystack, flags)
|
||||
finally:
|
||||
signal.setitimer(signal.ITIMER_REAL, 0)
|
||||
signal.signal(signal.SIGALRM, previous_handler)
|
||||
|
||||
|
||||
def _haystack(article: Article, match_field: str) -> str:
|
||||
if match_field == FilterRule.FIELD_TITLE:
|
||||
@@ -47,8 +84,12 @@ def rule_matches(rule: FilterRule, article: Article) -> bool:
|
||||
|
||||
if rule.match_type == FilterRule.MATCH_REGEX:
|
||||
flags = 0 if rule.case_sensitive else re.IGNORECASE
|
||||
haystack = raw_haystack[:MAX_REGEX_HAYSTACK_CHARS]
|
||||
try:
|
||||
return re.search(rule.match_value, raw_haystack, flags) is not None
|
||||
return _regex_search_with_timeout(rule.match_value, haystack, flags) is not None
|
||||
except _RegexTimeout:
|
||||
logger.warning("Regex timed out for rule %s", rule.pk)
|
||||
return False
|
||||
except re.error as exc:
|
||||
logger.warning("Invalid regex in rule %s: %s", rule.pk, exc)
|
||||
return False
|
||||
@@ -100,13 +141,36 @@ def apply_rule_action(rule: FilterRule, user, article: Article, state: UserArtic
|
||||
return False
|
||||
|
||||
|
||||
def evaluate_rules_for_article(user, article: Article, rules) -> None:
|
||||
"""Evaluates an ordered iterable of rules (already scoped/enabled)
|
||||
against one article for one user, applying actions as they match."""
|
||||
def rule_in_scope(rule: FilterRule, feed_id, subscription_tag_ids) -> bool:
|
||||
"""Whether `rule`'s own feed/tag scope (see FilterRule's docstring)
|
||||
covers an article from `feed_id`, given the user's subscription-tag
|
||||
ids for that feed.
|
||||
|
||||
Centralized here — rather than left for each caller to re-derive via
|
||||
its own query filters — so ingest-time dispatch and backlog re-apply
|
||||
can't drift into scoping rules differently (see apps.articles.tasks
|
||||
and apps.rules.tasks).
|
||||
"""
|
||||
if rule.feed_id is not None and rule.feed_id != feed_id:
|
||||
return False
|
||||
if rule.tag_id is not None and rule.tag_id not in subscription_tag_ids:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def evaluate_rules_for_article(user, article: Article, rules, subscription_tag_ids=()) -> None:
|
||||
"""Evaluates an ordered iterable of a user's enabled rules against one
|
||||
article, applying actions as they match. `rules` doesn't need to be
|
||||
pre-scoped to this article's feed/tags — out-of-scope rules (per
|
||||
rule_in_scope) are skipped here, so callers can pass one user's whole
|
||||
enabled ruleset regardless of which feed(s) the article(s) belong to.
|
||||
"""
|
||||
matched_any = False
|
||||
state = None
|
||||
|
||||
for rule in rules:
|
||||
if not rule_in_scope(rule, article.feed_id, subscription_tag_ids):
|
||||
continue
|
||||
if not rule_matches(rule, article):
|
||||
continue
|
||||
if state is None:
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
@@ -10,7 +12,7 @@ from apps.articles.models import (
|
||||
)
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
from .engine import evaluate_rules_for_article, rule_matches
|
||||
from .engine import evaluate_rules_for_article, rule_in_scope, rule_matches
|
||||
from .models import FilterRule
|
||||
|
||||
User = get_user_model()
|
||||
@@ -87,6 +89,22 @@ class TestRuleMatches:
|
||||
rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value="(unclosed")
|
||||
assert rule_matches(rule, article) is False
|
||||
|
||||
def test_catastrophic_backtracking_regex_times_out_instead_of_hanging(self, user, article):
|
||||
# (a+)+$ against a long run of a's followed by a non-matching
|
||||
# character is a textbook ReDoS pattern — left unguarded this
|
||||
# takes on the order of a minute (confirmed locally); the
|
||||
# REGEX_TIMEOUT_SECONDS guard in apps.rules.engine must interrupt
|
||||
# it well before that instead of hanging the caller.
|
||||
article.title = "a" * 30 + "!"
|
||||
rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value=r"(a+)+$")
|
||||
|
||||
start = time.monotonic()
|
||||
result = rule_matches(rule, article)
|
||||
elapsed = time.monotonic() - start
|
||||
|
||||
assert result is False
|
||||
assert elapsed < 5
|
||||
|
||||
def test_content_field_strips_html_tags(self, user, article):
|
||||
rule = make_rule(user, match_field=FilterRule.FIELD_CONTENT, match_value="Lots of details")
|
||||
assert rule_matches(rule, article) is True
|
||||
@@ -104,6 +122,24 @@ class TestRuleMatches:
|
||||
assert rule_matches(rule, article) is True
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestRuleInScope:
|
||||
def test_unscoped_rule_applies_to_any_feed(self, user):
|
||||
rule = make_rule(user)
|
||||
assert rule_in_scope(rule, feed_id=999, subscription_tag_ids=set()) is True
|
||||
|
||||
def test_feed_scoped_rule_only_applies_to_its_own_feed(self, user, feed):
|
||||
rule = make_rule(user, feed=feed)
|
||||
assert rule_in_scope(rule, feed_id=feed.id, subscription_tag_ids=set()) is True
|
||||
assert rule_in_scope(rule, feed_id=feed.id + 1, subscription_tag_ids=set()) is False
|
||||
|
||||
def test_tag_scoped_rule_only_applies_when_the_feed_carries_that_tag(self, user):
|
||||
tag = Tag.objects.create(user=user, name="work")
|
||||
rule = make_rule(user, tag=tag)
|
||||
assert rule_in_scope(rule, feed_id=1, subscription_tag_ids={tag.id}) is True
|
||||
assert rule_in_scope(rule, feed_id=1, subscription_tag_ids=set()) is False
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestApplyRuleAction:
|
||||
def test_star_action(self, user, article):
|
||||
|
||||
Reference in new issue
Block a user