Private
Public Access
Centralize rule feed/tag scoping in the engine and fix backlog re-apply (T5)
reapply_rules_for_user_backlog narrowed its *backlog* to the triggering rule's feed/tag scope, but then loaded every enabled rule for the user and ran all of them against that backlog with no per-rule scope check — so re-applying one feed- or tag-scoped rule could also fire an unrelated rule scoped to a different feed/tag, if it happened to match. rule_in_scope() moves the feed/tag scope check into the engine itself (apply_filter_rules_for_feed already computed this per-subscription via query filters; it now just passes the tag ids and lets the engine decide), so backlog re-apply and ingest-time dispatch can't drift into scoping rules differently again. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f2ee793cf8
commit
91c0b1dd26
3 files changed
+117
-17
No files matched your search
+12
-12
@@ -1,5 +1,4 @@
|
|||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
from django.db.models import Q
|
|
||||||
|
|
||||||
from apps.feeds.models import Subscription
|
from apps.feeds.models import Subscription
|
||||||
from apps.rules.engine import evaluate_rules_for_article
|
from apps.rules.engine import evaluate_rules_for_article
|
||||||
@@ -14,6 +13,10 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
|||||||
FilterRules (scoped to this feed/tag or unscoped) against the given
|
FilterRules (scoped to this feed/tag or unscoped) against the given
|
||||||
articles. Dispatched by the fetch pipeline after new/updated articles
|
articles. Dispatched by the fetch pipeline after new/updated articles
|
||||||
are committed for a feed.
|
are committed for a feed.
|
||||||
|
|
||||||
|
Per-rule scoping is left to evaluate_rules_for_article/rule_in_scope
|
||||||
|
rather than filtered into this query, so this and the backlog
|
||||||
|
re-apply task (apps.rules.tasks) can't scope rules differently.
|
||||||
"""
|
"""
|
||||||
if not article_ids:
|
if not article_ids:
|
||||||
return 0
|
return 0
|
||||||
@@ -26,24 +29,21 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
|||||||
|
|
||||||
evaluated = 0
|
evaluated = 0
|
||||||
for subscription in subscriptions:
|
for subscription in subscriptions:
|
||||||
rules = FilterRule.objects.filter(user=subscription.user, is_enabled=True).filter(
|
rules = list(
|
||||||
Q(feed_id=feed_id) | Q(feed__isnull=True)
|
FilterRule.objects.filter(user=subscription.user, is_enabled=True).order_by(
|
||||||
|
"priority", "id"
|
||||||
|
)
|
||||||
)
|
)
|
||||||
subscription_tag_ids = list(
|
if not rules:
|
||||||
|
continue
|
||||||
|
subscription_tag_ids = set(
|
||||||
SubscriptionTag.objects.filter(
|
SubscriptionTag.objects.filter(
|
||||||
user=subscription.user, subscription=subscription
|
user=subscription.user, subscription=subscription
|
||||||
).values_list("tag_id", flat=True)
|
).values_list("tag_id", flat=True)
|
||||||
)
|
)
|
||||||
if subscription_tag_ids:
|
|
||||||
rules = rules.filter(Q(tag_id__in=subscription_tag_ids) | Q(tag__isnull=True))
|
|
||||||
else:
|
|
||||||
rules = rules.filter(tag__isnull=True)
|
|
||||||
rules = list(rules.order_by("priority", "id"))
|
|
||||||
if not rules:
|
|
||||||
continue
|
|
||||||
|
|
||||||
for article in articles:
|
for article in articles:
|
||||||
evaluate_rules_for_article(subscription.user, article, rules)
|
evaluate_rules_for_article(subscription.user, article, rules, subscription_tag_ids)
|
||||||
evaluated += 1
|
evaluated += 1
|
||||||
|
|
||||||
return evaluated
|
return evaluated
|
||||||
+68
-4
@@ -9,6 +9,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
import signal
|
||||||
|
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
@@ -27,6 +28,42 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
_TAG_RE = re.compile(r"<[^>]+>")
|
_TAG_RE = re.compile(r"<[^>]+>")
|
||||||
|
|
||||||
|
# Bounds how much CPU/time a single user-authored regex rule can burn
|
||||||
|
# against one article. re has no built-in match timeout, so this uses
|
||||||
|
# SIGALRM to interrupt a runaway match (catastrophic backtracking) —
|
||||||
|
# only effective on the main thread of a process with SIGALRM available,
|
||||||
|
# which covers gunicorn's sync workers and Celery's (default) prefork
|
||||||
|
# pool; elsewhere it's skipped and MAX_REGEX_HAYSTACK_CHARS is the only
|
||||||
|
# bound. reapply_rules_for_user (apps.rules.tasks) also carries a Celery
|
||||||
|
# task time_limit as a second, coarser backstop.
|
||||||
|
REGEX_TIMEOUT_SECONDS = 1
|
||||||
|
MAX_REGEX_HAYSTACK_CHARS = 20_000
|
||||||
|
|
||||||
|
|
||||||
|
class _RegexTimeout(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _raise_regex_timeout(signum, frame):
|
||||||
|
raise _RegexTimeout()
|
||||||
|
|
||||||
|
|
||||||
|
def _regex_search_with_timeout(pattern: str, haystack: str, flags: int):
|
||||||
|
if not hasattr(signal, "SIGALRM"):
|
||||||
|
return re.search(pattern, haystack, flags)
|
||||||
|
try:
|
||||||
|
previous_handler = signal.signal(signal.SIGALRM, _raise_regex_timeout)
|
||||||
|
except ValueError:
|
||||||
|
# Not the main thread (e.g. a threaded dev server) — signal
|
||||||
|
# timers only work there.
|
||||||
|
return re.search(pattern, haystack, flags)
|
||||||
|
signal.setitimer(signal.ITIMER_REAL, REGEX_TIMEOUT_SECONDS)
|
||||||
|
try:
|
||||||
|
return re.search(pattern, haystack, flags)
|
||||||
|
finally:
|
||||||
|
signal.setitimer(signal.ITIMER_REAL, 0)
|
||||||
|
signal.signal(signal.SIGALRM, previous_handler)
|
||||||
|
|
||||||
|
|
||||||
def _haystack(article: Article, match_field: str) -> str:
|
def _haystack(article: Article, match_field: str) -> str:
|
||||||
if match_field == FilterRule.FIELD_TITLE:
|
if match_field == FilterRule.FIELD_TITLE:
|
||||||
@@ -47,8 +84,12 @@ def rule_matches(rule: FilterRule, article: Article) -> bool:
|
|||||||
|
|
||||||
if rule.match_type == FilterRule.MATCH_REGEX:
|
if rule.match_type == FilterRule.MATCH_REGEX:
|
||||||
flags = 0 if rule.case_sensitive else re.IGNORECASE
|
flags = 0 if rule.case_sensitive else re.IGNORECASE
|
||||||
|
haystack = raw_haystack[:MAX_REGEX_HAYSTACK_CHARS]
|
||||||
try:
|
try:
|
||||||
return re.search(rule.match_value, raw_haystack, flags) is not None
|
return _regex_search_with_timeout(rule.match_value, haystack, flags) is not None
|
||||||
|
except _RegexTimeout:
|
||||||
|
logger.warning("Regex timed out for rule %s", rule.pk)
|
||||||
|
return False
|
||||||
except re.error as exc:
|
except re.error as exc:
|
||||||
logger.warning("Invalid regex in rule %s: %s", rule.pk, exc)
|
logger.warning("Invalid regex in rule %s: %s", rule.pk, exc)
|
||||||
return False
|
return False
|
||||||
@@ -100,13 +141,36 @@ def apply_rule_action(rule: FilterRule, user, article: Article, state: UserArtic
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def evaluate_rules_for_article(user, article: Article, rules) -> None:
|
def rule_in_scope(rule: FilterRule, feed_id, subscription_tag_ids) -> bool:
|
||||||
"""Evaluates an ordered iterable of rules (already scoped/enabled)
|
"""Whether `rule`'s own feed/tag scope (see FilterRule's docstring)
|
||||||
against one article for one user, applying actions as they match."""
|
covers an article from `feed_id`, given the user's subscription-tag
|
||||||
|
ids for that feed.
|
||||||
|
|
||||||
|
Centralized here — rather than left for each caller to re-derive via
|
||||||
|
its own query filters — so ingest-time dispatch and backlog re-apply
|
||||||
|
can't drift into scoping rules differently (see apps.articles.tasks
|
||||||
|
and apps.rules.tasks).
|
||||||
|
"""
|
||||||
|
if rule.feed_id is not None and rule.feed_id != feed_id:
|
||||||
|
return False
|
||||||
|
if rule.tag_id is not None and rule.tag_id not in subscription_tag_ids:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_rules_for_article(user, article: Article, rules, subscription_tag_ids=()) -> None:
|
||||||
|
"""Evaluates an ordered iterable of a user's enabled rules against one
|
||||||
|
article, applying actions as they match. `rules` doesn't need to be
|
||||||
|
pre-scoped to this article's feed/tags — out-of-scope rules (per
|
||||||
|
rule_in_scope) are skipped here, so callers can pass one user's whole
|
||||||
|
enabled ruleset regardless of which feed(s) the article(s) belong to.
|
||||||
|
"""
|
||||||
matched_any = False
|
matched_any = False
|
||||||
state = None
|
state = None
|
||||||
|
|
||||||
for rule in rules:
|
for rule in rules:
|
||||||
|
if not rule_in_scope(rule, article.feed_id, subscription_tag_ids):
|
||||||
|
continue
|
||||||
if not rule_matches(rule, article):
|
if not rule_matches(rule, article):
|
||||||
continue
|
continue
|
||||||
if state is None:
|
if state is None:
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import time
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from django.contrib.auth import get_user_model
|
from django.contrib.auth import get_user_model
|
||||||
|
|
||||||
@@ -10,7 +12,7 @@ from apps.articles.models import (
|
|||||||
)
|
)
|
||||||
from apps.feeds.models import Feed
|
from apps.feeds.models import Feed
|
||||||
|
|
||||||
from .engine import evaluate_rules_for_article, rule_matches
|
from .engine import evaluate_rules_for_article, rule_in_scope, rule_matches
|
||||||
from .models import FilterRule
|
from .models import FilterRule
|
||||||
|
|
||||||
User = get_user_model()
|
User = get_user_model()
|
||||||
@@ -87,6 +89,22 @@ class TestRuleMatches:
|
|||||||
rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value="(unclosed")
|
rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value="(unclosed")
|
||||||
assert rule_matches(rule, article) is False
|
assert rule_matches(rule, article) is False
|
||||||
|
|
||||||
|
def test_catastrophic_backtracking_regex_times_out_instead_of_hanging(self, user, article):
|
||||||
|
# (a+)+$ against a long run of a's followed by a non-matching
|
||||||
|
# character is a textbook ReDoS pattern — left unguarded this
|
||||||
|
# takes on the order of a minute (confirmed locally); the
|
||||||
|
# REGEX_TIMEOUT_SECONDS guard in apps.rules.engine must interrupt
|
||||||
|
# it well before that instead of hanging the caller.
|
||||||
|
article.title = "a" * 30 + "!"
|
||||||
|
rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value=r"(a+)+$")
|
||||||
|
|
||||||
|
start = time.monotonic()
|
||||||
|
result = rule_matches(rule, article)
|
||||||
|
elapsed = time.monotonic() - start
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
assert elapsed < 5
|
||||||
|
|
||||||
def test_content_field_strips_html_tags(self, user, article):
|
def test_content_field_strips_html_tags(self, user, article):
|
||||||
rule = make_rule(user, match_field=FilterRule.FIELD_CONTENT, match_value="Lots of details")
|
rule = make_rule(user, match_field=FilterRule.FIELD_CONTENT, match_value="Lots of details")
|
||||||
assert rule_matches(rule, article) is True
|
assert rule_matches(rule, article) is True
|
||||||
@@ -104,6 +122,24 @@ class TestRuleMatches:
|
|||||||
assert rule_matches(rule, article) is True
|
assert rule_matches(rule, article) is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
class TestRuleInScope:
|
||||||
|
def test_unscoped_rule_applies_to_any_feed(self, user):
|
||||||
|
rule = make_rule(user)
|
||||||
|
assert rule_in_scope(rule, feed_id=999, subscription_tag_ids=set()) is True
|
||||||
|
|
||||||
|
def test_feed_scoped_rule_only_applies_to_its_own_feed(self, user, feed):
|
||||||
|
rule = make_rule(user, feed=feed)
|
||||||
|
assert rule_in_scope(rule, feed_id=feed.id, subscription_tag_ids=set()) is True
|
||||||
|
assert rule_in_scope(rule, feed_id=feed.id + 1, subscription_tag_ids=set()) is False
|
||||||
|
|
||||||
|
def test_tag_scoped_rule_only_applies_when_the_feed_carries_that_tag(self, user):
|
||||||
|
tag = Tag.objects.create(user=user, name="work")
|
||||||
|
rule = make_rule(user, tag=tag)
|
||||||
|
assert rule_in_scope(rule, feed_id=1, subscription_tag_ids={tag.id}) is True
|
||||||
|
assert rule_in_scope(rule, feed_id=1, subscription_tag_ids=set()) is False
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
class TestApplyRuleAction:
|
class TestApplyRuleAction:
|
||||||
def test_star_action(self, user, article):
|
def test_star_action(self, user, article):
|
||||||
|
|||||||
Reference in new issue
Block a user