diff --git a/apps/articles/tasks.py b/apps/articles/tasks.py index 32ba06a..818842f 100644 --- a/apps/articles/tasks.py +++ b/apps/articles/tasks.py @@ -1,5 +1,4 @@ from celery import shared_task -from django.db.models import Q from apps.feeds.models import Subscription from apps.rules.engine import evaluate_rules_for_article @@ -14,6 +13,10 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]): FilterRules (scoped to this feed/tag or unscoped) against the given articles. Dispatched by the fetch pipeline after new/updated articles are committed for a feed. + + Per-rule scoping is left to evaluate_rules_for_article/rule_in_scope + rather than filtered into this query, so this and the backlog + re-apply task (apps.rules.tasks) can't scope rules differently. """ if not article_ids: return 0 @@ -26,24 +29,21 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]): evaluated = 0 for subscription in subscriptions: - rules = FilterRule.objects.filter(user=subscription.user, is_enabled=True).filter( - Q(feed_id=feed_id) | Q(feed__isnull=True) + rules = list( + FilterRule.objects.filter(user=subscription.user, is_enabled=True).order_by( + "priority", "id" + ) ) - subscription_tag_ids = list( + if not rules: + continue + subscription_tag_ids = set( SubscriptionTag.objects.filter( user=subscription.user, subscription=subscription ).values_list("tag_id", flat=True) ) - if subscription_tag_ids: - rules = rules.filter(Q(tag_id__in=subscription_tag_ids) | Q(tag__isnull=True)) - else: - rules = rules.filter(tag__isnull=True) - rules = list(rules.order_by("priority", "id")) - if not rules: - continue for article in articles: - evaluate_rules_for_article(subscription.user, article, rules) + evaluate_rules_for_article(subscription.user, article, rules, subscription_tag_ids) evaluated += 1 return evaluated diff --git a/apps/rules/engine.py b/apps/rules/engine.py index e984d16..a311d78 100644 --- a/apps/rules/engine.py +++ b/apps/rules/engine.py @@ -9,6 +9,7 @@ from __future__ import annotations import logging import re +import signal from django.utils import timezone @@ -27,6 +28,42 @@ logger = logging.getLogger(__name__) _TAG_RE = re.compile(r"<[^>]+>") +# Bounds how much CPU/time a single user-authored regex rule can burn +# against one article. re has no built-in match timeout, so this uses +# SIGALRM to interrupt a runaway match (catastrophic backtracking) — +# only effective on the main thread of a process with SIGALRM available, +# which covers gunicorn's sync workers and Celery's (default) prefork +# pool; elsewhere it's skipped and MAX_REGEX_HAYSTACK_CHARS is the only +# bound. reapply_rules_for_user (apps.rules.tasks) also carries a Celery +# task time_limit as a second, coarser backstop. +REGEX_TIMEOUT_SECONDS = 1 +MAX_REGEX_HAYSTACK_CHARS = 20_000 + + +class _RegexTimeout(Exception): + pass + + +def _raise_regex_timeout(signum, frame): + raise _RegexTimeout() + + +def _regex_search_with_timeout(pattern: str, haystack: str, flags: int): + if not hasattr(signal, "SIGALRM"): + return re.search(pattern, haystack, flags) + try: + previous_handler = signal.signal(signal.SIGALRM, _raise_regex_timeout) + except ValueError: + # Not the main thread (e.g. a threaded dev server) — signal + # timers only work there. + return re.search(pattern, haystack, flags) + signal.setitimer(signal.ITIMER_REAL, REGEX_TIMEOUT_SECONDS) + try: + return re.search(pattern, haystack, flags) + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) + def _haystack(article: Article, match_field: str) -> str: if match_field == FilterRule.FIELD_TITLE: @@ -47,8 +84,12 @@ def rule_matches(rule: FilterRule, article: Article) -> bool: if rule.match_type == FilterRule.MATCH_REGEX: flags = 0 if rule.case_sensitive else re.IGNORECASE + haystack = raw_haystack[:MAX_REGEX_HAYSTACK_CHARS] try: - return re.search(rule.match_value, raw_haystack, flags) is not None + return _regex_search_with_timeout(rule.match_value, haystack, flags) is not None + except _RegexTimeout: + logger.warning("Regex timed out for rule %s", rule.pk) + return False except re.error as exc: logger.warning("Invalid regex in rule %s: %s", rule.pk, exc) return False @@ -100,13 +141,36 @@ def apply_rule_action(rule: FilterRule, user, article: Article, state: UserArtic return False -def evaluate_rules_for_article(user, article: Article, rules) -> None: - """Evaluates an ordered iterable of rules (already scoped/enabled) - against one article for one user, applying actions as they match.""" +def rule_in_scope(rule: FilterRule, feed_id, subscription_tag_ids) -> bool: + """Whether `rule`'s own feed/tag scope (see FilterRule's docstring) + covers an article from `feed_id`, given the user's subscription-tag + ids for that feed. + + Centralized here — rather than left for each caller to re-derive via + its own query filters — so ingest-time dispatch and backlog re-apply + can't drift into scoping rules differently (see apps.articles.tasks + and apps.rules.tasks). + """ + if rule.feed_id is not None and rule.feed_id != feed_id: + return False + if rule.tag_id is not None and rule.tag_id not in subscription_tag_ids: + return False + return True + + +def evaluate_rules_for_article(user, article: Article, rules, subscription_tag_ids=()) -> None: + """Evaluates an ordered iterable of a user's enabled rules against one + article, applying actions as they match. `rules` doesn't need to be + pre-scoped to this article's feed/tags — out-of-scope rules (per + rule_in_scope) are skipped here, so callers can pass one user's whole + enabled ruleset regardless of which feed(s) the article(s) belong to. + """ matched_any = False state = None for rule in rules: + if not rule_in_scope(rule, article.feed_id, subscription_tag_ids): + continue if not rule_matches(rule, article): continue if state is None: diff --git a/apps/rules/test_engine.py b/apps/rules/test_engine.py index 742ac6c..55c2617 100644 --- a/apps/rules/test_engine.py +++ b/apps/rules/test_engine.py @@ -1,3 +1,5 @@ +import time + import pytest from django.contrib.auth import get_user_model @@ -10,7 +12,7 @@ from apps.articles.models import ( ) from apps.feeds.models import Feed -from .engine import evaluate_rules_for_article, rule_matches +from .engine import evaluate_rules_for_article, rule_in_scope, rule_matches from .models import FilterRule User = get_user_model() @@ -87,6 +89,22 @@ class TestRuleMatches: rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value="(unclosed") assert rule_matches(rule, article) is False + def test_catastrophic_backtracking_regex_times_out_instead_of_hanging(self, user, article): + # (a+)+$ against a long run of a's followed by a non-matching + # character is a textbook ReDoS pattern — left unguarded this + # takes on the order of a minute (confirmed locally); the + # REGEX_TIMEOUT_SECONDS guard in apps.rules.engine must interrupt + # it well before that instead of hanging the caller. + article.title = "a" * 30 + "!" + rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value=r"(a+)+$") + + start = time.monotonic() + result = rule_matches(rule, article) + elapsed = time.monotonic() - start + + assert result is False + assert elapsed < 5 + def test_content_field_strips_html_tags(self, user, article): rule = make_rule(user, match_field=FilterRule.FIELD_CONTENT, match_value="Lots of details") assert rule_matches(rule, article) is True @@ -104,6 +122,24 @@ class TestRuleMatches: assert rule_matches(rule, article) is True +@pytest.mark.django_db +class TestRuleInScope: + def test_unscoped_rule_applies_to_any_feed(self, user): + rule = make_rule(user) + assert rule_in_scope(rule, feed_id=999, subscription_tag_ids=set()) is True + + def test_feed_scoped_rule_only_applies_to_its_own_feed(self, user, feed): + rule = make_rule(user, feed=feed) + assert rule_in_scope(rule, feed_id=feed.id, subscription_tag_ids=set()) is True + assert rule_in_scope(rule, feed_id=feed.id + 1, subscription_tag_ids=set()) is False + + def test_tag_scoped_rule_only_applies_when_the_feed_carries_that_tag(self, user): + tag = Tag.objects.create(user=user, name="work") + rule = make_rule(user, tag=tag) + assert rule_in_scope(rule, feed_id=1, subscription_tag_ids={tag.id}) is True + assert rule_in_scope(rule, feed_id=1, subscription_tag_ids=set()) is False + + @pytest.mark.django_db class TestApplyRuleAction: def test_star_action(self, user, article):