Private
Public Access
Batch rule/tag lookups and index FilterRule for the ingest query (C9)
FilterRule had no indexes beyond the default primary/foreign-key ones, and apply_filter_rules_for_feed issued 2 queries per subscriber (its enabled rules, its subscription tags) inside a loop over every subscriber of the feed — both cost grew with subscriber/rule count on every single feed poll. Adds a composite index on (user, is_enabled, priority), matching the exact filter+order shape every caller already uses (this task, apps.rules.tasks' backlog re-apply). apply_filter_rules_for_feed now fetches rules and subscription tags once for the whole feed's subscriber set, grouped by user/subscription in Python, instead of once per subscriber. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
abf4cef55b
commit
3d08d01015
4 files changed
+91
-11
No files matched your search
+21
-11
@@ -17,6 +17,10 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
|||||||
Per-rule scoping is left to evaluate_rules_for_article/rule_in_scope
|
Per-rule scoping is left to evaluate_rules_for_article/rule_in_scope
|
||||||
rather than filtered into this query, so this and the backlog
|
rather than filtered into this query, so this and the backlog
|
||||||
re-apply task (apps.rules.tasks) can't scope rules differently.
|
re-apply task (apps.rules.tasks) can't scope rules differently.
|
||||||
|
|
||||||
|
Rules and subscription tags are fetched once for every subscriber of
|
||||||
|
this feed, not once per subscriber (C9) — a popular feed with many
|
||||||
|
subscribers previously issued 2 queries per subscriber here.
|
||||||
"""
|
"""
|
||||||
if not article_ids:
|
if not article_ids:
|
||||||
return 0
|
return 0
|
||||||
@@ -25,22 +29,28 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
|||||||
if not articles:
|
if not articles:
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
subscriptions = Subscription.objects.filter(feed_id=feed_id).select_related("user")
|
subscriptions = list(Subscription.objects.filter(feed_id=feed_id).select_related("user"))
|
||||||
|
if not subscriptions:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
rules_by_user_id: dict[int, list[FilterRule]] = {}
|
||||||
|
for rule in FilterRule.objects.filter(
|
||||||
|
user_id__in={s.user_id for s in subscriptions}, is_enabled=True
|
||||||
|
).order_by("priority", "id"):
|
||||||
|
rules_by_user_id.setdefault(rule.user_id, []).append(rule)
|
||||||
|
|
||||||
|
tag_ids_by_subscription_id: dict[int, set[int]] = {}
|
||||||
|
for subscription_id, tag_id in SubscriptionTag.objects.filter(
|
||||||
|
subscription_id__in=[s.pk for s in subscriptions]
|
||||||
|
).values_list("subscription_id", "tag_id"):
|
||||||
|
tag_ids_by_subscription_id.setdefault(subscription_id, set()).add(tag_id)
|
||||||
|
|
||||||
evaluated = 0
|
evaluated = 0
|
||||||
for subscription in subscriptions:
|
for subscription in subscriptions:
|
||||||
rules = list(
|
rules = rules_by_user_id.get(subscription.user_id)
|
||||||
FilterRule.objects.filter(user=subscription.user, is_enabled=True).order_by(
|
|
||||||
"priority", "id"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if not rules:
|
if not rules:
|
||||||
continue
|
continue
|
||||||
subscription_tag_ids = set(
|
subscription_tag_ids = tag_ids_by_subscription_id.get(subscription.pk, set())
|
||||||
SubscriptionTag.objects.filter(
|
|
||||||
user=subscription.user, subscription=subscription
|
|
||||||
).values_list("tag_id", flat=True)
|
|
||||||
)
|
|
||||||
|
|
||||||
for article in articles:
|
for article in articles:
|
||||||
evaluate_rules_for_article(subscription.user, article, rules, subscription_tag_ids)
|
evaluate_rules_for_article(subscription.user, article, rules, subscription_tag_ids)
|
||||||
|
|||||||
@@ -150,3 +150,40 @@ def test_disabled_rule_does_not_apply(feed, alice):
|
|||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
def test_dispatch_with_empty_article_ids_is_a_noop(feed):
|
def test_dispatch_with_empty_article_ids_is_a_noop(feed):
|
||||||
assert apply_filter_rules_for_feed(feed.id, []) == 0
|
assert apply_filter_rules_for_feed(feed.id, []) == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_dispatch_fetches_rules_and_tags_once_per_feed_not_once_per_subscriber(feed):
|
||||||
|
"""Rules/subscription tags are batched across every subscriber of the
|
||||||
|
feed in one query each (C9), not re-queried per subscriber — assert
|
||||||
|
this by checking the query count stays flat as subscribers grow."""
|
||||||
|
from django.db import connection
|
||||||
|
from django.test.utils import CaptureQueriesContext
|
||||||
|
|
||||||
|
users = [User.objects.create_user(f"user{i}", password="pw-123456") for i in range(5)]
|
||||||
|
for user in users:
|
||||||
|
sub = Subscription.objects.create(user=user, feed=feed)
|
||||||
|
tag = Tag.objects.create(user=user, name="Tech")
|
||||||
|
SubscriptionTag.objects.create(user=user, subscription=sub, tag=tag)
|
||||||
|
FilterRule.objects.create(
|
||||||
|
user=user,
|
||||||
|
tag=tag,
|
||||||
|
match_field=FilterRule.FIELD_TITLE,
|
||||||
|
match_type=FilterRule.MATCH_CONTAINS,
|
||||||
|
match_value="Python",
|
||||||
|
action=FilterRule.ACTION_STAR,
|
||||||
|
)
|
||||||
|
article = Article.objects.create(feed=feed, guid="guid-1", title="Python news")
|
||||||
|
|
||||||
|
with CaptureQueriesContext(connection) as ctx:
|
||||||
|
apply_filter_rules_for_feed(feed.id, [article.pk])
|
||||||
|
|
||||||
|
# Exactly one SELECT against each table regardless of subscriber
|
||||||
|
# count (5 here) — a regression back to per-subscription querying
|
||||||
|
# would show up as 5 of each instead of 1.
|
||||||
|
rule_selects = [q for q in ctx.captured_queries if "rules_filterrule" in q["sql"]]
|
||||||
|
tag_selects = [q for q in ctx.captured_queries if "articles_subscriptiontag" in q["sql"]]
|
||||||
|
assert len(rule_selects) == 1
|
||||||
|
assert len(tag_selects) == 1
|
||||||
|
for user in users:
|
||||||
|
assert is_starred(user, article) is True
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# Generated by Django 5.2.18 on 2026-10-31 22:05
|
||||||
|
|
||||||
|
from django.conf import settings
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
("articles", "0004_starred_saved_as_system_tags"),
|
||||||
|
("feeds", "0006_backfill_feed_next_fetch_at"),
|
||||||
|
("rules", "0002_folder_scope_to_tag_scope"),
|
||||||
|
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddIndex(
|
||||||
|
model_name="filterrule",
|
||||||
|
index=models.Index(
|
||||||
|
fields=["user", "is_enabled", "priority"], name="rules_filte_user_id_dcb19b_idx"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
@@ -75,6 +75,16 @@ class FilterRule(models.Model):
|
|||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
ordering = ["priority", "id"]
|
ordering = ["priority", "id"]
|
||||||
|
indexes = [
|
||||||
|
# Matches the ingest-time query shape exactly (C9): every
|
||||||
|
# caller — apps.articles.tasks.apply_filter_rules_for_feed,
|
||||||
|
# apps.rules.tasks.reapply_rules_for_user_backlog — filters by
|
||||||
|
# (user, is_enabled) and orders by priority. `user` alone
|
||||||
|
# already has an index from the ForeignKey, but it doesn't
|
||||||
|
# cover is_enabled or let the database satisfy the ORDER BY
|
||||||
|
# from the index itself.
|
||||||
|
models.Index(fields=["user", "is_enabled", "priority"]),
|
||||||
|
]
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
label = self.name or f"{self.get_match_field_display()} {self.get_match_type_display()}"
|
label = self.name or f"{self.get_match_field_display()} {self.get_match_type_display()}"
|
||||||
|
|||||||
Reference in new issue
Block a user