Files
Evection/apps/feeds/models.py
T
ksmithandClaude Sonnet 5 b33fe905d2 Push due-feed scheduling into an indexed DB query (C1)
poll_due_feeds loaded every active feed's timing fields and recomputed
each one's error-backoff due-ness in Python on every beat tick (every
1-2 minutes) — scheduler CPU and DB reads grew linearly with total feed
count regardless of how many were actually due.

Feed.next_fetch_at (indexed together with is_active) is now set by
fetch_feed itself right after every fetch attempt, success or error,
using the same backoff formula that used to run per-feed in
poll_due_feeds. The scheduler's query is now a direct indexed range
scan (next_fetch_at <= now), ordered so the most overdue feeds claim
first, and bounded to POLL_BATCH_SIZE so a large backlog spreads across
several ticks instead of dispatching everything at once. A migration
backfills next_fetch_at for existing feeds using the same formula.
reset_feed_backoff also clears next_fetch_at so a manually-reset feed
is immediately due again, matching its prior behavior.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-31 16:09:18 -06:00

93 lines
3.7 KiB
Python

from django.conf import settings
from django.db import models
class Feed(models.Model):
"""A single polled RSS/Atom source, shared across every subscriber.
One row per feed_url regardless of how many users subscribe to it — the
fetch pipeline polls each feed exactly once and every Subscription below
just points at the same Feed/Article data. Never add per-user fetching.
"""
STATUS_PENDING = "pending"
STATUS_OK = "ok"
STATUS_NOT_MODIFIED = "not_modified"
STATUS_ERROR = "error"
STATUS_CHOICES = [
(STATUS_PENDING, "Pending"),
(STATUS_OK, "OK"),
(STATUS_NOT_MODIFIED, "Not modified"),
(STATUS_ERROR, "Error"),
]
feed_url = models.URLField(max_length=1000, unique=True)
site_url = models.URLField(max_length=1000, blank=True)
title = models.CharField(max_length=500, blank=True)
favicon_url = models.URLField(max_length=1000, blank=True)
etag = models.CharField(max_length=255, blank=True)
last_modified = models.CharField(max_length=255, blank=True)
last_fetched_at = models.DateTimeField(null=True, blank=True)
last_fetch_status = models.CharField(
max_length=20, choices=STATUS_CHOICES, default=STATUS_PENDING
)
last_error_message = models.TextField(blank=True)
# The poll interval itself is site-wide now (apps.core.models.
# SiteSettings), not per-feed — only the error-driven backoff on top
# of it still varies per feed (see apps.feeds.services.fetcher.
# effective_interval_minutes).
fetch_error_count = models.PositiveIntegerField(default=0)
is_active = models.BooleanField(default=True)
# Set by fetch_feed right after every fetch attempt (success or error)
# to last_fetched_at + effective_interval_minutes — precomputed once,
# at write time, instead of poll_due_feeds re-deriving due-ness (which
# depends on fetch_error_count's exponential backoff) for every active
# feed on every tick (C1). NULL means "due now", same as a feed that's
# never been fetched. Indexed with is_active below so the scheduler's
# query is a direct index range scan, not a full table scan.
next_fetch_at = models.DateTimeField(null=True, blank=True)
# Set by poll_due_feeds' atomic claim (an UPDATE ... WHERE clause, not
# select_for_update — Celery Beat and fetch_feed run in separate
# transactions) and cleared once fetch_feed finishes, so overlapping
# beat ticks or a fetch that runs long can't both dispatch the same
# feed. A claim older than CLAIM_STALE_MINUTES (tasks.py) is treated as
# abandoned and can be reclaimed, so a worker dying mid-fetch doesn't
# wedge the feed forever.
claimed_at = models.DateTimeField(null=True, blank=True)
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
indexes = [models.Index(fields=["is_active", "next_fetch_at"])]
def __str__(self):
return self.title or self.feed_url
class Subscription(models.Model):
"""Per-user join between a User and a shared Feed."""
user = models.ForeignKey(
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="subscriptions"
)
feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="subscriptions")
custom_title = models.CharField(max_length=500, blank=True)
order = models.PositiveIntegerField(default=0)
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
unique_together = [("user", "feed")]
ordering = ["order", "id"]
def __str__(self):
return self.custom_title or self.feed.title or self.feed.feed_url
@property
def display_title(self):
return self.custom_title or self.feed.title or self.feed.feed_url