Add an atomic per-feed claim to poll_due_feeds (T1)

poll_due_feeds scanned for due feeds and dispatched fetch_feed without
marking them claimed, so overlapping beat ticks or a slow-running
fetch could dispatch the same feed twice, racing on ETag/status/error-
count writes. Feed.claimed_at is now set via an UPDATE ... WHERE
claimed_at IS NULL OR claimed_at < stale-cutoff, so only one caller's
UPDATE can ever match a given row; fetch_feed clears it when done.
A claim older than CLAIM_STALE_MINUTES is treated as abandoned so a
worker dying mid-fetch doesn't wedge the feed forever.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-10-31 12:03:18 -06:00
1 parent d7b433e580
commit b7d9ee672c
4 files changed
+108 -4

No files matched your search

+29 -4
View File
@@ -4,6 +4,7 @@ from io import BytesIO
from celery import shared_task
from django.contrib.auth import get_user_model
from django.db.models import Q
from django.utils import timezone
from apps.articles.tasks import apply_filter_rules_for_feed
@@ -14,13 +15,23 @@ from .services.fetcher import effective_interval_minutes, fetch_and_ingest
logger = logging.getLogger(__name__)
# How long a claimed-but-unfinished fetch blocks the same feed from being
# claimed again — must comfortably exceed FETCH_TIMEOUT_SECONDS plus parse/
# ingest time, so a slow-but-healthy fetch is never raced by the next tick,
# while a worker that died mid-fetch doesn't wedge the feed forever.
CLAIM_STALE_MINUTES = 10
@shared_task
def poll_due_feeds():
"""Find every active Feed whose polling interval has elapsed and
dispatch a fetch_feed task for each — fan-out happens here, not via one
Celery Beat entry per feed.
"""Find every active Feed whose polling interval has elapsed, atomically
claim it (so overlapping beat ticks or a slow-running previous fetch
can't dispatch the same feed twice), and dispatch a fetch_feed task for
each claimed feed — fan-out happens here, not via one Celery Beat entry
per feed.
"""
now = timezone.now()
claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
due_feed_ids = [
feed.id
for feed in Feed.objects.filter(is_active=True).only(
@@ -28,9 +39,22 @@ def poll_due_feeds():
)
if _is_due(feed)
]
claimed_ids = []
for feed_id in due_feed_ids:
# The UPDATE's WHERE clause re-checks staleness at the database
# level: it only claims rows nobody else has already claimed more
# recently than CLAIM_STALE_MINUTES ago, so two workers racing this
# same loop can't both get rowcount=1 for the same feed.
claimed = (
Feed.objects.filter(id=feed_id, is_active=True)
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
.update(claimed_at=now)
)
if claimed:
claimed_ids.append(feed_id)
for feed_id in claimed_ids:
fetch_feed.delay(feed_id)
return len(due_feed_ids)
return len(claimed_ids)
def _is_due(feed: Feed) -> bool:
@@ -48,6 +72,7 @@ def fetch_feed(feed_id: int):
return None
result = fetch_and_ingest(feed)
feed.claimed_at = None
feed.save()
changed_article_ids = result.new_article_ids + result.updated_article_ids