Private
Public Access
Route every feed fetch through one atomic claim (R2-T1, #32)
Adding a feed and the admin retry queued fetch_feed directly without claiming the feed, so the periodic poller (which sees a never-fetched or just-reset feed as due) could claim and fetch it too: duplicate fetches, and one task overwriting another's claim/ETag/next-fetch state. The retry path also queued the backoff reset and the fetch as two independent tasks that could run in either order. claim_feed/enqueue_fetch in apps.feeds.tasks are now the only way a fetch gets dispatched — the poller, add-feed and retry all use them, so a feed can only have one fetch in flight. If queueing fails the claim is released (the poller recovers it), and add-feed still saves the subscription during a broker outage. The retry resets backoff synchronously before claiming, and says so when a fetch is already running. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0975a7634b
commit
85914f9107
5 files changed
+184
-33
No files matched your search
+55
-21
@@ -47,30 +47,57 @@ def poll_due_feeds():
|
||||
"""
|
||||
now = timezone.now()
|
||||
cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS)
|
||||
claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
|
||||
due_feed_ids = list(
|
||||
Feed.objects.filter(is_active=True)
|
||||
.filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now))
|
||||
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
|
||||
.filter(
|
||||
Q(claimed_at__isnull=True)
|
||||
| Q(claimed_at__lt=now - timedelta(minutes=CLAIM_STALE_MINUTES))
|
||||
)
|
||||
.order_by("next_fetch_at")
|
||||
.values_list("id", flat=True)[:POLL_BATCH_SIZE]
|
||||
)
|
||||
claimed_ids = []
|
||||
dispatched = 0
|
||||
for feed_id in due_feed_ids:
|
||||
# The UPDATE's WHERE clause re-checks staleness at the database
|
||||
# level: it only claims rows nobody else has already claimed more
|
||||
# recently than CLAIM_STALE_MINUTES ago, so two workers racing this
|
||||
# same loop can't both get rowcount=1 for the same feed.
|
||||
claimed = (
|
||||
Feed.objects.filter(id=feed_id, is_active=True)
|
||||
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
|
||||
.update(claimed_at=now)
|
||||
)
|
||||
if claimed:
|
||||
claimed_ids.append(feed_id)
|
||||
for feed_id in claimed_ids:
|
||||
try:
|
||||
if enqueue_fetch(feed_id):
|
||||
dispatched += 1
|
||||
except Exception:
|
||||
# Broker trouble: enqueue_fetch already released this feed's
|
||||
# claim, so it's simply picked up again next tick.
|
||||
logger.exception("Could not queue a fetch for feed %s", feed_id)
|
||||
return dispatched
|
||||
|
||||
|
||||
def claim_feed(feed_id: int) -> bool:
|
||||
"""Atomically marks a feed as having a fetch in flight (an UPDATE whose
|
||||
WHERE clause re-checks staleness, so two callers racing for the same
|
||||
feed can't both get rowcount=1). Every path that dispatches a fetch —
|
||||
the periodic poller, a newly added feed, an admin retry — goes through
|
||||
this, so they can never run the same feed concurrently."""
|
||||
now = timezone.now()
|
||||
stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
|
||||
return bool(
|
||||
Feed.objects.filter(id=feed_id, is_active=True)
|
||||
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=stale_before))
|
||||
.update(claimed_at=now)
|
||||
)
|
||||
|
||||
|
||||
def enqueue_fetch(feed_id: int) -> bool:
|
||||
"""Claim a feed and queue its fetch. False means another fetch of it is
|
||||
already in flight (nothing was queued — that one will do the work).
|
||||
If queueing itself fails the claim is released and the error re-raised,
|
||||
so a broker outage doesn't leave the feed claimed until it goes stale.
|
||||
"""
|
||||
if not claim_feed(feed_id):
|
||||
return False
|
||||
try:
|
||||
fetch_feed.delay(feed_id)
|
||||
return len(claimed_ids)
|
||||
except Exception:
|
||||
Feed.objects.filter(pk=feed_id).update(claimed_at=None)
|
||||
raise
|
||||
return True
|
||||
|
||||
|
||||
def _schedule_next_fetch(feed: Feed) -> None:
|
||||
@@ -148,13 +175,20 @@ def fetch_feed(feed_id: int):
|
||||
}
|
||||
|
||||
|
||||
def reset_backoff(feed_id: int) -> None:
|
||||
"""Clear a feed's error count and due-date backoff so it's retried at
|
||||
its normal configured interval instead of the backed-off one
|
||||
(next_fetch_at=None also makes it immediately due, same as a feed
|
||||
that's never been fetched). A plain synchronous function so a caller
|
||||
can finish the reset *before* dispatching the fetch — as an async task
|
||||
it could land after the fetch and undo its freshly scheduled
|
||||
next_fetch_at."""
|
||||
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
|
||||
|
||||
|
||||
@shared_task
|
||||
def reset_feed_backoff(feed_id: int):
|
||||
"""Admin-triggered: clear a feed's error count and due-date backoff so
|
||||
it's retried at its normal configured interval instead of the backed-
|
||||
off one (next_fetch_at=None also makes it immediately due for the
|
||||
next poll_due_feeds tick, same as a feed that's never been fetched)."""
|
||||
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
|
||||
reset_backoff(feed_id)
|
||||
|
||||
|
||||
@shared_task
|
||||
|
||||
Reference in new issue
Block a user