Add an atomic per-feed claim to poll_due_feeds (T1)

poll_due_feeds scanned for due feeds and dispatched fetch_feed without
marking them claimed, so overlapping beat ticks or a slow-running
fetch could dispatch the same feed twice, racing on ETag/status/error-
count writes. Feed.claimed_at is now set via an UPDATE ... WHERE
claimed_at IS NULL OR claimed_at < stale-cutoff, so only one caller's
UPDATE can ever match a given row; fetch_feed clears it when done.
A claim older than CLAIM_STALE_MINUTES is treated as abandoned so a
worker dying mid-fetch doesn't wedge the feed forever.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-10-31 12:03:18 -06:00
1 parent d7b433e580
commit b7d9ee672c
4 files changed
+108 -4

No files matched your search

@@ -0,0 +1,18 @@
# Generated by Django 5.2.18 on 2026-10-31 18:00
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("feeds", "0002_remove_folder"),
]
operations = [
migrations.AddField(
model_name="feed",
name="claimed_at",
field=models.DateTimeField(blank=True, null=True),
),
]
+9
View File
@@ -40,6 +40,15 @@ class Feed(models.Model):
fetch_error_count = models.PositiveIntegerField(default=0) fetch_error_count = models.PositiveIntegerField(default=0)
is_active = models.BooleanField(default=True) is_active = models.BooleanField(default=True)
# Set by poll_due_feeds' atomic claim (an UPDATE ... WHERE clause, not
# select_for_update — Celery Beat and fetch_feed run in separate
# transactions) and cleared once fetch_feed finishes, so overlapping
# beat ticks or a fetch that runs long can't both dispatch the same
# feed. A claim older than CLAIM_STALE_MINUTES (tasks.py) is treated as
# abandoned and can be reclaimed, so a worker dying mid-fetch doesn't
# wedge the feed forever.
claimed_at = models.DateTimeField(null=True, blank=True)
created_at = models.DateTimeField(auto_now_add=True) created_at = models.DateTimeField(auto_now_add=True)
def __str__(self): def __str__(self):
+29 -4
View File
@@ -4,6 +4,7 @@ from io import BytesIO
from celery import shared_task from celery import shared_task
from django.contrib.auth import get_user_model from django.contrib.auth import get_user_model
from django.db.models import Q
from django.utils import timezone from django.utils import timezone
from apps.articles.tasks import apply_filter_rules_for_feed from apps.articles.tasks import apply_filter_rules_for_feed
@@ -14,13 +15,23 @@ from .services.fetcher import effective_interval_minutes, fetch_and_ingest
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# How long a claimed-but-unfinished fetch blocks the same feed from being
# claimed again — must comfortably exceed FETCH_TIMEOUT_SECONDS plus parse/
# ingest time, so a slow-but-healthy fetch is never raced by the next tick,
# while a worker that died mid-fetch doesn't wedge the feed forever.
CLAIM_STALE_MINUTES = 10
@shared_task @shared_task
def poll_due_feeds(): def poll_due_feeds():
"""Find every active Feed whose polling interval has elapsed and """Find every active Feed whose polling interval has elapsed, atomically
dispatch a fetch_feed task for each — fan-out happens here, not via one claim it (so overlapping beat ticks or a slow-running previous fetch
Celery Beat entry per feed. can't dispatch the same feed twice), and dispatch a fetch_feed task for
each claimed feed — fan-out happens here, not via one Celery Beat entry
per feed.
""" """
now = timezone.now()
claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
due_feed_ids = [ due_feed_ids = [
feed.id feed.id
for feed in Feed.objects.filter(is_active=True).only( for feed in Feed.objects.filter(is_active=True).only(
@@ -28,9 +39,22 @@ def poll_due_feeds():
) )
if _is_due(feed) if _is_due(feed)
] ]
claimed_ids = []
for feed_id in due_feed_ids: for feed_id in due_feed_ids:
# The UPDATE's WHERE clause re-checks staleness at the database
# level: it only claims rows nobody else has already claimed more
# recently than CLAIM_STALE_MINUTES ago, so two workers racing this
# same loop can't both get rowcount=1 for the same feed.
claimed = (
Feed.objects.filter(id=feed_id, is_active=True)
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
.update(claimed_at=now)
)
if claimed:
claimed_ids.append(feed_id)
for feed_id in claimed_ids:
fetch_feed.delay(feed_id) fetch_feed.delay(feed_id)
return len(due_feed_ids) return len(claimed_ids)
def _is_due(feed: Feed) -> bool: def _is_due(feed: Feed) -> bool:
@@ -48,6 +72,7 @@ def fetch_feed(feed_id: int):
return None return None
result = fetch_and_ingest(feed) result = fetch_and_ingest(feed)
feed.claimed_at = None
feed.save() feed.save()
changed_article_ids = result.new_article_ids + result.updated_article_ids changed_article_ids = result.new_article_ids + result.updated_article_ids
+52
View File
@@ -40,6 +40,58 @@ def test_poll_due_feeds_dispatches_only_due_feeds():
assert inactive_but_overdue.id not in dispatched_ids assert inactive_but_overdue.id not in dispatched_ids
@pytest.mark.django_db
def test_poll_due_feeds_does_not_reclaim_a_feed_it_just_claimed():
# Models a second beat tick (or an overlapping worker) running
# poll_due_feeds before the first claimed fetch_feed has finished —
# the feed must only be dispatched once.
overdue = Feed.objects.create(
feed_url="https://c.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
fetch_interval_minutes=60,
)
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
first_count = poll_due_feeds()
second_count = poll_due_feeds()
assert first_count == 1
assert second_count == 0
assert mock_delay.call_count == 1
overdue.refresh_from_db()
assert overdue.claimed_at is not None
@pytest.mark.django_db
def test_poll_due_feeds_reclaims_a_stale_claim():
from apps.feeds.tasks import CLAIM_STALE_MINUTES
overdue = Feed.objects.create(
feed_url="https://c.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
fetch_interval_minutes=60,
claimed_at=timezone.now() - timedelta(minutes=CLAIM_STALE_MINUTES + 1),
)
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
dispatched_count = poll_due_feeds()
assert dispatched_count == 1
assert mock_delay.call_args.args[0] == overdue.id
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_task_clears_its_claim():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", claimed_at=timezone.now())
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
fetch_feed(feed.id)
feed.refresh_from_db()
assert feed.claimed_at is None
GOOD_FEED_XML = """<?xml version="1.0"?> GOOD_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel> <rss version="2.0"><channel>
<title>Example Blog</title> <title>Example Blog</title>