Private
Public Access
An unexpected exception while fetching was recorded and re-raised, then the feed waited for its next scheduled poll — which under error backoff can be hours — even for a locked SQLite file or a dropped database connection that clears in seconds. fetch_feed now retries OperationalError/InterfaceError up to three times with doubling delays (30s, 60s, 120s), holding the feed's claim so the poller can't also fetch it and without counting it against the feed. Anything else, and a transient error after the retries are spent, keep the existing path: record the error, back off, release the claim, re-raise. Time limits are unchanged. Verified with a real Celery worker against a real Redis broker, with the fetch rigged to fail twice: retries fired after 1s and 2s (delays shortened for the probe), the third attempt succeeded, and the feed ended healthy, unclaimed and scheduled. (Celery's eager test mode mishandles retries, so the unit tests mock retry() and assert the countdowns, claim and error-count behavior instead.) Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
244 lines
10 KiB
Python
244 lines
10 KiB
Python
import logging
|
|
from datetime import timedelta
|
|
from io import BytesIO
|
|
|
|
from celery import shared_task
|
|
from django.contrib.auth import get_user_model
|
|
from django.core.cache import cache
|
|
from django.db import InterfaceError, OperationalError
|
|
from django.db.models import Q
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.tasks import apply_filter_rules_for_feed
|
|
from apps.core.models import SiteSettings
|
|
|
|
from .models import Feed
|
|
from .opml import import_opml
|
|
from .services.fetcher import effective_interval_minutes, fetch_and_ingest, record_fetch_error
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# How long a claimed-but-unfinished fetch blocks the same feed from being
|
|
# claimed again — must comfortably exceed FETCH_TIMEOUT_SECONDS plus parse/
|
|
# ingest time, so a slow-but-healthy fetch is never raced by the next tick,
|
|
# while a worker that died mid-fetch doesn't wedge the feed forever.
|
|
CLAIM_STALE_MINUTES = 10
|
|
|
|
# Caps how many feeds one poll_due_feeds tick will claim/dispatch, so a
|
|
# large backlog (first run after a long outage, a huge import) spreads
|
|
# across several beat ticks instead of enqueuing everything — and every
|
|
# bounded Feed query below stays index-only work, not a full scan (C1).
|
|
POLL_BATCH_SIZE = 500
|
|
RULE_DISPATCH_CHUNK_SIZE = 50
|
|
|
|
# Written on every poll_due_feeds tick; apps.core.views.health_ready
|
|
# reads it to tell whether beat is scheduling and a worker is consuming.
|
|
POLL_HEARTBEAT_CACHE_KEY = "poll-due-feeds-heartbeat"
|
|
POLL_HEARTBEAT_TTL_SECONDS = 3600
|
|
|
|
|
|
@shared_task
|
|
def poll_due_feeds():
|
|
"""Find a bounded batch of active Feeds whose next_fetch_at has
|
|
elapsed (set by fetch_feed itself after every attempt — see Feed.
|
|
next_fetch_at), atomically claim them (so overlapping beat ticks or a
|
|
slow-running previous fetch can't dispatch the same feed twice), and
|
|
dispatch a fetch_feed task for each claimed feed — fan-out happens
|
|
here, not via one Celery Beat entry per feed.
|
|
"""
|
|
now = timezone.now()
|
|
cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS)
|
|
due_feed_ids = list(
|
|
Feed.objects.filter(is_active=True)
|
|
.filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now))
|
|
.filter(
|
|
Q(claimed_at__isnull=True)
|
|
| Q(claimed_at__lt=now - timedelta(minutes=CLAIM_STALE_MINUTES))
|
|
)
|
|
.order_by("next_fetch_at")
|
|
.values_list("id", flat=True)[:POLL_BATCH_SIZE]
|
|
)
|
|
dispatched = 0
|
|
for feed_id in due_feed_ids:
|
|
try:
|
|
if enqueue_fetch(feed_id):
|
|
dispatched += 1
|
|
except Exception:
|
|
# Broker trouble: enqueue_fetch already released this feed's
|
|
# claim, so it's simply picked up again next tick.
|
|
logger.exception("Could not queue a fetch for feed %s", feed_id)
|
|
return dispatched
|
|
|
|
|
|
def claim_feed(feed_id: int) -> bool:
|
|
"""Atomically marks a feed as having a fetch in flight (an UPDATE whose
|
|
WHERE clause re-checks staleness, so two callers racing for the same
|
|
feed can't both get rowcount=1). Every path that dispatches a fetch —
|
|
the periodic poller, a newly added feed, an admin retry — goes through
|
|
this, so they can never run the same feed concurrently."""
|
|
now = timezone.now()
|
|
stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
|
|
return bool(
|
|
Feed.objects.filter(id=feed_id, is_active=True)
|
|
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=stale_before))
|
|
.update(claimed_at=now)
|
|
)
|
|
|
|
|
|
def enqueue_fetch(feed_id: int) -> bool:
|
|
"""Claim a feed and queue its fetch. False means another fetch of it is
|
|
already in flight (nothing was queued — that one will do the work).
|
|
If queueing itself fails the claim is released and the error re-raised,
|
|
so a broker outage doesn't leave the feed claimed until it goes stale.
|
|
"""
|
|
if not claim_feed(feed_id):
|
|
return False
|
|
try:
|
|
fetch_feed.delay(feed_id)
|
|
except Exception:
|
|
Feed.objects.filter(pk=feed_id).update(claimed_at=None)
|
|
raise
|
|
return True
|
|
|
|
|
|
def _schedule_next_fetch(feed: Feed) -> None:
|
|
"""Sets feed.next_fetch_at from its (just-updated) fetch_error_count —
|
|
called right before every feed.save() in fetch_feed, so the next
|
|
poll_due_feeds tick can find it with a plain indexed range query
|
|
instead of recomputing every active feed's backoff in Python (C1)."""
|
|
configured_interval = SiteSettings.load().fetch_interval_minutes
|
|
interval = effective_interval_minutes(feed, configured_interval)
|
|
feed.next_fetch_at = timezone.now() + timedelta(minutes=interval)
|
|
|
|
|
|
# Bounds a single feed's fetch+parse+ingest, independent of
|
|
# FETCH_TIMEOUT_SECONDS (apps.feeds.services.fetcher) which only covers
|
|
# the HTTP request itself: parsing/sanitizing/writing a large-but-under-
|
|
# the-byte-cap feed could still run long. soft_time_limit lets the
|
|
# except-Exception handler below record a normal feed error (and clear
|
|
# the claim) before the hard time_limit kills the worker process outright.
|
|
FETCH_FEED_SOFT_TIME_LIMIT = 60
|
|
FETCH_FEED_TIME_LIMIT = 90
|
|
|
|
# Failures that say nothing about the feed and usually clear up in
|
|
# seconds — a locked SQLite file, a dropped database connection — are
|
|
# retried promptly (R2-T5) instead of waiting out the next poll interval,
|
|
# which under error backoff can be hours. Bounded: three attempts, 30s /
|
|
# 60s / 120s apart, then the normal "record the error and back off" path.
|
|
# Anything else (a parser bug, the soft time limit) is not retried: that
|
|
# would just repeat the same failure. The claim is held throughout so the
|
|
# poller can't also fetch the feed; it's far shorter than CLAIM_STALE_MINUTES.
|
|
TRANSIENT_FETCH_ERRORS = (OperationalError, InterfaceError)
|
|
FETCH_MAX_RETRIES = 3
|
|
FETCH_RETRY_BASE_DELAY_SECONDS = 30
|
|
|
|
|
|
def _record_unexpected_failure(feed: Feed) -> None:
|
|
logger.exception("Feed %s: unexpected error during fetch", feed.feed_url)
|
|
record_fetch_error(feed, "Unexpected error — see worker logs")
|
|
feed.last_fetched_at = timezone.now()
|
|
feed.claimed_at = None
|
|
_schedule_next_fetch(feed)
|
|
feed.save()
|
|
|
|
|
|
@shared_task(
|
|
bind=True, time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT
|
|
)
|
|
def fetch_feed(self, feed_id: int):
|
|
try:
|
|
feed = Feed.objects.get(pk=feed_id, is_active=True)
|
|
except Feed.DoesNotExist:
|
|
return None
|
|
|
|
try:
|
|
result = fetch_and_ingest(feed)
|
|
except TRANSIENT_FETCH_ERRORS as exc:
|
|
if self.request.retries < FETCH_MAX_RETRIES:
|
|
delay = FETCH_RETRY_BASE_DELAY_SECONDS * 2**self.request.retries
|
|
logger.warning(
|
|
"Feed %s: transient error (%s); retry %d/%d in %ds",
|
|
feed.feed_url,
|
|
type(exc).__name__,
|
|
self.request.retries + 1,
|
|
FETCH_MAX_RETRIES,
|
|
delay,
|
|
)
|
|
raise self.retry(exc=exc, countdown=delay, max_retries=FETCH_MAX_RETRIES)
|
|
_record_unexpected_failure(feed)
|
|
raise
|
|
except Exception:
|
|
# fetch_and_ingest already handles the expected failure modes
|
|
# (network errors, SSRF-blocked targets, unparseable feeds) by
|
|
# returning an error FetchResult rather than raising — anything
|
|
# that reaches here is unexpected (a parser/sanitizer bug, a DB
|
|
# error, this task's own soft time limit), so it's recorded the
|
|
# same way a handled failure would be rather than silently
|
|
# leaving the feed's status stale, the claim held for the full
|
|
# CLAIM_STALE_MINUTES, and no record of what happened. Re-raised
|
|
# afterward so Celery still marks the task FAILED (visible in
|
|
# worker logs/monitoring) instead of swallowing it.
|
|
_record_unexpected_failure(feed)
|
|
raise
|
|
|
|
feed.claimed_at = None
|
|
_schedule_next_fetch(feed)
|
|
feed.save()
|
|
|
|
changed_article_ids = result.new_article_ids + result.updated_article_ids
|
|
if changed_article_ids:
|
|
logger.info(
|
|
"Feed %s: %d new article(s), %d updated",
|
|
feed.feed_url,
|
|
len(result.new_article_ids),
|
|
len(result.updated_article_ids),
|
|
)
|
|
# Chunked so one huge poll (a first fetch of a long archive feed)
|
|
# becomes several bounded rule-evaluation tasks, each well inside
|
|
# the worker's time limits, instead of one unbounded one (C2).
|
|
for start in range(0, len(changed_article_ids), RULE_DISPATCH_CHUNK_SIZE):
|
|
apply_filter_rules_for_feed.delay(
|
|
feed_id, changed_article_ids[start : start + RULE_DISPATCH_CHUNK_SIZE]
|
|
)
|
|
|
|
return {
|
|
"status": result.status,
|
|
"new": len(result.new_article_ids),
|
|
"updated": len(result.updated_article_ids),
|
|
}
|
|
|
|
|
|
def reset_backoff(feed_id: int) -> None:
|
|
"""Clear a feed's error count and due-date backoff so it's retried at
|
|
its normal configured interval instead of the backed-off one
|
|
(next_fetch_at=None also makes it immediately due, same as a feed
|
|
that's never been fetched). A plain synchronous function so a caller
|
|
can finish the reset *before* dispatching the fetch — as an async task
|
|
it could land after the fetch and undo its freshly scheduled
|
|
next_fetch_at."""
|
|
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
|
|
|
|
|
|
@shared_task
|
|
def reset_feed_backoff(feed_id: int):
|
|
reset_backoff(feed_id)
|
|
|
|
|
|
@shared_task
|
|
def import_opml_task(user_id: int, upload_key: str):
|
|
"""Background path for large OPML imports — see OPML_ASYNC_THRESHOLD
|
|
in apps.feeds.views. The document itself waits in the cache under
|
|
upload_key (set by the view) rather than travelling in the broker
|
|
message; it's removed once read, and expires on its own if this task
|
|
never runs."""
|
|
content = cache.get(upload_key)
|
|
if content is None:
|
|
logger.warning("OPML import %s expired or missing; nothing imported", upload_key)
|
|
return None
|
|
cache.delete(upload_key)
|
|
try:
|
|
user = get_user_model().objects.get(pk=user_id)
|
|
except get_user_model().DoesNotExist:
|
|
return None
|
|
return import_opml(user, BytesIO(content))
|