import logging from datetime import timedelta from io import BytesIO from celery import shared_task from django.contrib.auth import get_user_model from django.core.cache import cache from django.db import InterfaceError, OperationalError from django.db.models import Q from django.utils import timezone from apps.articles.tasks import apply_filter_rules_for_feed from apps.core.cache_utils import invalidate_feed_subscribers_sidebar_cache from apps.core.models import SiteSettings from .models import Feed from .opml import import_opml from .services.fetcher import effective_interval_minutes, fetch_and_ingest, record_fetch_error logger = logging.getLogger(__name__) # How long a claimed-but-unfinished fetch blocks the same feed from being # claimed again — must comfortably exceed FETCH_TIMEOUT_SECONDS plus parse/ # ingest time, so a slow-but-healthy fetch is never raced by the next tick, # while a worker that died mid-fetch doesn't wedge the feed forever. CLAIM_STALE_MINUTES = 10 # Caps how many feeds one poll_due_feeds tick will claim/dispatch, so a # large backlog (first run after a long outage, a huge import) spreads # across several beat ticks instead of enqueuing everything — and every # bounded Feed query below stays index-only work, not a full scan (C1). POLL_BATCH_SIZE = 500 RULE_DISPATCH_CHUNK_SIZE = 50 # Written on every poll_due_feeds tick; apps.core.views.health_ready # reads it to tell whether beat is scheduling and a worker is consuming. POLL_HEARTBEAT_CACHE_KEY = "poll-due-feeds-heartbeat" POLL_HEARTBEAT_TTL_SECONDS = 3600 @shared_task def poll_due_feeds(): """Find a bounded batch of active Feeds whose next_fetch_at has elapsed (set by fetch_feed itself after every attempt — see Feed. next_fetch_at), atomically claim them (so overlapping beat ticks or a slow-running previous fetch can't dispatch the same feed twice), and dispatch a fetch_feed task for each claimed feed — fan-out happens here, not via one Celery Beat entry per feed. """ now = timezone.now() cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS) due_feed_ids = list( Feed.objects.filter(is_active=True) .filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now)) .filter( Q(claimed_at__isnull=True) | Q(claimed_at__lt=now - timedelta(minutes=CLAIM_STALE_MINUTES)) ) .order_by("next_fetch_at") .values_list("id", flat=True)[:POLL_BATCH_SIZE] ) dispatched = 0 for feed_id in due_feed_ids: try: if enqueue_fetch(feed_id): dispatched += 1 except Exception: # Broker trouble: enqueue_fetch already released this feed's # claim, so it's simply picked up again next tick. logger.exception("Could not queue a fetch for feed %s", feed_id) return dispatched def claim_feed(feed_id: int) -> bool: """Atomically marks a feed as having a fetch in flight (an UPDATE whose WHERE clause re-checks staleness, so two callers racing for the same feed can't both get rowcount=1). Every path that dispatches a fetch — the periodic poller, a newly added feed, an admin retry — goes through this, so they can never run the same feed concurrently.""" now = timezone.now() stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES) return bool( Feed.objects.filter(id=feed_id, is_active=True) .filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=stale_before)) .update(claimed_at=now) ) def enqueue_fetch(feed_id: int) -> bool: """Claim a feed and queue its fetch. False means another fetch of it is already in flight (nothing was queued — that one will do the work). If queueing itself fails the claim is released and the error re-raised, so a broker outage doesn't leave the feed claimed until it goes stale. """ if not claim_feed(feed_id): return False try: fetch_feed.delay(feed_id) except Exception: Feed.objects.filter(pk=feed_id).update(claimed_at=None) raise return True def _schedule_next_fetch(feed: Feed) -> None: """Sets feed.next_fetch_at from its (just-updated) fetch_error_count — called right before every feed.save() in fetch_feed, so the next poll_due_feeds tick can find it with a plain indexed range query instead of recomputing every active feed's backoff in Python (C1).""" configured_interval = SiteSettings.load().fetch_interval_minutes interval = effective_interval_minutes(feed, configured_interval) feed.next_fetch_at = timezone.now() + timedelta(minutes=interval) # Bounds a single feed's fetch+parse+ingest, independent of # FETCH_TIMEOUT_SECONDS (apps.feeds.services.fetcher) which only covers # the HTTP request itself: parsing/sanitizing/writing a large-but-under- # the-byte-cap feed could still run long. soft_time_limit lets the # except-Exception handler below record a normal feed error (and clear # the claim) before the hard time_limit kills the worker process outright. FETCH_FEED_SOFT_TIME_LIMIT = 60 FETCH_FEED_TIME_LIMIT = 90 # Failures that say nothing about the feed and usually clear up in # seconds — a locked SQLite file, a dropped database connection — are # retried promptly (R2-T5) instead of waiting out the next poll interval, # which under error backoff can be hours. Bounded: three attempts, 30s / # 60s / 120s apart, then the normal "record the error and back off" path. # Anything else (a parser bug, the soft time limit) is not retried: that # would just repeat the same failure. The claim is held throughout so the # poller can't also fetch the feed; it's far shorter than CLAIM_STALE_MINUTES. TRANSIENT_FETCH_ERRORS = (OperationalError, InterfaceError) FETCH_MAX_RETRIES = 3 FETCH_RETRY_BASE_DELAY_SECONDS = 30 def _record_unexpected_failure(feed: Feed) -> None: logger.exception("Feed %s: unexpected error during fetch", feed.feed_url) record_fetch_error(feed, "Unexpected error — see worker logs") feed.last_fetched_at = timezone.now() feed.claimed_at = None _schedule_next_fetch(feed) feed.save() @shared_task( bind=True, time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT ) def fetch_feed(self, feed_id: int): try: feed = Feed.objects.get(pk=feed_id, is_active=True) except Feed.DoesNotExist: return None try: result = fetch_and_ingest(feed) except TRANSIENT_FETCH_ERRORS as exc: if self.request.retries < FETCH_MAX_RETRIES: delay = FETCH_RETRY_BASE_DELAY_SECONDS * 2**self.request.retries logger.warning( "Feed %s: transient error (%s); retry %d/%d in %ds", feed.feed_url, type(exc).__name__, self.request.retries + 1, FETCH_MAX_RETRIES, delay, ) raise self.retry(exc=exc, countdown=delay, max_retries=FETCH_MAX_RETRIES) _record_unexpected_failure(feed) raise except Exception: # fetch_and_ingest already handles the expected failure modes # (network errors, SSRF-blocked targets, unparseable feeds) by # returning an error FetchResult rather than raising — anything # that reaches here is unexpected (a parser/sanitizer bug, a DB # error, this task's own soft time limit), so it's recorded the # same way a handled failure would be rather than silently # leaving the feed's status stale, the claim held for the full # CLAIM_STALE_MINUTES, and no record of what happened. Re-raised # afterward so Celery still marks the task FAILED (visible in # worker logs/monitoring) instead of swallowing it. _record_unexpected_failure(feed) raise feed.claimed_at = None _schedule_next_fetch(feed) feed.save() changed_article_ids = result.new_article_ids + result.updated_article_ids if changed_article_ids: invalidate_feed_subscribers_sidebar_cache(feed_id) logger.info( "Feed %s: %d new article(s), %d updated", feed.feed_url, len(result.new_article_ids), len(result.updated_article_ids), ) # Chunked so one huge poll (a first fetch of a long archive feed) # becomes several bounded rule-evaluation tasks, each well inside # the worker's time limits, instead of one unbounded one (C2). for start in range(0, len(changed_article_ids), RULE_DISPATCH_CHUNK_SIZE): apply_filter_rules_for_feed.delay( feed_id, changed_article_ids[start : start + RULE_DISPATCH_CHUNK_SIZE] ) return { "status": result.status, "new": len(result.new_article_ids), "updated": len(result.updated_article_ids), } def reset_backoff(feed_id: int) -> None: """Clear a feed's error count and due-date backoff so it's retried at its normal configured interval instead of the backed-off one (next_fetch_at=None also makes it immediately due, same as a feed that's never been fetched). A plain synchronous function so a caller can finish the reset *before* dispatching the fetch — as an async task it could land after the fetch and undo its freshly scheduled next_fetch_at.""" Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None) @shared_task def reset_feed_backoff(feed_id: int): reset_backoff(feed_id) @shared_task def import_opml_task(user_id: int, upload_key: str): """Background path for large OPML imports — see OPML_ASYNC_THRESHOLD in apps.feeds.views. The document itself waits in the cache under upload_key (set by the view) rather than travelling in the broker message; it's removed once read, and expires on its own if this task never runs.""" content = cache.get(upload_key) if content is None: logger.warning("OPML import %s expired or missing; nothing imported", upload_key) return None cache.delete(upload_key) try: user = get_user_model().objects.get(pk=user_id) except get_user_model().DoesNotExist: return None return import_opml(user, BytesIO(content))