import logging from datetime import timedelta from io import BytesIO from celery import shared_task from django.contrib.auth import get_user_model from django.db.models import Q from django.utils import timezone from apps.articles.tasks import apply_filter_rules_for_feed from apps.core.models import SiteSettings from .models import Feed from .opml import import_opml from .services.fetcher import effective_interval_minutes, fetch_and_ingest, record_fetch_error logger = logging.getLogger(__name__) # How long a claimed-but-unfinished fetch blocks the same feed from being # claimed again — must comfortably exceed FETCH_TIMEOUT_SECONDS plus parse/ # ingest time, so a slow-but-healthy fetch is never raced by the next tick, # while a worker that died mid-fetch doesn't wedge the feed forever. CLAIM_STALE_MINUTES = 10 # Caps how many feeds one poll_due_feeds tick will claim/dispatch, so a # large backlog (first run after a long outage, a huge import) spreads # across several beat ticks instead of enqueuing everything — and every # bounded Feed query below stays index-only work, not a full scan (C1). POLL_BATCH_SIZE = 500 @shared_task def poll_due_feeds(): """Find a bounded batch of active Feeds whose next_fetch_at has elapsed (set by fetch_feed itself after every attempt — see Feed. next_fetch_at), atomically claim them (so overlapping beat ticks or a slow-running previous fetch can't dispatch the same feed twice), and dispatch a fetch_feed task for each claimed feed — fan-out happens here, not via one Celery Beat entry per feed. """ now = timezone.now() claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES) due_feed_ids = list( Feed.objects.filter(is_active=True) .filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now)) .filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before)) .order_by("next_fetch_at") .values_list("id", flat=True)[:POLL_BATCH_SIZE] ) claimed_ids = [] for feed_id in due_feed_ids: # The UPDATE's WHERE clause re-checks staleness at the database # level: it only claims rows nobody else has already claimed more # recently than CLAIM_STALE_MINUTES ago, so two workers racing this # same loop can't both get rowcount=1 for the same feed. claimed = ( Feed.objects.filter(id=feed_id, is_active=True) .filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before)) .update(claimed_at=now) ) if claimed: claimed_ids.append(feed_id) for feed_id in claimed_ids: fetch_feed.delay(feed_id) return len(claimed_ids) def _schedule_next_fetch(feed: Feed) -> None: """Sets feed.next_fetch_at from its (just-updated) fetch_error_count — called right before every feed.save() in fetch_feed, so the next poll_due_feeds tick can find it with a plain indexed range query instead of recomputing every active feed's backoff in Python (C1).""" configured_interval = SiteSettings.load().fetch_interval_minutes interval = effective_interval_minutes(feed, configured_interval) feed.next_fetch_at = timezone.now() + timedelta(minutes=interval) # Bounds a single feed's fetch+parse+ingest, independent of # FETCH_TIMEOUT_SECONDS (apps.feeds.services.fetcher) which only covers # the HTTP request itself: parsing/sanitizing/writing a large-but-under- # the-byte-cap feed could still run long. soft_time_limit lets the # except-Exception handler below record a normal feed error (and clear # the claim) before the hard time_limit kills the worker process outright. FETCH_FEED_SOFT_TIME_LIMIT = 60 FETCH_FEED_TIME_LIMIT = 90 @shared_task(time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT) def fetch_feed(feed_id: int): try: feed = Feed.objects.get(pk=feed_id, is_active=True) except Feed.DoesNotExist: return None try: result = fetch_and_ingest(feed) except Exception: # fetch_and_ingest already handles the expected failure modes # (network errors, SSRF-blocked targets, unparseable feeds) by # returning an error FetchResult rather than raising — anything # that reaches here is unexpected (a parser/sanitizer bug, a DB # error, this task's own soft time limit), so it's recorded the # same way a handled failure would be rather than silently # leaving the feed's status stale, the claim held for the full # CLAIM_STALE_MINUTES, and no record of what happened. Re-raised # afterward so Celery still marks the task FAILED (visible in # worker logs/monitoring) instead of swallowing it. logger.exception("Feed %s: unexpected error during fetch", feed.feed_url) record_fetch_error(feed, "Unexpected error — see worker logs") feed.last_fetched_at = timezone.now() feed.claimed_at = None _schedule_next_fetch(feed) feed.save() raise feed.claimed_at = None _schedule_next_fetch(feed) feed.save() changed_article_ids = result.new_article_ids + result.updated_article_ids if changed_article_ids: logger.info( "Feed %s: %d new article(s), %d updated", feed.feed_url, len(result.new_article_ids), len(result.updated_article_ids), ) apply_filter_rules_for_feed.delay(feed_id, changed_article_ids) return { "status": result.status, "new": len(result.new_article_ids), "updated": len(result.updated_article_ids), } @shared_task def reset_feed_backoff(feed_id: int): """Admin-triggered: clear a feed's error count and due-date backoff so it's retried at its normal configured interval instead of the backed- off one (next_fetch_at=None also makes it immediately due for the next poll_due_feeds tick, same as a feed that's never been fetched).""" Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None) @shared_task def import_opml_task(user_id: int, opml_content: str): """Background path for large OPML imports — see OPML_ASYNC_THRESHOLD in apps.feeds.views. opml_content is passed as text rather than a file handle since Celery tasks must be JSON-serializable.""" try: user = get_user_model().objects.get(pk=user_id) except get_user_model().DoesNotExist: return None return import_opml(user, BytesIO(opml_content.encode("utf-8")))