import logging from datetime import timedelta from io import BytesIO from celery import shared_task from django.contrib.auth import get_user_model from django.db.models import Q from django.utils import timezone from apps.articles.tasks import apply_filter_rules_for_feed from .models import Feed from .opml import import_opml from .services.fetcher import effective_interval_minutes, fetch_and_ingest, record_fetch_error logger = logging.getLogger(__name__) # How long a claimed-but-unfinished fetch blocks the same feed from being # claimed again — must comfortably exceed FETCH_TIMEOUT_SECONDS plus parse/ # ingest time, so a slow-but-healthy fetch is never raced by the next tick, # while a worker that died mid-fetch doesn't wedge the feed forever. CLAIM_STALE_MINUTES = 10 @shared_task def poll_due_feeds(): """Find every active Feed whose polling interval has elapsed, atomically claim it (so overlapping beat ticks or a slow-running previous fetch can't dispatch the same feed twice), and dispatch a fetch_feed task for each claimed feed — fan-out happens here, not via one Celery Beat entry per feed. """ now = timezone.now() claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES) due_feed_ids = [ feed.id for feed in Feed.objects.filter(is_active=True).only( "id", "last_fetched_at", "fetch_interval_minutes", "fetch_error_count" ) if _is_due(feed) ] claimed_ids = [] for feed_id in due_feed_ids: # The UPDATE's WHERE clause re-checks staleness at the database # level: it only claims rows nobody else has already claimed more # recently than CLAIM_STALE_MINUTES ago, so two workers racing this # same loop can't both get rowcount=1 for the same feed. claimed = ( Feed.objects.filter(id=feed_id, is_active=True) .filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before)) .update(claimed_at=now) ) if claimed: claimed_ids.append(feed_id) for feed_id in claimed_ids: fetch_feed.delay(feed_id) return len(claimed_ids) def _is_due(feed: Feed) -> bool: if feed.last_fetched_at is None: return True interval = effective_interval_minutes(feed) return feed.last_fetched_at <= timezone.now() - timedelta(minutes=interval) # Bounds a single feed's fetch+parse+ingest, independent of # FETCH_TIMEOUT_SECONDS (apps.feeds.services.fetcher) which only covers # the HTTP request itself: parsing/sanitizing/writing a large-but-under- # the-byte-cap feed could still run long. soft_time_limit lets the # except-Exception handler below record a normal feed error (and clear # the claim) before the hard time_limit kills the worker process outright. FETCH_FEED_SOFT_TIME_LIMIT = 60 FETCH_FEED_TIME_LIMIT = 90 @shared_task(time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT) def fetch_feed(feed_id: int): try: feed = Feed.objects.get(pk=feed_id, is_active=True) except Feed.DoesNotExist: return None try: result = fetch_and_ingest(feed) except Exception: # fetch_and_ingest already handles the expected failure modes # (network errors, SSRF-blocked targets, unparseable feeds) by # returning an error FetchResult rather than raising — anything # that reaches here is unexpected (a parser/sanitizer bug, a DB # error, this task's own soft time limit), so it's recorded the # same way a handled failure would be rather than silently # leaving the feed's status stale, the claim held for the full # CLAIM_STALE_MINUTES, and no record of what happened. Re-raised # afterward so Celery still marks the task FAILED (visible in # worker logs/monitoring) instead of swallowing it. logger.exception("Feed %s: unexpected error during fetch", feed.feed_url) record_fetch_error(feed, "Unexpected error — see worker logs") feed.last_fetched_at = timezone.now() feed.claimed_at = None feed.save() raise feed.claimed_at = None feed.save() changed_article_ids = result.new_article_ids + result.updated_article_ids if changed_article_ids: logger.info( "Feed %s: %d new article(s), %d updated", feed.feed_url, len(result.new_article_ids), len(result.updated_article_ids), ) apply_filter_rules_for_feed.delay(feed_id, changed_article_ids) return { "status": result.status, "new": len(result.new_article_ids), "updated": len(result.updated_article_ids), } @shared_task def reset_feed_backoff(feed_id: int): """Admin-triggered: clear a feed's error count so it's retried at its normal configured interval instead of the backed-off one.""" Feed.objects.filter(pk=feed_id).update(fetch_error_count=0) @shared_task def import_opml_task(user_id: int, opml_content: str): """Background path for large OPML imports — see OPML_ASYNC_THRESHOLD in apps.feeds.views. opml_content is passed as text rather than a file handle since Celery tasks must be JSON-serializable.""" try: user = get_user_model().objects.get(pk=user_id) except get_user_model().DoesNotExist: return None return import_opml(user, BytesIO(opml_content.encode("utf-8")))