import logging from datetime import timedelta from io import BytesIO from celery import shared_task from django.contrib.auth import get_user_model from django.db.models import Q from django.utils import timezone from apps.articles.tasks import apply_filter_rules_for_feed from .models import Feed from .opml import import_opml from .services.fetcher import effective_interval_minutes, fetch_and_ingest logger = logging.getLogger(__name__) # How long a claimed-but-unfinished fetch blocks the same feed from being # claimed again — must comfortably exceed FETCH_TIMEOUT_SECONDS plus parse/ # ingest time, so a slow-but-healthy fetch is never raced by the next tick, # while a worker that died mid-fetch doesn't wedge the feed forever. CLAIM_STALE_MINUTES = 10 @shared_task def poll_due_feeds(): """Find every active Feed whose polling interval has elapsed, atomically claim it (so overlapping beat ticks or a slow-running previous fetch can't dispatch the same feed twice), and dispatch a fetch_feed task for each claimed feed — fan-out happens here, not via one Celery Beat entry per feed. """ now = timezone.now() claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES) due_feed_ids = [ feed.id for feed in Feed.objects.filter(is_active=True).only( "id", "last_fetched_at", "fetch_interval_minutes", "fetch_error_count" ) if _is_due(feed) ] claimed_ids = [] for feed_id in due_feed_ids: # The UPDATE's WHERE clause re-checks staleness at the database # level: it only claims rows nobody else has already claimed more # recently than CLAIM_STALE_MINUTES ago, so two workers racing this # same loop can't both get rowcount=1 for the same feed. claimed = ( Feed.objects.filter(id=feed_id, is_active=True) .filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before)) .update(claimed_at=now) ) if claimed: claimed_ids.append(feed_id) for feed_id in claimed_ids: fetch_feed.delay(feed_id) return len(claimed_ids) def _is_due(feed: Feed) -> bool: if feed.last_fetched_at is None: return True interval = effective_interval_minutes(feed) return feed.last_fetched_at <= timezone.now() - timedelta(minutes=interval) @shared_task def fetch_feed(feed_id: int): try: feed = Feed.objects.get(pk=feed_id, is_active=True) except Feed.DoesNotExist: return None result = fetch_and_ingest(feed) feed.claimed_at = None feed.save() changed_article_ids = result.new_article_ids + result.updated_article_ids if changed_article_ids: logger.info( "Feed %s: %d new article(s), %d updated", feed.feed_url, len(result.new_article_ids), len(result.updated_article_ids), ) apply_filter_rules_for_feed.delay(feed_id, changed_article_ids) return { "status": result.status, "new": len(result.new_article_ids), "updated": len(result.updated_article_ids), } @shared_task def reset_feed_backoff(feed_id: int): """Admin-triggered: clear a feed's error count so it's retried at its normal configured interval instead of the backed-off one.""" Feed.objects.filter(pk=feed_id).update(fetch_error_count=0) @shared_task def import_opml_task(user_id: int, opml_content: str): """Background path for large OPML imports — see OPML_ASYNC_THRESHOLD in apps.feeds.views. opml_content is passed as text rather than a file handle since Celery tasks must be JSON-serializable.""" try: user = get_user_model().objects.get(pk=user_id) except get_user_model().DoesNotExist: return None return import_opml(user, BytesIO(opml_content.encode("utf-8")))