from datetime import timedelta import redis from django.conf import settings from django.db.models import Max, Min from django.utils import timezone from apps.feeds.models import Feed from .models import SiteSettings # More than this many tasks waiting means workers aren't keeping up (or # aren't running) — fetches are piling up faster than they complete. MAX_QUEUE_DEPTH = 1000 # Past this share of active feeds failing, it's far more likely something # on our side (DNS, outbound network, a bad deploy) than every publisher # breaking at once. Needs a handful of feeds first so one broken feed out # of two doesn't read as an outage. MAX_FAILING_RATIO = 0.5 MIN_FEEDS_FOR_RATIO = 4 MIN_SUCCESS_WINDOW_MINUTES = 30 def queue_depth() -> int | None: """Tasks waiting in Celery's default queue, or None if the broker can't be reached. Celery's Redis transport keeps each queue as a Redis list named after the queue.""" try: client = redis.Redis.from_url( settings.CELERY_BROKER_URL, socket_connect_timeout=2, socket_timeout=2 ) return int(client.llen("celery")) except Exception: return None def _success_window() -> timedelta: floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15) base = max(SiteSettings.load().fetch_interval_minutes, floor) return timedelta(minutes=max(3 * base, MIN_SUCCESS_WINDOW_MINUTES)) def fetch_pipeline_status(now=None) -> tuple[dict[str, bool], dict]: """Checks that fetching is actually *succeeding*, not just scheduled (the poll heartbeat only proves beat and a worker are alive): returns ({check: passed}, detail) covering queue depth, whether any feed has been fetched successfully recently, and the share of feeds failing. """ now = now or timezone.now() detail: dict = {} depth = queue_depth() detail["queue_depth"] = depth checks = {"queue_depth": depth is not None and depth <= MAX_QUEUE_DEPTH} active = Feed.objects.filter(is_active=True) active_count = active.count() failing = active.filter(fetch_error_count__gt=0).count() detail["active_feeds"] = active_count detail["failing_feeds"] = failing window = _success_window() last_success = active.filter( last_fetch_status__in=[Feed.STATUS_OK, Feed.STATUS_NOT_MODIFIED] ).aggregate(latest=Max("last_fetched_at"))["latest"] if last_success is not None: detail["last_successful_fetch_age_seconds"] = int((now - last_success).total_seconds()) recent_success = now - last_success <= window elif active_count == 0: recent_success = True else: # Nothing has ever fetched successfully — only a problem once # the oldest feed has had a full window to be fetched. oldest = active.aggregate(oldest=Min("created_at"))["oldest"] recent_success = now - oldest <= window checks["recent_fetch_success"] = recent_success checks["feed_failures"] = not ( active_count >= MIN_FEEDS_FOR_RATIO and failing / active_count > MAX_FAILING_RATIO ) return checks, detail