from datetime import timedelta import redis from django.conf import settings from django.core.cache import cache from django.db.models import Max, Min from django.utils import timezone from apps.feeds.models import Feed from .models import SiteSettings # More than this many tasks waiting means workers aren't keeping up (or # aren't running) — fetches are piling up faster than they complete. MAX_QUEUE_DEPTH = 1000 # Past this share of active feeds failing, it's far more likely something # on our side (DNS, outbound network, a bad deploy) than every publisher # breaking at once. Needs a handful of feeds first so one broken feed out # of two doesn't read as an outage. MAX_FAILING_RATIO = 0.5 MIN_FEEDS_FOR_RATIO = 4 MIN_SUCCESS_WINDOW_MINUTES = 30 def queue_depth() -> int | None: """Tasks waiting in Celery's default queue, or None if the broker can't be reached. Celery's Redis transport keeps each queue as a Redis list named after the queue.""" try: client = redis.Redis.from_url( settings.CELERY_BROKER_URL, socket_connect_timeout=2, socket_timeout=2 ) return int(client.llen("celery")) except Exception: return None def _success_window() -> timedelta: floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15) base = max(SiteSettings.load().fetch_interval_minutes, floor) return timedelta(minutes=max(3 * base, MIN_SUCCESS_WINDOW_MINUTES)) PIPELINE_CACHE_KEY = "health-ready-pipeline" def cached_fetch_pipeline_status() -> tuple[dict[str, bool], dict]: """fetch_pipeline_status, reused for EVECTION_HEALTH_CACHE_SECONDS (R2-C5): the queue/feed aggregates behind it cost real queries, and a monitor that polls /health/ready/ every second shouldn't turn that into recurring database and Redis load. The cheap liveness probes (database, Redis, poll heartbeat) are not cached — those stay live. A cache failure just means computing it fresh.""" ttl = settings.EVECTION_HEALTH_CACHE_SECONDS if ttl > 0: try: cached = cache.get(PIPELINE_CACHE_KEY) except Exception: cached = None if cached is not None: return cached result = fetch_pipeline_status() if ttl > 0: try: cache.set(PIPELINE_CACHE_KEY, result, timeout=ttl) except Exception: pass return result def fetch_pipeline_status(now=None) -> tuple[dict[str, bool], dict]: """Checks that fetching is actually *succeeding*, not just scheduled (the poll heartbeat only proves beat and a worker are alive): returns ({check: passed}, detail) covering queue depth, whether any feed has been fetched successfully recently, and the share of feeds failing. """ now = now or timezone.now() detail: dict = {} depth = queue_depth() detail["queue_depth"] = depth checks = {"queue_depth": depth is not None and depth <= MAX_QUEUE_DEPTH} active = Feed.objects.filter(is_active=True) active_count = active.count() failing = active.filter(fetch_error_count__gt=0).count() detail["active_feeds"] = active_count detail["failing_feeds"] = failing window = _success_window() last_success = active.filter( last_fetch_status__in=[Feed.STATUS_OK, Feed.STATUS_NOT_MODIFIED] ).aggregate(latest=Max("last_fetched_at"))["latest"] if last_success is not None: detail["last_successful_fetch_age_seconds"] = int((now - last_success).total_seconds()) recent_success = now - last_success <= window elif active_count == 0: recent_success = True else: # Nothing has ever fetched successfully — only a problem once # the oldest feed has had a full window to be fetched. oldest = active.aggregate(oldest=Min("created_at"))["oldest"] recent_success = now - oldest <= window checks["recent_fetch_success"] = recent_success checks["feed_failures"] = not ( active_count >= MIN_FEEDS_FOR_RATIO and failing / active_count > MAX_FAILING_RATIO ) return checks, detail