Private
Public Access
An OPML upload was read fully into memory, parsed into a whole tree, walked recursively, and (for large imports) embedded in a Celery message. Nothing bounded raw size, ZIP entries, nesting depth, outline count or feed count. Uploads are capped at 5 MB (checked on the reported size before reading) and ZIPs at 50 entries, with the decompressed member bounded by what it actually yields rather than only its header. Parsing now streams with iterparse and aborts the moment a limit is crossed — 24 levels deep, 20,000 outlines, 5,000 feeds — so a hostile file never builds a huge or deep tree; a 50,000-level nesting bomb is rejected cleanly. Limit errors are shown to the user. Large imports go to the worker by cache reference (deleted once read, expires after an hour) instead of as the whole document inside the broker message. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
211 lines
8.5 KiB
Python
211 lines
8.5 KiB
Python
import logging
|
|
from datetime import timedelta
|
|
from io import BytesIO
|
|
|
|
from celery import shared_task
|
|
from django.contrib.auth import get_user_model
|
|
from django.core.cache import cache
|
|
from django.db.models import Q
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.tasks import apply_filter_rules_for_feed
|
|
from apps.core.models import SiteSettings
|
|
|
|
from .models import Feed
|
|
from .opml import import_opml
|
|
from .services.fetcher import effective_interval_minutes, fetch_and_ingest, record_fetch_error
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# How long a claimed-but-unfinished fetch blocks the same feed from being
|
|
# claimed again — must comfortably exceed FETCH_TIMEOUT_SECONDS plus parse/
|
|
# ingest time, so a slow-but-healthy fetch is never raced by the next tick,
|
|
# while a worker that died mid-fetch doesn't wedge the feed forever.
|
|
CLAIM_STALE_MINUTES = 10
|
|
|
|
# Caps how many feeds one poll_due_feeds tick will claim/dispatch, so a
|
|
# large backlog (first run after a long outage, a huge import) spreads
|
|
# across several beat ticks instead of enqueuing everything — and every
|
|
# bounded Feed query below stays index-only work, not a full scan (C1).
|
|
POLL_BATCH_SIZE = 500
|
|
RULE_DISPATCH_CHUNK_SIZE = 50
|
|
|
|
# Written on every poll_due_feeds tick; apps.core.views.health_ready
|
|
# reads it to tell whether beat is scheduling and a worker is consuming.
|
|
POLL_HEARTBEAT_CACHE_KEY = "poll-due-feeds-heartbeat"
|
|
POLL_HEARTBEAT_TTL_SECONDS = 3600
|
|
|
|
|
|
@shared_task
|
|
def poll_due_feeds():
|
|
"""Find a bounded batch of active Feeds whose next_fetch_at has
|
|
elapsed (set by fetch_feed itself after every attempt — see Feed.
|
|
next_fetch_at), atomically claim them (so overlapping beat ticks or a
|
|
slow-running previous fetch can't dispatch the same feed twice), and
|
|
dispatch a fetch_feed task for each claimed feed — fan-out happens
|
|
here, not via one Celery Beat entry per feed.
|
|
"""
|
|
now = timezone.now()
|
|
cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS)
|
|
due_feed_ids = list(
|
|
Feed.objects.filter(is_active=True)
|
|
.filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now))
|
|
.filter(
|
|
Q(claimed_at__isnull=True)
|
|
| Q(claimed_at__lt=now - timedelta(minutes=CLAIM_STALE_MINUTES))
|
|
)
|
|
.order_by("next_fetch_at")
|
|
.values_list("id", flat=True)[:POLL_BATCH_SIZE]
|
|
)
|
|
dispatched = 0
|
|
for feed_id in due_feed_ids:
|
|
try:
|
|
if enqueue_fetch(feed_id):
|
|
dispatched += 1
|
|
except Exception:
|
|
# Broker trouble: enqueue_fetch already released this feed's
|
|
# claim, so it's simply picked up again next tick.
|
|
logger.exception("Could not queue a fetch for feed %s", feed_id)
|
|
return dispatched
|
|
|
|
|
|
def claim_feed(feed_id: int) -> bool:
|
|
"""Atomically marks a feed as having a fetch in flight (an UPDATE whose
|
|
WHERE clause re-checks staleness, so two callers racing for the same
|
|
feed can't both get rowcount=1). Every path that dispatches a fetch —
|
|
the periodic poller, a newly added feed, an admin retry — goes through
|
|
this, so they can never run the same feed concurrently."""
|
|
now = timezone.now()
|
|
stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
|
|
return bool(
|
|
Feed.objects.filter(id=feed_id, is_active=True)
|
|
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=stale_before))
|
|
.update(claimed_at=now)
|
|
)
|
|
|
|
|
|
def enqueue_fetch(feed_id: int) -> bool:
|
|
"""Claim a feed and queue its fetch. False means another fetch of it is
|
|
already in flight (nothing was queued — that one will do the work).
|
|
If queueing itself fails the claim is released and the error re-raised,
|
|
so a broker outage doesn't leave the feed claimed until it goes stale.
|
|
"""
|
|
if not claim_feed(feed_id):
|
|
return False
|
|
try:
|
|
fetch_feed.delay(feed_id)
|
|
except Exception:
|
|
Feed.objects.filter(pk=feed_id).update(claimed_at=None)
|
|
raise
|
|
return True
|
|
|
|
|
|
def _schedule_next_fetch(feed: Feed) -> None:
|
|
"""Sets feed.next_fetch_at from its (just-updated) fetch_error_count —
|
|
called right before every feed.save() in fetch_feed, so the next
|
|
poll_due_feeds tick can find it with a plain indexed range query
|
|
instead of recomputing every active feed's backoff in Python (C1)."""
|
|
configured_interval = SiteSettings.load().fetch_interval_minutes
|
|
interval = effective_interval_minutes(feed, configured_interval)
|
|
feed.next_fetch_at = timezone.now() + timedelta(minutes=interval)
|
|
|
|
|
|
# Bounds a single feed's fetch+parse+ingest, independent of
|
|
# FETCH_TIMEOUT_SECONDS (apps.feeds.services.fetcher) which only covers
|
|
# the HTTP request itself: parsing/sanitizing/writing a large-but-under-
|
|
# the-byte-cap feed could still run long. soft_time_limit lets the
|
|
# except-Exception handler below record a normal feed error (and clear
|
|
# the claim) before the hard time_limit kills the worker process outright.
|
|
FETCH_FEED_SOFT_TIME_LIMIT = 60
|
|
FETCH_FEED_TIME_LIMIT = 90
|
|
|
|
|
|
@shared_task(time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT)
|
|
def fetch_feed(feed_id: int):
|
|
try:
|
|
feed = Feed.objects.get(pk=feed_id, is_active=True)
|
|
except Feed.DoesNotExist:
|
|
return None
|
|
|
|
try:
|
|
result = fetch_and_ingest(feed)
|
|
except Exception:
|
|
# fetch_and_ingest already handles the expected failure modes
|
|
# (network errors, SSRF-blocked targets, unparseable feeds) by
|
|
# returning an error FetchResult rather than raising — anything
|
|
# that reaches here is unexpected (a parser/sanitizer bug, a DB
|
|
# error, this task's own soft time limit), so it's recorded the
|
|
# same way a handled failure would be rather than silently
|
|
# leaving the feed's status stale, the claim held for the full
|
|
# CLAIM_STALE_MINUTES, and no record of what happened. Re-raised
|
|
# afterward so Celery still marks the task FAILED (visible in
|
|
# worker logs/monitoring) instead of swallowing it.
|
|
logger.exception("Feed %s: unexpected error during fetch", feed.feed_url)
|
|
record_fetch_error(feed, "Unexpected error — see worker logs")
|
|
feed.last_fetched_at = timezone.now()
|
|
feed.claimed_at = None
|
|
_schedule_next_fetch(feed)
|
|
feed.save()
|
|
raise
|
|
|
|
feed.claimed_at = None
|
|
_schedule_next_fetch(feed)
|
|
feed.save()
|
|
|
|
changed_article_ids = result.new_article_ids + result.updated_article_ids
|
|
if changed_article_ids:
|
|
logger.info(
|
|
"Feed %s: %d new article(s), %d updated",
|
|
feed.feed_url,
|
|
len(result.new_article_ids),
|
|
len(result.updated_article_ids),
|
|
)
|
|
# Chunked so one huge poll (a first fetch of a long archive feed)
|
|
# becomes several bounded rule-evaluation tasks, each well inside
|
|
# the worker's time limits, instead of one unbounded one (C2).
|
|
for start in range(0, len(changed_article_ids), RULE_DISPATCH_CHUNK_SIZE):
|
|
apply_filter_rules_for_feed.delay(
|
|
feed_id, changed_article_ids[start : start + RULE_DISPATCH_CHUNK_SIZE]
|
|
)
|
|
|
|
return {
|
|
"status": result.status,
|
|
"new": len(result.new_article_ids),
|
|
"updated": len(result.updated_article_ids),
|
|
}
|
|
|
|
|
|
def reset_backoff(feed_id: int) -> None:
|
|
"""Clear a feed's error count and due-date backoff so it's retried at
|
|
its normal configured interval instead of the backed-off one
|
|
(next_fetch_at=None also makes it immediately due, same as a feed
|
|
that's never been fetched). A plain synchronous function so a caller
|
|
can finish the reset *before* dispatching the fetch — as an async task
|
|
it could land after the fetch and undo its freshly scheduled
|
|
next_fetch_at."""
|
|
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
|
|
|
|
|
|
@shared_task
|
|
def reset_feed_backoff(feed_id: int):
|
|
reset_backoff(feed_id)
|
|
|
|
|
|
@shared_task
|
|
def import_opml_task(user_id: int, upload_key: str):
|
|
"""Background path for large OPML imports — see OPML_ASYNC_THRESHOLD
|
|
in apps.feeds.views. The document itself waits in the cache under
|
|
upload_key (set by the view) rather than travelling in the broker
|
|
message; it's removed once read, and expires on its own if this task
|
|
never runs."""
|
|
content = cache.get(upload_key)
|
|
if content is None:
|
|
logger.warning("OPML import %s expired or missing; nothing imported", upload_key)
|
|
return None
|
|
cache.delete(upload_key)
|
|
try:
|
|
user = get_user_model().objects.get(pk=user_id)
|
|
except get_user_model().DoesNotExist:
|
|
return None
|
|
return import_opml(user, BytesIO(content))
|