Bound OPML uploads in size, depth, count and broker payload (R2-S5, #37)

An OPML upload was read fully into memory, parsed into a whole tree,
walked recursively, and (for large imports) embedded in a Celery
message. Nothing bounded raw size, ZIP entries, nesting depth, outline
count or feed count.

Uploads are capped at 5 MB (checked on the reported size before reading)
and ZIPs at 50 entries, with the decompressed member bounded by what it
actually yields rather than only its header. Parsing now streams with
iterparse and aborts the moment a limit is crossed — 24 levels deep,
20,000 outlines, 5,000 feeds — so a hostile file never builds a huge or
deep tree; a 50,000-level nesting bomb is rejected cleanly. Limit errors
are shown to the user. Large imports go to the worker by cache reference
(deleted once read, expires after an hour) instead of as the whole
document inside the broker message.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5.5 committed 2026-11-01 11:58:49 -07:00
1 parent 41f99e9405
commit 698940763f
6 files changed
+240 -40

No files matched your search

+74 -30
View File
@@ -26,6 +26,14 @@ from .models import Feed, Subscription
# that directly instead of making people unzip it themselves first.
MAX_ZIPPED_OPML_SIZE = 20 * 1024 * 1024
# An upload is untrusted: bound everything an attacker controls (R2-S5).
# A real export of thousands of feeds is well under a megabyte.
MAX_UPLOAD_BYTES = 5 * 1024 * 1024
MAX_ZIP_ENTRIES = 50
MAX_OPML_DEPTH = 24
MAX_OPML_OUTLINES = 20_000
MAX_OPML_FEEDS = 5_000
def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
"""If the upload is a zip (e.g. FreshRSS's export), returns the bytes
@@ -35,6 +43,8 @@ def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
return uploaded_bytes
with zipfile.ZipFile(BytesIO(uploaded_bytes)) as archive:
if len(archive.infolist()) > MAX_ZIP_ENTRIES:
raise ValueError("The uploaded zip contains too many files.")
candidates = [
info
for info in archive.infolist()
@@ -45,7 +55,13 @@ def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
member = min(candidates, key=lambda info: info.file_size)
if member.file_size > MAX_ZIPPED_OPML_SIZE:
raise ValueError("The .opml file inside the zip is implausibly large.")
return archive.read(member)
# file_size is just a header field an attacker controls, so bound
# what actually decompresses too, not only what it claims.
with archive.open(member) as handle:
data = handle.read(MAX_ZIPPED_OPML_SIZE + 1)
if len(data) > MAX_ZIPPED_OPML_SIZE:
raise ValueError("The .opml file inside the zip is implausibly large.")
return data
def parse_opml(file_obj) -> list[dict]:
@@ -58,40 +74,68 @@ def parse_opml(file_obj) -> list[dict]:
on the feed's own outline, the convention export_opml below writes and
many other readers use for a flat multi-category representation.
"""
tree = DefusedET.parse(file_obj)
root = tree.getroot()
body = root.find("body")
entries: list[dict] = []
if body is None:
return entries
# Streamed with iterparse instead of building the whole tree first, so
# a hostile file (absurd nesting, millions of outlines) is rejected as
# soon as it crosses a limit rather than after it has used the memory.
in_body = False
depth = 0
outline_count = 0
open_feed_outlines = 0 # >0 while inside a feed outline: its children are ignored
pushed_name: list[bool] = [] # per open category outline: did it add a tag name?
ancestor_names: list[str] = []
def walk(container, ancestor_names):
for outline in container.findall("outline"):
xml_url = http_url_or_empty(outline.get("xmlUrl"))
if outline.get("xmlUrl") and not xml_url:
continue # a feed URL that isn't http(s) can never be fetched
if xml_url:
tag_names = list(ancestor_names)
category_attr = outline.get("category")
if category_attr:
for name in category_attr.split(","):
for event, elem in DefusedET.iterparse(file_obj, events=("start", "end")):
if event == "start":
depth += 1
if depth > MAX_OPML_DEPTH:
raise ValueError("The OPML file is nested too deeply.")
if elem.tag == "body":
in_body = True
elif elem.tag == "outline" and in_body:
outline_count += 1
if outline_count > MAX_OPML_OUTLINES:
raise ValueError("The OPML file contains too many entries.")
if open_feed_outlines:
open_feed_outlines += 1
continue
raw_xml_url = elem.get("xmlUrl")
if raw_xml_url:
open_feed_outlines = 1
xml_url = http_url_or_empty(raw_xml_url)
if not xml_url:
continue # a feed URL that isn't http(s) can never be fetched
tag_names = list(ancestor_names)
for name in (elem.get("category") or "").split(","):
name = name.strip()
if name and name not in tag_names:
tag_names.append(name)
entries.append(
{
"feed_url": xml_url,
"title": outline.get("title") or outline.get("text") or "",
"site_url": http_url_or_empty(outline.get("htmlUrl")),
"tag_names": tag_names,
}
)
else:
nested_name = outline.get("title") or outline.get("text")
next_ancestors = ancestor_names + [nested_name] if nested_name else ancestor_names
walk(outline, next_ancestors)
walk(body, [])
entries.append(
{
"feed_url": xml_url,
"title": elem.get("title") or elem.get("text") or "",
"site_url": http_url_or_empty(elem.get("htmlUrl")),
"tag_names": tag_names,
}
)
if len(entries) > MAX_OPML_FEEDS:
raise ValueError("The OPML file contains too many feeds.")
else:
nested_name = elem.get("title") or elem.get("text")
pushed_name.append(bool(nested_name))
if nested_name:
ancestor_names.append(nested_name)
else:
depth -= 1
if elem.tag == "body":
in_body = False
elif elem.tag == "outline" and in_body:
if open_feed_outlines:
open_feed_outlines -= 1
elif pushed_name:
if pushed_name.pop():
ancestor_names.pop()
elem.clear()
return entries