Bound OPML uploads in size, depth, count and broker payload (R2-S5, #37)

An OPML upload was read fully into memory, parsed into a whole tree,
walked recursively, and (for large imports) embedded in a Celery
message. Nothing bounded raw size, ZIP entries, nesting depth, outline
count or feed count.

Uploads are capped at 5 MB (checked on the reported size before reading)
and ZIPs at 50 entries, with the decompressed member bounded by what it
actually yields rather than only its header. Parsing now streams with
iterparse and aborts the moment a limit is crossed — 24 levels deep,
20,000 outlines, 5,000 feeds — so a hostile file never builds a huge or
deep tree; a 50,000-level nesting bomb is rejected cleanly. Limit errors
are shown to the user. Large imports go to the worker by cache reference
(deleted once read, expires after an hour) instead of as the whole
document inside the broker message.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5.5 committed 2026-11-01 11:58:49 -07:00
1 parent 41f99e9405
commit 698940763f
6 files changed
+240 -40

No files matched your search

+12
View File
@@ -3,6 +3,7 @@ from django import forms
from apps.core.urlutils import http_url_or_empty
from .models import Subscription
from .opml import MAX_UPLOAD_BYTES
class AddFeedForm(forms.Form):
@@ -37,3 +38,14 @@ class EditSubscriptionForm(forms.ModelForm):
class OPMLImportForm(forms.Form):
opml_file = forms.FileField(label="OPML file")
def clean_opml_file(self):
uploaded = self.cleaned_data["opml_file"]
# Checked on the upload's reported size before anything reads it
# into memory (R2-S5); the reverse proxy's client_max_body_size is
# a second, coarser limit.
if uploaded.size > MAX_UPLOAD_BYTES:
raise forms.ValidationError(
f"That file is too large (limit {MAX_UPLOAD_BYTES // (1024 * 1024)} MB)."
)
return uploaded
+74 -30
View File
@@ -26,6 +26,14 @@ from .models import Feed, Subscription
# that directly instead of making people unzip it themselves first.
MAX_ZIPPED_OPML_SIZE = 20 * 1024 * 1024
# An upload is untrusted: bound everything an attacker controls (R2-S5).
# A real export of thousands of feeds is well under a megabyte.
MAX_UPLOAD_BYTES = 5 * 1024 * 1024
MAX_ZIP_ENTRIES = 50
MAX_OPML_DEPTH = 24
MAX_OPML_OUTLINES = 20_000
MAX_OPML_FEEDS = 5_000
def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
"""If the upload is a zip (e.g. FreshRSS's export), returns the bytes
@@ -35,6 +43,8 @@ def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
return uploaded_bytes
with zipfile.ZipFile(BytesIO(uploaded_bytes)) as archive:
if len(archive.infolist()) > MAX_ZIP_ENTRIES:
raise ValueError("The uploaded zip contains too many files.")
candidates = [
info
for info in archive.infolist()
@@ -45,7 +55,13 @@ def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
member = min(candidates, key=lambda info: info.file_size)
if member.file_size > MAX_ZIPPED_OPML_SIZE:
raise ValueError("The .opml file inside the zip is implausibly large.")
return archive.read(member)
# file_size is just a header field an attacker controls, so bound
# what actually decompresses too, not only what it claims.
with archive.open(member) as handle:
data = handle.read(MAX_ZIPPED_OPML_SIZE + 1)
if len(data) > MAX_ZIPPED_OPML_SIZE:
raise ValueError("The .opml file inside the zip is implausibly large.")
return data
def parse_opml(file_obj) -> list[dict]:
@@ -58,40 +74,68 @@ def parse_opml(file_obj) -> list[dict]:
on the feed's own outline, the convention export_opml below writes and
many other readers use for a flat multi-category representation.
"""
tree = DefusedET.parse(file_obj)
root = tree.getroot()
body = root.find("body")
entries: list[dict] = []
if body is None:
return entries
# Streamed with iterparse instead of building the whole tree first, so
# a hostile file (absurd nesting, millions of outlines) is rejected as
# soon as it crosses a limit rather than after it has used the memory.
in_body = False
depth = 0
outline_count = 0
open_feed_outlines = 0 # >0 while inside a feed outline: its children are ignored
pushed_name: list[bool] = [] # per open category outline: did it add a tag name?
ancestor_names: list[str] = []
def walk(container, ancestor_names):
for outline in container.findall("outline"):
xml_url = http_url_or_empty(outline.get("xmlUrl"))
if outline.get("xmlUrl") and not xml_url:
continue # a feed URL that isn't http(s) can never be fetched
if xml_url:
tag_names = list(ancestor_names)
category_attr = outline.get("category")
if category_attr:
for name in category_attr.split(","):
for event, elem in DefusedET.iterparse(file_obj, events=("start", "end")):
if event == "start":
depth += 1
if depth > MAX_OPML_DEPTH:
raise ValueError("The OPML file is nested too deeply.")
if elem.tag == "body":
in_body = True
elif elem.tag == "outline" and in_body:
outline_count += 1
if outline_count > MAX_OPML_OUTLINES:
raise ValueError("The OPML file contains too many entries.")
if open_feed_outlines:
open_feed_outlines += 1
continue
raw_xml_url = elem.get("xmlUrl")
if raw_xml_url:
open_feed_outlines = 1
xml_url = http_url_or_empty(raw_xml_url)
if not xml_url:
continue # a feed URL that isn't http(s) can never be fetched
tag_names = list(ancestor_names)
for name in (elem.get("category") or "").split(","):
name = name.strip()
if name and name not in tag_names:
tag_names.append(name)
entries.append(
{
"feed_url": xml_url,
"title": outline.get("title") or outline.get("text") or "",
"site_url": http_url_or_empty(outline.get("htmlUrl")),
"tag_names": tag_names,
}
)
else:
nested_name = outline.get("title") or outline.get("text")
next_ancestors = ancestor_names + [nested_name] if nested_name else ancestor_names
walk(outline, next_ancestors)
walk(body, [])
entries.append(
{
"feed_url": xml_url,
"title": elem.get("title") or elem.get("text") or "",
"site_url": http_url_or_empty(elem.get("htmlUrl")),
"tag_names": tag_names,
}
)
if len(entries) > MAX_OPML_FEEDS:
raise ValueError("The OPML file contains too many feeds.")
else:
nested_name = elem.get("title") or elem.get("text")
pushed_name.append(bool(nested_name))
if nested_name:
ancestor_names.append(nested_name)
else:
depth -= 1
if elem.tag == "body":
in_body = False
elif elem.tag == "outline" and in_body:
if open_feed_outlines:
open_feed_outlines -= 1
elif pushed_name:
if pushed_name.pop():
ancestor_names.pop()
elem.clear()
return entries
+11 -4
View File
@@ -192,12 +192,19 @@ def reset_feed_backoff(feed_id: int):
@shared_task
def import_opml_task(user_id: int, opml_content: str):
def import_opml_task(user_id: int, upload_key: str):
"""Background path for large OPML imports — see OPML_ASYNC_THRESHOLD
in apps.feeds.views. opml_content is passed as text rather than a
file handle since Celery tasks must be JSON-serializable."""
in apps.feeds.views. The document itself waits in the cache under
upload_key (set by the view) rather than travelling in the broker
message; it's removed once read, and expires on its own if this task
never runs."""
content = cache.get(upload_key)
if content is None:
logger.warning("OPML import %s expired or missing; nothing imported", upload_key)
return None
cache.delete(upload_key)
try:
user = get_user_model().objects.get(pk=user_id)
except get_user_model().DoesNotExist:
return None
return import_opml(user, BytesIO(opml_content.encode("utf-8")))
return import_opml(user, BytesIO(content))
+74
View File
@@ -210,3 +210,77 @@ class TestUnsafeOPMLUrls:
assert [e["feed_url"] for e in entries] == ["https://example.com/feed.xml"]
assert entries[0]["site_url"] == ""
class TestOPMLLimits:
def _opml(self, body):
from io import BytesIO
return BytesIO(
f'<?xml version="1.0"?><opml version="2.0"><head/><body>{body}</body></opml>'.encode()
)
def test_deep_nesting_is_rejected_without_recursing(self):
from apps.feeds.opml import MAX_OPML_DEPTH, parse_opml
depth = MAX_OPML_DEPTH + 50
body = '<outline text="x">' * depth + "</outline>" * depth
with pytest.raises(ValueError, match="nested too deeply"):
parse_opml(self._opml(body))
def test_a_pathologically_deep_file_does_not_crash_the_parser(self):
from apps.feeds.opml import parse_opml
depth = 50_000
body = "<outline>" * depth + "</outline>" * depth
with pytest.raises(ValueError):
parse_opml(self._opml(body))
def test_too_many_outlines_is_rejected(self, monkeypatch):
from apps.feeds import opml
monkeypatch.setattr(opml, "MAX_OPML_OUTLINES", 5)
body = '<outline text="folder"/>' * 6
with pytest.raises(ValueError, match="too many entries"):
opml.parse_opml(self._opml(body))
def test_too_many_feeds_is_rejected(self, monkeypatch):
from apps.feeds import opml
monkeypatch.setattr(opml, "MAX_OPML_FEEDS", 2)
body = "".join(f'<outline xmlUrl="https://e.example.com/{i}.xml"/>' for i in range(3))
with pytest.raises(ValueError, match="too many feeds"):
opml.parse_opml(self._opml(body))
def test_a_zip_with_too_many_files_is_rejected(self, monkeypatch):
import zipfile
from io import BytesIO
from apps.feeds import opml
monkeypatch.setattr(opml, "MAX_ZIP_ENTRIES", 3)
buffer = BytesIO()
with zipfile.ZipFile(buffer, "w") as archive:
for i in range(5):
archive.writestr(f"f{i}.txt", "x")
archive.writestr("subs.opml", "<opml/>")
with pytest.raises(ValueError, match="too many files"):
opml.extract_opml_bytes(buffer.getvalue())
def test_a_feed_outlines_children_are_not_imported_as_separate_feeds(self):
from apps.feeds.opml import parse_opml
body = (
'<outline xmlUrl="https://a.example.com/f.xml">'
'<outline xmlUrl="https://child.example.com/f.xml"/></outline>'
'<outline xmlUrl="https://b.example.com/f.xml"/>'
)
urls = [e["feed_url"] for e in parse_opml(self._opml(body))]
assert urls == ["https://a.example.com/f.xml", "https://b.example.com/f.xml"]
+50 -3
View File
@@ -173,9 +173,9 @@ def test_import_opml_dispatches_background_task_above_threshold(client, alice, m
dispatched = {}
def fake_delay(user_id, content):
def fake_delay(user_id, upload_key):
dispatched["user_id"] = user_id
dispatched["content"] = content
dispatched["upload_key"] = upload_key
from . import views as feeds_views
@@ -187,6 +187,11 @@ def test_import_opml_dispatches_background_task_above_threshold(client, alice, m
assert response.status_code == 302
assert dispatched["user_id"] == alice.id
# The task gets a reference to the document, not the document itself.
from django.core.cache import cache
assert dispatched["upload_key"].startswith("opml-import:")
assert cache.get(dispatched["upload_key"]) == SIMPLE_OPML
# The background task was dispatched instead of a synchronous import.
assert not Subscription.objects.filter(user=alice).exists()
@@ -204,9 +209,51 @@ def test_export_opml_view_returns_downloadable_file(client, alice, alices_subscr
@pytest.mark.django_db
def test_import_opml_task_runs_import(alice):
from django.core.cache import cache
from .tasks import import_opml_task
result = import_opml_task(alice.id, SIMPLE_OPML.decode("utf-8"))
cache.set("opml-import:test", SIMPLE_OPML, timeout=60)
result = import_opml_task(alice.id, "opml-import:test")
assert result["created"] == 2
assert Subscription.objects.filter(user=alice).count() == 2
assert cache.get("opml-import:test") is None, "the stored upload is removed once read"
@pytest.mark.django_db
def test_import_opml_task_does_nothing_if_the_upload_expired(alice):
from .tasks import import_opml_task
assert import_opml_task(alice.id, "opml-import:gone") is None
assert not Subscription.objects.filter(user=alice).exists()
@pytest.mark.django_db
def test_import_opml_rejects_an_oversized_upload(client, alice, monkeypatch):
from . import forms
monkeypatch.setattr(forms, "MAX_UPLOAD_BYTES", 100)
client.login(username="alice", password="pw-alice-123")
upload = SimpleUploadedFile("big.opml", SIMPLE_OPML, content_type="text/x-opml")
response = client.post(reverse("feeds:import_opml"), {"opml_file": upload})
assert response.status_code == 200
assert "too large" in response.content.decode()
assert not Subscription.objects.filter(user=alice).exists()
@pytest.mark.django_db
def test_import_opml_reports_a_limit_error_instead_of_a_generic_one(client, alice, monkeypatch):
from . import opml
monkeypatch.setattr(opml, "MAX_OPML_FEEDS", 1)
client.login(username="alice", password="pw-alice-123")
upload = SimpleUploadedFile("subs.opml", SIMPLE_OPML, content_type="text/x-opml")
response = client.post(reverse("feeds:import_opml"), {"opml_file": upload}, follow=True)
assert "too many feeds" in response.content.decode()
assert not Subscription.objects.filter(user=alice).exists()
+19 -3
View File
@@ -1,8 +1,10 @@
import logging
import uuid
from io import BytesIO
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.core.cache import cache
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, redirect, render
from django.views.decorators.http import require_POST
@@ -12,12 +14,13 @@ from apps.articles.models import SubscriptionTag, Tag
from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm
from .models import Feed, Subscription
from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml
from .opml import MAX_UPLOAD_BYTES, export_opml, extract_opml_bytes, import_opml, parse_opml
from .tasks import enqueue_fetch, import_opml_task
logger = logging.getLogger(__name__)
OPML_ASYNC_THRESHOLD = 50
OPML_UPLOAD_TTL_SECONDS = 60 * 60
@login_required
@@ -149,16 +152,29 @@ def unsubscribe(request, subscription_id):
def import_opml_view(request):
form = OPMLImportForm(request.POST or None, request.FILES or None)
if request.method == "POST" and form.is_valid():
uploaded = form.cleaned_data["opml_file"].read()
uploaded = form.cleaned_data["opml_file"].read(MAX_UPLOAD_BYTES + 1)
try:
content = extract_opml_bytes(uploaded)
entry_count = len(parse_opml(BytesIO(content)))
except ValueError as exc:
# Our own limit/format errors carry a user-presentable message.
messages.error(request, str(exc))
return redirect("feeds:import_opml")
except Exception:
messages.error(request, "That file doesn't look like a valid OPML export.")
return redirect("feeds:import_opml")
if entry_count > OPML_ASYNC_THRESHOLD:
import_opml_task.delay(request.user.id, content.decode("utf-8"))
# Hand the worker a reference, not the document: a multi-MB
# file doesn't belong inside a broker message.
upload_key = f"opml-import:{uuid.uuid4().hex}"
try:
cache.set(upload_key, content, timeout=OPML_UPLOAD_TTL_SECONDS)
import_opml_task.delay(request.user.id, upload_key)
except Exception:
logger.exception("Could not queue the OPML import")
messages.error(request, "Couldn't queue that import right now — try again shortly.")
return redirect("feeds:import_opml")
messages.success(request, f"Importing {entry_count} feeds in the background.")
else:
result = import_opml(request.user, BytesIO(content))