diff --git a/apps/feeds/forms.py b/apps/feeds/forms.py
index e85cc05..19215e9 100644
--- a/apps/feeds/forms.py
+++ b/apps/feeds/forms.py
@@ -3,6 +3,7 @@ from django import forms
from apps.core.urlutils import http_url_or_empty
from .models import Subscription
+from .opml import MAX_UPLOAD_BYTES
class AddFeedForm(forms.Form):
@@ -37,3 +38,14 @@ class EditSubscriptionForm(forms.ModelForm):
class OPMLImportForm(forms.Form):
opml_file = forms.FileField(label="OPML file")
+
+ def clean_opml_file(self):
+ uploaded = self.cleaned_data["opml_file"]
+ # Checked on the upload's reported size before anything reads it
+ # into memory (R2-S5); the reverse proxy's client_max_body_size is
+ # a second, coarser limit.
+ if uploaded.size > MAX_UPLOAD_BYTES:
+ raise forms.ValidationError(
+ f"That file is too large (limit {MAX_UPLOAD_BYTES // (1024 * 1024)} MB)."
+ )
+ return uploaded
diff --git a/apps/feeds/opml.py b/apps/feeds/opml.py
index caa7713..d3c62b9 100644
--- a/apps/feeds/opml.py
+++ b/apps/feeds/opml.py
@@ -26,6 +26,14 @@ from .models import Feed, Subscription
# that directly instead of making people unzip it themselves first.
MAX_ZIPPED_OPML_SIZE = 20 * 1024 * 1024
+# An upload is untrusted: bound everything an attacker controls (R2-S5).
+# A real export of thousands of feeds is well under a megabyte.
+MAX_UPLOAD_BYTES = 5 * 1024 * 1024
+MAX_ZIP_ENTRIES = 50
+MAX_OPML_DEPTH = 24
+MAX_OPML_OUTLINES = 20_000
+MAX_OPML_FEEDS = 5_000
+
def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
"""If the upload is a zip (e.g. FreshRSS's export), returns the bytes
@@ -35,6 +43,8 @@ def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
return uploaded_bytes
with zipfile.ZipFile(BytesIO(uploaded_bytes)) as archive:
+ if len(archive.infolist()) > MAX_ZIP_ENTRIES:
+ raise ValueError("The uploaded zip contains too many files.")
candidates = [
info
for info in archive.infolist()
@@ -45,7 +55,13 @@ def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
member = min(candidates, key=lambda info: info.file_size)
if member.file_size > MAX_ZIPPED_OPML_SIZE:
raise ValueError("The .opml file inside the zip is implausibly large.")
- return archive.read(member)
+ # file_size is just a header field an attacker controls, so bound
+ # what actually decompresses too, not only what it claims.
+ with archive.open(member) as handle:
+ data = handle.read(MAX_ZIPPED_OPML_SIZE + 1)
+ if len(data) > MAX_ZIPPED_OPML_SIZE:
+ raise ValueError("The .opml file inside the zip is implausibly large.")
+ return data
def parse_opml(file_obj) -> list[dict]:
@@ -58,40 +74,68 @@ def parse_opml(file_obj) -> list[dict]:
on the feed's own outline, the convention export_opml below writes and
many other readers use for a flat multi-category representation.
"""
- tree = DefusedET.parse(file_obj)
- root = tree.getroot()
- body = root.find("body")
entries: list[dict] = []
- if body is None:
- return entries
+ # Streamed with iterparse instead of building the whole tree first, so
+ # a hostile file (absurd nesting, millions of outlines) is rejected as
+ # soon as it crosses a limit rather than after it has used the memory.
+ in_body = False
+ depth = 0
+ outline_count = 0
+ open_feed_outlines = 0 # >0 while inside a feed outline: its children are ignored
+ pushed_name: list[bool] = [] # per open category outline: did it add a tag name?
+ ancestor_names: list[str] = []
- def walk(container, ancestor_names):
- for outline in container.findall("outline"):
- xml_url = http_url_or_empty(outline.get("xmlUrl"))
- if outline.get("xmlUrl") and not xml_url:
- continue # a feed URL that isn't http(s) can never be fetched
- if xml_url:
- tag_names = list(ancestor_names)
- category_attr = outline.get("category")
- if category_attr:
- for name in category_attr.split(","):
+ for event, elem in DefusedET.iterparse(file_obj, events=("start", "end")):
+ if event == "start":
+ depth += 1
+ if depth > MAX_OPML_DEPTH:
+ raise ValueError("The OPML file is nested too deeply.")
+ if elem.tag == "body":
+ in_body = True
+ elif elem.tag == "outline" and in_body:
+ outline_count += 1
+ if outline_count > MAX_OPML_OUTLINES:
+ raise ValueError("The OPML file contains too many entries.")
+ if open_feed_outlines:
+ open_feed_outlines += 1
+ continue
+ raw_xml_url = elem.get("xmlUrl")
+ if raw_xml_url:
+ open_feed_outlines = 1
+ xml_url = http_url_or_empty(raw_xml_url)
+ if not xml_url:
+ continue # a feed URL that isn't http(s) can never be fetched
+ tag_names = list(ancestor_names)
+ for name in (elem.get("category") or "").split(","):
name = name.strip()
if name and name not in tag_names:
tag_names.append(name)
- entries.append(
- {
- "feed_url": xml_url,
- "title": outline.get("title") or outline.get("text") or "",
- "site_url": http_url_or_empty(outline.get("htmlUrl")),
- "tag_names": tag_names,
- }
- )
- else:
- nested_name = outline.get("title") or outline.get("text")
- next_ancestors = ancestor_names + [nested_name] if nested_name else ancestor_names
- walk(outline, next_ancestors)
-
- walk(body, [])
+ entries.append(
+ {
+ "feed_url": xml_url,
+ "title": elem.get("title") or elem.get("text") or "",
+ "site_url": http_url_or_empty(elem.get("htmlUrl")),
+ "tag_names": tag_names,
+ }
+ )
+ if len(entries) > MAX_OPML_FEEDS:
+ raise ValueError("The OPML file contains too many feeds.")
+ else:
+ nested_name = elem.get("title") or elem.get("text")
+ pushed_name.append(bool(nested_name))
+ if nested_name:
+ ancestor_names.append(nested_name)
+ else:
+ depth -= 1
+ if elem.tag == "body":
+ in_body = False
+ elif elem.tag == "outline" and in_body:
+ if open_feed_outlines:
+ open_feed_outlines -= 1
+ elif pushed_name:
+ if pushed_name.pop():
+ ancestor_names.pop()
+ elem.clear()
return entries
diff --git a/apps/feeds/tasks.py b/apps/feeds/tasks.py
index e8f6bc1..5d25fa4 100644
--- a/apps/feeds/tasks.py
+++ b/apps/feeds/tasks.py
@@ -192,12 +192,19 @@ def reset_feed_backoff(feed_id: int):
@shared_task
-def import_opml_task(user_id: int, opml_content: str):
+def import_opml_task(user_id: int, upload_key: str):
"""Background path for large OPML imports — see OPML_ASYNC_THRESHOLD
- in apps.feeds.views. opml_content is passed as text rather than a
- file handle since Celery tasks must be JSON-serializable."""
+ in apps.feeds.views. The document itself waits in the cache under
+ upload_key (set by the view) rather than travelling in the broker
+ message; it's removed once read, and expires on its own if this task
+ never runs."""
+ content = cache.get(upload_key)
+ if content is None:
+ logger.warning("OPML import %s expired or missing; nothing imported", upload_key)
+ return None
+ cache.delete(upload_key)
try:
user = get_user_model().objects.get(pk=user_id)
except get_user_model().DoesNotExist:
return None
- return import_opml(user, BytesIO(opml_content.encode("utf-8")))
+ return import_opml(user, BytesIO(content))
diff --git a/apps/feeds/test_opml.py b/apps/feeds/test_opml.py
index 64da460..29da302 100644
--- a/apps/feeds/test_opml.py
+++ b/apps/feeds/test_opml.py
@@ -210,3 +210,77 @@ class TestUnsafeOPMLUrls:
assert [e["feed_url"] for e in entries] == ["https://example.com/feed.xml"]
assert entries[0]["site_url"] == ""
+
+
+class TestOPMLLimits:
+ def _opml(self, body):
+ from io import BytesIO
+
+ return BytesIO(
+ f'