diff --git a/apps/feeds/forms.py b/apps/feeds/forms.py index e85cc05..19215e9 100644 --- a/apps/feeds/forms.py +++ b/apps/feeds/forms.py @@ -3,6 +3,7 @@ from django import forms from apps.core.urlutils import http_url_or_empty from .models import Subscription +from .opml import MAX_UPLOAD_BYTES class AddFeedForm(forms.Form): @@ -37,3 +38,14 @@ class EditSubscriptionForm(forms.ModelForm): class OPMLImportForm(forms.Form): opml_file = forms.FileField(label="OPML file") + + def clean_opml_file(self): + uploaded = self.cleaned_data["opml_file"] + # Checked on the upload's reported size before anything reads it + # into memory (R2-S5); the reverse proxy's client_max_body_size is + # a second, coarser limit. + if uploaded.size > MAX_UPLOAD_BYTES: + raise forms.ValidationError( + f"That file is too large (limit {MAX_UPLOAD_BYTES // (1024 * 1024)} MB)." + ) + return uploaded diff --git a/apps/feeds/opml.py b/apps/feeds/opml.py index caa7713..d3c62b9 100644 --- a/apps/feeds/opml.py +++ b/apps/feeds/opml.py @@ -26,6 +26,14 @@ from .models import Feed, Subscription # that directly instead of making people unzip it themselves first. MAX_ZIPPED_OPML_SIZE = 20 * 1024 * 1024 +# An upload is untrusted: bound everything an attacker controls (R2-S5). +# A real export of thousands of feeds is well under a megabyte. +MAX_UPLOAD_BYTES = 5 * 1024 * 1024 +MAX_ZIP_ENTRIES = 50 +MAX_OPML_DEPTH = 24 +MAX_OPML_OUTLINES = 20_000 +MAX_OPML_FEEDS = 5_000 + def extract_opml_bytes(uploaded_bytes: bytes) -> bytes: """If the upload is a zip (e.g. FreshRSS's export), returns the bytes @@ -35,6 +43,8 @@ def extract_opml_bytes(uploaded_bytes: bytes) -> bytes: return uploaded_bytes with zipfile.ZipFile(BytesIO(uploaded_bytes)) as archive: + if len(archive.infolist()) > MAX_ZIP_ENTRIES: + raise ValueError("The uploaded zip contains too many files.") candidates = [ info for info in archive.infolist() @@ -45,7 +55,13 @@ def extract_opml_bytes(uploaded_bytes: bytes) -> bytes: member = min(candidates, key=lambda info: info.file_size) if member.file_size > MAX_ZIPPED_OPML_SIZE: raise ValueError("The .opml file inside the zip is implausibly large.") - return archive.read(member) + # file_size is just a header field an attacker controls, so bound + # what actually decompresses too, not only what it claims. + with archive.open(member) as handle: + data = handle.read(MAX_ZIPPED_OPML_SIZE + 1) + if len(data) > MAX_ZIPPED_OPML_SIZE: + raise ValueError("The .opml file inside the zip is implausibly large.") + return data def parse_opml(file_obj) -> list[dict]: @@ -58,40 +74,68 @@ def parse_opml(file_obj) -> list[dict]: on the feed's own outline, the convention export_opml below writes and many other readers use for a flat multi-category representation. """ - tree = DefusedET.parse(file_obj) - root = tree.getroot() - body = root.find("body") entries: list[dict] = [] - if body is None: - return entries + # Streamed with iterparse instead of building the whole tree first, so + # a hostile file (absurd nesting, millions of outlines) is rejected as + # soon as it crosses a limit rather than after it has used the memory. + in_body = False + depth = 0 + outline_count = 0 + open_feed_outlines = 0 # >0 while inside a feed outline: its children are ignored + pushed_name: list[bool] = [] # per open category outline: did it add a tag name? + ancestor_names: list[str] = [] - def walk(container, ancestor_names): - for outline in container.findall("outline"): - xml_url = http_url_or_empty(outline.get("xmlUrl")) - if outline.get("xmlUrl") and not xml_url: - continue # a feed URL that isn't http(s) can never be fetched - if xml_url: - tag_names = list(ancestor_names) - category_attr = outline.get("category") - if category_attr: - for name in category_attr.split(","): + for event, elem in DefusedET.iterparse(file_obj, events=("start", "end")): + if event == "start": + depth += 1 + if depth > MAX_OPML_DEPTH: + raise ValueError("The OPML file is nested too deeply.") + if elem.tag == "body": + in_body = True + elif elem.tag == "outline" and in_body: + outline_count += 1 + if outline_count > MAX_OPML_OUTLINES: + raise ValueError("The OPML file contains too many entries.") + if open_feed_outlines: + open_feed_outlines += 1 + continue + raw_xml_url = elem.get("xmlUrl") + if raw_xml_url: + open_feed_outlines = 1 + xml_url = http_url_or_empty(raw_xml_url) + if not xml_url: + continue # a feed URL that isn't http(s) can never be fetched + tag_names = list(ancestor_names) + for name in (elem.get("category") or "").split(","): name = name.strip() if name and name not in tag_names: tag_names.append(name) - entries.append( - { - "feed_url": xml_url, - "title": outline.get("title") or outline.get("text") or "", - "site_url": http_url_or_empty(outline.get("htmlUrl")), - "tag_names": tag_names, - } - ) - else: - nested_name = outline.get("title") or outline.get("text") - next_ancestors = ancestor_names + [nested_name] if nested_name else ancestor_names - walk(outline, next_ancestors) - - walk(body, []) + entries.append( + { + "feed_url": xml_url, + "title": elem.get("title") or elem.get("text") or "", + "site_url": http_url_or_empty(elem.get("htmlUrl")), + "tag_names": tag_names, + } + ) + if len(entries) > MAX_OPML_FEEDS: + raise ValueError("The OPML file contains too many feeds.") + else: + nested_name = elem.get("title") or elem.get("text") + pushed_name.append(bool(nested_name)) + if nested_name: + ancestor_names.append(nested_name) + else: + depth -= 1 + if elem.tag == "body": + in_body = False + elif elem.tag == "outline" and in_body: + if open_feed_outlines: + open_feed_outlines -= 1 + elif pushed_name: + if pushed_name.pop(): + ancestor_names.pop() + elem.clear() return entries diff --git a/apps/feeds/tasks.py b/apps/feeds/tasks.py index e8f6bc1..5d25fa4 100644 --- a/apps/feeds/tasks.py +++ b/apps/feeds/tasks.py @@ -192,12 +192,19 @@ def reset_feed_backoff(feed_id: int): @shared_task -def import_opml_task(user_id: int, opml_content: str): +def import_opml_task(user_id: int, upload_key: str): """Background path for large OPML imports — see OPML_ASYNC_THRESHOLD - in apps.feeds.views. opml_content is passed as text rather than a - file handle since Celery tasks must be JSON-serializable.""" + in apps.feeds.views. The document itself waits in the cache under + upload_key (set by the view) rather than travelling in the broker + message; it's removed once read, and expires on its own if this task + never runs.""" + content = cache.get(upload_key) + if content is None: + logger.warning("OPML import %s expired or missing; nothing imported", upload_key) + return None + cache.delete(upload_key) try: user = get_user_model().objects.get(pk=user_id) except get_user_model().DoesNotExist: return None - return import_opml(user, BytesIO(opml_content.encode("utf-8"))) + return import_opml(user, BytesIO(content)) diff --git a/apps/feeds/test_opml.py b/apps/feeds/test_opml.py index 64da460..29da302 100644 --- a/apps/feeds/test_opml.py +++ b/apps/feeds/test_opml.py @@ -210,3 +210,77 @@ class TestUnsafeOPMLUrls: assert [e["feed_url"] for e in entries] == ["https://example.com/feed.xml"] assert entries[0]["site_url"] == "" + + +class TestOPMLLimits: + def _opml(self, body): + from io import BytesIO + + return BytesIO( + f'{body}'.encode() + ) + + def test_deep_nesting_is_rejected_without_recursing(self): + from apps.feeds.opml import MAX_OPML_DEPTH, parse_opml + + depth = MAX_OPML_DEPTH + 50 + body = '' * depth + "" * depth + + with pytest.raises(ValueError, match="nested too deeply"): + parse_opml(self._opml(body)) + + def test_a_pathologically_deep_file_does_not_crash_the_parser(self): + from apps.feeds.opml import parse_opml + + depth = 50_000 + body = "" * depth + "" * depth + + with pytest.raises(ValueError): + parse_opml(self._opml(body)) + + def test_too_many_outlines_is_rejected(self, monkeypatch): + from apps.feeds import opml + + monkeypatch.setattr(opml, "MAX_OPML_OUTLINES", 5) + body = '' * 6 + + with pytest.raises(ValueError, match="too many entries"): + opml.parse_opml(self._opml(body)) + + def test_too_many_feeds_is_rejected(self, monkeypatch): + from apps.feeds import opml + + monkeypatch.setattr(opml, "MAX_OPML_FEEDS", 2) + body = "".join(f'' for i in range(3)) + + with pytest.raises(ValueError, match="too many feeds"): + opml.parse_opml(self._opml(body)) + + def test_a_zip_with_too_many_files_is_rejected(self, monkeypatch): + import zipfile + from io import BytesIO + + from apps.feeds import opml + + monkeypatch.setattr(opml, "MAX_ZIP_ENTRIES", 3) + buffer = BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for i in range(5): + archive.writestr(f"f{i}.txt", "x") + archive.writestr("subs.opml", "") + + with pytest.raises(ValueError, match="too many files"): + opml.extract_opml_bytes(buffer.getvalue()) + + def test_a_feed_outlines_children_are_not_imported_as_separate_feeds(self): + from apps.feeds.opml import parse_opml + + body = ( + '' + '' + '' + ) + + urls = [e["feed_url"] for e in parse_opml(self._opml(body))] + + assert urls == ["https://a.example.com/f.xml", "https://b.example.com/f.xml"] diff --git a/apps/feeds/test_subscription_views.py b/apps/feeds/test_subscription_views.py index 8b561cc..cefe4ca 100644 --- a/apps/feeds/test_subscription_views.py +++ b/apps/feeds/test_subscription_views.py @@ -173,9 +173,9 @@ def test_import_opml_dispatches_background_task_above_threshold(client, alice, m dispatched = {} - def fake_delay(user_id, content): + def fake_delay(user_id, upload_key): dispatched["user_id"] = user_id - dispatched["content"] = content + dispatched["upload_key"] = upload_key from . import views as feeds_views @@ -187,6 +187,11 @@ def test_import_opml_dispatches_background_task_above_threshold(client, alice, m assert response.status_code == 302 assert dispatched["user_id"] == alice.id + # The task gets a reference to the document, not the document itself. + from django.core.cache import cache + + assert dispatched["upload_key"].startswith("opml-import:") + assert cache.get(dispatched["upload_key"]) == SIMPLE_OPML # The background task was dispatched instead of a synchronous import. assert not Subscription.objects.filter(user=alice).exists() @@ -204,9 +209,51 @@ def test_export_opml_view_returns_downloadable_file(client, alice, alices_subscr @pytest.mark.django_db def test_import_opml_task_runs_import(alice): + from django.core.cache import cache + from .tasks import import_opml_task - result = import_opml_task(alice.id, SIMPLE_OPML.decode("utf-8")) + cache.set("opml-import:test", SIMPLE_OPML, timeout=60) + + result = import_opml_task(alice.id, "opml-import:test") assert result["created"] == 2 assert Subscription.objects.filter(user=alice).count() == 2 + assert cache.get("opml-import:test") is None, "the stored upload is removed once read" + + +@pytest.mark.django_db +def test_import_opml_task_does_nothing_if_the_upload_expired(alice): + from .tasks import import_opml_task + + assert import_opml_task(alice.id, "opml-import:gone") is None + assert not Subscription.objects.filter(user=alice).exists() + + +@pytest.mark.django_db +def test_import_opml_rejects_an_oversized_upload(client, alice, monkeypatch): + from . import forms + + monkeypatch.setattr(forms, "MAX_UPLOAD_BYTES", 100) + client.login(username="alice", password="pw-alice-123") + + upload = SimpleUploadedFile("big.opml", SIMPLE_OPML, content_type="text/x-opml") + response = client.post(reverse("feeds:import_opml"), {"opml_file": upload}) + + assert response.status_code == 200 + assert "too large" in response.content.decode() + assert not Subscription.objects.filter(user=alice).exists() + + +@pytest.mark.django_db +def test_import_opml_reports_a_limit_error_instead_of_a_generic_one(client, alice, monkeypatch): + from . import opml + + monkeypatch.setattr(opml, "MAX_OPML_FEEDS", 1) + client.login(username="alice", password="pw-alice-123") + + upload = SimpleUploadedFile("subs.opml", SIMPLE_OPML, content_type="text/x-opml") + response = client.post(reverse("feeds:import_opml"), {"opml_file": upload}, follow=True) + + assert "too many feeds" in response.content.decode() + assert not Subscription.objects.filter(user=alice).exists() diff --git a/apps/feeds/views.py b/apps/feeds/views.py index f8567ca..0bf0469 100644 --- a/apps/feeds/views.py +++ b/apps/feeds/views.py @@ -1,8 +1,10 @@ import logging +import uuid from io import BytesIO from django.contrib import messages from django.contrib.auth.decorators import login_required +from django.core.cache import cache from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.views.decorators.http import require_POST @@ -12,12 +14,13 @@ from apps.articles.models import SubscriptionTag, Tag from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm from .models import Feed, Subscription -from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml +from .opml import MAX_UPLOAD_BYTES, export_opml, extract_opml_bytes, import_opml, parse_opml from .tasks import enqueue_fetch, import_opml_task logger = logging.getLogger(__name__) OPML_ASYNC_THRESHOLD = 50 +OPML_UPLOAD_TTL_SECONDS = 60 * 60 @login_required @@ -149,16 +152,29 @@ def unsubscribe(request, subscription_id): def import_opml_view(request): form = OPMLImportForm(request.POST or None, request.FILES or None) if request.method == "POST" and form.is_valid(): - uploaded = form.cleaned_data["opml_file"].read() + uploaded = form.cleaned_data["opml_file"].read(MAX_UPLOAD_BYTES + 1) try: content = extract_opml_bytes(uploaded) entry_count = len(parse_opml(BytesIO(content))) + except ValueError as exc: + # Our own limit/format errors carry a user-presentable message. + messages.error(request, str(exc)) + return redirect("feeds:import_opml") except Exception: messages.error(request, "That file doesn't look like a valid OPML export.") return redirect("feeds:import_opml") if entry_count > OPML_ASYNC_THRESHOLD: - import_opml_task.delay(request.user.id, content.decode("utf-8")) + # Hand the worker a reference, not the document: a multi-MB + # file doesn't belong inside a broker message. + upload_key = f"opml-import:{uuid.uuid4().hex}" + try: + cache.set(upload_key, content, timeout=OPML_UPLOAD_TTL_SECONDS) + import_opml_task.delay(request.user.id, upload_key) + except Exception: + logger.exception("Could not queue the OPML import") + messages.error(request, "Couldn't queue that import right now — try again shortly.") + return redirect("feeds:import_opml") messages.success(request, f"Importing {entry_count} feeds in the background.") else: result = import_opml(request.user, BytesIO(content))