From 9439f9cc595654da7e9193a621fade8a7f36ada3 Mon Sep 17 00:00:00 2001 From: Keith Smith Date: Sat, 31 Oct 2026 12:25:20 -0600 Subject: [PATCH] Add SSRF protection to feed fetching: validate scheme/address, bound redirects and response size (S1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetch_and_ingest fetched a feed_url supplied straight from the "add feed" form or an uploaded OPML file, with requests following redirects on its own and no check that the target (or a redirect's target) was actually a public, fetchable feed rather than an internal service, localhost, or a cloud metadata endpoint — reachable from wherever this app runs, on any authenticated user's behalf. apps.feeds.services.ssrf_guard.safe_get() replaces the direct requests.get() call: it resolves and validates the hostname (rejecting anything that isn't a globally routable address) before the initial request and before following each redirect hop manually, caps the number of hops, and bounds the response body size by streaming it instead of buffering it unbounded. OPML-imported feeds go through the same check the first time poll_due_feeds fetches them, since this is the one chokepoint every feed URL passes through regardless of which path created the Feed row. Known residual gap, documented in ssrf_guard.py: this validates then connects rather than pinning the TCP connection to the exact address it validated, so a narrow DNS-rebinding race isn't closed. Fixing that needs a custom transport and felt like more machinery than this self-hosted app's threat model currently justifies. Co-Authored-By: Claude Sonnet 5 --- apps/feeds/conftest.py | 21 +++++ apps/feeds/services/fetcher.py | 7 +- apps/feeds/services/ssrf_guard.py | 121 ++++++++++++++++++++++++++ apps/feeds/test_ssrf_guard.py | 138 ++++++++++++++++++++++++++++++ 4 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 apps/feeds/conftest.py create mode 100644 apps/feeds/services/ssrf_guard.py create mode 100644 apps/feeds/test_ssrf_guard.py diff --git a/apps/feeds/conftest.py b/apps/feeds/conftest.py new file mode 100644 index 0000000..b9c3c1e --- /dev/null +++ b/apps/feeds/conftest.py @@ -0,0 +1,21 @@ +import socket + +import pytest + + +@pytest.fixture(autouse=True) +def _fake_public_dns(monkeypatch): + """apps.feeds.services.ssrf_guard resolves a feed's real hostname via + socket.getaddrinfo before every fetch. Tests use made-up hostnames + like example.com/a.example.com purely as fixture data — they must + not depend on real DNS/network access to pass that check, and must + resolve deterministically regardless of this sandbox's network + access. Every lookup here resolves to a fixed, definitely-public IP; + tests that specifically exercise the "resolves to a non-public + address" rejection path patch socket.getaddrinfo themselves instead. + """ + + def _fake_getaddrinfo(host, *args, **kwargs): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))] + + monkeypatch.setattr(socket, "getaddrinfo", _fake_getaddrinfo) diff --git a/apps/feeds/services/fetcher.py b/apps/feeds/services/fetcher.py index 017b715..cfb7919 100644 --- a/apps/feeds/services/fetcher.py +++ b/apps/feeds/services/fetcher.py @@ -23,6 +23,8 @@ from django.utils import timezone from apps.articles.models import Article from apps.feeds.models import Feed +from .ssrf_guard import SSRFBlocked, safe_get + logger = logging.getLogger(__name__) FETCH_TIMEOUT_SECONDS = 15 @@ -139,7 +141,10 @@ def fetch_and_ingest(feed: Feed) -> FetchResult: headers["If-Modified-Since"] = feed.last_modified try: - response = requests.get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS) + response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS) + except SSRFBlocked as exc: + feed.last_fetched_at = timezone.now() + return _record_error(feed, f"Refused to fetch: {exc}") except requests.RequestException as exc: feed.last_fetched_at = timezone.now() return _record_error(feed, f"Request failed: {exc}") diff --git a/apps/feeds/services/ssrf_guard.py b/apps/feeds/services/ssrf_guard.py new file mode 100644 index 0000000..494945a --- /dev/null +++ b/apps/feeds/services/ssrf_guard.py @@ -0,0 +1,121 @@ +"""Rejects feed-fetch targets that could turn this server/worker's own +network access into an SSRF vector. + +Feed URLs are entirely attacker-controlled (any authenticated user via +"add feed", or an uploaded OPML file — see apps.feeds.opml) and this app +fetches them from the server's own network position. fetch_and_ingest +(apps.feeds.services.fetcher) is the single chokepoint every feed URL +passes through before any HTTP request is made, regardless of which +path created the Feed row, so enforcing this here covers all of them. + +This validates the resolved address before connecting and again before +following each redirect hop, denying by default anything that isn't a +globally routable address (ip.is_global) — covering loopback, RFC1918 +private ranges, link-local (including the 169.254.169.254 cloud +metadata address), and so on, for both IPv4 and IPv6. + +What this does *not* close: the TCP connection isn't pinned to the +specific IP that was validated, so a DNS answer that changes between +the validation lookup and the actual connect (classic DNS rebinding) +could still slip through. Closing that fully needs a custom transport +that connects to the exact validated IP while still verifying TLS +against the original hostname — more machinery than this self-hosted +app's threat model currently justifies; this stops the much more +common case of a feed URL (or a redirect) simply pointing straight at +an internal address. +""" + +from __future__ import annotations + +import ipaddress +import socket +from dataclasses import dataclass +from urllib.parse import urljoin, urlparse + +import requests +from requests.structures import CaseInsensitiveDict + +ALLOWED_SCHEMES = {"http", "https"} +MAX_REDIRECTS = 5 +MAX_RESPONSE_BYTES = 10 * 1024 * 1024 # 10 MB — generous for an RSS/Atom feed +READ_CHUNK_BYTES = 65536 + + +class SSRFBlocked(Exception): + """Raised when a fetch target — or one of its redirects — isn't safe + to connect to on a user's behalf.""" + + +@dataclass +class BoundedResponse: + status_code: int + # requests' own CaseInsensitiveDict — server header casing varies + # (ETag/Etag/etag) and fetcher.py looks these up by a fixed-case key. + headers: CaseInsensitiveDict + content: bytes + + +def validate_fetch_url(url: str) -> None: + """Raises SSRFBlocked unless `url` has an http(s) scheme, a hostname, + and every address that hostname resolves to is globally routable.""" + parsed = urlparse(url) + if parsed.scheme not in ALLOWED_SCHEMES: + raise SSRFBlocked(f"Unsupported URL scheme: {parsed.scheme!r}") + if not parsed.hostname: + raise SSRFBlocked("URL has no hostname") + + try: + addrinfo = socket.getaddrinfo(parsed.hostname, None) + except OSError as exc: + raise SSRFBlocked(f"Could not resolve {parsed.hostname!r}: {exc}") from exc + + for *_, sockaddr in addrinfo: + ip = ipaddress.ip_address(sockaddr[0]) + if not ip.is_global: + raise SSRFBlocked(f"{parsed.hostname!r} resolves to a non-public address: {ip}") + + +def safe_get(url: str, *, headers: dict, timeout: float) -> BoundedResponse: + """GETs `url` the way fetch_and_ingest needs it: every redirect hop is + validated and followed manually (never left to requests/urllib3, so + a redirect can't bypass validate_fetch_url), capped at MAX_REDIRECTS, + and the body is read in bounded chunks rather than trusting + Content-Length or buffering an unbounded response. + """ + current_url = url + with requests.Session() as session: + for _ in range(MAX_REDIRECTS + 1): + validate_fetch_url(current_url) + response = session.get( + current_url, + headers=headers, + timeout=timeout, + stream=True, + allow_redirects=False, + ) + try: + if response.is_redirect or response.is_permanent_redirect: + location = response.headers.get("Location") + if not location: + raise SSRFBlocked("Redirect response had no Location header") + current_url = urljoin(current_url, location) + continue + return BoundedResponse( + status_code=response.status_code, + headers=response.headers, + content=_read_bounded(response), + ) + finally: + response.close() + raise SSRFBlocked(f"Too many redirects (> {MAX_REDIRECTS})") + + +def _read_bounded(response: requests.Response) -> bytes: + chunks = [] + total = 0 + for chunk in response.iter_content(chunk_size=READ_CHUNK_BYTES): + total += len(chunk) + if total > MAX_RESPONSE_BYTES: + raise SSRFBlocked(f"Response exceeded {MAX_RESPONSE_BYTES} bytes") + chunks.append(chunk) + return b"".join(chunks) diff --git a/apps/feeds/test_ssrf_guard.py b/apps/feeds/test_ssrf_guard.py new file mode 100644 index 0000000..f5461c4 --- /dev/null +++ b/apps/feeds/test_ssrf_guard.py @@ -0,0 +1,138 @@ +import socket + +import pytest +import responses + +from apps.feeds.services.ssrf_guard import ( + MAX_REDIRECTS, + MAX_RESPONSE_BYTES, + SSRFBlocked, + safe_get, + validate_fetch_url, +) + +# apps/feeds/conftest.py's autouse _fake_public_dns fixture pins every +# hostname to this address unless a test below overrides it directly. +PUBLIC_IP = "93.184.216.34" + + +def _patch_dns(monkeypatch, ip: str): + def _fake_getaddrinfo(host, *args, **kwargs): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, 0))] + + monkeypatch.setattr(socket, "getaddrinfo", _fake_getaddrinfo) + + +class TestValidateFetchUrl: + def test_allows_a_public_https_url(self): + validate_fetch_url("https://example.com/feed.xml") + + def test_rejects_a_non_http_scheme(self): + with pytest.raises(SSRFBlocked, match="scheme"): + validate_fetch_url("file:///etc/passwd") + + def test_rejects_a_url_with_no_hostname(self): + with pytest.raises(SSRFBlocked, match="hostname"): + validate_fetch_url("https:///feed.xml") + + @pytest.mark.parametrize( + "ip", + [ + "127.0.0.1", # loopback + "10.0.0.5", # RFC1918 private + "192.168.1.1", # RFC1918 private + "169.254.169.254", # link-local / cloud metadata + "0.0.0.0", # unspecified + "::1", # IPv6 loopback + "fc00::1", # IPv6 unique local + ], + ) + def test_rejects_a_url_resolving_to_a_non_public_address(self, monkeypatch, ip): + _patch_dns(monkeypatch, ip) + with pytest.raises(SSRFBlocked, match="non-public"): + validate_fetch_url("https://internal.example.com/feed.xml") + + def test_rejects_a_url_whose_hostname_does_not_resolve(self, monkeypatch): + def _raise(*args, **kwargs): + raise socket.gaierror("nope") + + monkeypatch.setattr(socket, "getaddrinfo", _raise) + with pytest.raises(SSRFBlocked, match="Could not resolve"): + validate_fetch_url("https://nowhere.example.com/feed.xml") + + +class TestSafeGet: + @responses.activate + def test_returns_the_response_body_for_a_normal_feed(self): + responses.add(responses.GET, "https://example.com/feed.xml", body="") + response = safe_get("https://example.com/feed.xml", headers={}, timeout=5) + assert response.status_code == 200 + assert response.content == b"" + + @responses.activate + def test_follows_a_safe_redirect(self): + responses.add( + responses.GET, + "https://example.com/feed.xml", + status=302, + headers={"Location": "https://example.com/final.xml"}, + ) + responses.add(responses.GET, "https://example.com/final.xml", body="final") + + response = safe_get("https://example.com/feed.xml", headers={}, timeout=5) + assert response.content == b"final" + + @responses.activate + def test_blocks_a_redirect_to_a_private_address(self, monkeypatch): + responses.add( + responses.GET, + "https://example.com/feed.xml", + status=302, + headers={"Location": "http://internal.example.com/secret"}, + ) + + real_getaddrinfo = socket.getaddrinfo + + def _fake_getaddrinfo(host, *args, **kwargs): + if host == "internal.example.com": + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))] + return real_getaddrinfo(host, *args, **kwargs) + + monkeypatch.setattr(socket, "getaddrinfo", _fake_getaddrinfo) + + with pytest.raises(SSRFBlocked, match="non-public"): + safe_get("https://example.com/feed.xml", headers={}, timeout=5) + + @responses.activate + def test_blocks_a_redirect_loop_past_max_redirects(self): + for _ in range(MAX_REDIRECTS + 2): + responses.add( + responses.GET, + "https://example.com/feed.xml", + status=302, + headers={"Location": "https://example.com/feed.xml"}, + ) + + with pytest.raises(SSRFBlocked, match="Too many redirects"): + safe_get("https://example.com/feed.xml", headers={}, timeout=5) + + @responses.activate + def test_blocks_a_response_over_the_size_cap(self): + responses.add( + responses.GET, + "https://example.com/feed.xml", + body=b"x" * (MAX_RESPONSE_BYTES + 1), + ) + with pytest.raises(SSRFBlocked, match="exceeded"): + safe_get("https://example.com/feed.xml", headers={}, timeout=5) + + @responses.activate + def test_preserves_case_insensitive_header_lookup(self): + responses.add( + responses.GET, + "https://example.com/feed.xml", + body="", + headers={"etag": "abc123"}, + ) + response = safe_get("https://example.com/feed.xml", headers={}, timeout=5) + assert response.headers.get("ETag") == "abc123"