diff --git a/apps/feeds/conftest.py b/apps/feeds/conftest.py new file mode 100644 index 0000000..b9c3c1e --- /dev/null +++ b/apps/feeds/conftest.py @@ -0,0 +1,21 @@ +import socket + +import pytest + + +@pytest.fixture(autouse=True) +def _fake_public_dns(monkeypatch): + """apps.feeds.services.ssrf_guard resolves a feed's real hostname via + socket.getaddrinfo before every fetch. Tests use made-up hostnames + like example.com/a.example.com purely as fixture data — they must + not depend on real DNS/network access to pass that check, and must + resolve deterministically regardless of this sandbox's network + access. Every lookup here resolves to a fixed, definitely-public IP; + tests that specifically exercise the "resolves to a non-public + address" rejection path patch socket.getaddrinfo themselves instead. + """ + + def _fake_getaddrinfo(host, *args, **kwargs): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))] + + monkeypatch.setattr(socket, "getaddrinfo", _fake_getaddrinfo) diff --git a/apps/feeds/services/fetcher.py b/apps/feeds/services/fetcher.py index 017b715..cfb7919 100644 --- a/apps/feeds/services/fetcher.py +++ b/apps/feeds/services/fetcher.py @@ -23,6 +23,8 @@ from django.utils import timezone from apps.articles.models import Article from apps.feeds.models import Feed +from .ssrf_guard import SSRFBlocked, safe_get + logger = logging.getLogger(__name__) FETCH_TIMEOUT_SECONDS = 15 @@ -139,7 +141,10 @@ def fetch_and_ingest(feed: Feed) -> FetchResult: headers["If-Modified-Since"] = feed.last_modified try: - response = requests.get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS) + response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS) + except SSRFBlocked as exc: + feed.last_fetched_at = timezone.now() + return _record_error(feed, f"Refused to fetch: {exc}") except requests.RequestException as exc: feed.last_fetched_at = timezone.now() return _record_error(feed, f"Request failed: {exc}") diff --git a/apps/feeds/services/ssrf_guard.py b/apps/feeds/services/ssrf_guard.py new file mode 100644 index 0000000..494945a --- /dev/null +++ b/apps/feeds/services/ssrf_guard.py @@ -0,0 +1,121 @@ +"""Rejects feed-fetch targets that could turn this server/worker's own +network access into an SSRF vector. + +Feed URLs are entirely attacker-controlled (any authenticated user via +"add feed", or an uploaded OPML file — see apps.feeds.opml) and this app +fetches them from the server's own network position. fetch_and_ingest +(apps.feeds.services.fetcher) is the single chokepoint every feed URL +passes through before any HTTP request is made, regardless of which +path created the Feed row, so enforcing this here covers all of them. + +This validates the resolved address before connecting and again before +following each redirect hop, denying by default anything that isn't a +globally routable address (ip.is_global) — covering loopback, RFC1918 +private ranges, link-local (including the 169.254.169.254 cloud +metadata address), and so on, for both IPv4 and IPv6. + +What this does *not* close: the TCP connection isn't pinned to the +specific IP that was validated, so a DNS answer that changes between +the validation lookup and the actual connect (classic DNS rebinding) +could still slip through. Closing that fully needs a custom transport +that connects to the exact validated IP while still verifying TLS +against the original hostname — more machinery than this self-hosted +app's threat model currently justifies; this stops the much more +common case of a feed URL (or a redirect) simply pointing straight at +an internal address. +""" + +from __future__ import annotations + +import ipaddress +import socket +from dataclasses import dataclass +from urllib.parse import urljoin, urlparse + +import requests +from requests.structures import CaseInsensitiveDict + +ALLOWED_SCHEMES = {"http", "https"} +MAX_REDIRECTS = 5 +MAX_RESPONSE_BYTES = 10 * 1024 * 1024 # 10 MB — generous for an RSS/Atom feed +READ_CHUNK_BYTES = 65536 + + +class SSRFBlocked(Exception): + """Raised when a fetch target — or one of its redirects — isn't safe + to connect to on a user's behalf.""" + + +@dataclass +class BoundedResponse: + status_code: int + # requests' own CaseInsensitiveDict — server header casing varies + # (ETag/Etag/etag) and fetcher.py looks these up by a fixed-case key. + headers: CaseInsensitiveDict + content: bytes + + +def validate_fetch_url(url: str) -> None: + """Raises SSRFBlocked unless `url` has an http(s) scheme, a hostname, + and every address that hostname resolves to is globally routable.""" + parsed = urlparse(url) + if parsed.scheme not in ALLOWED_SCHEMES: + raise SSRFBlocked(f"Unsupported URL scheme: {parsed.scheme!r}") + if not parsed.hostname: + raise SSRFBlocked("URL has no hostname") + + try: + addrinfo = socket.getaddrinfo(parsed.hostname, None) + except OSError as exc: + raise SSRFBlocked(f"Could not resolve {parsed.hostname!r}: {exc}") from exc + + for *_, sockaddr in addrinfo: + ip = ipaddress.ip_address(sockaddr[0]) + if not ip.is_global: + raise SSRFBlocked(f"{parsed.hostname!r} resolves to a non-public address: {ip}") + + +def safe_get(url: str, *, headers: dict, timeout: float) -> BoundedResponse: + """GETs `url` the way fetch_and_ingest needs it: every redirect hop is + validated and followed manually (never left to requests/urllib3, so + a redirect can't bypass validate_fetch_url), capped at MAX_REDIRECTS, + and the body is read in bounded chunks rather than trusting + Content-Length or buffering an unbounded response. + """ + current_url = url + with requests.Session() as session: + for _ in range(MAX_REDIRECTS + 1): + validate_fetch_url(current_url) + response = session.get( + current_url, + headers=headers, + timeout=timeout, + stream=True, + allow_redirects=False, + ) + try: + if response.is_redirect or response.is_permanent_redirect: + location = response.headers.get("Location") + if not location: + raise SSRFBlocked("Redirect response had no Location header") + current_url = urljoin(current_url, location) + continue + return BoundedResponse( + status_code=response.status_code, + headers=response.headers, + content=_read_bounded(response), + ) + finally: + response.close() + raise SSRFBlocked(f"Too many redirects (> {MAX_REDIRECTS})") + + +def _read_bounded(response: requests.Response) -> bytes: + chunks = [] + total = 0 + for chunk in response.iter_content(chunk_size=READ_CHUNK_BYTES): + total += len(chunk) + if total > MAX_RESPONSE_BYTES: + raise SSRFBlocked(f"Response exceeded {MAX_RESPONSE_BYTES} bytes") + chunks.append(chunk) + return b"".join(chunks) diff --git a/apps/feeds/test_ssrf_guard.py b/apps/feeds/test_ssrf_guard.py new file mode 100644 index 0000000..f5461c4 --- /dev/null +++ b/apps/feeds/test_ssrf_guard.py @@ -0,0 +1,138 @@ +import socket + +import pytest +import responses + +from apps.feeds.services.ssrf_guard import ( + MAX_REDIRECTS, + MAX_RESPONSE_BYTES, + SSRFBlocked, + safe_get, + validate_fetch_url, +) + +# apps/feeds/conftest.py's autouse _fake_public_dns fixture pins every +# hostname to this address unless a test below overrides it directly. +PUBLIC_IP = "93.184.216.34" + + +def _patch_dns(monkeypatch, ip: str): + def _fake_getaddrinfo(host, *args, **kwargs): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, 0))] + + monkeypatch.setattr(socket, "getaddrinfo", _fake_getaddrinfo) + + +class TestValidateFetchUrl: + def test_allows_a_public_https_url(self): + validate_fetch_url("https://example.com/feed.xml") + + def test_rejects_a_non_http_scheme(self): + with pytest.raises(SSRFBlocked, match="scheme"): + validate_fetch_url("file:///etc/passwd") + + def test_rejects_a_url_with_no_hostname(self): + with pytest.raises(SSRFBlocked, match="hostname"): + validate_fetch_url("https:///feed.xml") + + @pytest.mark.parametrize( + "ip", + [ + "127.0.0.1", # loopback + "10.0.0.5", # RFC1918 private + "192.168.1.1", # RFC1918 private + "169.254.169.254", # link-local / cloud metadata + "0.0.0.0", # unspecified + "::1", # IPv6 loopback + "fc00::1", # IPv6 unique local + ], + ) + def test_rejects_a_url_resolving_to_a_non_public_address(self, monkeypatch, ip): + _patch_dns(monkeypatch, ip) + with pytest.raises(SSRFBlocked, match="non-public"): + validate_fetch_url("https://internal.example.com/feed.xml") + + def test_rejects_a_url_whose_hostname_does_not_resolve(self, monkeypatch): + def _raise(*args, **kwargs): + raise socket.gaierror("nope") + + monkeypatch.setattr(socket, "getaddrinfo", _raise) + with pytest.raises(SSRFBlocked, match="Could not resolve"): + validate_fetch_url("https://nowhere.example.com/feed.xml") + + +class TestSafeGet: + @responses.activate + def test_returns_the_response_body_for_a_normal_feed(self): + responses.add(responses.GET, "https://example.com/feed.xml", body="") + response = safe_get("https://example.com/feed.xml", headers={}, timeout=5) + assert response.status_code == 200 + assert response.content == b"" + + @responses.activate + def test_follows_a_safe_redirect(self): + responses.add( + responses.GET, + "https://example.com/feed.xml", + status=302, + headers={"Location": "https://example.com/final.xml"}, + ) + responses.add(responses.GET, "https://example.com/final.xml", body="final") + + response = safe_get("https://example.com/feed.xml", headers={}, timeout=5) + assert response.content == b"final" + + @responses.activate + def test_blocks_a_redirect_to_a_private_address(self, monkeypatch): + responses.add( + responses.GET, + "https://example.com/feed.xml", + status=302, + headers={"Location": "http://internal.example.com/secret"}, + ) + + real_getaddrinfo = socket.getaddrinfo + + def _fake_getaddrinfo(host, *args, **kwargs): + if host == "internal.example.com": + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))] + return real_getaddrinfo(host, *args, **kwargs) + + monkeypatch.setattr(socket, "getaddrinfo", _fake_getaddrinfo) + + with pytest.raises(SSRFBlocked, match="non-public"): + safe_get("https://example.com/feed.xml", headers={}, timeout=5) + + @responses.activate + def test_blocks_a_redirect_loop_past_max_redirects(self): + for _ in range(MAX_REDIRECTS + 2): + responses.add( + responses.GET, + "https://example.com/feed.xml", + status=302, + headers={"Location": "https://example.com/feed.xml"}, + ) + + with pytest.raises(SSRFBlocked, match="Too many redirects"): + safe_get("https://example.com/feed.xml", headers={}, timeout=5) + + @responses.activate + def test_blocks_a_response_over_the_size_cap(self): + responses.add( + responses.GET, + "https://example.com/feed.xml", + body=b"x" * (MAX_RESPONSE_BYTES + 1), + ) + with pytest.raises(SSRFBlocked, match="exceeded"): + safe_get("https://example.com/feed.xml", headers={}, timeout=5) + + @responses.activate + def test_preserves_case_insensitive_header_lookup(self): + responses.add( + responses.GET, + "https://example.com/feed.xml", + body="", + headers={"etag": "abc123"}, + ) + response = safe_get("https://example.com/feed.xml", headers={}, timeout=5) + assert response.headers.get("ETag") == "abc123"