Add SSRF protection to feed fetching: validate scheme/address, bound redirects and response size (S1)

fetch_and_ingest fetched a feed_url supplied straight from the "add
feed" form or an uploaded OPML file, with requests following redirects
on its own and no check that the target (or a redirect's target) was
actually a public, fetchable feed rather than an internal service,
localhost, or a cloud metadata endpoint — reachable from wherever this
app runs, on any authenticated user's behalf.

apps.feeds.services.ssrf_guard.safe_get() replaces the direct
requests.get() call: it resolves and validates the hostname (rejecting
anything that isn't a globally routable address) before the initial
request and before following each redirect hop manually, caps the
number of hops, and bounds the response body size by streaming it
instead of buffering it unbounded. OPML-imported feeds go through the
same check the first time poll_due_feeds fetches them, since this is
the one chokepoint every feed URL passes through regardless of which
path created the Feed row.

Known residual gap, documented in ssrf_guard.py: this validates then
connects rather than pinning the TCP connection to the exact address it
validated, so a narrow DNS-rebinding race isn't closed. Fixing that
needs a custom transport and felt like more machinery than this
self-hosted app's threat model currently justifies.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-10-31 12:25:20 -06:00
1 parent fd8eeb605e
commit 9439f9cc59
4 files changed
+286 -1

No files matched your search

+21
View File
@@ -0,0 +1,21 @@
import socket
import pytest
@pytest.fixture(autouse=True)
def _fake_public_dns(monkeypatch):
"""apps.feeds.services.ssrf_guard resolves a feed's real hostname via
socket.getaddrinfo before every fetch. Tests use made-up hostnames
like example.com/a.example.com purely as fixture data — they must
not depend on real DNS/network access to pass that check, and must
resolve deterministically regardless of this sandbox's network
access. Every lookup here resolves to a fixed, definitely-public IP;
tests that specifically exercise the "resolves to a non-public
address" rejection path patch socket.getaddrinfo themselves instead.
"""
def _fake_getaddrinfo(host, *args, **kwargs):
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))]
monkeypatch.setattr(socket, "getaddrinfo", _fake_getaddrinfo)
+6 -1
View File
@@ -23,6 +23,8 @@ from django.utils import timezone
from apps.articles.models import Article from apps.articles.models import Article
from apps.feeds.models import Feed from apps.feeds.models import Feed
from .ssrf_guard import SSRFBlocked, safe_get
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
FETCH_TIMEOUT_SECONDS = 15 FETCH_TIMEOUT_SECONDS = 15
@@ -139,7 +141,10 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
headers["If-Modified-Since"] = feed.last_modified headers["If-Modified-Since"] = feed.last_modified
try: try:
response = requests.get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS) response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
except SSRFBlocked as exc:
feed.last_fetched_at = timezone.now()
return _record_error(feed, f"Refused to fetch: {exc}")
except requests.RequestException as exc: except requests.RequestException as exc:
feed.last_fetched_at = timezone.now() feed.last_fetched_at = timezone.now()
return _record_error(feed, f"Request failed: {exc}") return _record_error(feed, f"Request failed: {exc}")
+121
View File
@@ -0,0 +1,121 @@
"""Rejects feed-fetch targets that could turn this server/worker's own
network access into an SSRF vector.
Feed URLs are entirely attacker-controlled (any authenticated user via
"add feed", or an uploaded OPML file — see apps.feeds.opml) and this app
fetches them from the server's own network position. fetch_and_ingest
(apps.feeds.services.fetcher) is the single chokepoint every feed URL
passes through before any HTTP request is made, regardless of which
path created the Feed row, so enforcing this here covers all of them.
This validates the resolved address before connecting and again before
following each redirect hop, denying by default anything that isn't a
globally routable address (ip.is_global) — covering loopback, RFC1918
private ranges, link-local (including the 169.254.169.254 cloud
metadata address), and so on, for both IPv4 and IPv6.
What this does *not* close: the TCP connection isn't pinned to the
specific IP that was validated, so a DNS answer that changes between
the validation lookup and the actual connect (classic DNS rebinding)
could still slip through. Closing that fully needs a custom transport
that connects to the exact validated IP while still verifying TLS
against the original hostname — more machinery than this self-hosted
app's threat model currently justifies; this stops the much more
common case of a feed URL (or a redirect) simply pointing straight at
an internal address.
"""
from __future__ import annotations
import ipaddress
import socket
from dataclasses import dataclass
from urllib.parse import urljoin, urlparse
import requests
from requests.structures import CaseInsensitiveDict
ALLOWED_SCHEMES = {"http", "https"}
MAX_REDIRECTS = 5
MAX_RESPONSE_BYTES = 10 * 1024 * 1024 # 10 MB — generous for an RSS/Atom feed
READ_CHUNK_BYTES = 65536
class SSRFBlocked(Exception):
"""Raised when a fetch target — or one of its redirects — isn't safe
to connect to on a user's behalf."""
@dataclass
class BoundedResponse:
status_code: int
# requests' own CaseInsensitiveDict — server header casing varies
# (ETag/Etag/etag) and fetcher.py looks these up by a fixed-case key.
headers: CaseInsensitiveDict
content: bytes
def validate_fetch_url(url: str) -> None:
"""Raises SSRFBlocked unless `url` has an http(s) scheme, a hostname,
and every address that hostname resolves to is globally routable."""
parsed = urlparse(url)
if parsed.scheme not in ALLOWED_SCHEMES:
raise SSRFBlocked(f"Unsupported URL scheme: {parsed.scheme!r}")
if not parsed.hostname:
raise SSRFBlocked("URL has no hostname")
try:
addrinfo = socket.getaddrinfo(parsed.hostname, None)
except OSError as exc:
raise SSRFBlocked(f"Could not resolve {parsed.hostname!r}: {exc}") from exc
for *_, sockaddr in addrinfo:
ip = ipaddress.ip_address(sockaddr[0])
if not ip.is_global:
raise SSRFBlocked(f"{parsed.hostname!r} resolves to a non-public address: {ip}")
def safe_get(url: str, *, headers: dict, timeout: float) -> BoundedResponse:
"""GETs `url` the way fetch_and_ingest needs it: every redirect hop is
validated and followed manually (never left to requests/urllib3, so
a redirect can't bypass validate_fetch_url), capped at MAX_REDIRECTS,
and the body is read in bounded chunks rather than trusting
Content-Length or buffering an unbounded response.
"""
current_url = url
with requests.Session() as session:
for _ in range(MAX_REDIRECTS + 1):
validate_fetch_url(current_url)
response = session.get(
current_url,
headers=headers,
timeout=timeout,
stream=True,
allow_redirects=False,
)
try:
if response.is_redirect or response.is_permanent_redirect:
location = response.headers.get("Location")
if not location:
raise SSRFBlocked("Redirect response had no Location header")
current_url = urljoin(current_url, location)
continue
return BoundedResponse(
status_code=response.status_code,
headers=response.headers,
content=_read_bounded(response),
)
finally:
response.close()
raise SSRFBlocked(f"Too many redirects (> {MAX_REDIRECTS})")
def _read_bounded(response: requests.Response) -> bytes:
chunks = []
total = 0
for chunk in response.iter_content(chunk_size=READ_CHUNK_BYTES):
total += len(chunk)
if total > MAX_RESPONSE_BYTES:
raise SSRFBlocked(f"Response exceeded {MAX_RESPONSE_BYTES} bytes")
chunks.append(chunk)
return b"".join(chunks)
+138
View File
@@ -0,0 +1,138 @@
import socket
import pytest
import responses
from apps.feeds.services.ssrf_guard import (
MAX_REDIRECTS,
MAX_RESPONSE_BYTES,
SSRFBlocked,
safe_get,
validate_fetch_url,
)
# apps/feeds/conftest.py's autouse _fake_public_dns fixture pins every
# hostname to this address unless a test below overrides it directly.
PUBLIC_IP = "93.184.216.34"
def _patch_dns(monkeypatch, ip: str):
def _fake_getaddrinfo(host, *args, **kwargs):
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, 0))]
monkeypatch.setattr(socket, "getaddrinfo", _fake_getaddrinfo)
class TestValidateFetchUrl:
def test_allows_a_public_https_url(self):
validate_fetch_url("https://example.com/feed.xml")
def test_rejects_a_non_http_scheme(self):
with pytest.raises(SSRFBlocked, match="scheme"):
validate_fetch_url("file:///etc/passwd")
def test_rejects_a_url_with_no_hostname(self):
with pytest.raises(SSRFBlocked, match="hostname"):
validate_fetch_url("https:///feed.xml")
@pytest.mark.parametrize(
"ip",
[
"127.0.0.1", # loopback
"10.0.0.5", # RFC1918 private
"192.168.1.1", # RFC1918 private
"169.254.169.254", # link-local / cloud metadata
"0.0.0.0", # unspecified
"::1", # IPv6 loopback
"fc00::1", # IPv6 unique local
],
)
def test_rejects_a_url_resolving_to_a_non_public_address(self, monkeypatch, ip):
_patch_dns(monkeypatch, ip)
with pytest.raises(SSRFBlocked, match="non-public"):
validate_fetch_url("https://internal.example.com/feed.xml")
def test_rejects_a_url_whose_hostname_does_not_resolve(self, monkeypatch):
def _raise(*args, **kwargs):
raise socket.gaierror("nope")
monkeypatch.setattr(socket, "getaddrinfo", _raise)
with pytest.raises(SSRFBlocked, match="Could not resolve"):
validate_fetch_url("https://nowhere.example.com/feed.xml")
class TestSafeGet:
@responses.activate
def test_returns_the_response_body_for_a_normal_feed(self):
responses.add(responses.GET, "https://example.com/feed.xml", body="<rss></rss>")
response = safe_get("https://example.com/feed.xml", headers={}, timeout=5)
assert response.status_code == 200
assert response.content == b"<rss></rss>"
@responses.activate
def test_follows_a_safe_redirect(self):
responses.add(
responses.GET,
"https://example.com/feed.xml",
status=302,
headers={"Location": "https://example.com/final.xml"},
)
responses.add(responses.GET, "https://example.com/final.xml", body="<rss>final</rss>")
response = safe_get("https://example.com/feed.xml", headers={}, timeout=5)
assert response.content == b"<rss>final</rss>"
@responses.activate
def test_blocks_a_redirect_to_a_private_address(self, monkeypatch):
responses.add(
responses.GET,
"https://example.com/feed.xml",
status=302,
headers={"Location": "http://internal.example.com/secret"},
)
real_getaddrinfo = socket.getaddrinfo
def _fake_getaddrinfo(host, *args, **kwargs):
if host == "internal.example.com":
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))]
return real_getaddrinfo(host, *args, **kwargs)
monkeypatch.setattr(socket, "getaddrinfo", _fake_getaddrinfo)
with pytest.raises(SSRFBlocked, match="non-public"):
safe_get("https://example.com/feed.xml", headers={}, timeout=5)
@responses.activate
def test_blocks_a_redirect_loop_past_max_redirects(self):
for _ in range(MAX_REDIRECTS + 2):
responses.add(
responses.GET,
"https://example.com/feed.xml",
status=302,
headers={"Location": "https://example.com/feed.xml"},
)
with pytest.raises(SSRFBlocked, match="Too many redirects"):
safe_get("https://example.com/feed.xml", headers={}, timeout=5)
@responses.activate
def test_blocks_a_response_over_the_size_cap(self):
responses.add(
responses.GET,
"https://example.com/feed.xml",
body=b"x" * (MAX_RESPONSE_BYTES + 1),
)
with pytest.raises(SSRFBlocked, match="exceeded"):
safe_get("https://example.com/feed.xml", headers={}, timeout=5)
@responses.activate
def test_preserves_case_insensitive_header_lookup(self):
responses.add(
responses.GET,
"https://example.com/feed.xml",
body="<rss></rss>",
headers={"etag": "abc123"},
)
response = safe_get("https://example.com/feed.xml", headers={}, timeout=5)
assert response.headers.get("ETag") == "abc123"