Private
Public Access
Pin feed connections to the address that was validated (R2-S2, #34)
The SSRF guard checked that a hostname resolved to public addresses, then let requests resolve it again to connect — so a DNS answer that changed in between (DNS rebinding) could point the connection at an internal address. Connections now go through a pinned adapter whose socket factory resolves, checks and connects inside one function: whatever the name resolves to at connect time is what gets validated and used. Only the socket is replaced, so TLS still verifies against the original hostname. The up-front validate_fetch_url check stays as defense in depth. Tests cover a rebinding answer at connect time being refused and a real loopback round trip with the original Host header preserved; checked live: a real HTTPS feed fetches, 127.0.0.1/169.254.169.254/ localhost are blocked, and a wrong-hostname certificate is still rejected. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
10ba491e10
commit
2e1c0e31a8
2 files changed
+163
-10
No files matched your search
@@ -14,15 +14,15 @@ globally routable address (ip.is_global) — covering loopback, RFC1918
|
|||||||
private ranges, link-local (including the 169.254.169.254 cloud
|
private ranges, link-local (including the 169.254.169.254 cloud
|
||||||
metadata address), and so on, for both IPv4 and IPv6.
|
metadata address), and so on, for both IPv4 and IPv6.
|
||||||
|
|
||||||
What this does *not* close: the TCP connection isn't pinned to the
|
The connection is pinned too (R2-S2): the address is resolved, checked and
|
||||||
specific IP that was validated, so a DNS answer that changes between
|
connected to inside one function (_connect_validated, used by the adapter
|
||||||
the validation lookup and the actual connect (classic DNS rebinding)
|
below as the connection's socket factory), so a DNS answer that changes
|
||||||
could still slip through. Closing that fully needs a custom transport
|
between a separate validation lookup and the connect (DNS rebinding) can't
|
||||||
that connects to the exact validated IP while still verifying TLS
|
steer the connection to an internal address — whatever it resolves to *at
|
||||||
against the original hostname — more machinery than this self-hosted
|
connect time* is what gets validated. TLS still verifies against the
|
||||||
app's threat model currently justifies; this stops the much more
|
original hostname, because only the socket is replaced, not the host the
|
||||||
common case of a feed URL (or a redirect) simply pointing straight at
|
TLS layer is told about. The up-front validate_fetch_url check stays as
|
||||||
an internal address.
|
defense in depth and gives a clear error before any connection attempt.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -33,7 +33,10 @@ from dataclasses import dataclass
|
|||||||
from urllib.parse import urljoin, urlparse
|
from urllib.parse import urljoin, urlparse
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
from requests.adapters import HTTPAdapter
|
||||||
from requests.structures import CaseInsensitiveDict
|
from requests.structures import CaseInsensitiveDict
|
||||||
|
from urllib3.connection import HTTPConnection, HTTPSConnection
|
||||||
|
from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
||||||
|
|
||||||
ALLOWED_SCHEMES = {"http", "https"}
|
ALLOWED_SCHEMES = {"http", "https"}
|
||||||
MAX_REDIRECTS = 5
|
MAX_REDIRECTS = 5
|
||||||
@@ -55,6 +58,10 @@ class BoundedResponse:
|
|||||||
content: bytes
|
content: bytes
|
||||||
|
|
||||||
|
|
||||||
|
def _is_allowed_ip(ip) -> bool:
|
||||||
|
return ip.is_global
|
||||||
|
|
||||||
|
|
||||||
def validate_fetch_url(url: str) -> None:
|
def validate_fetch_url(url: str) -> None:
|
||||||
"""Raises SSRFBlocked unless `url` has an http(s) scheme, a hostname,
|
"""Raises SSRFBlocked unless `url` has an http(s) scheme, a hostname,
|
||||||
and every address that hostname resolves to is globally routable."""
|
and every address that hostname resolves to is globally routable."""
|
||||||
@@ -71,10 +78,82 @@ def validate_fetch_url(url: str) -> None:
|
|||||||
|
|
||||||
for *_, sockaddr in addrinfo:
|
for *_, sockaddr in addrinfo:
|
||||||
ip = ipaddress.ip_address(sockaddr[0])
|
ip = ipaddress.ip_address(sockaddr[0])
|
||||||
if not ip.is_global:
|
if not _is_allowed_ip(ip):
|
||||||
raise SSRFBlocked(f"{parsed.hostname!r} resolves to a non-public address: {ip}")
|
raise SSRFBlocked(f"{parsed.hostname!r} resolves to a non-public address: {ip}")
|
||||||
|
|
||||||
|
|
||||||
|
def _connect_validated(host, port, timeout, source_address, socket_options):
|
||||||
|
"""Resolves `host`, requires every address to be globally routable, and
|
||||||
|
connects to one of those exact addresses — the same lookup result is
|
||||||
|
both validated and used, so there is nothing for a rebinding DNS
|
||||||
|
answer to change in between."""
|
||||||
|
try:
|
||||||
|
addrinfo = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
||||||
|
except OSError as exc:
|
||||||
|
raise SSRFBlocked(f"Could not resolve {host!r}: {exc}") from exc
|
||||||
|
|
||||||
|
candidates = []
|
||||||
|
for family, socktype, proto, _canonname, sockaddr in addrinfo:
|
||||||
|
ip = ipaddress.ip_address(sockaddr[0])
|
||||||
|
if not _is_allowed_ip(ip):
|
||||||
|
raise SSRFBlocked(f"{host!r} resolves to a non-public address: {ip}")
|
||||||
|
candidates.append((family, socktype, proto, sockaddr))
|
||||||
|
if not candidates:
|
||||||
|
raise SSRFBlocked(f"{host!r} did not resolve to any address")
|
||||||
|
|
||||||
|
last_error: OSError | None = None
|
||||||
|
for family, socktype, proto, sockaddr in candidates:
|
||||||
|
sock = None
|
||||||
|
try:
|
||||||
|
sock = socket.socket(family, socktype, proto)
|
||||||
|
for level, option, value in socket_options or ():
|
||||||
|
sock.setsockopt(level, option, value)
|
||||||
|
if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
|
||||||
|
sock.settimeout(timeout)
|
||||||
|
if source_address:
|
||||||
|
sock.bind(source_address)
|
||||||
|
sock.connect(sockaddr)
|
||||||
|
return sock
|
||||||
|
except OSError as exc:
|
||||||
|
last_error = exc
|
||||||
|
if sock is not None:
|
||||||
|
sock.close()
|
||||||
|
raise last_error or OSError(f"Could not connect to {host!r}")
|
||||||
|
|
||||||
|
|
||||||
|
class _PinnedHTTPConnection(HTTPConnection):
|
||||||
|
def _new_conn(self):
|
||||||
|
return _connect_validated(
|
||||||
|
self._dns_host, self.port, self.timeout, self.source_address, self.socket_options
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _PinnedHTTPSConnection(HTTPSConnection):
|
||||||
|
def _new_conn(self):
|
||||||
|
return _connect_validated(
|
||||||
|
self._dns_host, self.port, self.timeout, self.source_address, self.socket_options
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _PinnedHTTPConnectionPool(HTTPConnectionPool):
|
||||||
|
ConnectionCls = _PinnedHTTPConnection
|
||||||
|
|
||||||
|
|
||||||
|
class _PinnedHTTPSConnectionPool(HTTPSConnectionPool):
|
||||||
|
ConnectionCls = _PinnedHTTPSConnection
|
||||||
|
|
||||||
|
|
||||||
|
class PinnedAdapter(HTTPAdapter):
|
||||||
|
"""requests adapter whose connections go through _connect_validated."""
|
||||||
|
|
||||||
|
def init_poolmanager(self, *args, **kwargs):
|
||||||
|
super().init_poolmanager(*args, **kwargs)
|
||||||
|
self.poolmanager.pool_classes_by_scheme = {
|
||||||
|
"http": _PinnedHTTPConnectionPool,
|
||||||
|
"https": _PinnedHTTPSConnectionPool,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def safe_get(url: str, *, headers: dict, timeout: float) -> BoundedResponse:
|
def safe_get(url: str, *, headers: dict, timeout: float) -> BoundedResponse:
|
||||||
"""GETs `url` the way fetch_and_ingest needs it: every redirect hop is
|
"""GETs `url` the way fetch_and_ingest needs it: every redirect hop is
|
||||||
validated and followed manually (never left to requests/urllib3, so
|
validated and followed manually (never left to requests/urllib3, so
|
||||||
@@ -84,6 +163,9 @@ def safe_get(url: str, *, headers: dict, timeout: float) -> BoundedResponse:
|
|||||||
"""
|
"""
|
||||||
current_url = url
|
current_url = url
|
||||||
with requests.Session() as session:
|
with requests.Session() as session:
|
||||||
|
adapter = PinnedAdapter()
|
||||||
|
session.mount("http://", adapter)
|
||||||
|
session.mount("https://", adapter)
|
||||||
for _ in range(MAX_REDIRECTS + 1):
|
for _ in range(MAX_REDIRECTS + 1):
|
||||||
validate_fetch_url(current_url)
|
validate_fetch_url(current_url)
|
||||||
response = session.get(
|
response = session.get(
|
||||||
|
|||||||
@@ -136,3 +136,74 @@ class TestSafeGet:
|
|||||||
)
|
)
|
||||||
response = safe_get("https://example.com/feed.xml", headers={}, timeout=5)
|
response = safe_get("https://example.com/feed.xml", headers={}, timeout=5)
|
||||||
assert response.headers.get("ETag") == "abc123"
|
assert response.headers.get("ETag") == "abc123"
|
||||||
|
|
||||||
|
|
||||||
|
class TestConnectionPinning:
|
||||||
|
"""R2-S2: the address that's validated is the address that's connected
|
||||||
|
to — a hostname that resolves differently at connect time can't slip an
|
||||||
|
internal address past a pre-check."""
|
||||||
|
|
||||||
|
def test_a_rebinding_dns_answer_at_connect_time_is_blocked(self, monkeypatch):
|
||||||
|
import socket
|
||||||
|
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def rebinding_getaddrinfo(host, *args, **kwargs):
|
||||||
|
calls.append(host)
|
||||||
|
ip = "93.184.216.34" if len(calls) == 1 else "127.0.0.1"
|
||||||
|
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, 0))]
|
||||||
|
|
||||||
|
monkeypatch.setattr(socket, "getaddrinfo", rebinding_getaddrinfo)
|
||||||
|
|
||||||
|
# The up-front check sees the public answer...
|
||||||
|
validate_fetch_url("http://rebind.example.com/feed.xml")
|
||||||
|
# ...but the connection's own lookup gets the internal one and must refuse.
|
||||||
|
with pytest.raises(SSRFBlocked, match="non-public"):
|
||||||
|
safe_get("http://rebind.example.com/feed.xml", headers={}, timeout=2)
|
||||||
|
|
||||||
|
def test_connects_to_the_validated_address_and_keeps_the_original_host_header(
|
||||||
|
self, monkeypatch
|
||||||
|
):
|
||||||
|
import socket
|
||||||
|
import threading
|
||||||
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||||
|
|
||||||
|
from apps.feeds.services import ssrf_guard
|
||||||
|
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self):
|
||||||
|
seen["host"] = self.headers["Host"]
|
||||||
|
body = b"<rss/>"
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def log_message(self, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
server = HTTPServer(("127.0.0.1", 0), Handler)
|
||||||
|
port = server.server_address[1]
|
||||||
|
threading.Thread(target=server.handle_request, daemon=True).start()
|
||||||
|
|
||||||
|
real_getaddrinfo = socket.getaddrinfo
|
||||||
|
monkeypatch.setattr(
|
||||||
|
socket,
|
||||||
|
"getaddrinfo",
|
||||||
|
lambda host, *a, **k: (
|
||||||
|
[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", port))]
|
||||||
|
if host == "feed.test"
|
||||||
|
else real_getaddrinfo(host, *a, **k)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
# Allow loopback for this test only, standing in for a public address.
|
||||||
|
monkeypatch.setattr(ssrf_guard, "_is_allowed_ip", lambda ip: True)
|
||||||
|
|
||||||
|
response = safe_get(f"http://feed.test:{port}/feed.xml", headers={}, timeout=5)
|
||||||
|
server.server_close()
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.content == b"<rss/>"
|
||||||
|
assert seen["host"] == f"feed.test:{port}"
|
||||||
Reference in new issue
Block a user