DarkSingularity/orbithub
Internal
Public Access
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 7 Wiki Activity

Add Apple Screen Sharing authentication for VNC (security type 30)

Browse Source 
Apple's macOS Screen Sharing server doesn't speak standard VNC
Authentication -- it uses a Diffie-Hellman key exchange followed by
AES-128-ECB-encrypted credentials, security type 30. Apple never
published this scheme (it's not part of RFC 6143); this implements
the well-established reverse-engineered wire format: the server sends
a generator, prime, and its own DH public key; the client generates
an ephemeral keypair, derives the shared secret, MD5-hashes it into
an AES key, and sends back its public key plus a 128-byte encrypted
username+password buffer.

The DH/AES math lives in new src/vnc_apple_dh_auth.h/.cpp as a pure,
socket-free helper (mirroring vncAuthResponse()'s shape for standard
VNC Auth), built entirely on modern EVP_PKEY-based OpenSSL 3.0 APIs --
no deprecated low-level DH_* calls, unlike VNC Authentication's
necessary use of classic DES. Reuses Profile::username (already a
shared field) since Apple's scheme needs an actual macOS account name,
unlike password-only VNC Authentication.

Security-type preference when multiple are offered is now None >
AppleDH > VNCAuth, since DH+AES is strictly stronger than static-
challenge DES. Adds a DH round-trip test (generates a real 512-bit
group at test time, computes the response, then independently
re-derives the shared secret as the server would and decrypts the
credentials back out -- proving self-consistency without needing a
hand-computed expected value), a fake-server integration test for the
full RFB 3.8 handshake sequencing, and a preference-order test.

Not yet live-verified against a real macOS Screen Sharing server --
that's next.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmith
2026-09-15 21:27:17 -06:00
co-authored by Claude Sonnet 5
parent 2fe2022182
commit 1f026dde70
7 changed files with 579 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vnc_session_backend.cpp
+97 -10
View File
@@ -1,5 +1,6 @@
#include "vnc_session_backend.h"
#include "vnc_apple_dh_auth.h"
#include "vnc_pixel_codecs.h"
#include <QPainter>
@@ -49,6 +50,12 @@ constexpr qint32 kEncTight = 7;
// the framebuffer.
constexpr qint32 kEncCursor = -239;
// Apple's Screen Sharing authentication (Diffie-Hellman + AES). Not part
// of RFC 6143 -- see vnc_apple_dh_auth.h.
constexpr quint8 kSecurityTypeNone = 1;
constexpr quint8 kSecurityTypeVncAuth = 2;
constexpr quint8 kSecurityTypeAppleDh = 30;
constexpr std::array<qint32, 6> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile,
kEncZRLE, kEncTight, kEncCursor };
@@ -184,7 +191,8 @@ VncSessionBackend::VncSessionBackend(const Profile& profile, QObject* parent)
m_tightInflateInitialized{ false, false, false, false },
m_tightCompressionMode(0),
m_tightFilterId(0),
m_tightLengthByteIndex(0)
m_tightLengthByteIndex(0),
m_appleAuthPrimeLength(0)
{
std::memset(m_zrleInflateStream, 0, sizeof(z_stream_s));
for (z_stream_s* stream : m_tightInflateStreams) {
@@ -517,6 +525,8 @@ void VncSessionBackend::resetProtocolState()
m_tightInflateInitialized[i] = false;
}
}
m_appleAuthGenerator.clear();
m_appleAuthPrimeLength = 0;
}
bool VncSessionBackend::haveBytes(int count) const
@@ -781,11 +791,18 @@ void VncSessionBackend::processReceiveBuffer()
m_offeredSecurityTypes = m_recvBuffer.left(m_securityTypeCount);
m_recvBuffer.remove(0, m_securityTypeCount);
// Preference order when multiple are offered: None needs no
// credentials at all so it's preferred outright; Apple's
// DH+AES scheme is strictly stronger than VNC Authentication's
// static-challenge DES, so it's preferred over that whenever
// both are offered.
quint8 chosen = 0;
if (m_offeredSecurityTypes.contains(char(1))) {
chosen = 1;
} else if (m_offeredSecurityTypes.contains(char(2))) {
chosen = 2;
if (m_offeredSecurityTypes.contains(static_cast<char>(kSecurityTypeNone))) {
chosen = kSecurityTypeNone;
} else if (m_offeredSecurityTypes.contains(static_cast<char>(kSecurityTypeAppleDh))) {
chosen = kSecurityTypeAppleDh;
} else if (m_offeredSecurityTypes.contains(static_cast<char>(kSecurityTypeVncAuth))) {
chosen = kSecurityTypeVncAuth;
}
if (chosen == 0) {
QStringList offered;
@@ -803,8 +820,10 @@ void VncSessionBackend::processReceiveBuffer()
m_chosenSecurityType = chosen;
m_socket->write(QByteArray(1, static_cast<char>(chosen)));
if (chosen == 2) {
if (chosen == kSecurityTypeVncAuth) {
m_rfbState = RfbState::WaitingVncAuthChallenge;
} else if (chosen == kSecurityTypeAppleDh) {
m_rfbState = RfbState::WaitingAppleAuthGeneratorLength;
} else if (m_negotiatedMinorVersion >= 8) {
m_rfbState = RfbState::WaitingSecurityResult;
} else {
@@ -823,13 +842,16 @@ void VncSessionBackend::processReceiveBuffer()
if (type == 0) {
m_rfbState = RfbState::WaitingSecurityFailureReasonLength;
} else if (type == 1) {
m_chosenSecurityType = 1;
} else if (type == kSecurityTypeNone) {
m_chosenSecurityType = kSecurityTypeNone;
sendClientInit();
m_rfbState = RfbState::WaitingServerInitHeader;
} else if (type == 2) {
m_chosenSecurityType = 2;
} else if (type == kSecurityTypeVncAuth) {
m_chosenSecurityType = kSecurityTypeVncAuth;
m_rfbState = RfbState::WaitingVncAuthChallenge;
} else if (type == kSecurityTypeAppleDh) {
m_chosenSecurityType = kSecurityTypeAppleDh;
m_rfbState = RfbState::WaitingAppleAuthGeneratorLength;
} else {
failConnection(
QStringLiteral(
@@ -888,6 +910,71 @@ void VncSessionBackend::processReceiveBuffer()
break;
}
case RfbState::WaitingAppleAuthGeneratorLength: {
if (!haveBytes(2)) {
return;
}
m_pendingLength = readU16BE(m_recvBuffer, 0);
m_recvBuffer.remove(0, 2);
m_rfbState = RfbState::WaitingAppleAuthGeneratorBytes;
break;
}
case RfbState::WaitingAppleAuthGeneratorBytes: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
m_appleAuthGenerator = m_recvBuffer.left(static_cast<int>(m_pendingLength));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
m_rfbState = RfbState::WaitingAppleAuthPrimeLength;
break;
}
case RfbState::WaitingAppleAuthPrimeLength: {
if (!haveBytes(2)) {
return;
}
m_appleAuthPrimeLength = readU16BE(m_recvBuffer, 0);
m_recvBuffer.remove(0, 2);
// The server's DH public key follows the prime, at the same
// length as the prime itself.
m_pendingLength = m_appleAuthPrimeLength * 2;
m_rfbState = RfbState::WaitingAppleAuthPrimeAndServerKey;
break;
}
case RfbState::WaitingAppleAuthPrimeAndServerKey: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const int primeLength = static_cast<int>(m_appleAuthPrimeLength);
const QByteArray prime = m_recvBuffer.left(primeLength);
const QByteArray serverPublicKey = m_recvBuffer.mid(primeLength, primeLength);
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
const VncAppleDhAuth::Response response = VncAppleDhAuth::computeResponse(
m_appleAuthGenerator, prime, serverPublicKey, profile().username,
m_activeOptions.password);
if (response.clientPublicKey.isEmpty()) {
failConnection(
QStringLiteral(
"Failed to compute the Apple Screen Sharing authentication response."),
QStringLiteral("VncAppleDhAuth::computeResponse() failed"));
return;
}
m_socket->write(response.clientPublicKey);
m_socket->write(response.encryptedCredentials);
if (m_negotiatedMinorVersion >= 8) {
m_rfbState = RfbState::WaitingSecurityResult;
} else {
sendClientInit();
m_rfbState = RfbState::WaitingServerInitHeader;
}
break;
}
case RfbState::WaitingSecurityResult: {
if (!haveBytes(4)) {
return;