Add Apple Screen Sharing authentication for VNC (security type 30)

Apple's macOS Screen Sharing server doesn't speak standard VNC
Authentication -- it uses a Diffie-Hellman key exchange followed by
AES-128-ECB-encrypted credentials, security type 30. Apple never
published this scheme (it's not part of RFC 6143); this implements
the well-established reverse-engineered wire format: the server sends
a generator, prime, and its own DH public key; the client generates
an ephemeral keypair, derives the shared secret, MD5-hashes it into
an AES key, and sends back its public key plus a 128-byte encrypted
username+password buffer.

The DH/AES math lives in new src/vnc_apple_dh_auth.h/.cpp as a pure,
socket-free helper (mirroring vncAuthResponse()'s shape for standard
VNC Auth), built entirely on modern EVP_PKEY-based OpenSSL 3.0 APIs --
no deprecated low-level DH_* calls, unlike VNC Authentication's
necessary use of classic DES. Reuses Profile::username (already a
shared field) since Apple's scheme needs an actual macOS account name,
unlike password-only VNC Authentication.

Security-type preference when multiple are offered is now None >
AppleDH > VNCAuth, since DH+AES is strictly stronger than static-
challenge DES. Adds a DH round-trip test (generates a real 512-bit
group at test time, computes the response, then independently
re-derives the shared secret as the server would and decrypts the
credentials back out -- proving self-consistency without needing a
hand-computed expected value), a fake-server integration test for the
full RFB 3.8 handshake sequencing, and a preference-order test.

Not yet live-verified against a real macOS Screen Sharing server --
that's next.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

CMakeLists.txt

@@ -132,6 +132,8 @@ set(ORBITHUB_SOURCES
     src/vnc_session_backend.h
     src/vnc_pixel_codecs.cpp
     src/vnc_pixel_codecs.h
+    src/vnc_apple_dh_auth.cpp
+    src/vnc_apple_dh_auth.h
     src/unsupported_session_backend.cpp
     src/unsupported_session_backend.h
 )

src/vnc_apple_dh_auth.cpp

@@ -0,0 +1,219 @@
+#include "vnc_apple_dh_auth.h"
+
+#include <openssl/bn.h>
+#include <openssl/core_names.h>
+#include <openssl/evp.h>
+#include <openssl/param_build.h>
+
+#include <cstring>
+#include <memory>
+
+namespace VncAppleDhAuth {
+
+namespace {
+
+struct EvpPkeyDeleter {
+    void operator()(EVP_PKEY* key) const { EVP_PKEY_free(key); }
+};
+struct EvpPkeyCtxDeleter {
+    void operator()(EVP_PKEY_CTX* ctx) const { EVP_PKEY_CTX_free(ctx); }
+};
+struct BnDeleter {
+    void operator()(BIGNUM* bn) const { BN_free(bn); }
+};
+struct ParamBldDeleter {
+    void operator()(OSSL_PARAM_BLD* bld) const { OSSL_PARAM_BLD_free(bld); }
+};
+struct ParamDeleter {
+    void operator()(OSSL_PARAM* params) const { OSSL_PARAM_free(params); }
+};
+struct CipherCtxDeleter {
+    void operator()(EVP_CIPHER_CTX* ctx) const { EVP_CIPHER_CTX_free(ctx); }
+};
+
+using EvpPkeyPtr = std::unique_ptr<EVP_PKEY, EvpPkeyDeleter>;
+using EvpPkeyCtxPtr = std::unique_ptr<EVP_PKEY_CTX, EvpPkeyCtxDeleter>;
+using BnPtr = std::unique_ptr<BIGNUM, BnDeleter>;
+using ParamBldPtr = std::unique_ptr<OSSL_PARAM_BLD, ParamBldDeleter>;
+using ParamPtr = std::unique_ptr<OSSL_PARAM, ParamDeleter>;
+using CipherCtxPtr = std::unique_ptr<EVP_CIPHER_CTX, CipherCtxDeleter>;
+
+BnPtr bnFromBytes(const QByteArray& bytes)
+{
+    return BnPtr(
+        BN_bin2bn(reinterpret_cast<const unsigned char*>(bytes.constData()), bytes.size(), nullptr));
+}
+
+// Builds an EVP_PKEY holding just the DH domain parameters (generator,
+// prime) -- used both as the basis for our own keygen and, with a public
+// value added, to represent the server's public key for derive().
+EvpPkeyPtr buildDomainParams(const BIGNUM* g, const BIGNUM* p)
+{
+    ParamBldPtr bld(OSSL_PARAM_BLD_new());
+    if (!bld) {
+        return nullptr;
+    }
+    if (OSSL_PARAM_BLD_push_BN(bld.get(), OSSL_PKEY_PARAM_FFC_P, p) <= 0
+        || OSSL_PARAM_BLD_push_BN(bld.get(), OSSL_PKEY_PARAM_FFC_G, g) <= 0) {
+        return nullptr;
+    }
+    ParamPtr params(OSSL_PARAM_BLD_to_param(bld.get()));
+    if (!params) {
+        return nullptr;
+    }
+
+    EvpPkeyCtxPtr ctx(EVP_PKEY_CTX_new_from_name(nullptr, "DH", nullptr));
+    if (!ctx || EVP_PKEY_fromdata_init(ctx.get()) <= 0) {
+        return nullptr;
+    }
+    EVP_PKEY* rawKey = nullptr;
+    if (EVP_PKEY_fromdata(ctx.get(), &rawKey, EVP_PKEY_KEY_PARAMETERS, params.get()) <= 0) {
+        return nullptr;
+    }
+    return EvpPkeyPtr(rawKey);
+}
+
+EvpPkeyPtr buildPeerPublicKey(const BIGNUM* g, const BIGNUM* p, const BIGNUM* pub)
+{
+    ParamBldPtr bld(OSSL_PARAM_BLD_new());
+    if (!bld) {
+        return nullptr;
+    }
+    if (OSSL_PARAM_BLD_push_BN(bld.get(), OSSL_PKEY_PARAM_FFC_P, p) <= 0
+        || OSSL_PARAM_BLD_push_BN(bld.get(), OSSL_PKEY_PARAM_FFC_G, g) <= 0
+        || OSSL_PARAM_BLD_push_BN(bld.get(), OSSL_PKEY_PARAM_PUB_KEY, pub) <= 0) {
+        return nullptr;
+    }
+    ParamPtr params(OSSL_PARAM_BLD_to_param(bld.get()));
+    if (!params) {
+        return nullptr;
+    }
+
+    EvpPkeyCtxPtr ctx(EVP_PKEY_CTX_new_from_name(nullptr, "DH", nullptr));
+    if (!ctx || EVP_PKEY_fromdata_init(ctx.get()) <= 0) {
+        return nullptr;
+    }
+    EVP_PKEY* rawKey = nullptr;
+    if (EVP_PKEY_fromdata(ctx.get(), &rawKey, EVP_PKEY_PUBLIC_KEY, params.get()) <= 0) {
+        return nullptr;
+    }
+    return EvpPkeyPtr(rawKey);
+}
+
+}
+
+Response computeResponse(const QByteArray& generator, const QByteArray& prime,
+                         const QByteArray& serverPublicKey, const QString& username,
+                         const QString& password)
+{
+    Response response;
+    if (generator.isEmpty() || prime.isEmpty() || serverPublicKey.isEmpty()) {
+        return response;
+    }
+
+    BnPtr g = bnFromBytes(generator);
+    BnPtr p = bnFromBytes(prime);
+    BnPtr serverPub = bnFromBytes(serverPublicKey);
+    if (!g || !p || !serverPub) {
+        return response;
+    }
+
+    EvpPkeyPtr domainParams = buildDomainParams(g.get(), p.get());
+    if (!domainParams) {
+        return response;
+    }
+
+    // Generate our own ephemeral DH keypair against the same domain
+    // parameters the server offered.
+    EvpPkeyCtxPtr keygenCtx(EVP_PKEY_CTX_new_from_pkey(nullptr, domainParams.get(), nullptr));
+    if (!keygenCtx || EVP_PKEY_keygen_init(keygenCtx.get()) <= 0) {
+        return response;
+    }
+    EVP_PKEY* rawOurKey = nullptr;
+    if (EVP_PKEY_keygen(keygenCtx.get(), &rawOurKey) <= 0) {
+        return response;
+    }
+    EvpPkeyPtr ourKey(rawOurKey);
+
+    BIGNUM* ourPubRaw = nullptr;
+    if (EVP_PKEY_get_bn_param(ourKey.get(), OSSL_PKEY_PARAM_PUB_KEY, &ourPubRaw) <= 0
+        || ourPubRaw == nullptr) {
+        return response;
+    }
+    BnPtr ourPub(ourPubRaw);
+
+    QByteArray clientPublicKey(prime.size(), char(0));
+    if (BN_bn2binpad(ourPub.get(), reinterpret_cast<unsigned char*>(clientPublicKey.data()),
+                     prime.size())
+        < 0) {
+        return response;
+    }
+
+    EvpPkeyPtr peerKey = buildPeerPublicKey(g.get(), p.get(), serverPub.get());
+    if (!peerKey) {
+        return response;
+    }
+
+    EvpPkeyCtxPtr deriveCtx(EVP_PKEY_CTX_new_from_pkey(nullptr, ourKey.get(), nullptr));
+    if (!deriveCtx || EVP_PKEY_derive_init(deriveCtx.get()) <= 0
+        || EVP_PKEY_derive_set_peer(deriveCtx.get(), peerKey.get()) <= 0) {
+        return response;
+    }
+
+    size_t secretLen = 0;
+    if (EVP_PKEY_derive(deriveCtx.get(), nullptr, &secretLen) <= 0 || secretLen == 0) {
+        return response;
+    }
+    QByteArray secret(static_cast<int>(secretLen), char(0));
+    if (EVP_PKEY_derive(deriveCtx.get(), reinterpret_cast<unsigned char*>(secret.data()),
+                        &secretLen)
+        <= 0) {
+        return response;
+    }
+    secret.resize(static_cast<int>(secretLen));
+
+    unsigned char aesKey[16];
+    if (EVP_Digest(secret.constData(), static_cast<size_t>(secret.size()), aesKey, nullptr,
+                   EVP_md5(), nullptr)
+        <= 0) {
+        return response;
+    }
+
+    // 64 bytes username + 64 bytes password, NUL-padded/truncated.
+    QByteArray credentials(128, char(0));
+    const QByteArray userBytes = username.toLatin1().left(64);
+    const QByteArray passBytes = password.toLatin1().left(64);
+    std::memcpy(credentials.data(), userBytes.constData(),
+               static_cast<size_t>(userBytes.size()));
+    std::memcpy(credentials.data() + 64, passBytes.constData(),
+               static_cast<size_t>(passBytes.size()));
+
+    CipherCtxPtr cipherCtx(EVP_CIPHER_CTX_new());
+    if (!cipherCtx
+        || EVP_EncryptInit_ex(cipherCtx.get(), EVP_aes_128_ecb(), nullptr, aesKey, nullptr) <= 0) {
+        return response;
+    }
+    EVP_CIPHER_CTX_set_padding(cipherCtx.get(), 0);
+
+    QByteArray ciphertext(credentials.size() + EVP_MAX_BLOCK_LENGTH, char(0));
+    int outLen1 = 0;
+    if (EVP_EncryptUpdate(cipherCtx.get(), reinterpret_cast<unsigned char*>(ciphertext.data()),
+                          &outLen1, reinterpret_cast<const unsigned char*>(credentials.constData()),
+                          credentials.size())
+        <= 0) {
+        return response;
+    }
+    int outLen2 = 0;
+    if (EVP_EncryptFinal_ex(cipherCtx.get(),
+                            reinterpret_cast<unsigned char*>(ciphertext.data()) + outLen1, &outLen2)
+        <= 0) {
+        return response;
+    }
+    ciphertext.resize(outLen1 + outLen2);
+
+    response.clientPublicKey = clientPublicKey;
+    response.encryptedCredentials = ciphertext;
+    return response;
+}
+
+}

src/vnc_apple_dh_auth.h

@@ -0,0 +1,44 @@
+#ifndef ORBITHUB_VNC_APPLE_DH_AUTH_H
+#define ORBITHUB_VNC_APPLE_DH_AUTH_H
+
+#include <QByteArray>
+#include <QString>
+
+// Apple's Screen Sharing authentication scheme (RFB security type 30):
+// Diffie-Hellman key exchange followed by AES-128-ECB-encrypted
+// credentials. Apple never published this officially -- it's not part of
+// RFC 6143 -- so this implements the well-established reverse-engineered
+// wire format used by several independent VNC clients, not a primary
+// spec. Kept as a pure, state-free helper (no socket access) so it's
+// unit-testable without a live connection, mirroring
+// VncSessionBackend::vncAuthResponse()'s shape for standard VNC
+// Authentication.
+namespace VncAppleDhAuth {
+
+struct Response {
+    // Same byte length as the server's prime, big-endian, zero-padded.
+    // Empty on any failure (malformed input, an OpenSSL operation
+    // failing) -- callers should treat an empty clientPublicKey as "could
+    // not compute a response" rather than send a degenerate one.
+    QByteArray clientPublicKey;
+    // Always exactly 128 bytes on success (16 AES blocks): a 64-byte
+    // NUL-padded/truncated username followed by a 64-byte NUL-padded/
+    // truncated password, AES-128-ECB encrypted (no padding, since the
+    // plaintext is already an exact multiple of the block size) with a
+    // key derived as MD5(sharedSecret).
+    QByteArray encryptedCredentials;
+};
+
+// Computes the DH keypair, the shared secret, the derived AES key, and
+// the encrypted credential blob, using modern EVP-based OpenSSL 3.0 APIs
+// throughout (no deprecated low-level DH_*/legacy calls -- unlike VNC
+// Authentication's classic DES usage, none of MD5/AES-ECB/the EVP_PKEY DH
+// APIs are deprecated, so no compatibility pragma is needed here).
+// Returns a Response with an empty clientPublicKey on any failure.
+Response computeResponse(const QByteArray& generator, const QByteArray& prime,
+                         const QByteArray& serverPublicKey, const QString& username,
+                         const QString& password);
+
+}
+
+#endif

src/vnc_session_backend.cpp

@@ -1,5 +1,6 @@
 #include "vnc_session_backend.h"
 
+#include "vnc_apple_dh_auth.h"
 #include "vnc_pixel_codecs.h"
 
 #include <QPainter>
@@ -49,6 +50,12 @@ constexpr qint32 kEncTight = 7;
 // the framebuffer.
 constexpr qint32 kEncCursor = -239;
 
+// Apple's Screen Sharing authentication (Diffie-Hellman + AES). Not part
+// of RFC 6143 -- see vnc_apple_dh_auth.h.
+constexpr quint8 kSecurityTypeNone = 1;
+constexpr quint8 kSecurityTypeVncAuth = 2;
+constexpr quint8 kSecurityTypeAppleDh = 30;
+
 constexpr std::array<qint32, 6> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile,
                                                         kEncZRLE, kEncTight, kEncCursor };
 
@@ -184,7 +191,8 @@ VncSessionBackend::VncSessionBackend(const Profile& profile, QObject* parent)
       m_tightInflateInitialized{ false, false, false, false },
       m_tightCompressionMode(0),
       m_tightFilterId(0),
-      m_tightLengthByteIndex(0)
+      m_tightLengthByteIndex(0),
+      m_appleAuthPrimeLength(0)
 {
     std::memset(m_zrleInflateStream, 0, sizeof(z_stream_s));
     for (z_stream_s* stream : m_tightInflateStreams) {
@@ -517,6 +525,8 @@ void VncSessionBackend::resetProtocolState()
             m_tightInflateInitialized[i] = false;
         }
     }
+    m_appleAuthGenerator.clear();
+    m_appleAuthPrimeLength = 0;
 }
 
 bool VncSessionBackend::haveBytes(int count) const
@@ -781,11 +791,18 @@ void VncSessionBackend::processReceiveBuffer()
             m_offeredSecurityTypes = m_recvBuffer.left(m_securityTypeCount);
             m_recvBuffer.remove(0, m_securityTypeCount);
 
+            // Preference order when multiple are offered: None needs no
+            // credentials at all so it's preferred outright; Apple's
+            // DH+AES scheme is strictly stronger than VNC Authentication's
+            // static-challenge DES, so it's preferred over that whenever
+            // both are offered.
             quint8 chosen = 0;
-            if (m_offeredSecurityTypes.contains(char(1))) {
-                chosen = 1;
-            } else if (m_offeredSecurityTypes.contains(char(2))) {
-                chosen = 2;
+            if (m_offeredSecurityTypes.contains(static_cast<char>(kSecurityTypeNone))) {
+                chosen = kSecurityTypeNone;
+            } else if (m_offeredSecurityTypes.contains(static_cast<char>(kSecurityTypeAppleDh))) {
+                chosen = kSecurityTypeAppleDh;
+            } else if (m_offeredSecurityTypes.contains(static_cast<char>(kSecurityTypeVncAuth))) {
+                chosen = kSecurityTypeVncAuth;
             }
             if (chosen == 0) {
                 QStringList offered;
@@ -803,8 +820,10 @@ void VncSessionBackend::processReceiveBuffer()
             m_chosenSecurityType = chosen;
             m_socket->write(QByteArray(1, static_cast<char>(chosen)));
 
-            if (chosen == 2) {
+            if (chosen == kSecurityTypeVncAuth) {
                 m_rfbState = RfbState::WaitingVncAuthChallenge;
+            } else if (chosen == kSecurityTypeAppleDh) {
+                m_rfbState = RfbState::WaitingAppleAuthGeneratorLength;
             } else if (m_negotiatedMinorVersion >= 8) {
                 m_rfbState = RfbState::WaitingSecurityResult;
             } else {
@@ -823,13 +842,16 @@ void VncSessionBackend::processReceiveBuffer()
 
             if (type == 0) {
                 m_rfbState = RfbState::WaitingSecurityFailureReasonLength;
-            } else if (type == 1) {
-                m_chosenSecurityType = 1;
+            } else if (type == kSecurityTypeNone) {
+                m_chosenSecurityType = kSecurityTypeNone;
                 sendClientInit();
                 m_rfbState = RfbState::WaitingServerInitHeader;
-            } else if (type == 2) {
-                m_chosenSecurityType = 2;
+            } else if (type == kSecurityTypeVncAuth) {
+                m_chosenSecurityType = kSecurityTypeVncAuth;
                 m_rfbState = RfbState::WaitingVncAuthChallenge;
+            } else if (type == kSecurityTypeAppleDh) {
+                m_chosenSecurityType = kSecurityTypeAppleDh;
+                m_rfbState = RfbState::WaitingAppleAuthGeneratorLength;
             } else {
                 failConnection(
                     QStringLiteral(
@@ -888,6 +910,71 @@ void VncSessionBackend::processReceiveBuffer()
             break;
         }
 
+        case RfbState::WaitingAppleAuthGeneratorLength: {
+            if (!haveBytes(2)) {
+                return;
+            }
+            m_pendingLength = readU16BE(m_recvBuffer, 0);
+            m_recvBuffer.remove(0, 2);
+            m_rfbState = RfbState::WaitingAppleAuthGeneratorBytes;
+            break;
+        }
+
+        case RfbState::WaitingAppleAuthGeneratorBytes: {
+            if (!haveBytes(static_cast<int>(m_pendingLength))) {
+                return;
+            }
+            m_appleAuthGenerator = m_recvBuffer.left(static_cast<int>(m_pendingLength));
+            m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
+            m_rfbState = RfbState::WaitingAppleAuthPrimeLength;
+            break;
+        }
+
+        case RfbState::WaitingAppleAuthPrimeLength: {
+            if (!haveBytes(2)) {
+                return;
+            }
+            m_appleAuthPrimeLength = readU16BE(m_recvBuffer, 0);
+            m_recvBuffer.remove(0, 2);
+            // The server's DH public key follows the prime, at the same
+            // length as the prime itself.
+            m_pendingLength = m_appleAuthPrimeLength * 2;
+            m_rfbState = RfbState::WaitingAppleAuthPrimeAndServerKey;
+            break;
+        }
+
+        case RfbState::WaitingAppleAuthPrimeAndServerKey: {
+            if (!haveBytes(static_cast<int>(m_pendingLength))) {
+                return;
+            }
+            const int primeLength = static_cast<int>(m_appleAuthPrimeLength);
+            const QByteArray prime = m_recvBuffer.left(primeLength);
+            const QByteArray serverPublicKey = m_recvBuffer.mid(primeLength, primeLength);
+            m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
+
+            const VncAppleDhAuth::Response response = VncAppleDhAuth::computeResponse(
+                m_appleAuthGenerator, prime, serverPublicKey, profile().username,
+                m_activeOptions.password);
+            if (response.clientPublicKey.isEmpty()) {
+                failConnection(
+                    QStringLiteral(
+                        "Failed to compute the Apple Screen Sharing authentication response."),
+                    QStringLiteral("VncAppleDhAuth::computeResponse() failed"));
+                return;
+            }
+
+            m_socket->write(response.clientPublicKey);
+            m_socket->write(response.encryptedCredentials);
+
+            if (m_negotiatedMinorVersion >= 8) {
+                m_rfbState = RfbState::WaitingSecurityResult;
+            } else {
+                sendClientInit();
+                m_rfbState = RfbState::WaitingServerInitHeader;
+            }
+            break;
+        }
+
         case RfbState::WaitingSecurityResult: {
             if (!haveBytes(4)) {
                 return;

src/vnc_session_backend.h

@@ -24,11 +24,13 @@ struct z_stream_s;
 // C API to wrangle here.
 //
 // Scope (see plan / issue #3 for the full rationale): standard VNC
-// Authentication (security type 2) and no-auth (type 1) only -- not
-// Apple's Screen Sharing scheme (type 30). Raw + CopyRect + Hextile + ZRLE
-// + Tight encodings. No dynamic resize. Clipboard sync (Latin-1 only, per
-// RFB's ServerCutText/ClientCutText) and remote cursor shape sync (the
-// Cursor pseudo-encoding) are supported.
+// Authentication (security type 2), no-auth (type 1), and Apple's Screen
+// Sharing scheme (type 30, Diffie-Hellman + AES -- see
+// vnc_apple_dh_auth.h; not part of RFC 6143, implemented against the
+// well-established reverse-engineered wire format). Raw + CopyRect +
+// Hextile + ZRLE + Tight encodings. No dynamic resize. Clipboard sync
+// (Latin-1 only, per RFB's ServerCutText/ClientCutText) and remote cursor
+// shape sync (the Cursor pseudo-encoding) are supported.
 //
 // Tight decoding gap: the real protocol allows the server to skip zlib
 // compression entirely for very small Basic-mode payloads; this decoder
@@ -109,6 +111,10 @@ private:
         WaitingTightFilterId,
         WaitingTightLengthByte,
         WaitingTightPayload,
+        WaitingAppleAuthGeneratorLength,
+        WaitingAppleAuthGeneratorBytes,
+        WaitingAppleAuthPrimeLength,
+        WaitingAppleAuthPrimeAndServerKey,
         WaitingSetColourMapHeader,
         WaitingSetColourMapData,
         WaitingServerCutTextHeader,
@@ -178,6 +184,16 @@ private:
     quint8 m_tightFilterId;
     int m_tightLengthByteIndex;
 
+    // Apple Screen Sharing authentication state (security type 30, not
+    // part of RFC 6143 -- see vnc_apple_dh_auth.h). m_appleAuthGenerator
+    // must persist across the generator-length/generator-bytes state
+    // hop; m_appleAuthPrimeLength must persist from when it's first read
+    // until the combined prime+server-public-key buffer (2x that length)
+    // has fully arrived, since m_pendingLength gets reused to track that
+    // combined byte count in the meantime.
+    QByteArray m_appleAuthGenerator;
+    quint32 m_appleAuthPrimeLength;
+
     void setState(SessionState state, const QString& message);
     void resetProtocolState();
     void processReceiveBuffer();

tests/CMakeLists.txt

@@ -30,6 +30,7 @@ add_executable(test_vnc_session_backend
     test_vnc_session_backend.cpp
     ${CMAKE_SOURCE_DIR}/src/vnc_session_backend.cpp
     ${CMAKE_SOURCE_DIR}/src/vnc_pixel_codecs.cpp
+    ${CMAKE_SOURCE_DIR}/src/vnc_apple_dh_auth.cpp
     ${CMAKE_SOURCE_DIR}/src/session_backend.h
 )
 target_include_directories(test_vnc_session_backend PRIVATE ${CMAKE_SOURCE_DIR}/src)

tests/test_vnc_session_backend.cpp

@@ -1,9 +1,12 @@
 #include "vnc_session_backend.h"
+#include "vnc_apple_dh_auth.h"
 
 #include <QTcpServer>
 #include <QTcpSocket>
 #include <QTest>
 
+#include <openssl/bn.h>
+#include <openssl/evp.h>
 #include <zlib.h>
 
 extern "C" {
@@ -11,6 +14,7 @@ extern "C" {
 }
 
 #include <cstdlib>
+#include <cstring>
 
 namespace {
 // Independently documented bit-reversal example for VNC Authentication's
@@ -297,6 +301,65 @@ QByteArray encodeJpegForTest(int width, int height, QRgb color)
     std::free(buffer);
     return result;
 }
+
+// A small (512-bit) DH group generated fresh at test-run time -- rather
+// than a hardcoded literal prime, to avoid any transcription risk -- plus
+// one keypair against it, used to exercise VncAppleDhAuth::computeResponse()
+// both directly and via a fake server acting as an Apple Screen Sharing
+// (security type 30) endpoint. 512 bits keeps generation fast while still
+// avoiding any "key too small" policy rejection from OpenSSL's default
+// provider that a truly tiny hand-picked prime might trigger.
+struct ToyDhKeypair {
+    QByteArray generatorBytes;
+    QByteArray primeBytes;
+    QByteArray publicKeyBytes;
+    BIGNUM* privateExponent = nullptr; // caller must BN_free
+    BIGNUM* prime = nullptr; // caller must BN_free
+};
+
+ToyDhKeypair generateToyDhKeypair()
+{
+    ToyDhKeypair result;
+    BIGNUM* p = BN_new();
+    BN_generate_prime_ex(p, 512, 0, nullptr, nullptr, nullptr);
+    BIGNUM* g = BN_new();
+    BN_set_word(g, 2);
+
+    BN_CTX* ctx = BN_CTX_new();
+    BIGNUM* priv = BN_new();
+    BN_rand(priv, 256, -1, 0);
+    BIGNUM* pub = BN_new();
+    BN_mod_exp(pub, g, priv, p, ctx);
+
+    const int primeLen = BN_num_bytes(p);
+    result.primeBytes = QByteArray(primeLen, char(0));
+    BN_bn2binpad(p, reinterpret_cast<unsigned char*>(result.primeBytes.data()), primeLen);
+    result.generatorBytes = QByteArray(1, char(2));
+    result.publicKeyBytes = QByteArray(primeLen, char(0));
+    BN_bn2binpad(pub, reinterpret_cast<unsigned char*>(result.publicKeyBytes.data()), primeLen);
+    result.privateExponent = priv;
+    result.prime = p;
+
+    BN_free(g);
+    BN_free(pub);
+    BN_CTX_free(ctx);
+    return result;
+}
+
+// A FramebufferUpdate-adjacent helper isn't needed here -- these bytes are
+// sent immediately after the client selects security type 30, before
+// ClientInit/ServerInit even happen.
+QByteArray appleDhAuthServerMessage(const QByteArray& generator, const QByteArray& prime,
+                                    const QByteArray& serverPublicKey)
+{
+    QByteArray msg;
+    appendU16(msg, static_cast<quint16>(generator.size()));
+    msg += generator;
+    appendU16(msg, static_cast<quint16>(prime.size()));
+    msg += prime;
+    msg += serverPublicKey;
+    return msg;
+}
 }
 
 class TestVncSessionBackend : public QObject
@@ -348,6 +411,9 @@ private slots:
     void tightBasicPaletteFilterProducesExpectedPixels();
     void tightJpegProducesExpectedPixels();
     void tightStreamResetFlagAllowsIndependentDecoding();
+    void appleDhAuthRoundTripDecryptsToOriginalCredentials();
+    void connectsWithAppleDhAuthenticationRfb38();
+    void appleDhAuthIsPreferredOverVncAuthWhenBothOffered();
 
 private:
     std::unique_ptr<FakeVncServer> m_server;
@@ -626,7 +692,7 @@ void TestVncSessionBackend::unsupportedSecurityTypeReachesFailedState()
         if (m_server->nextStep() == 0) {
             QByteArray securityTypes;
             securityTypes.append(char(1));
-            securityTypes.append(char(30)); // Apple's scheme -- unsupported here
+            securityTypes.append(char(19)); // VeNCrypt (TLS) -- unsupported here
             m_server->sendWhenConnected(securityTypes);
         }
     });
@@ -1862,5 +1928,133 @@ void TestVncSessionBackend::tightStreamResetFlagAllowsIndependentDecoding()
     QCOMPARE(frames.at(1).pixelColor(0, 0), QColor(200, 210, 220));
 }
 
+void TestVncSessionBackend::appleDhAuthRoundTripDecryptsToOriginalCredentials()
+{
+    ToyDhKeypair serverKeypair = generateToyDhKeypair();
+
+    const QString username = QStringLiteral("tester");
+    const QString password = QStringLiteral("s3cret-pass");
+
+    const VncAppleDhAuth::Response response = VncAppleDhAuth::computeResponse(
+        serverKeypair.generatorBytes, serverKeypair.primeBytes, serverKeypair.publicKeyBytes,
+        username, password);
+
+    QVERIFY(!response.clientPublicKey.isEmpty());
+    QCOMPARE(response.clientPublicKey.size(), serverKeypair.primeBytes.size());
+    QCOMPARE(response.encryptedCredentials.size(), 128);
+
+    // Derive the shared secret the way the *server* would, using the
+    // client's returned public key and the server's own private exponent
+    // -- Diffie-Hellman's commutativity means this must equal whatever
+    // computeResponse() used internally to derive its AES key.
+    BIGNUM* clientPub = BN_bin2bn(
+        reinterpret_cast<const unsigned char*>(response.clientPublicKey.constData()),
+        response.clientPublicKey.size(), nullptr);
+    BN_CTX* ctx = BN_CTX_new();
+    BIGNUM* sharedSecret = BN_new();
+    BN_mod_exp(sharedSecret, clientPub, serverKeypair.privateExponent, serverKeypair.prime, ctx);
+
+    const int secretLen = BN_num_bytes(sharedSecret);
+    QByteArray secretBytes(secretLen, char(0));
+    BN_bn2binpad(sharedSecret, reinterpret_cast<unsigned char*>(secretBytes.data()), secretLen);
+
+    unsigned char aesKey[16];
+    EVP_Digest(secretBytes.constData(), static_cast<size_t>(secretBytes.size()), aesKey, nullptr,
+              EVP_md5(), nullptr);
+
+    EVP_CIPHER_CTX* decCtx = EVP_CIPHER_CTX_new();
+    EVP_DecryptInit_ex(decCtx, EVP_aes_128_ecb(), nullptr, aesKey, nullptr);
+    EVP_CIPHER_CTX_set_padding(decCtx, 0);
+    QByteArray plain(response.encryptedCredentials.size() + 16, char(0));
+    int outLen1 = 0;
+    EVP_DecryptUpdate(
+        decCtx, reinterpret_cast<unsigned char*>(plain.data()), &outLen1,
+        reinterpret_cast<const unsigned char*>(response.encryptedCredentials.constData()),
+        response.encryptedCredentials.size());
+    int outLen2 = 0;
+    EVP_DecryptFinal_ex(decCtx, reinterpret_cast<unsigned char*>(plain.data()) + outLen1, &outLen2);
+    plain.resize(outLen1 + outLen2);
+    EVP_CIPHER_CTX_free(decCtx);
+
+    QByteArray expected(128, char(0));
+    const QByteArray userBytes = username.toLatin1().left(64);
+    const QByteArray passBytes = password.toLatin1().left(64);
+    std::memcpy(expected.data(), userBytes.constData(), static_cast<size_t>(userBytes.size()));
+    std::memcpy(expected.data() + 64, passBytes.constData(), static_cast<size_t>(passBytes.size()));
+    QCOMPARE(plain, expected);
+
+    BN_free(clientPub);
+    BN_free(sharedSecret);
+    BN_CTX_free(ctx);
+    BN_free(serverKeypair.privateExponent);
+    BN_free(serverKeypair.prime);
+}
+
+void TestVncSessionBackend::connectsWithAppleDhAuthenticationRfb38()
+{
+    ToyDhKeypair serverKeypair = generateToyDhKeypair();
+    const QByteArray authMessage = appleDhAuthServerMessage(
+        serverKeypair.generatorBytes, serverKeypair.primeBytes, serverKeypair.publicKeyBytes);
+
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, authMessage]() {
+        switch (m_server->nextStep()) {
+        case 0: { // version reply
+            QByteArray securityTypes;
+            securityTypes.append(char(1));
+            securityTypes.append(char(30)); // Apple's scheme
+            m_server->sendWhenConnected(securityTypes);
+            break;
+        }
+        case 1: // security-type selection (byte value 30)
+            m_server->sendWhenConnected(authMessage);
+            break;
+        case 2: // client's DH public key + encrypted credentials -- accept unconditionally
+            m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
+            break;
+        case 3: { // ClientInit
+            QByteArray serverInit;
+            serverInit.append(char(0)); serverInit.append(char(1));
+            serverInit.append(char(0)); serverInit.append(char(1));
+            serverInit.append(QByteArray(16, char(0)));
+            serverInit.append(QByteArray(4, char(0)));
+            m_server->sendWhenConnected(serverInit);
+            break;
+        }
+        default:
+            break;
+        }
+    });
+
+    m_backend->connectSession(makeOptions(QStringLiteral("irrelevant-password")));
+    QTRY_COMPARE(m_lastState, SessionState::Connected);
+
+    BN_free(serverKeypair.privateExponent);
+    BN_free(serverKeypair.prime);
+}
+
+void TestVncSessionBackend::appleDhAuthIsPreferredOverVncAuthWhenBothOffered()
+{
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
+        if (m_server->nextStep() == 0) {
+            m_server->received.clear();
+            QByteArray securityTypes;
+            securityTypes.append(char(2)); // count = 2
+            securityTypes.append(char(2)); // VNC Authentication
+            securityTypes.append(char(30)); // Apple DH
+            m_server->sendWhenConnected(securityTypes);
+        }
+    });
+
+    m_backend->connectSession(makeOptions(QStringLiteral("whatever")));
+    QTRY_VERIFY(m_server->received.size() >= 1);
+    QCOMPARE(static_cast<quint8>(m_server->received.at(0)), quint8(30));
+}
+
 QTEST_GUILESS_MAIN(TestVncSessionBackend)
 #include "test_vnc_session_backend.moc"