diff --git a/CMakeLists.txt b/CMakeLists.txt index ab37502..46718ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,6 +132,8 @@ set(ORBITHUB_SOURCES src/vnc_session_backend.h src/vnc_pixel_codecs.cpp src/vnc_pixel_codecs.h + src/vnc_apple_dh_auth.cpp + src/vnc_apple_dh_auth.h src/unsupported_session_backend.cpp src/unsupported_session_backend.h ) diff --git a/src/vnc_apple_dh_auth.cpp b/src/vnc_apple_dh_auth.cpp new file mode 100644 index 0000000..bd274cc --- /dev/null +++ b/src/vnc_apple_dh_auth.cpp @@ -0,0 +1,219 @@ +#include "vnc_apple_dh_auth.h" + +#include +#include +#include +#include + +#include +#include + +namespace VncAppleDhAuth { + +namespace { + +struct EvpPkeyDeleter { + void operator()(EVP_PKEY* key) const { EVP_PKEY_free(key); } +}; +struct EvpPkeyCtxDeleter { + void operator()(EVP_PKEY_CTX* ctx) const { EVP_PKEY_CTX_free(ctx); } +}; +struct BnDeleter { + void operator()(BIGNUM* bn) const { BN_free(bn); } +}; +struct ParamBldDeleter { + void operator()(OSSL_PARAM_BLD* bld) const { OSSL_PARAM_BLD_free(bld); } +}; +struct ParamDeleter { + void operator()(OSSL_PARAM* params) const { OSSL_PARAM_free(params); } +}; +struct CipherCtxDeleter { + void operator()(EVP_CIPHER_CTX* ctx) const { EVP_CIPHER_CTX_free(ctx); } +}; + +using EvpPkeyPtr = std::unique_ptr; +using EvpPkeyCtxPtr = std::unique_ptr; +using BnPtr = std::unique_ptr; +using ParamBldPtr = std::unique_ptr; +using ParamPtr = std::unique_ptr; +using CipherCtxPtr = std::unique_ptr; + +BnPtr bnFromBytes(const QByteArray& bytes) +{ + return BnPtr( + BN_bin2bn(reinterpret_cast(bytes.constData()), bytes.size(), nullptr)); +} + +// Builds an EVP_PKEY holding just the DH domain parameters (generator, +// prime) -- used both as the basis for our own keygen and, with a public +// value added, to represent the server's public key for derive(). +EvpPkeyPtr buildDomainParams(const BIGNUM* g, const BIGNUM* p) +{ + ParamBldPtr bld(OSSL_PARAM_BLD_new()); + if (!bld) { + return nullptr; + } + if (OSSL_PARAM_BLD_push_BN(bld.get(), OSSL_PKEY_PARAM_FFC_P, p) <= 0 + || OSSL_PARAM_BLD_push_BN(bld.get(), OSSL_PKEY_PARAM_FFC_G, g) <= 0) { + return nullptr; + } + ParamPtr params(OSSL_PARAM_BLD_to_param(bld.get())); + if (!params) { + return nullptr; + } + + EvpPkeyCtxPtr ctx(EVP_PKEY_CTX_new_from_name(nullptr, "DH", nullptr)); + if (!ctx || EVP_PKEY_fromdata_init(ctx.get()) <= 0) { + return nullptr; + } + EVP_PKEY* rawKey = nullptr; + if (EVP_PKEY_fromdata(ctx.get(), &rawKey, EVP_PKEY_KEY_PARAMETERS, params.get()) <= 0) { + return nullptr; + } + return EvpPkeyPtr(rawKey); +} + +EvpPkeyPtr buildPeerPublicKey(const BIGNUM* g, const BIGNUM* p, const BIGNUM* pub) +{ + ParamBldPtr bld(OSSL_PARAM_BLD_new()); + if (!bld) { + return nullptr; + } + if (OSSL_PARAM_BLD_push_BN(bld.get(), OSSL_PKEY_PARAM_FFC_P, p) <= 0 + || OSSL_PARAM_BLD_push_BN(bld.get(), OSSL_PKEY_PARAM_FFC_G, g) <= 0 + || OSSL_PARAM_BLD_push_BN(bld.get(), OSSL_PKEY_PARAM_PUB_KEY, pub) <= 0) { + return nullptr; + } + ParamPtr params(OSSL_PARAM_BLD_to_param(bld.get())); + if (!params) { + return nullptr; + } + + EvpPkeyCtxPtr ctx(EVP_PKEY_CTX_new_from_name(nullptr, "DH", nullptr)); + if (!ctx || EVP_PKEY_fromdata_init(ctx.get()) <= 0) { + return nullptr; + } + EVP_PKEY* rawKey = nullptr; + if (EVP_PKEY_fromdata(ctx.get(), &rawKey, EVP_PKEY_PUBLIC_KEY, params.get()) <= 0) { + return nullptr; + } + return EvpPkeyPtr(rawKey); +} + +} + +Response computeResponse(const QByteArray& generator, const QByteArray& prime, + const QByteArray& serverPublicKey, const QString& username, + const QString& password) +{ + Response response; + if (generator.isEmpty() || prime.isEmpty() || serverPublicKey.isEmpty()) { + return response; + } + + BnPtr g = bnFromBytes(generator); + BnPtr p = bnFromBytes(prime); + BnPtr serverPub = bnFromBytes(serverPublicKey); + if (!g || !p || !serverPub) { + return response; + } + + EvpPkeyPtr domainParams = buildDomainParams(g.get(), p.get()); + if (!domainParams) { + return response; + } + + // Generate our own ephemeral DH keypair against the same domain + // parameters the server offered. + EvpPkeyCtxPtr keygenCtx(EVP_PKEY_CTX_new_from_pkey(nullptr, domainParams.get(), nullptr)); + if (!keygenCtx || EVP_PKEY_keygen_init(keygenCtx.get()) <= 0) { + return response; + } + EVP_PKEY* rawOurKey = nullptr; + if (EVP_PKEY_keygen(keygenCtx.get(), &rawOurKey) <= 0) { + return response; + } + EvpPkeyPtr ourKey(rawOurKey); + + BIGNUM* ourPubRaw = nullptr; + if (EVP_PKEY_get_bn_param(ourKey.get(), OSSL_PKEY_PARAM_PUB_KEY, &ourPubRaw) <= 0 + || ourPubRaw == nullptr) { + return response; + } + BnPtr ourPub(ourPubRaw); + + QByteArray clientPublicKey(prime.size(), char(0)); + if (BN_bn2binpad(ourPub.get(), reinterpret_cast(clientPublicKey.data()), + prime.size()) + < 0) { + return response; + } + + EvpPkeyPtr peerKey = buildPeerPublicKey(g.get(), p.get(), serverPub.get()); + if (!peerKey) { + return response; + } + + EvpPkeyCtxPtr deriveCtx(EVP_PKEY_CTX_new_from_pkey(nullptr, ourKey.get(), nullptr)); + if (!deriveCtx || EVP_PKEY_derive_init(deriveCtx.get()) <= 0 + || EVP_PKEY_derive_set_peer(deriveCtx.get(), peerKey.get()) <= 0) { + return response; + } + + size_t secretLen = 0; + if (EVP_PKEY_derive(deriveCtx.get(), nullptr, &secretLen) <= 0 || secretLen == 0) { + return response; + } + QByteArray secret(static_cast(secretLen), char(0)); + if (EVP_PKEY_derive(deriveCtx.get(), reinterpret_cast(secret.data()), + &secretLen) + <= 0) { + return response; + } + secret.resize(static_cast(secretLen)); + + unsigned char aesKey[16]; + if (EVP_Digest(secret.constData(), static_cast(secret.size()), aesKey, nullptr, + EVP_md5(), nullptr) + <= 0) { + return response; + } + + // 64 bytes username + 64 bytes password, NUL-padded/truncated. + QByteArray credentials(128, char(0)); + const QByteArray userBytes = username.toLatin1().left(64); + const QByteArray passBytes = password.toLatin1().left(64); + std::memcpy(credentials.data(), userBytes.constData(), + static_cast(userBytes.size())); + std::memcpy(credentials.data() + 64, passBytes.constData(), + static_cast(passBytes.size())); + + CipherCtxPtr cipherCtx(EVP_CIPHER_CTX_new()); + if (!cipherCtx + || EVP_EncryptInit_ex(cipherCtx.get(), EVP_aes_128_ecb(), nullptr, aesKey, nullptr) <= 0) { + return response; + } + EVP_CIPHER_CTX_set_padding(cipherCtx.get(), 0); + + QByteArray ciphertext(credentials.size() + EVP_MAX_BLOCK_LENGTH, char(0)); + int outLen1 = 0; + if (EVP_EncryptUpdate(cipherCtx.get(), reinterpret_cast(ciphertext.data()), + &outLen1, reinterpret_cast(credentials.constData()), + credentials.size()) + <= 0) { + return response; + } + int outLen2 = 0; + if (EVP_EncryptFinal_ex(cipherCtx.get(), + reinterpret_cast(ciphertext.data()) + outLen1, &outLen2) + <= 0) { + return response; + } + ciphertext.resize(outLen1 + outLen2); + + response.clientPublicKey = clientPublicKey; + response.encryptedCredentials = ciphertext; + return response; +} + +} diff --git a/src/vnc_apple_dh_auth.h b/src/vnc_apple_dh_auth.h new file mode 100644 index 0000000..2702857 --- /dev/null +++ b/src/vnc_apple_dh_auth.h @@ -0,0 +1,44 @@ +#ifndef ORBITHUB_VNC_APPLE_DH_AUTH_H +#define ORBITHUB_VNC_APPLE_DH_AUTH_H + +#include +#include + +// Apple's Screen Sharing authentication scheme (RFB security type 30): +// Diffie-Hellman key exchange followed by AES-128-ECB-encrypted +// credentials. Apple never published this officially -- it's not part of +// RFC 6143 -- so this implements the well-established reverse-engineered +// wire format used by several independent VNC clients, not a primary +// spec. Kept as a pure, state-free helper (no socket access) so it's +// unit-testable without a live connection, mirroring +// VncSessionBackend::vncAuthResponse()'s shape for standard VNC +// Authentication. +namespace VncAppleDhAuth { + +struct Response { + // Same byte length as the server's prime, big-endian, zero-padded. + // Empty on any failure (malformed input, an OpenSSL operation + // failing) -- callers should treat an empty clientPublicKey as "could + // not compute a response" rather than send a degenerate one. + QByteArray clientPublicKey; + // Always exactly 128 bytes on success (16 AES blocks): a 64-byte + // NUL-padded/truncated username followed by a 64-byte NUL-padded/ + // truncated password, AES-128-ECB encrypted (no padding, since the + // plaintext is already an exact multiple of the block size) with a + // key derived as MD5(sharedSecret). + QByteArray encryptedCredentials; +}; + +// Computes the DH keypair, the shared secret, the derived AES key, and +// the encrypted credential blob, using modern EVP-based OpenSSL 3.0 APIs +// throughout (no deprecated low-level DH_*/legacy calls -- unlike VNC +// Authentication's classic DES usage, none of MD5/AES-ECB/the EVP_PKEY DH +// APIs are deprecated, so no compatibility pragma is needed here). +// Returns a Response with an empty clientPublicKey on any failure. +Response computeResponse(const QByteArray& generator, const QByteArray& prime, + const QByteArray& serverPublicKey, const QString& username, + const QString& password); + +} + +#endif diff --git a/src/vnc_session_backend.cpp b/src/vnc_session_backend.cpp index 20f3d90..e4268b2 100644 --- a/src/vnc_session_backend.cpp +++ b/src/vnc_session_backend.cpp @@ -1,5 +1,6 @@ #include "vnc_session_backend.h" +#include "vnc_apple_dh_auth.h" #include "vnc_pixel_codecs.h" #include @@ -49,6 +50,12 @@ constexpr qint32 kEncTight = 7; // the framebuffer. constexpr qint32 kEncCursor = -239; +// Apple's Screen Sharing authentication (Diffie-Hellman + AES). Not part +// of RFC 6143 -- see vnc_apple_dh_auth.h. +constexpr quint8 kSecurityTypeNone = 1; +constexpr quint8 kSecurityTypeVncAuth = 2; +constexpr quint8 kSecurityTypeAppleDh = 30; + constexpr std::array kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile, kEncZRLE, kEncTight, kEncCursor }; @@ -184,7 +191,8 @@ VncSessionBackend::VncSessionBackend(const Profile& profile, QObject* parent) m_tightInflateInitialized{ false, false, false, false }, m_tightCompressionMode(0), m_tightFilterId(0), - m_tightLengthByteIndex(0) + m_tightLengthByteIndex(0), + m_appleAuthPrimeLength(0) { std::memset(m_zrleInflateStream, 0, sizeof(z_stream_s)); for (z_stream_s* stream : m_tightInflateStreams) { @@ -517,6 +525,8 @@ void VncSessionBackend::resetProtocolState() m_tightInflateInitialized[i] = false; } } + m_appleAuthGenerator.clear(); + m_appleAuthPrimeLength = 0; } bool VncSessionBackend::haveBytes(int count) const @@ -781,11 +791,18 @@ void VncSessionBackend::processReceiveBuffer() m_offeredSecurityTypes = m_recvBuffer.left(m_securityTypeCount); m_recvBuffer.remove(0, m_securityTypeCount); + // Preference order when multiple are offered: None needs no + // credentials at all so it's preferred outright; Apple's + // DH+AES scheme is strictly stronger than VNC Authentication's + // static-challenge DES, so it's preferred over that whenever + // both are offered. quint8 chosen = 0; - if (m_offeredSecurityTypes.contains(char(1))) { - chosen = 1; - } else if (m_offeredSecurityTypes.contains(char(2))) { - chosen = 2; + if (m_offeredSecurityTypes.contains(static_cast(kSecurityTypeNone))) { + chosen = kSecurityTypeNone; + } else if (m_offeredSecurityTypes.contains(static_cast(kSecurityTypeAppleDh))) { + chosen = kSecurityTypeAppleDh; + } else if (m_offeredSecurityTypes.contains(static_cast(kSecurityTypeVncAuth))) { + chosen = kSecurityTypeVncAuth; } if (chosen == 0) { QStringList offered; @@ -803,8 +820,10 @@ void VncSessionBackend::processReceiveBuffer() m_chosenSecurityType = chosen; m_socket->write(QByteArray(1, static_cast(chosen))); - if (chosen == 2) { + if (chosen == kSecurityTypeVncAuth) { m_rfbState = RfbState::WaitingVncAuthChallenge; + } else if (chosen == kSecurityTypeAppleDh) { + m_rfbState = RfbState::WaitingAppleAuthGeneratorLength; } else if (m_negotiatedMinorVersion >= 8) { m_rfbState = RfbState::WaitingSecurityResult; } else { @@ -823,13 +842,16 @@ void VncSessionBackend::processReceiveBuffer() if (type == 0) { m_rfbState = RfbState::WaitingSecurityFailureReasonLength; - } else if (type == 1) { - m_chosenSecurityType = 1; + } else if (type == kSecurityTypeNone) { + m_chosenSecurityType = kSecurityTypeNone; sendClientInit(); m_rfbState = RfbState::WaitingServerInitHeader; - } else if (type == 2) { - m_chosenSecurityType = 2; + } else if (type == kSecurityTypeVncAuth) { + m_chosenSecurityType = kSecurityTypeVncAuth; m_rfbState = RfbState::WaitingVncAuthChallenge; + } else if (type == kSecurityTypeAppleDh) { + m_chosenSecurityType = kSecurityTypeAppleDh; + m_rfbState = RfbState::WaitingAppleAuthGeneratorLength; } else { failConnection( QStringLiteral( @@ -888,6 +910,71 @@ void VncSessionBackend::processReceiveBuffer() break; } + case RfbState::WaitingAppleAuthGeneratorLength: { + if (!haveBytes(2)) { + return; + } + m_pendingLength = readU16BE(m_recvBuffer, 0); + m_recvBuffer.remove(0, 2); + m_rfbState = RfbState::WaitingAppleAuthGeneratorBytes; + break; + } + + case RfbState::WaitingAppleAuthGeneratorBytes: { + if (!haveBytes(static_cast(m_pendingLength))) { + return; + } + m_appleAuthGenerator = m_recvBuffer.left(static_cast(m_pendingLength)); + m_recvBuffer.remove(0, static_cast(m_pendingLength)); + m_rfbState = RfbState::WaitingAppleAuthPrimeLength; + break; + } + + case RfbState::WaitingAppleAuthPrimeLength: { + if (!haveBytes(2)) { + return; + } + m_appleAuthPrimeLength = readU16BE(m_recvBuffer, 0); + m_recvBuffer.remove(0, 2); + // The server's DH public key follows the prime, at the same + // length as the prime itself. + m_pendingLength = m_appleAuthPrimeLength * 2; + m_rfbState = RfbState::WaitingAppleAuthPrimeAndServerKey; + break; + } + + case RfbState::WaitingAppleAuthPrimeAndServerKey: { + if (!haveBytes(static_cast(m_pendingLength))) { + return; + } + const int primeLength = static_cast(m_appleAuthPrimeLength); + const QByteArray prime = m_recvBuffer.left(primeLength); + const QByteArray serverPublicKey = m_recvBuffer.mid(primeLength, primeLength); + m_recvBuffer.remove(0, static_cast(m_pendingLength)); + + const VncAppleDhAuth::Response response = VncAppleDhAuth::computeResponse( + m_appleAuthGenerator, prime, serverPublicKey, profile().username, + m_activeOptions.password); + if (response.clientPublicKey.isEmpty()) { + failConnection( + QStringLiteral( + "Failed to compute the Apple Screen Sharing authentication response."), + QStringLiteral("VncAppleDhAuth::computeResponse() failed")); + return; + } + + m_socket->write(response.clientPublicKey); + m_socket->write(response.encryptedCredentials); + + if (m_negotiatedMinorVersion >= 8) { + m_rfbState = RfbState::WaitingSecurityResult; + } else { + sendClientInit(); + m_rfbState = RfbState::WaitingServerInitHeader; + } + break; + } + case RfbState::WaitingSecurityResult: { if (!haveBytes(4)) { return; diff --git a/src/vnc_session_backend.h b/src/vnc_session_backend.h index 1a52c1e..0536f87 100644 --- a/src/vnc_session_backend.h +++ b/src/vnc_session_backend.h @@ -24,11 +24,13 @@ struct z_stream_s; // C API to wrangle here. // // Scope (see plan / issue #3 for the full rationale): standard VNC -// Authentication (security type 2) and no-auth (type 1) only -- not -// Apple's Screen Sharing scheme (type 30). Raw + CopyRect + Hextile + ZRLE -// + Tight encodings. No dynamic resize. Clipboard sync (Latin-1 only, per -// RFB's ServerCutText/ClientCutText) and remote cursor shape sync (the -// Cursor pseudo-encoding) are supported. +// Authentication (security type 2), no-auth (type 1), and Apple's Screen +// Sharing scheme (type 30, Diffie-Hellman + AES -- see +// vnc_apple_dh_auth.h; not part of RFC 6143, implemented against the +// well-established reverse-engineered wire format). Raw + CopyRect + +// Hextile + ZRLE + Tight encodings. No dynamic resize. Clipboard sync +// (Latin-1 only, per RFB's ServerCutText/ClientCutText) and remote cursor +// shape sync (the Cursor pseudo-encoding) are supported. // // Tight decoding gap: the real protocol allows the server to skip zlib // compression entirely for very small Basic-mode payloads; this decoder @@ -109,6 +111,10 @@ private: WaitingTightFilterId, WaitingTightLengthByte, WaitingTightPayload, + WaitingAppleAuthGeneratorLength, + WaitingAppleAuthGeneratorBytes, + WaitingAppleAuthPrimeLength, + WaitingAppleAuthPrimeAndServerKey, WaitingSetColourMapHeader, WaitingSetColourMapData, WaitingServerCutTextHeader, @@ -178,6 +184,16 @@ private: quint8 m_tightFilterId; int m_tightLengthByteIndex; + // Apple Screen Sharing authentication state (security type 30, not + // part of RFC 6143 -- see vnc_apple_dh_auth.h). m_appleAuthGenerator + // must persist across the generator-length/generator-bytes state + // hop; m_appleAuthPrimeLength must persist from when it's first read + // until the combined prime+server-public-key buffer (2x that length) + // has fully arrived, since m_pendingLength gets reused to track that + // combined byte count in the meantime. + QByteArray m_appleAuthGenerator; + quint32 m_appleAuthPrimeLength; + void setState(SessionState state, const QString& message); void resetProtocolState(); void processReceiveBuffer(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c4bc562..a16dce7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -30,6 +30,7 @@ add_executable(test_vnc_session_backend test_vnc_session_backend.cpp ${CMAKE_SOURCE_DIR}/src/vnc_session_backend.cpp ${CMAKE_SOURCE_DIR}/src/vnc_pixel_codecs.cpp + ${CMAKE_SOURCE_DIR}/src/vnc_apple_dh_auth.cpp ${CMAKE_SOURCE_DIR}/src/session_backend.h ) target_include_directories(test_vnc_session_backend PRIVATE ${CMAKE_SOURCE_DIR}/src) diff --git a/tests/test_vnc_session_backend.cpp b/tests/test_vnc_session_backend.cpp index 1f0d317..3cdf33a 100644 --- a/tests/test_vnc_session_backend.cpp +++ b/tests/test_vnc_session_backend.cpp @@ -1,9 +1,12 @@ #include "vnc_session_backend.h" +#include "vnc_apple_dh_auth.h" #include #include #include +#include +#include #include extern "C" { @@ -11,6 +14,7 @@ extern "C" { } #include +#include namespace { // Independently documented bit-reversal example for VNC Authentication's @@ -297,6 +301,65 @@ QByteArray encodeJpegForTest(int width, int height, QRgb color) std::free(buffer); return result; } + +// A small (512-bit) DH group generated fresh at test-run time -- rather +// than a hardcoded literal prime, to avoid any transcription risk -- plus +// one keypair against it, used to exercise VncAppleDhAuth::computeResponse() +// both directly and via a fake server acting as an Apple Screen Sharing +// (security type 30) endpoint. 512 bits keeps generation fast while still +// avoiding any "key too small" policy rejection from OpenSSL's default +// provider that a truly tiny hand-picked prime might trigger. +struct ToyDhKeypair { + QByteArray generatorBytes; + QByteArray primeBytes; + QByteArray publicKeyBytes; + BIGNUM* privateExponent = nullptr; // caller must BN_free + BIGNUM* prime = nullptr; // caller must BN_free +}; + +ToyDhKeypair generateToyDhKeypair() +{ + ToyDhKeypair result; + BIGNUM* p = BN_new(); + BN_generate_prime_ex(p, 512, 0, nullptr, nullptr, nullptr); + BIGNUM* g = BN_new(); + BN_set_word(g, 2); + + BN_CTX* ctx = BN_CTX_new(); + BIGNUM* priv = BN_new(); + BN_rand(priv, 256, -1, 0); + BIGNUM* pub = BN_new(); + BN_mod_exp(pub, g, priv, p, ctx); + + const int primeLen = BN_num_bytes(p); + result.primeBytes = QByteArray(primeLen, char(0)); + BN_bn2binpad(p, reinterpret_cast(result.primeBytes.data()), primeLen); + result.generatorBytes = QByteArray(1, char(2)); + result.publicKeyBytes = QByteArray(primeLen, char(0)); + BN_bn2binpad(pub, reinterpret_cast(result.publicKeyBytes.data()), primeLen); + result.privateExponent = priv; + result.prime = p; + + BN_free(g); + BN_free(pub); + BN_CTX_free(ctx); + return result; +} + +// A FramebufferUpdate-adjacent helper isn't needed here -- these bytes are +// sent immediately after the client selects security type 30, before +// ClientInit/ServerInit even happen. +QByteArray appleDhAuthServerMessage(const QByteArray& generator, const QByteArray& prime, + const QByteArray& serverPublicKey) +{ + QByteArray msg; + appendU16(msg, static_cast(generator.size())); + msg += generator; + appendU16(msg, static_cast(prime.size())); + msg += prime; + msg += serverPublicKey; + return msg; +} } class TestVncSessionBackend : public QObject @@ -348,6 +411,9 @@ private slots: void tightBasicPaletteFilterProducesExpectedPixels(); void tightJpegProducesExpectedPixels(); void tightStreamResetFlagAllowsIndependentDecoding(); + void appleDhAuthRoundTripDecryptsToOriginalCredentials(); + void connectsWithAppleDhAuthenticationRfb38(); + void appleDhAuthIsPreferredOverVncAuthWhenBothOffered(); private: std::unique_ptr m_server; @@ -626,7 +692,7 @@ void TestVncSessionBackend::unsupportedSecurityTypeReachesFailedState() if (m_server->nextStep() == 0) { QByteArray securityTypes; securityTypes.append(char(1)); - securityTypes.append(char(30)); // Apple's scheme -- unsupported here + securityTypes.append(char(19)); // VeNCrypt (TLS) -- unsupported here m_server->sendWhenConnected(securityTypes); } }); @@ -1862,5 +1928,133 @@ void TestVncSessionBackend::tightStreamResetFlagAllowsIndependentDecoding() QCOMPARE(frames.at(1).pixelColor(0, 0), QColor(200, 210, 220)); } +void TestVncSessionBackend::appleDhAuthRoundTripDecryptsToOriginalCredentials() +{ + ToyDhKeypair serverKeypair = generateToyDhKeypair(); + + const QString username = QStringLiteral("tester"); + const QString password = QStringLiteral("s3cret-pass"); + + const VncAppleDhAuth::Response response = VncAppleDhAuth::computeResponse( + serverKeypair.generatorBytes, serverKeypair.primeBytes, serverKeypair.publicKeyBytes, + username, password); + + QVERIFY(!response.clientPublicKey.isEmpty()); + QCOMPARE(response.clientPublicKey.size(), serverKeypair.primeBytes.size()); + QCOMPARE(response.encryptedCredentials.size(), 128); + + // Derive the shared secret the way the *server* would, using the + // client's returned public key and the server's own private exponent + // -- Diffie-Hellman's commutativity means this must equal whatever + // computeResponse() used internally to derive its AES key. + BIGNUM* clientPub = BN_bin2bn( + reinterpret_cast(response.clientPublicKey.constData()), + response.clientPublicKey.size(), nullptr); + BN_CTX* ctx = BN_CTX_new(); + BIGNUM* sharedSecret = BN_new(); + BN_mod_exp(sharedSecret, clientPub, serverKeypair.privateExponent, serverKeypair.prime, ctx); + + const int secretLen = BN_num_bytes(sharedSecret); + QByteArray secretBytes(secretLen, char(0)); + BN_bn2binpad(sharedSecret, reinterpret_cast(secretBytes.data()), secretLen); + + unsigned char aesKey[16]; + EVP_Digest(secretBytes.constData(), static_cast(secretBytes.size()), aesKey, nullptr, + EVP_md5(), nullptr); + + EVP_CIPHER_CTX* decCtx = EVP_CIPHER_CTX_new(); + EVP_DecryptInit_ex(decCtx, EVP_aes_128_ecb(), nullptr, aesKey, nullptr); + EVP_CIPHER_CTX_set_padding(decCtx, 0); + QByteArray plain(response.encryptedCredentials.size() + 16, char(0)); + int outLen1 = 0; + EVP_DecryptUpdate( + decCtx, reinterpret_cast(plain.data()), &outLen1, + reinterpret_cast(response.encryptedCredentials.constData()), + response.encryptedCredentials.size()); + int outLen2 = 0; + EVP_DecryptFinal_ex(decCtx, reinterpret_cast(plain.data()) + outLen1, &outLen2); + plain.resize(outLen1 + outLen2); + EVP_CIPHER_CTX_free(decCtx); + + QByteArray expected(128, char(0)); + const QByteArray userBytes = username.toLatin1().left(64); + const QByteArray passBytes = password.toLatin1().left(64); + std::memcpy(expected.data(), userBytes.constData(), static_cast(userBytes.size())); + std::memcpy(expected.data() + 64, passBytes.constData(), static_cast(passBytes.size())); + QCOMPARE(plain, expected); + + BN_free(clientPub); + BN_free(sharedSecret); + BN_CTX_free(ctx); + BN_free(serverKeypair.privateExponent); + BN_free(serverKeypair.prime); +} + +void TestVncSessionBackend::connectsWithAppleDhAuthenticationRfb38() +{ + ToyDhKeypair serverKeypair = generateToyDhKeypair(); + const QByteArray authMessage = appleDhAuthServerMessage( + serverKeypair.generatorBytes, serverKeypair.primeBytes, serverKeypair.publicKeyBytes); + + connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() { + m_server->sendWhenConnected(QByteArray("RFB 003.008\n")); + }); + connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, authMessage]() { + switch (m_server->nextStep()) { + case 0: { // version reply + QByteArray securityTypes; + securityTypes.append(char(1)); + securityTypes.append(char(30)); // Apple's scheme + m_server->sendWhenConnected(securityTypes); + break; + } + case 1: // security-type selection (byte value 30) + m_server->sendWhenConnected(authMessage); + break; + case 2: // client's DH public key + encrypted credentials -- accept unconditionally + m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK + break; + case 3: { // ClientInit + QByteArray serverInit; + serverInit.append(char(0)); serverInit.append(char(1)); + serverInit.append(char(0)); serverInit.append(char(1)); + serverInit.append(QByteArray(16, char(0))); + serverInit.append(QByteArray(4, char(0))); + m_server->sendWhenConnected(serverInit); + break; + } + default: + break; + } + }); + + m_backend->connectSession(makeOptions(QStringLiteral("irrelevant-password"))); + QTRY_COMPARE(m_lastState, SessionState::Connected); + + BN_free(serverKeypair.privateExponent); + BN_free(serverKeypair.prime); +} + +void TestVncSessionBackend::appleDhAuthIsPreferredOverVncAuthWhenBothOffered() +{ + connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() { + m_server->sendWhenConnected(QByteArray("RFB 003.008\n")); + }); + connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() { + if (m_server->nextStep() == 0) { + m_server->received.clear(); + QByteArray securityTypes; + securityTypes.append(char(2)); // count = 2 + securityTypes.append(char(2)); // VNC Authentication + securityTypes.append(char(30)); // Apple DH + m_server->sendWhenConnected(securityTypes); + } + }); + + m_backend->connectSession(makeOptions(QStringLiteral("whatever"))); + QTRY_VERIFY(m_server->received.size() >= 1); + QCOMPARE(static_cast(m_server->received.at(0)), quint8(30)); +} + QTEST_GUILESS_MAIN(TestVncSessionBackend) #include "test_vnc_session_backend.moc"