DarkSingularity/orbithub
Internal
Public Access
0
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases 7 Wiki Activity

Add Apple Screen Sharing authentication for VNC (security type 30)

Browse Source 
Apple's macOS Screen Sharing server doesn't speak standard VNC
Authentication -- it uses a Diffie-Hellman key exchange followed by
AES-128-ECB-encrypted credentials, security type 30. Apple never
published this scheme (it's not part of RFC 6143); this implements
the well-established reverse-engineered wire format: the server sends
a generator, prime, and its own DH public key; the client generates
an ephemeral keypair, derives the shared secret, MD5-hashes it into
an AES key, and sends back its public key plus a 128-byte encrypted
username+password buffer.

The DH/AES math lives in new src/vnc_apple_dh_auth.h/.cpp as a pure,
socket-free helper (mirroring vncAuthResponse()'s shape for standard
VNC Auth), built entirely on modern EVP_PKEY-based OpenSSL 3.0 APIs --
no deprecated low-level DH_* calls, unlike VNC Authentication's
necessary use of classic DES. Reuses Profile::username (already a
shared field) since Apple's scheme needs an actual macOS account name,
unlike password-only VNC Authentication.

Security-type preference when multiple are offered is now None >
AppleDH > VNCAuth, since DH+AES is strictly stronger than static-
challenge DES. Adds a DH round-trip test (generates a real 512-bit
group at test time, computes the response, then independently
re-derives the shared secret as the server would and decrypts the
credentials back out -- proving self-consistency without needing a
hand-computed expected value), a fake-server integration test for the
full RFB 3.8 handshake sequencing, and a preference-order test.

Not yet live-verified against a real macOS Screen Sharing server --
that's next.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmith
2026-09-15 21:27:17 -06:00
co-authored by Claude Sonnet 5
parent 2fe2022182
commit 1f026dde70
7 changed files with 579 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/CMakeLists.txt
+1
View File
@@ -30,6 +30,7 @@ add_executable(test_vnc_session_backend
test_vnc_session_backend.cpp
${CMAKE_SOURCE_DIR}/src/vnc_session_backend.cpp
${CMAKE_SOURCE_DIR}/src/vnc_pixel_codecs.cpp
${CMAKE_SOURCE_DIR}/src/vnc_apple_dh_auth.cpp
${CMAKE_SOURCE_DIR}/src/session_backend.h
)
target_include_directories(test_vnc_session_backend PRIVATE ${CMAKE_SOURCE_DIR}/src)
tests/test_vnc_session_backend.cpp
+195 -1
View File
@@ -1,9 +1,12 @@
#include "vnc_session_backend.h"
#include "vnc_apple_dh_auth.h"
#include <QTcpServer>
#include <QTcpSocket>
#include <QTest>
#include <openssl/bn.h>
#include <openssl/evp.h>
#include <zlib.h>
extern "C" {
@@ -11,6 +14,7 @@ extern "C" {
}
#include <cstdlib>
#include <cstring>
namespace {
// Independently documented bit-reversal example for VNC Authentication's
@@ -297,6 +301,65 @@ QByteArray encodeJpegForTest(int width, int height, QRgb color)
std::free(buffer);
return result;
}
// A small (512-bit) DH group generated fresh at test-run time -- rather
// than a hardcoded literal prime, to avoid any transcription risk -- plus
// one keypair against it, used to exercise VncAppleDhAuth::computeResponse()
// both directly and via a fake server acting as an Apple Screen Sharing
// (security type 30) endpoint. 512 bits keeps generation fast while still
// avoiding any "key too small" policy rejection from OpenSSL's default
// provider that a truly tiny hand-picked prime might trigger.
struct ToyDhKeypair {
QByteArray generatorBytes;
QByteArray primeBytes;
QByteArray publicKeyBytes;
BIGNUM* privateExponent = nullptr; // caller must BN_free
BIGNUM* prime = nullptr; // caller must BN_free
};
ToyDhKeypair generateToyDhKeypair()
{
ToyDhKeypair result;
BIGNUM* p = BN_new();
BN_generate_prime_ex(p, 512, 0, nullptr, nullptr, nullptr);
BIGNUM* g = BN_new();
BN_set_word(g, 2);
BN_CTX* ctx = BN_CTX_new();
BIGNUM* priv = BN_new();
BN_rand(priv, 256, -1, 0);
BIGNUM* pub = BN_new();
BN_mod_exp(pub, g, priv, p, ctx);
const int primeLen = BN_num_bytes(p);
result.primeBytes = QByteArray(primeLen, char(0));
BN_bn2binpad(p, reinterpret_cast<unsigned char*>(result.primeBytes.data()), primeLen);
result.generatorBytes = QByteArray(1, char(2));
result.publicKeyBytes = QByteArray(primeLen, char(0));
BN_bn2binpad(pub, reinterpret_cast<unsigned char*>(result.publicKeyBytes.data()), primeLen);
result.privateExponent = priv;
result.prime = p;
BN_free(g);
BN_free(pub);
BN_CTX_free(ctx);
return result;
}
// A FramebufferUpdate-adjacent helper isn't needed here -- these bytes are
// sent immediately after the client selects security type 30, before
// ClientInit/ServerInit even happen.
QByteArray appleDhAuthServerMessage(const QByteArray& generator, const QByteArray& prime,
const QByteArray& serverPublicKey)
{
QByteArray msg;
appendU16(msg, static_cast<quint16>(generator.size()));
msg += generator;
appendU16(msg, static_cast<quint16>(prime.size()));
msg += prime;
msg += serverPublicKey;
return msg;
}
}
class TestVncSessionBackend : public QObject
@@ -348,6 +411,9 @@ private slots:
void tightBasicPaletteFilterProducesExpectedPixels();
void tightJpegProducesExpectedPixels();
void tightStreamResetFlagAllowsIndependentDecoding();
void appleDhAuthRoundTripDecryptsToOriginalCredentials();
void connectsWithAppleDhAuthenticationRfb38();
void appleDhAuthIsPreferredOverVncAuthWhenBothOffered();
private:
std::unique_ptr<FakeVncServer> m_server;
@@ -626,7 +692,7 @@ void TestVncSessionBackend::unsupportedSecurityTypeReachesFailedState()
if (m_server->nextStep() == 0) {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(30)); // Apple's scheme -- unsupported here
securityTypes.append(char(19)); // VeNCrypt (TLS) -- unsupported here
m_server->sendWhenConnected(securityTypes);
}
});
@@ -1862,5 +1928,133 @@ void TestVncSessionBackend::tightStreamResetFlagAllowsIndependentDecoding()
QCOMPARE(frames.at(1).pixelColor(0, 0), QColor(200, 210, 220));
}
void TestVncSessionBackend::appleDhAuthRoundTripDecryptsToOriginalCredentials()
{
ToyDhKeypair serverKeypair = generateToyDhKeypair();
const QString username = QStringLiteral("tester");
const QString password = QStringLiteral("s3cret-pass");
const VncAppleDhAuth::Response response = VncAppleDhAuth::computeResponse(
serverKeypair.generatorBytes, serverKeypair.primeBytes, serverKeypair.publicKeyBytes,
username, password);
QVERIFY(!response.clientPublicKey.isEmpty());
QCOMPARE(response.clientPublicKey.size(), serverKeypair.primeBytes.size());
QCOMPARE(response.encryptedCredentials.size(), 128);
// Derive the shared secret the way the *server* would, using the
// client's returned public key and the server's own private exponent
// -- Diffie-Hellman's commutativity means this must equal whatever
// computeResponse() used internally to derive its AES key.
BIGNUM* clientPub = BN_bin2bn(
reinterpret_cast<const unsigned char*>(response.clientPublicKey.constData()),
response.clientPublicKey.size(), nullptr);
BN_CTX* ctx = BN_CTX_new();
BIGNUM* sharedSecret = BN_new();
BN_mod_exp(sharedSecret, clientPub, serverKeypair.privateExponent, serverKeypair.prime, ctx);
const int secretLen = BN_num_bytes(sharedSecret);
QByteArray secretBytes(secretLen, char(0));
BN_bn2binpad(sharedSecret, reinterpret_cast<unsigned char*>(secretBytes.data()), secretLen);
unsigned char aesKey[16];
EVP_Digest(secretBytes.constData(), static_cast<size_t>(secretBytes.size()), aesKey, nullptr,
EVP_md5(), nullptr);
EVP_CIPHER_CTX* decCtx = EVP_CIPHER_CTX_new();
EVP_DecryptInit_ex(decCtx, EVP_aes_128_ecb(), nullptr, aesKey, nullptr);
EVP_CIPHER_CTX_set_padding(decCtx, 0);
QByteArray plain(response.encryptedCredentials.size() + 16, char(0));
int outLen1 = 0;
EVP_DecryptUpdate(
decCtx, reinterpret_cast<unsigned char*>(plain.data()), &outLen1,
reinterpret_cast<const unsigned char*>(response.encryptedCredentials.constData()),
response.encryptedCredentials.size());
int outLen2 = 0;
EVP_DecryptFinal_ex(decCtx, reinterpret_cast<unsigned char*>(plain.data()) + outLen1, &outLen2);
plain.resize(outLen1 + outLen2);
EVP_CIPHER_CTX_free(decCtx);
QByteArray expected(128, char(0));
const QByteArray userBytes = username.toLatin1().left(64);
const QByteArray passBytes = password.toLatin1().left(64);
std::memcpy(expected.data(), userBytes.constData(), static_cast<size_t>(userBytes.size()));
std::memcpy(expected.data() + 64, passBytes.constData(), static_cast<size_t>(passBytes.size()));
QCOMPARE(plain, expected);
BN_free(clientPub);
BN_free(sharedSecret);
BN_CTX_free(ctx);
BN_free(serverKeypair.privateExponent);
BN_free(serverKeypair.prime);
}
void TestVncSessionBackend::connectsWithAppleDhAuthenticationRfb38()
{
ToyDhKeypair serverKeypair = generateToyDhKeypair();
const QByteArray authMessage = appleDhAuthServerMessage(
serverKeypair.generatorBytes, serverKeypair.primeBytes, serverKeypair.publicKeyBytes);
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, authMessage]() {
switch (m_server->nextStep()) {
case 0: { // version reply
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(30)); // Apple's scheme
m_server->sendWhenConnected(securityTypes);
break;
}
case 1: // security-type selection (byte value 30)
m_server->sendWhenConnected(authMessage);
break;
case 2: // client's DH public key + encrypted credentials -- accept unconditionally
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 3: { // ClientInit
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(1));
serverInit.append(char(0)); serverInit.append(char(1));
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
m_server->sendWhenConnected(serverInit);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions(QStringLiteral("irrelevant-password")));
QTRY_COMPARE(m_lastState, SessionState::Connected);
BN_free(serverKeypair.privateExponent);
BN_free(serverKeypair.prime);
}
void TestVncSessionBackend::appleDhAuthIsPreferredOverVncAuthWhenBothOffered()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
if (m_server->nextStep() == 0) {
m_server->received.clear();
QByteArray securityTypes;
securityTypes.append(char(2)); // count = 2
securityTypes.append(char(2)); // VNC Authentication
securityTypes.append(char(30)); // Apple DH
m_server->sendWhenConnected(securityTypes);
}
});
m_backend->connectSession(makeOptions(QStringLiteral("whatever")));
QTRY_VERIFY(m_server->received.size() >= 1);
QCOMPARE(static_cast<quint8>(m_server->received.at(0)), quint8(30));
}
QTEST_GUILESS_MAIN(TestVncSessionBackend)
#include "test_vnc_session_backend.moc"