DarkSingularity/orbithub
Internal
Public Access
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity
Files
d7f9d4966be064685d7a1ec3c1f3e9dcbf3457fe
orbithub/tests/test_vnc_session_backend.cpp
T
ksmithandClaude Sonnet 5 e80fe7d634 Fix Apple DH auth wire-order bug, add type 33 fallback, fix missing VNC password prompt 
Live testing against a real macOS Screen Sharing server surfaced two
real bugs, independent of each other:

1. SessionTab::requestConnectOptions() never prompted for a password
   on VNC profiles (only SSH/RDP) -- every VNC connection went out
   with an empty password regardless of what the server needed. VNC
   now gets its own prompt; an empty password is allowed through
   (unlike RDP's hard requirement) since no-auth VNC servers exist and
   there's no way to know client-side before the security-type
   negotiation happens.

2. VncSessionBackend's Apple DH (type 30) response sent the client's
   public key before the encrypted credentials. Cross-checking against
   neatvnc's rfb-proto.h (an independent, authoritative reference: both
   the wire struct definitions and the full server-side verification
   code, matched field-by-field against this implementation) showed
   the correct order is credentials first, then public key -- exactly
   backwards from what was implemented. Fixed, with a new regression
   test that decrypts the credentials back out using the trailing
   public-key bytes to derive the shared secret, which would fail if
   the fields were swapped again.

Also adds security type 33 (RSA + AES, src/vnc_apple_rsa_auth.h) as a
fallback Apple auth scheme, sourced from the `asyncvnc` PyPI package.
Preference when multiple are offered: None > AppleDH(30) >
AppleRSA(33) > VNCAuth(2).

Neither scheme has been gotten working live yet against the specific
macOS Tahoe (26.6.2) server available for testing -- type 30's wire
format is now verified correct byte-for-byte against the independent
reference above, but the server still rejects it with a generic
"Authentication or authorization failure"; type 33 is rejected even
earlier, right after the initial host-key request. macOS Tahoe was
released after this assistant's knowledge cutoff, so there may be a
protocol or permission-model change specific to it that isn't
reflected in either reference. Documented as an open issue in
docs/PROGRESS.md rather than claimed as working.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-16 03:49:06 -06:00

2177 lines
88 KiB
C++
Raw Blame History

#include "vnc_session_backend.h"
#include "vnc_apple_dh_auth.h"
#include <QTcpServer>
#include <QTcpSocket>
#include <QTest>
#include <openssl/bn.h>
#include <openssl/evp.h>
#include <zlib.h>
extern "C" {
#include <jpeglib.h>
}
#include <cstdlib>
#include <cstring>
namespace {
// Independently documented bit-reversal example for VNC Authentication's
// DES key prep (password "COW"): 'C'=0x43, 'O'=0x4F, 'W'=0x57, each
// reversed bit-by-bit -> 0xC2, 0xF2, 0xEA, padded with zeros to 8 bytes.
// Cross-checked against multiple independent VNC client implementations'
// documented behavior, not just re-derived from this code's own logic.
QByteArray expectedCowKey()
{
return QByteArray::fromHex("c2f2ea0000000000");
}
// A tiny scripted RFB "server" for state-machine tests: accepts exactly one
// connection on 127.0.0.1 and lets the test drive the byte sequence it
// sends, while capturing whatever the real VncSessionBackend under test
// writes back.
class FakeVncServer : public QObject
{
Q_OBJECT
public:
FakeVncServer()
{
server.listen(QHostAddress::LocalHost);
connect(&server, &QTcpServer::newConnection, this, [this]() {
connection = server.nextPendingConnection();
connect(connection, &QTcpSocket::readyRead, this, [this]() {
received.append(connection->readAll());
emit dataReceived();
});
emit clientConnected();
});
}
quint16 port() const { return server.serverPort(); }
void sendWhenConnected(const QByteArray& bytes)
{
if (connection != nullptr) {
connection->write(bytes);
}
}
// Scripted steps are matched by ordinal position, not by pattern-
// matching received byte content: several distinct RFB messages (e.g.
// the 1-byte security-type selection and the 1-byte ClientInit
// shared-flag) are indistinguishable by content alone, so content
// matching is genuinely ambiguous here.
int nextStep() { return step++; }
QTcpServer server;
QTcpSocket* connection = nullptr;
QByteArray received;
int step = 0;
signals:
void clientConnected();
void dataReceived();
};
Profile makeVncProfile(quint16 port)
{
Profile profile;
profile.name = QStringLiteral("Test VNC");
profile.host = QStringLiteral("127.0.0.1");
profile.port = port;
profile.protocol = QStringLiteral("VNC");
return profile;
}
SessionConnectOptions makeOptions(const QString& password = QString())
{
SessionConnectOptions options;
options.password = password;
return options;
}
void appendU16(QByteArray& buf, quint16 value)
{
buf.append(static_cast<char>((value >> 8) & 0xFF));
buf.append(static_cast<char>(value & 0xFF));
}
// Pixel bytes as expected under our negotiated SetPixelFormat: 32bpp
// little-endian, byte order B,G,R,pad.
QByteArray pixelBytes(int r, int g, int b)
{
QByteArray bytes;
bytes.append(static_cast<char>(b));
bytes.append(static_cast<char>(g));
bytes.append(static_cast<char>(r));
bytes.append(char(0));
return bytes;
}
QByteArray serverInitBytes(int width, int height)
{
QByteArray serverInit;
appendU16(serverInit, static_cast<quint16>(width));
appendU16(serverInit, static_cast<quint16>(height));
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
return serverInit;
}
// A FramebufferUpdate message with exactly one Hextile-encoded rectangle
// (RFC 6143 encoding type 5), given the already fully-formed tile byte
// stream to follow the rectangle header.
QByteArray hextileFramebufferUpdate(int x, int y, int width, int height, const QByteArray& tileData)
{
QByteArray update;
update.append(char(0)); update.append(char(0));
update.append(char(0)); update.append(char(1)); // 1 rectangle
appendU16(update, static_cast<quint16>(x));
appendU16(update, static_cast<quint16>(y));
appendU16(update, static_cast<quint16>(width));
appendU16(update, static_cast<quint16>(height));
update.append(char(0)); update.append(char(0)); update.append(char(0));
update.append(char(5)); // encoding = Hextile
update.append(tileData);
return update;
}
void appendU32(QByteArray& buf, quint32 value)
{
buf.append(static_cast<char>((value >> 24) & 0xFF));
buf.append(static_cast<char>((value >> 16) & 0xFF));
buf.append(static_cast<char>((value >> 8) & 0xFF));
buf.append(static_cast<char>(value & 0xFF));
}
// ZRLE's CPIXEL: 3 bytes, B,G,R order (the pad byte a full 32bpp pixel
// would have is simply omitted) -- matches VncPixelCodecs::rgbFromPixelBytes.
QByteArray cpixelBytes(int r, int g, int b)
{
QByteArray bytes;
bytes.append(static_cast<char>(b));
bytes.append(static_cast<char>(g));
bytes.append(static_cast<char>(r));
return bytes;
}
// A FramebufferUpdate message with exactly one ZRLE-encoded rectangle
// (RFC 6143 encoding type 16): a 4-byte compressed length followed by that
// many already-zlib-compressed bytes.
QByteArray zrleFramebufferUpdate(int x, int y, int width, int height, const QByteArray& compressed)
{
QByteArray update;
update.append(char(0)); update.append(char(0));
update.append(char(0)); update.append(char(1)); // 1 rectangle
appendU16(update, static_cast<quint16>(x));
appendU16(update, static_cast<quint16>(y));
appendU16(update, static_cast<quint16>(width));
appendU16(update, static_cast<quint16>(height));
update.append(char(0)); update.append(char(0)); update.append(char(0));
update.append(char(16)); // encoding = ZRLE
appendU32(update, static_cast<quint32>(compressed.size()));
update.append(compressed);
return update;
}
// Compresses `input` as a single, complete, self-contained zlib stream
// (its own header + Z_FINISH) -- correct for tests where each
// VncSessionBackend under test lazily starts its own fresh inflate stream
// on the first ZRLE rectangle it ever sees.
QByteArray zlibCompressWhole(const QByteArray& input)
{
z_stream stream{};
deflateInit(&stream, Z_DEFAULT_COMPRESSION);
QByteArray output(static_cast<int>(deflateBound(&stream, static_cast<uLong>(input.size()))),
char(0));
stream.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(input.constData()));
stream.avail_in = static_cast<uInt>(input.size());
stream.next_out = reinterpret_cast<Bytef*>(output.data());
stream.avail_out = static_cast<uInt>(output.size());
deflate(&stream, Z_FINISH);
output.resize(output.size() - static_cast<int>(stream.avail_out));
deflateEnd(&stream);
return output;
}
// Compresses `input` as one chunk of an ongoing (not yet finished, unless
// `finish` is true) zlib stream carried in `stream` across calls -- used to
// build two separately-decodable compressed chunks that only both decode
// correctly against a single persistent inflate stream, for testing that
// ZRLE's zlib stream is retained across rectangles/updates rather than
// reset each time.
QByteArray zlibCompressChunk(z_stream& stream, const QByteArray& input, bool finish)
{
QByteArray output(static_cast<int>(deflateBound(&stream, static_cast<uLong>(input.size()))) + 64,
char(0));
stream.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(input.constData()));
stream.avail_in = static_cast<uInt>(input.size());
const int flushMode = finish ? Z_FINISH : Z_SYNC_FLUSH;
int totalOut = 0;
do {
stream.next_out = reinterpret_cast<Bytef*>(output.data() + totalOut);
stream.avail_out = static_cast<uInt>(output.size() - totalOut);
deflate(&stream, flushMode);
totalOut = output.size() - static_cast<int>(stream.avail_out);
} while (stream.avail_out == 0);
output.resize(totalOut);
return output;
}
// Mirrors VncSessionBackend's compact-length decode in reverse, for
// building test fixtures.
void appendTightCompactLength(QByteArray& buf, quint32 length)
{
buf.append(static_cast<char>((length & 0x7F) | ((length > 0x7F) ? 0x80 : 0)));
if (length <= 0x7F) {
return;
}
length >>= 7;
buf.append(static_cast<char>((length & 0x7F) | ((length > 0x7F) ? 0x80 : 0)));
if (length <= 0x7F) {
return;
}
length >>= 7;
buf.append(static_cast<char>(length & 0xFF));
}
// A FramebufferUpdate message with exactly one Tight-encoded rectangle
// (RFC 6143 encoding type 7), given the already fully-formed byte stream
// to follow the rectangle header (starting with the compression-control
// byte).
QByteArray tightFramebufferUpdate(int x, int y, int width, int height, const QByteArray& tightData)
{
QByteArray update;
update.append(char(0)); update.append(char(0));
update.append(char(0)); update.append(char(1)); // 1 rectangle
appendU16(update, static_cast<quint16>(x));
appendU16(update, static_cast<quint16>(y));
appendU16(update, static_cast<quint16>(width));
appendU16(update, static_cast<quint16>(height));
update.append(char(0)); update.append(char(0)); update.append(char(0));
update.append(char(7)); // encoding = Tight
update.append(tightData);
return update;
}
// Encodes a solid-colored width x height baseline JPEG using libjpeg-turbo
// directly (mirroring the decode side's use of the same library), for
// exercising Tight's JPEG sub-mode against real, valid JPEG bytes.
QByteArray encodeJpegForTest(int width, int height, QRgb color)
{
jpeg_compress_struct cinfo;
jpeg_error_mgr jerr;
cinfo.err = jpeg_std_error(&jerr);
jpeg_create_compress(&cinfo);
unsigned char* buffer = nullptr;
unsigned long bufferSize = 0;
jpeg_mem_dest(&cinfo, &buffer, &bufferSize);
cinfo.image_width = static_cast<JDIMENSION>(width);
cinfo.image_height = static_cast<JDIMENSION>(height);
cinfo.input_components = 3;
cinfo.in_color_space = JCS_RGB;
jpeg_set_defaults(&cinfo);
jpeg_set_quality(&cinfo, 100, TRUE);
// Disable chroma subsampling so a solid color round-trips as exactly
// as JPEG's DCT quantization allows, for a tighter test tolerance.
for (int i = 0; i < cinfo.num_components; ++i) {
cinfo.comp_info[i].h_samp_factor = 1;
cinfo.comp_info[i].v_samp_factor = 1;
}
jpeg_start_compress(&cinfo, TRUE);
QVector<unsigned char> row(width * 3);
for (int x = 0; x < width; ++x) {
row[(x * 3) + 0] = static_cast<unsigned char>(qRed(color));
row[(x * 3) + 1] = static_cast<unsigned char>(qGreen(color));
row[(x * 3) + 2] = static_cast<unsigned char>(qBlue(color));
}
JSAMPROW rowPointer[1] = { row.data() };
for (int y = 0; y < height; ++y) {
jpeg_write_scanlines(&cinfo, rowPointer, 1);
}
jpeg_finish_compress(&cinfo);
const QByteArray result(reinterpret_cast<const char*>(buffer), static_cast<int>(bufferSize));
jpeg_destroy_compress(&cinfo);
std::free(buffer);
return result;
}
// A small (512-bit) DH group generated fresh at test-run time -- rather
// than a hardcoded literal prime, to avoid any transcription risk -- plus
// one keypair against it, used to exercise VncAppleDhAuth::computeResponse()
// both directly and via a fake server acting as an Apple Screen Sharing
// (security type 30) endpoint. 512 bits keeps generation fast while still
// avoiding any "key too small" policy rejection from OpenSSL's default
// provider that a truly tiny hand-picked prime might trigger.
struct ToyDhKeypair {
QByteArray generatorBytes;
QByteArray primeBytes;
QByteArray publicKeyBytes;
BIGNUM* privateExponent = nullptr; // caller must BN_free
BIGNUM* prime = nullptr; // caller must BN_free
};
ToyDhKeypair generateToyDhKeypair()
{
ToyDhKeypair result;
BIGNUM* p = BN_new();
BN_generate_prime_ex(p, 512, 0, nullptr, nullptr, nullptr);
BIGNUM* g = BN_new();
BN_set_word(g, 2);
BN_CTX* ctx = BN_CTX_new();
BIGNUM* priv = BN_new();
BN_rand(priv, 256, -1, 0);
BIGNUM* pub = BN_new();
BN_mod_exp(pub, g, priv, p, ctx);
const int primeLen = BN_num_bytes(p);
result.primeBytes = QByteArray(primeLen, char(0));
BN_bn2binpad(p, reinterpret_cast<unsigned char*>(result.primeBytes.data()), primeLen);
// The real wire format's generator field is a literal 2 bytes (see
// appleDhAuthServerMessage()), so keep this the same size here too.
result.generatorBytes = QByteArray::fromHex("0002");
result.publicKeyBytes = QByteArray(primeLen, char(0));
BN_bn2binpad(pub, reinterpret_cast<unsigned char*>(result.publicKeyBytes.data()), primeLen);
result.privateExponent = priv;
result.prime = p;
BN_free(g);
BN_free(pub);
BN_CTX_free(ctx);
return result;
}
// The bytes a real Apple Screen Sharing server sends immediately after the
// client selects security type 30, before ClientInit/ServerInit even
// happen. Format confirmed empirically against a real macOS server: a
// literal 2-byte generator (NOT length-prefixed -- there is no separate
// generator-length field, unlike a first guess at this scheme might
// assume), then a 2-byte key length applying to both the prime and the
// server's public key that follow.
QByteArray appleDhAuthServerMessage(const QByteArray& generator, const QByteArray& prime,
const QByteArray& serverPublicKey)
{
QByteArray msg;
msg += generator;
appendU16(msg, static_cast<quint16>(prime.size()));
msg += prime;
msg += serverPublicKey;
return msg;
}
}
class TestVncSessionBackend : public QObject
{
Q_OBJECT
private slots:
// Pure-function coverage.
void desKeyFromPasswordMatchesKnownVector();
void desKeyFromPasswordPadsShortPasswords();
void desKeyFromPasswordTruncatesLongPasswords();
void vncAuthResponseIsSixteenBytesAndDeterministic();
void vncAuthResponseRejectsWrongChallengeSize();
void keysymForQtKeyMapsNamedKeys();
void keysymForQtKeyMapsFunctionKeys();
void keysymForQtKeyPassesThroughPrintableText();
void keysymForQtKeyUsesUnicodeConventionBeyondLatin1();
void keysymForQtKeyReturnsZeroForUnmapped();
void mapSocketErrorCoversCommonCases();
// State-machine coverage against a scripted in-process fake server.
void init();
void cleanup();
void connectsWithNoAuthRfb38();
void connectsWithVncAuthenticationRfb38();
void authFailureRfb38ReachesFailedState();
void unsupportedSecurityTypeReachesFailedState();
void rfb33ServerWithNoAuthConnectsWithoutSecurityResult();
void rawFramebufferUpdateProducesExpectedPixels();
void copyRectEncodingDoesNotFailConnection();
void unannouncedEncodingFailsConnectionWithClearMessage();
void serverCutTextEmitsRemoteClipboardTextChanged();
void setClipboardTextSendsClientCutText();
void cursorPseudoEncodingProducesExpectedImageAndHotspot();
void zeroSizeCursorPseudoEncodingHidesCursor();
void hextileRawTileProducesExpectedPixels();
void hextileBackgroundOnlyTileFillsSolidColor();
void hextileUncolouredSubrectUsesForeground();
void hextileColouredSubrectUsesOwnColor();
void hextileMultiTilePersistsBackgroundAcrossTiles();
void zrleRawTileProducesExpectedPixels();
void zrleSolidTileFillsColor();
void zrlePackedPaletteTileProducesExpectedPixels();
void zrlePlainRleTileProducesExpectedPixels();
void zrlePaletteRleTileProducesExpectedPixels();
void zrleStreamPersistsAcrossTwoFramebufferUpdates();
void tightFillProducesExpectedPixels();
void tightBasicCopyFilterProducesExpectedPixels();
void tightBasicPaletteFilterProducesExpectedPixels();
void tightJpegProducesExpectedPixels();
void tightStreamResetFlagAllowsIndependentDecoding();
void appleDhAuthRoundTripDecryptsToOriginalCredentials();
void connectsWithAppleDhAuthenticationRfb38();
void appleDhAuthIsPreferredOverVncAuthWhenBothOffered();
void appleDhAuthAcceptsRealCapturedMacOsServerParameters();
private:
std::unique_ptr<FakeVncServer> m_server;
std::unique_ptr<VncSessionBackend> m_backend;
SessionState m_lastState = SessionState::Disconnected;
QString m_lastErrorDisplay;
QString m_lastErrorRaw;
QImage m_lastFrame;
bool m_gotFrame = false;
};
void TestVncSessionBackend::desKeyFromPasswordMatchesKnownVector()
{
QCOMPARE(VncSessionBackend::desKeyFromPassword(QStringLiteral("COW")), expectedCowKey());
}
void TestVncSessionBackend::desKeyFromPasswordPadsShortPasswords()
{
const QByteArray key = VncSessionBackend::desKeyFromPassword(QStringLiteral(""));
QCOMPARE(key, QByteArray(8, char(0)));
}
void TestVncSessionBackend::desKeyFromPasswordTruncatesLongPasswords()
{
// Only the first 8 characters are ever used as the DES key.
const QByteArray key1 = VncSessionBackend::desKeyFromPassword(QStringLiteral("12345678"));
const QByteArray key2 = VncSessionBackend::desKeyFromPassword(QStringLiteral("12345678ignored"));
QCOMPARE(key1, key2);
QCOMPARE(key1.size(), 8);
}
void TestVncSessionBackend::vncAuthResponseIsSixteenBytesAndDeterministic()
{
const QByteArray challenge(16, char(0x42));
const QByteArray response1 = VncSessionBackend::vncAuthResponse(challenge, QStringLiteral("secret"));
const QByteArray response2 = VncSessionBackend::vncAuthResponse(challenge, QStringLiteral("secret"));
QCOMPARE(response1.size(), 16);
QCOMPARE(response1, response2);
const QByteArray differentPassword =
VncSessionBackend::vncAuthResponse(challenge, QStringLiteral("other"));
QVERIFY(response1 != differentPassword);
}
void TestVncSessionBackend::vncAuthResponseRejectsWrongChallengeSize()
{
QVERIFY(VncSessionBackend::vncAuthResponse(QByteArray(15, char(0)), QStringLiteral("x")).isEmpty());
QVERIFY(VncSessionBackend::vncAuthResponse(QByteArray(), QStringLiteral("x")).isEmpty());
}
void TestVncSessionBackend::keysymForQtKeyMapsNamedKeys()
{
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Backspace, QString()), quint32(0xff08));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Tab, QString()), quint32(0xff09));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Return, QString()), quint32(0xff0d));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Escape, QString()), quint32(0xff1b));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Delete, QString()), quint32(0xffff));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Left, QString()), quint32(0xff51));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Up, QString()), quint32(0xff52));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Right, QString()), quint32(0xff53));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Down, QString()), quint32(0xff54));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Shift, QString()), quint32(0xffe1));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Control, QString()), quint32(0xffe3));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Alt, QString()), quint32(0xffe9));
}
void TestVncSessionBackend::keysymForQtKeyMapsFunctionKeys()
{
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_F1, QString()), quint32(0xffbe));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_F12, QString()), quint32(0xffc9));
}
void TestVncSessionBackend::keysymForQtKeyPassesThroughPrintableText()
{
// Printable ASCII/Latin-1 keysyms are just the codepoint itself.
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_A, QStringLiteral("a")), quint32('a'));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_5, QStringLiteral("5")), quint32('5'));
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_Space, QStringLiteral(" ")), quint32(' '));
}
void TestVncSessionBackend::keysymForQtKeyUsesUnicodeConventionBeyondLatin1()
{
// X11's convention for Unicode codepoints beyond Latin-1: keysym =
// 0x01000000 + codepoint. Euro sign U+20AC as an example.
const QString euro = QString::fromUtf8("\xE2\x82\xAC");
QCOMPARE(VncSessionBackend::keysymForQtKey(0, euro), quint32(0x01000000u + 0x20ACu));
}
void TestVncSessionBackend::keysymForQtKeyReturnsZeroForUnmapped()
{
QCOMPARE(VncSessionBackend::keysymForQtKey(Qt::Key_MediaPlay, QString()), quint32(0));
}
void TestVncSessionBackend::mapSocketErrorCoversCommonCases()
{
QCOMPARE(VncSessionBackend::mapSocketError(QAbstractSocket::ConnectionRefusedError, QString()),
QStringLiteral("Connection refused by remote host."));
QCOMPARE(VncSessionBackend::mapSocketError(QAbstractSocket::HostNotFoundError, QString()),
QStringLiteral("Host could not be resolved."));
QCOMPARE(VncSessionBackend::mapSocketError(QAbstractSocket::SocketTimeoutError, QString()),
QStringLiteral("Connection timed out."));
QVERIFY(!VncSessionBackend::mapSocketError(QAbstractSocket::UnknownSocketError,
QStringLiteral("raw detail"))
.isEmpty());
}
void TestVncSessionBackend::init()
{
m_server = std::make_unique<FakeVncServer>();
m_backend =
std::make_unique<VncSessionBackend>(makeVncProfile(m_server->port()), nullptr);
m_lastState = SessionState::Disconnected;
m_lastErrorDisplay.clear();
m_lastErrorRaw.clear();
m_lastFrame = QImage();
m_gotFrame = false;
connect(m_backend.get(), &SessionBackend::stateChanged, this,
[this](SessionState state, const QString&) { m_lastState = state; });
connect(m_backend.get(), &SessionBackend::connectionError, this,
[this](const QString& display, const QString& raw) {
m_lastErrorDisplay = display;
m_lastErrorRaw = raw;
});
connect(m_backend.get(), &SessionBackend::frameUpdated, this, [this](const QImage& frame) {
m_lastFrame = frame;
m_gotFrame = true;
});
}
void TestVncSessionBackend::cleanup()
{
if (m_backend) {
m_backend->disconnectSession();
}
m_backend.reset();
m_server.reset();
}
void TestVncSessionBackend::connectsWithNoAuthRfb38()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
m_server->received.clear();
switch (m_server->nextStep()) {
case 0: { // client's version reply
QByteArray securityTypes;
securityTypes.append(char(1)); // count = 1
securityTypes.append(char(1)); // type 1 = None
m_server->sendWhenConnected(securityTypes);
break;
}
case 1: // client's security-type selection (byte value 1)
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2: { // ClientInit (shared-flag byte)
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(4)); // width = 4
serverInit.append(char(0)); serverInit.append(char(2)); // height = 2
serverInit.append(QByteArray(16, char(0))); // pixel format (ignored by client)
serverInit.append(QByteArray(4, char(0))); // name length = 0
m_server->sendWhenConnected(serverInit);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_COMPARE(m_lastState, SessionState::Connected);
}
void TestVncSessionBackend::connectsWithVncAuthenticationRfb38()
{
const QString password = QStringLiteral("secret1");
const QByteArray challenge(16, char(0x11));
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, &challenge]() {
m_server->received.clear();
switch (m_server->nextStep()) {
case 0: { // version reply
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(2)); // type 2 = VNC Authentication
m_server->sendWhenConnected(securityTypes);
break;
}
case 1: // security-type selection
m_server->sendWhenConnected(challenge);
break;
case 2: // 16-byte DES response (content itself checked separately below)
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 3: { // ClientInit
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(2));
serverInit.append(char(0)); serverInit.append(char(2));
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
m_server->sendWhenConnected(serverInit);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions(password));
QTRY_COMPARE(m_lastState, SessionState::Connected);
// Independently verify the response the fake server actually received
// at step 2 was the correct DES-encrypted challenge -- can't capture it
// mid-script above without complicating the dispatch, so just replay
// the expected computation here for comparison purposes is redundant;
// instead this is implicitly proven by reaching Connected at all, since
// a real VncSessionBackend only proceeds past WaitingSecurityResult
// (here, an explicit `OK`) after sending *some* response and the
// server unconditionally accepts it in this script. The actual byte
// correctness of vncAuthResponse() is covered directly by
// vncAuthResponseIsSixteenBytesAndDeterministic() and the "COW" key
// vector above.
}
void TestVncSessionBackend::authFailureRfb38ReachesFailedState()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
m_server->received.clear();
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(2));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(16, char(0x22))); // challenge
break;
case 2: {
QByteArray result;
result.append(char(0)); result.append(char(0)); result.append(char(0));
result.append(char(1)); // SecurityResult: failed
const QByteArray reason = QByteArray("bad password");
result.append(char(0)); result.append(char(0)); result.append(char(0));
result.append(static_cast<char>(reason.size()));
result.append(reason);
m_server->sendWhenConnected(result);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions(QStringLiteral("wrong")));
QTRY_COMPARE(m_lastState, SessionState::Failed);
QVERIFY(m_lastErrorDisplay.contains(QStringLiteral("bad password")));
}
void TestVncSessionBackend::unsupportedSecurityTypeReachesFailedState()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
m_server->received.clear();
if (m_server->nextStep() == 0) {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(19)); // VeNCrypt (TLS) -- unsupported here
m_server->sendWhenConnected(securityTypes);
}
});
m_backend->connectSession(makeOptions());
QTRY_COMPARE(m_lastState, SessionState::Failed);
QVERIFY(m_lastErrorDisplay.contains(QStringLiteral("doesn't support")));
}
void TestVncSessionBackend::rfb33ServerWithNoAuthConnectsWithoutSecurityResult()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.003\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
m_server->received.clear();
switch (m_server->nextStep()) {
case 0: { // version reply
QByteArray securityType(4, char(0));
securityType[3] = char(1); // type 1 = None, sent directly (3.3 style)
m_server->sendWhenConnected(securityType);
break;
}
case 1: { // ClientInit arrives directly -- 3.3 has no SecurityResult at all
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(1));
serverInit.append(char(0)); serverInit.append(char(1));
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
m_server->sendWhenConnected(serverInit);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_COMPARE(m_lastState, SessionState::Connected);
}
void TestVncSessionBackend::rawFramebufferUpdateProducesExpectedPixels()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
m_server->received.clear();
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2: { // ClientInit
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(2)); // width = 2
serverInit.append(char(0)); serverInit.append(char(1)); // height = 1
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
m_server->sendWhenConnected(serverInit);
break;
}
case 3: {
// First FramebufferUpdateRequest -- reply with a single Raw
// rectangle covering the whole 2x1 framebuffer: one red pixel,
// one green pixel (as bytes matching the requested 32bpp
// little-endian R@16/G@8/B@0 format: B,G,R,pad per pixel).
QByteArray update;
update.append(char(0)); // message-type: FramebufferUpdate
update.append(char(0)); // padding
update.append(char(0)); update.append(char(1)); // 1 rectangle
update.append(char(0)); update.append(char(0)); // x = 0
update.append(char(0)); update.append(char(0)); // y = 0
update.append(char(0)); update.append(char(2)); // width = 2
update.append(char(0)); update.append(char(1)); // height = 1
update.append(char(0)); update.append(char(0)); update.append(char(0));
update.append(char(0)); // encoding = 0 (Raw)
// pixel 0: red (B=0,G=0,R=255,pad=0)
update.append(char(0)); update.append(char(0)); update.append(char(0xff));
update.append(char(0));
// pixel 1: green (B=0,G=255,R=0,pad=0)
update.append(char(0)); update.append(char(0xff)); update.append(char(0));
update.append(char(0));
m_server->sendWhenConnected(update);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(2, 1));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(255, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(0, 255, 0));
}
// Regression guard for the WaitingRectangleHeader dispatch: every encoding
// this backend announces via SetEncodings must have a working decode path.
// Raw is already exercised by rawFramebufferUpdateProducesExpectedPixels();
// this covers CopyRect. Whenever a new encoding is added to
// kAnnouncedEncodings in vnc_session_backend.cpp, a matching test belongs
// here (or nearby) so the announced list and the dispatch switch can never
// silently drift apart.
void TestVncSessionBackend::copyRectEncodingDoesNotFailConnection()
{
int frameCount = 0;
connect(m_backend.get(), &SessionBackend::frameUpdated, this,
[&frameCount](const QImage&) { ++frameCount; });
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
m_server->received.clear();
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2: { // ClientInit
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(2)); // width = 2
serverInit.append(char(0)); serverInit.append(char(2)); // height = 2
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
m_server->sendWhenConnected(serverInit);
break;
}
case 3: { // initial non-incremental request -> seed the framebuffer with Raw
QByteArray update;
update.append(char(0)); update.append(char(0));
update.append(char(0)); update.append(char(1)); // 1 rectangle
update.append(char(0)); update.append(char(0)); // x
update.append(char(0)); update.append(char(0)); // y
update.append(char(0)); update.append(char(2)); // width
update.append(char(0)); update.append(char(2)); // height
update.append(char(0)); update.append(char(0)); update.append(char(0));
update.append(char(0)); // encoding = Raw
update.append(QByteArray(2 * 2 * 4, char(0x11)));
m_server->sendWhenConnected(update);
break;
}
case 4: { // next incremental request -> a CopyRect rectangle
QByteArray update;
update.append(char(0)); update.append(char(0));
update.append(char(0)); update.append(char(1)); // 1 rectangle
update.append(char(0)); update.append(char(0)); // x
update.append(char(0)); update.append(char(0)); // y
update.append(char(0)); update.append(char(2)); // width
update.append(char(0)); update.append(char(2)); // height
update.append(char(0)); update.append(char(0)); update.append(char(0));
update.append(char(1)); // encoding = CopyRect
update.append(char(0)); update.append(char(0)); // src x = 0
update.append(char(0)); update.append(char(0)); // src y = 0
m_server->sendWhenConnected(update);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(frameCount >= 2);
QCOMPARE(m_lastState, SessionState::Connected);
}
void TestVncSessionBackend::unannouncedEncodingFailsConnectionWithClearMessage()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
m_server->received.clear();
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2: { // ClientInit
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(1)); // width = 1
serverInit.append(char(0)); serverInit.append(char(1)); // height = 1
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
m_server->sendWhenConnected(serverInit);
break;
}
case 3: { // a rectangle whose encoding was never announced via SetEncodings
QByteArray update;
update.append(char(0)); update.append(char(0));
update.append(char(0)); update.append(char(1)); // 1 rectangle
update.append(char(0)); update.append(char(0)); // x
update.append(char(0)); update.append(char(0)); // y
update.append(char(0)); update.append(char(1)); // width
update.append(char(0)); update.append(char(1)); // height
update.append(char(0)); update.append(char(0)); update.append(char(0));
update.append(char(99)); // encoding = 99, never announced
m_server->sendWhenConnected(update);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_COMPARE(m_lastState, SessionState::Failed);
QVERIFY(m_lastErrorDisplay.contains(QStringLiteral("non-compliant")));
}
void TestVncSessionBackend::serverCutTextEmitsRemoteClipboardTextChanged()
{
QString lastClipboardText;
connect(m_backend.get(), &SessionBackend::remoteClipboardTextChanged, this,
[&lastClipboardText](const QString& text) { lastClipboardText = text; });
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2: { // ClientInit -- reply with ServerInit, then a ServerCutText message
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(1));
serverInit.append(char(0)); serverInit.append(char(1));
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
QByteArray cutText;
cutText.append(char(3)); // message-type: ServerCutText
cutText.append(3, char(0)); // padding
const QByteArray text = QByteArray("hello clipboard");
cutText.append(char(0)); cutText.append(char(0)); cutText.append(char(0));
cutText.append(static_cast<char>(text.size())); // length (BE, fits one byte here)
m_server->sendWhenConnected(serverInit + cutText + text);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_COMPARE(lastClipboardText, QStringLiteral("hello clipboard"));
}
void TestVncSessionBackend::setClipboardTextSendsClientCutText()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2: { // ClientInit
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(1));
serverInit.append(char(0)); serverInit.append(char(1));
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
m_server->sendWhenConnected(serverInit);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_COMPARE(m_lastState, SessionState::Connected);
m_backend->setClipboardText(QStringLiteral("hi"));
// The exact ClientCutText bytes must eventually appear at the tail of
// whatever the fake server received -- checked with endsWith() rather
// than an exact match on the whole buffer, since exactly how the
// preceding handshake/SetEncodings/FramebufferUpdateRequest bytes get
// chunked into TCP reads isn't guaranteed and doesn't matter here.
QByteArray expected;
expected.append(char(6)); // message-type: ClientCutText
expected.append(3, char(0)); // padding
expected.append(char(0)); expected.append(char(0)); expected.append(char(0));
expected.append(char(2)); // length = 2
expected.append("hi");
QTRY_VERIFY(m_server->received.endsWith(expected));
}
void TestVncSessionBackend::cursorPseudoEncodingProducesExpectedImageAndHotspot()
{
QImage lastCursorImage;
QPoint lastHotspot;
bool gotCursor = false;
connect(m_backend.get(), &SessionBackend::cursorImageChanged, this,
[&](const QImage& image, const QPoint& hotspot) {
lastCursorImage = image;
lastHotspot = hotspot;
gotCursor = true;
});
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2: { // ClientInit
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(4)); // width = 4
serverInit.append(char(0)); serverInit.append(char(4)); // height = 4
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
m_server->sendWhenConnected(serverInit);
break;
}
case 3: { // first FramebufferUpdateRequest -> a Cursor pseudo-encoding rectangle
QByteArray update;
update.append(char(0)); update.append(char(0));
update.append(char(0)); update.append(char(1)); // 1 rectangle
update.append(char(0)); update.append(char(1)); // x = hotspot x = 1
update.append(char(0)); update.append(char(1)); // y = hotspot y = 1
update.append(char(0)); update.append(char(2)); // width = 2
update.append(char(0)); update.append(char(1)); // height = 1
update.append(char(0xff)); update.append(char(0xff));
update.append(char(0xff)); update.append(char(0x11)); // encoding = -239 (Cursor)
// pixel 0: red, opaque (byte order B,G,R,pad, per our negotiated format)
update.append(char(0)); update.append(char(0)); update.append(char(0xff));
update.append(char(0));
// pixel 1: blue, will be masked transparent
update.append(char(0xff)); update.append(char(0)); update.append(char(0));
update.append(char(0));
// mask: ceil(2/8)=1 byte/row * height 1 -- bit7=pixel0 opaque, bit6=pixel1 transparent
update.append(char(0x80));
m_server->sendWhenConnected(update);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(gotCursor);
QCOMPARE(lastCursorImage.size(), QSize(2, 1));
QCOMPARE(lastHotspot, QPoint(1, 1));
QCOMPARE(lastCursorImage.pixelColor(0, 0), QColor(255, 0, 0, 255));
QCOMPARE(lastCursorImage.pixelColor(1, 0).alpha(), 0);
}
void TestVncSessionBackend::zeroSizeCursorPseudoEncodingHidesCursor()
{
bool gotHidden = false;
connect(m_backend.get(), &SessionBackend::cursorHidden, this,
[&gotHidden]() { gotHidden = true; });
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2: { // ClientInit
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(1));
serverInit.append(char(0)); serverInit.append(char(1));
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
m_server->sendWhenConnected(serverInit);
break;
}
case 3: { // a 0x0 Cursor rectangle means "hide the cursor"
QByteArray update;
update.append(char(0)); update.append(char(0));
update.append(char(0)); update.append(char(1)); // 1 rectangle
update.append(char(0)); update.append(char(0)); // x (hotspot, unused here)
update.append(char(0)); update.append(char(0)); // y
update.append(char(0)); update.append(char(0)); // width = 0
update.append(char(0)); update.append(char(0)); // height = 0
update.append(char(0xff)); update.append(char(0xff));
update.append(char(0xff)); update.append(char(0x11)); // encoding = -239 (Cursor)
m_server->sendWhenConnected(update);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(gotHidden);
}
void TestVncSessionBackend::hextileRawTileProducesExpectedPixels()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(2, 2));
break;
case 3: { // a single Raw-subencoding Hextile tile covering the whole 2x2 rectangle
QByteArray tile;
tile.append(char(0x01)); // HextileFlags::kRaw
tile += pixelBytes(255, 0, 0);
tile += pixelBytes(0, 255, 0);
tile += pixelBytes(0, 0, 255);
tile += pixelBytes(255, 255, 255);
m_server->sendWhenConnected(hextileFramebufferUpdate(0, 0, 2, 2, tile));
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(2, 2));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(255, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(0, 255, 0));
QCOMPARE(m_lastFrame.pixelColor(0, 1), QColor(0, 0, 255));
QCOMPARE(m_lastFrame.pixelColor(1, 1), QColor(255, 255, 255));
}
void TestVncSessionBackend::hextileBackgroundOnlyTileFillsSolidColor()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(3, 3));
break;
case 3: { // BackgroundSpecified only, no subrects -- solid fill
QByteArray tile;
tile.append(char(0x02)); // HextileFlags::kBackgroundSpecified
tile += pixelBytes(0, 128, 255);
m_server->sendWhenConnected(hextileFramebufferUpdate(0, 0, 3, 3, tile));
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(3, 3));
for (int y = 0; y < 3; ++y) {
for (int x = 0; x < 3; ++x) {
QCOMPARE(m_lastFrame.pixelColor(x, y), QColor(0, 128, 255));
}
}
}
void TestVncSessionBackend::hextileUncolouredSubrectUsesForeground()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(4, 4));
break;
case 3: {
// Background(black) + Foreground(red) + one uncoloured 2x2
// subrect at tile-local (1,1) -- must render in the foreground
// color, everywhere else in the background color.
QByteArray tile;
tile.append(char(0x02 | 0x04 | 0x08)); // Background|Foreground|AnySubrects
tile += pixelBytes(0, 0, 0); // background
tile += pixelBytes(255, 0, 0); // foreground
tile.append(char(1)); // subrect count
tile.append(char(0x11)); // xy: x=1, y=1
tile.append(char(0x11)); // wh: width=2, height=2
m_server->sendWhenConnected(hextileFramebufferUpdate(0, 0, 4, 4, tile));
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(4, 4));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(0, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(1, 1), QColor(255, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(2, 2), QColor(255, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(3, 3), QColor(0, 0, 0));
}
void TestVncSessionBackend::hextileColouredSubrectUsesOwnColor()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(3, 3));
break;
case 3: {
// No background/foreground specified this tile -- both stay at
// the rectangle-start default (black). One individually-
// coloured 1x1 subrect at tile-local (0,0).
QByteArray tile;
tile.append(char(0x08 | 0x10)); // AnySubrects|SubrectsColoured
tile.append(char(1)); // subrect count
tile += pixelBytes(0, 255, 0); // this subrect's own color
tile.append(char(0x00)); // xy: x=0, y=0
tile.append(char(0x00)); // wh: width=1, height=1
m_server->sendWhenConnected(hextileFramebufferUpdate(0, 0, 3, 3, tile));
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(3, 3));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(0, 255, 0));
QCOMPARE(m_lastFrame.pixelColor(1, 1), QColor(0, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(2, 2), QColor(0, 0, 0));
}
void TestVncSessionBackend::hextileMultiTilePersistsBackgroundAcrossTiles()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(20, 20));
break;
case 3: {
// A 20x20 rectangle tiles as (0,0) 16x16, (16,0) 4x16,
// (0,16) 16x4, (16,16) 4x4 -- only the first tile specifies a
// background color; the other three specify nothing at all
// (subencoding 0x00) and must inherit it.
QByteArray tile1;
tile1.append(char(0x02)); // BackgroundSpecified
tile1 += pixelBytes(0, 0, 255);
QByteArray tile2(1, char(0x00));
QByteArray tile3(1, char(0x00));
QByteArray tile4(1, char(0x00));
m_server->sendWhenConnected(
hextileFramebufferUpdate(0, 0, 20, 20, tile1 + tile2 + tile3 + tile4));
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(20, 20));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(0, 0, 255));
QCOMPARE(m_lastFrame.pixelColor(19, 0), QColor(0, 0, 255));
QCOMPARE(m_lastFrame.pixelColor(0, 19), QColor(0, 0, 255));
QCOMPARE(m_lastFrame.pixelColor(19, 19), QColor(0, 0, 255));
QCOMPARE(m_lastFrame.pixelColor(10, 10), QColor(0, 0, 255));
}
void TestVncSessionBackend::zrleRawTileProducesExpectedPixels()
{
QByteArray tile;
tile.append(char(0)); // Raw
tile += cpixelBytes(255, 0, 0);
tile += cpixelBytes(0, 255, 0);
const QByteArray compressed = zlibCompressWhole(tile);
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(2, 1));
break;
case 3:
m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 2, 1, compressed));
break;
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(2, 1));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(255, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(0, 255, 0));
}
void TestVncSessionBackend::zrleSolidTileFillsColor()
{
QByteArray tile;
tile.append(char(1)); // Solid
tile += cpixelBytes(0, 128, 255);
const QByteArray compressed = zlibCompressWhole(tile);
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(2, 2));
break;
case 3:
m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 2, 2, compressed));
break;
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(2, 2));
for (int y = 0; y < 2; ++y) {
for (int x = 0; x < 2; ++x) {
QCOMPARE(m_lastFrame.pixelColor(x, y), QColor(0, 128, 255));
}
}
}
void TestVncSessionBackend::zrlePackedPaletteTileProducesExpectedPixels()
{
// 4 pixels, 2-color palette, 1 bit/pixel, packed MSB-first: indices
// [0,1,1,0] -> byte 0b0110_0000.
QByteArray tile;
tile.append(char(2)); // packed palette, paletteSize = 2
tile += cpixelBytes(10, 20, 30); // palette[0]
tile += cpixelBytes(200, 210, 220); // palette[1]
tile.append(char(0x60));
const QByteArray compressed = zlibCompressWhole(tile);
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(4, 1));
break;
case 3:
m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 4, 1, compressed));
break;
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(4, 1));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(10, 20, 30));
QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(200, 210, 220));
QCOMPARE(m_lastFrame.pixelColor(2, 0), QColor(200, 210, 220));
QCOMPARE(m_lastFrame.pixelColor(3, 0), QColor(10, 20, 30));
}
void TestVncSessionBackend::zrlePlainRleTileProducesExpectedPixels()
{
// One run of length 3 (encoded as total=2, single continuation byte
// since 2 != 255) covering the whole 3x1 tile.
QByteArray tile;
tile.append(char(128)); // Plain RLE
tile += cpixelBytes(50, 60, 70);
tile.append(char(2));
const QByteArray compressed = zlibCompressWhole(tile);
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(3, 1));
break;
case 3:
m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 3, 1, compressed));
break;
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(3, 1));
for (int x = 0; x < 3; ++x) {
QCOMPARE(m_lastFrame.pixelColor(x, 0), QColor(50, 60, 70));
}
}
void TestVncSessionBackend::zrlePaletteRleTileProducesExpectedPixels()
{
// palette = [colorA, colorB]. Entries: index 0, run length 1 (index
// byte < 128, no run-length byte follows); then index 1, run length 2
// (index byte 0x81 = 128 + 1, followed by a run-length byte encoding
// total=1 -> actual length 2).
QByteArray tile;
tile.append(char(130)); // Palette RLE, paletteSize = 2
tile += cpixelBytes(1, 2, 3); // palette[0] = colorA
tile += cpixelBytes(4, 5, 6); // palette[1] = colorB
tile.append(char(0x00));
tile.append(char(0x81));
tile.append(char(0x01));
const QByteArray compressed = zlibCompressWhole(tile);
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(3, 1));
break;
case 3:
m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 3, 1, compressed));
break;
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(3, 1));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(1, 2, 3));
QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(4, 5, 6));
QCOMPARE(m_lastFrame.pixelColor(2, 0), QColor(4, 5, 6));
}
void TestVncSessionBackend::zrleStreamPersistsAcrossTwoFramebufferUpdates()
{
QByteArray tile1;
tile1.append(char(0)); // Raw
tile1 += cpixelBytes(255, 0, 0);
tile1 += cpixelBytes(0, 255, 0);
QByteArray tile2;
tile2.append(char(1)); // Solid
tile2 += cpixelBytes(10, 20, 30);
// One continuous deflate stream split into two chunks -- the second
// chunk is only decodable by an inflate stream that already consumed
// the first, which is exactly what should happen if
// VncSessionBackend's ZRLE stream persists across separate
// FramebufferUpdate messages instead of being reset each time.
z_stream deflateStream{};
deflateInit(&deflateStream, Z_DEFAULT_COMPRESSION);
const QByteArray compressed1 = zlibCompressChunk(deflateStream, tile1, false);
const QByteArray compressed2 = zlibCompressChunk(deflateStream, tile2, true);
deflateEnd(&deflateStream);
// Capture each frame as it arrives, rather than polling m_lastFrame
// between separately-timed QTRY_VERIFY checks -- the two updates can
// both land before the first check ever gets scheduled to run, so only
// recording state exactly when each signal fires is race-free here.
QVector<QImage> frames;
connect(m_backend.get(), &SessionBackend::frameUpdated, this,
[&frames](const QImage& frame) { frames.append(frame.copy()); });
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this,
[this, compressed1, compressed2]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(2, 1));
break;
case 3: // first (non-incremental) FramebufferUpdateRequest
m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 2, 1, compressed1));
break;
case 4: // next (incremental) FramebufferUpdateRequest
m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 2, 1, compressed2));
break;
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(frames.size() >= 2);
QCOMPARE(m_lastState, SessionState::Connected);
QCOMPARE(frames.at(0).pixelColor(0, 0), QColor(255, 0, 0));
QCOMPARE(frames.at(0).pixelColor(1, 0), QColor(0, 255, 0));
QCOMPARE(frames.at(1).pixelColor(0, 0), QColor(10, 20, 30));
QCOMPARE(frames.at(1).pixelColor(1, 0), QColor(10, 20, 30));
}
void TestVncSessionBackend::tightFillProducesExpectedPixels()
{
QByteArray tight;
tight.append(char(0x80)); // mode = Fill, no stream-reset bits
tight += cpixelBytes(10, 200, 30);
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, tight]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(3, 3));
break;
case 3:
m_server->sendWhenConnected(tightFramebufferUpdate(0, 0, 3, 3, tight));
break;
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(3, 3));
for (int y = 0; y < 3; ++y) {
for (int x = 0; x < 3; ++x) {
QCOMPARE(m_lastFrame.pixelColor(x, y), QColor(10, 200, 30));
}
}
}
void TestVncSessionBackend::tightBasicCopyFilterProducesExpectedPixels()
{
QByteArray filtered;
filtered += cpixelBytes(255, 0, 0);
filtered += cpixelBytes(0, 255, 0);
filtered += cpixelBytes(0, 0, 255);
filtered += cpixelBytes(255, 255, 255);
const QByteArray compressed = zlibCompressWhole(filtered);
QByteArray tight;
tight.append(char(0x00)); // mode = Basic, stream 0, Copy implied
appendTightCompactLength(tight, static_cast<quint32>(compressed.size()));
tight += compressed;
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, tight]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(2, 2));
break;
case 3:
m_server->sendWhenConnected(tightFramebufferUpdate(0, 0, 2, 2, tight));
break;
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(2, 2));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(255, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(0, 255, 0));
QCOMPARE(m_lastFrame.pixelColor(0, 1), QColor(0, 0, 255));
QCOMPARE(m_lastFrame.pixelColor(1, 1), QColor(255, 255, 255));
}
void TestVncSessionBackend::tightBasicPaletteFilterProducesExpectedPixels()
{
// 4 pixels, 2-color palette, 1 bit/pixel, packed MSB-first: indices
// [0,1,1,0] -> byte 0b0110_0000 (same packing convention as ZRLE's
// packed palette; with only one row here, continuous vs row-padded
// bit-packing happen to coincide).
QByteArray filtered;
filtered.append(char(0x01)); // paletteSize - 1 = 1 -> paletteSize = 2
filtered += cpixelBytes(10, 20, 30); // palette[0]
filtered += cpixelBytes(200, 210, 220); // palette[1]
filtered.append(char(0x60));
const QByteArray compressed = zlibCompressWhole(filtered);
QByteArray tight;
tight.append(char(0x40)); // mode = Basic, stream 0, explicit filter follows
tight.append(char(1)); // filter id = Palette
appendTightCompactLength(tight, static_cast<quint32>(compressed.size()));
tight += compressed;
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, tight]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(4, 1));
break;
case 3:
m_server->sendWhenConnected(tightFramebufferUpdate(0, 0, 4, 1, tight));
break;
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(4, 1));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(10, 20, 30));
QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(200, 210, 220));
QCOMPARE(m_lastFrame.pixelColor(2, 0), QColor(200, 210, 220));
QCOMPARE(m_lastFrame.pixelColor(3, 0), QColor(10, 20, 30));
}
void TestVncSessionBackend::tightJpegProducesExpectedPixels()
{
// 8x8 (one full JPEG MCU at 1x1 chroma sampling) solid red.
const QByteArray jpegBytes = encodeJpegForTest(8, 8, qRgb(220, 20, 20));
QByteArray tight;
tight.append(char(0x90)); // mode = JPEG
appendTightCompactLength(tight, static_cast<quint32>(jpegBytes.size()));
tight += jpegBytes;
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, tight]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(8, 8));
break;
case 3:
m_server->sendWhenConnected(tightFramebufferUpdate(0, 0, 8, 8, tight));
break;
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(8, 8));
// JPEG is lossy even at quality 100 -- assert closeness, not equality.
const QColor decoded = m_lastFrame.pixelColor(4, 4);
QVERIFY(qAbs(decoded.red() - 220) <= 8);
QVERIFY(qAbs(decoded.green() - 20) <= 8);
QVERIFY(qAbs(decoded.blue() - 20) <= 8);
}
void TestVncSessionBackend::tightStreamResetFlagAllowsIndependentDecoding()
{
QByteArray filteredA;
filteredA += cpixelBytes(10, 20, 30);
filteredA += cpixelBytes(10, 20, 30);
filteredA += cpixelBytes(10, 20, 30);
filteredA += cpixelBytes(10, 20, 30);
QByteArray filteredB;
filteredB += cpixelBytes(200, 210, 220);
filteredB += cpixelBytes(200, 210, 220);
filteredB += cpixelBytes(200, 210, 220);
filteredB += cpixelBytes(200, 210, 220);
// Each independently a complete, self-contained zlib stream (its own
// deflateInit + Z_FINISH) -- decoding the second one correctly after
// the first, on the same persistent stream slot, requires that
// stream-reset bit 0 in rectangle B's control byte actually tore down
// and re-initialized stream 0 rather than trying to keep feeding an
// already-finished stream more data.
const QByteArray compressedA = zlibCompressWhole(filteredA);
const QByteArray compressedB = zlibCompressWhole(filteredB);
QByteArray tightA;
tightA.append(char(0x00)); // mode = Basic, stream 0, Copy, no reset
appendTightCompactLength(tightA, static_cast<quint32>(compressedA.size()));
tightA += compressedA;
QByteArray tightB;
tightB.append(char(0x01)); // mode = Basic, stream 0, Copy, reset stream 0
appendTightCompactLength(tightB, static_cast<quint32>(compressedB.size()));
tightB += compressedB;
QVector<QImage> frames;
connect(m_backend.get(), &SessionBackend::frameUpdated, this,
[&frames](const QImage& frame) { frames.append(frame.copy()); });
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, tightA, tightB]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(2, 2));
break;
case 3: // first (non-incremental) FramebufferUpdateRequest
m_server->sendWhenConnected(tightFramebufferUpdate(0, 0, 2, 2, tightA));
break;
case 4: // next (incremental) FramebufferUpdateRequest
m_server->sendWhenConnected(tightFramebufferUpdate(0, 0, 2, 2, tightB));
break;
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(frames.size() >= 2);
QCOMPARE(m_lastState, SessionState::Connected);
QCOMPARE(frames.at(0).pixelColor(0, 0), QColor(10, 20, 30));
QCOMPARE(frames.at(1).pixelColor(0, 0), QColor(200, 210, 220));
}
void TestVncSessionBackend::appleDhAuthRoundTripDecryptsToOriginalCredentials()
{
ToyDhKeypair serverKeypair = generateToyDhKeypair();
const QString username = QStringLiteral("tester");
const QString password = QStringLiteral("s3cret-pass");
const VncAppleDhAuth::Response response = VncAppleDhAuth::computeResponse(
serverKeypair.generatorBytes, serverKeypair.primeBytes, serverKeypair.publicKeyBytes,
username, password);
QVERIFY(!response.clientPublicKey.isEmpty());
QCOMPARE(response.clientPublicKey.size(), serverKeypair.primeBytes.size());
QCOMPARE(response.encryptedCredentials.size(), 128);
// Derive the shared secret the way the *server* would, using the
// client's returned public key and the server's own private exponent
// -- Diffie-Hellman's commutativity means this must equal whatever
// computeResponse() used internally to derive its AES key.
BIGNUM* clientPub = BN_bin2bn(
reinterpret_cast<const unsigned char*>(response.clientPublicKey.constData()),
response.clientPublicKey.size(), nullptr);
BN_CTX* ctx = BN_CTX_new();
BIGNUM* sharedSecret = BN_new();
BN_mod_exp(sharedSecret, clientPub, serverKeypair.privateExponent, serverKeypair.prime, ctx);
const int secretLen = BN_num_bytes(sharedSecret);
QByteArray secretBytes(secretLen, char(0));
BN_bn2binpad(sharedSecret, reinterpret_cast<unsigned char*>(secretBytes.data()), secretLen);
unsigned char aesKey[16];
EVP_Digest(secretBytes.constData(), static_cast<size_t>(secretBytes.size()), aesKey, nullptr,
EVP_md5(), nullptr);
EVP_CIPHER_CTX* decCtx = EVP_CIPHER_CTX_new();
EVP_DecryptInit_ex(decCtx, EVP_aes_128_ecb(), nullptr, aesKey, nullptr);
EVP_CIPHER_CTX_set_padding(decCtx, 0);
QByteArray plain(response.encryptedCredentials.size() + 16, char(0));
int outLen1 = 0;
EVP_DecryptUpdate(
decCtx, reinterpret_cast<unsigned char*>(plain.data()), &outLen1,
reinterpret_cast<const unsigned char*>(response.encryptedCredentials.constData()),
response.encryptedCredentials.size());
int outLen2 = 0;
EVP_DecryptFinal_ex(decCtx, reinterpret_cast<unsigned char*>(plain.data()) + outLen1, &outLen2);
plain.resize(outLen1 + outLen2);
EVP_CIPHER_CTX_free(decCtx);
QByteArray expected(128, char(0));
const QByteArray userBytes = username.toLatin1().left(64);
const QByteArray passBytes = password.toLatin1().left(64);
std::memcpy(expected.data(), userBytes.constData(), static_cast<size_t>(userBytes.size()));
std::memcpy(expected.data() + 64, passBytes.constData(), static_cast<size_t>(passBytes.size()));
QCOMPARE(plain, expected);
BN_free(clientPub);
BN_free(sharedSecret);
BN_CTX_free(ctx);
BN_free(serverKeypair.privateExponent);
BN_free(serverKeypair.prime);
}
void TestVncSessionBackend::connectsWithAppleDhAuthenticationRfb38()
{
ToyDhKeypair serverKeypair = generateToyDhKeypair();
const QByteArray authMessage = appleDhAuthServerMessage(
serverKeypair.generatorBytes, serverKeypair.primeBytes, serverKeypair.publicKeyBytes);
const int keyLength = serverKeypair.primeBytes.size();
const QString password = QStringLiteral("s3cret-pass");
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, authMessage]() {
switch (m_server->nextStep()) {
case 0: { // version reply
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(30)); // Apple's scheme
m_server->sendWhenConnected(securityTypes);
break;
}
case 1: // security-type selection (byte value 30)
m_server->sendWhenConnected(authMessage);
break;
case 2: // client's response -- accept unconditionally, checked below
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 3: { // ClientInit
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(1));
serverInit.append(char(0)); serverInit.append(char(1));
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
m_server->sendWhenConnected(serverInit);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions(password));
QTRY_COMPARE(m_lastState, SessionState::Connected);
// Verify wire order: encrypted credentials (128 bytes) MUST come
// before the client's public key, per neatvnc's authoritative
// rfb_apple_dh_client_msg struct layout (this is exactly the ordering
// bug that made real-world testing fail before it was found and
// fixed). Proven by actually deriving the shared secret from the
// trailing public-key bytes -- as the server would -- and decrypting
// the leading 128 bytes with it; a swap would make this decrypt to
// garbage instead of the real credentials.
// 12 bytes for the client's version-reply line, 1 byte for its
// security-type selection, precede the response itself; more bytes
// (ClientInit, SetPixelFormat, ...) may already follow it by the time
// Connected is observed, so slice by offset rather than assuming
// total size.
const int responseOffset = 13;
QVERIFY(m_server->received.size() >= responseOffset + 128 + keyLength);
const QByteArray encryptedCredentials = m_server->received.mid(responseOffset, 128);
const QByteArray clientPublicKeyBytes =
m_server->received.mid(responseOffset + 128, keyLength);
BIGNUM* clientPub = BN_bin2bn(
reinterpret_cast<const unsigned char*>(clientPublicKeyBytes.constData()),
clientPublicKeyBytes.size(), nullptr);
BN_CTX* ctx = BN_CTX_new();
BIGNUM* sharedSecret = BN_new();
BN_mod_exp(sharedSecret, clientPub, serverKeypair.privateExponent, serverKeypair.prime, ctx);
QByteArray secretBytes(keyLength, char(0));
BN_bn2binpad(sharedSecret, reinterpret_cast<unsigned char*>(secretBytes.data()), keyLength);
unsigned char aesKey[16];
EVP_Digest(secretBytes.constData(), static_cast<size_t>(secretBytes.size()), aesKey, nullptr,
EVP_md5(), nullptr);
EVP_CIPHER_CTX* decCtx = EVP_CIPHER_CTX_new();
EVP_DecryptInit_ex(decCtx, EVP_aes_128_ecb(), nullptr, aesKey, nullptr);
EVP_CIPHER_CTX_set_padding(decCtx, 0);
QByteArray plain(encryptedCredentials.size() + 16, char(0));
int outLen1 = 0;
EVP_DecryptUpdate(decCtx, reinterpret_cast<unsigned char*>(plain.data()), &outLen1,
reinterpret_cast<const unsigned char*>(encryptedCredentials.constData()),
encryptedCredentials.size());
int outLen2 = 0;
EVP_DecryptFinal_ex(decCtx, reinterpret_cast<unsigned char*>(plain.data()) + outLen1, &outLen2);
plain.resize(outLen1 + outLen2);
EVP_CIPHER_CTX_free(decCtx);
QByteArray expected(128, char(0));
const QByteArray passBytes = password.toLatin1().left(64);
std::memcpy(expected.data() + 64, passBytes.constData(), static_cast<size_t>(passBytes.size()));
QCOMPARE(plain, expected);
BN_free(clientPub);
BN_free(sharedSecret);
BN_CTX_free(ctx);
BN_free(serverKeypair.privateExponent);
BN_free(serverKeypair.prime);
}
void TestVncSessionBackend::appleDhAuthIsPreferredOverVncAuthWhenBothOffered()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
if (m_server->nextStep() == 0) {
m_server->received.clear();
QByteArray securityTypes;
securityTypes.append(char(2)); // count = 2
securityTypes.append(char(2)); // VNC Authentication
securityTypes.append(char(30)); // Apple DH
m_server->sendWhenConnected(securityTypes);
}
});
m_backend->connectSession(makeOptions(QStringLiteral("whatever")));
QTRY_VERIFY(m_server->received.size() >= 1);
QCOMPARE(static_cast<quint8>(m_server->received.at(0)), quint8(30));
}
void TestVncSessionBackend::appleDhAuthAcceptsRealCapturedMacOsServerParameters()
{
// The exact generator/prime/server-public-key bytes captured from a
// real macOS Screen Sharing server's security-type-30 auth message
// (generator=5, a 512-byte/4096-bit RFC 3526 Group 16-family prime,
// and its server public key) -- this is what caught the original
// implementation's wrong assumption that the generator was itself
// length-prefixed like the prime is; it wasn't (it's a literal 2
// bytes), and this pins that fix against the real bytes that exposed
// the bug rather than only against freshly-generated toy data.
const QByteArray captured = QByteArray::fromHex(
"00050200"
"ffffffffffffffffc90fdaa22168c234c4c6628b80dc1cd129024e088a67cc74020bbea63b139b22514a0"
"8798e3404ddef9519b3cd3a431b302b0a6df25f14374fe1356d6d51c245e485b576625e7ec6f44c42e9a6"
"37ed6b0bff5cb6f406b7edee386bfb5a899fa5ae9f24117c4b1fe649286651ece45b3dc2007cb8a163bf0"
"598da48361c55d39a69163fa8fd24cf5f83655d23dca3ad961c62f356208552bb9ed529077096966d670c"
"354e4abc9804f1746c08ca18217c32905e462e36ce3be39e772c180e86039b2783a2ec07a28fb5c55df06"
"f4c52c9de2bcbf6955817183995497cea956ae515d2261898fa051015728e5a8aaac42dad33170d04507a"
"33a85521abdf1cba64ecfb850458dbef0a8aea71575d060c7db3970f85a6e1e4c7abf5ae8cdb0933d71e8"
"c94e04a25619dcee3d2261ad2ee6bf12ffa06d98a0864d87602733ec86a64521f2b18177b200cbbe11757"
"7a615d6c770988c0bad946e208e24fa074e5ab3143db5bfce0fd108e4b82d120a92108011a723c12a787e"
"6d788719a10bdba5b2699c327186af4e23c1a946834b6150bda2583e9ca2ad44ce8dbbbc2db04de8ef92e"
"8efc141fbecaa6287c59474e6bc05d99b2964fa090c3a2233ba186515be7ed1f612970cee2d7afb81bdd7"
"62170481cd0069127d5b05aa993b4ea988d8fddc186ffb7dc90a6c08f4df435c934063199"
"ffffffffffffffff"
"91c11c23a5b54389a05127d7fa94e681b7884667d770cd646fa46583d73e0a5ad09e361c30a0f9a9b765a"
"b291e2b8a13ba0088ba4bdb4a838120e649d49ed78db7fdb3397b907d1a42c0531d16796229d7b3a5a369"
"bfc639771ab1dea093ca7756b2b4a4a98f296478b331538533faead12342d6f1fe92817690ae8e2c51cbf"
"dc96c819e480c2170e6be12149691cb2bab218528c40cc5729b8a303bbe95d77eba288f3c4137d316c5b1"
"d9004c8fe3d034b1284d39e57133972700d1933465b4cc656d2dcbef67dc7eeb464f197bd5187abce304b"
"c972768dcaaa703e287f4f1e375c287b474fe0e3df9897a24bdaf06466976becbc8908fd81211537d1a44"
"e63e705d69b5c90f375fa717c978b27a21384769da67cd97e5134f71aef420dd4b36a2c4e05f80cee043f"
"ab873b478804dc126495d35f0a1df041c4b2bf0926e35ae033f6f6808bdc8a5792b7d380e7c7fc95fc80c"
"fea44c92842ed75757230f590b7dd1d9beadbabb78a96c47f21e65f6ea8566820ff9c59d078870fa1d367"
"2ff0b7f9a3d600f3479232e533bc8b64a3909fd61e070b19ac98d50a7c2a33c885294183e82a4e7777f2a"
"423002219a6094fdf51651d48550f771ede87eb95592eda604b85058cd0d8db9a92670e6e0d74400f7cd4"
"b48c8dfbace5c92c39775a60bb4e59d70878a50b609b15ae4a930ee5eb0494902d44869089349fa2ab343"
"028e");
QCOMPARE(captured.size(), 1028);
const QByteArray generator = captured.left(2);
const int keyLength = (static_cast<quint8>(captured.at(2)) << 8) | static_cast<quint8>(captured.at(3));
QCOMPARE(keyLength, 512);
const QByteArray prime = captured.mid(4, keyLength);
const QByteArray serverPublicKey = captured.mid(4 + keyLength, keyLength);
QCOMPARE(prime.size(), 512);
QCOMPARE(serverPublicKey.size(), 512);
const VncAppleDhAuth::Response response = VncAppleDhAuth::computeResponse(
generator, prime, serverPublicKey, QStringLiteral("tester"), QStringLiteral("s3cret"));
QVERIFY(!response.clientPublicKey.isEmpty());
QCOMPARE(response.clientPublicKey.size(), 512);
QCOMPARE(response.encryptedCredentials.size(), 128);
}
QTEST_GUILESS_MAIN(TestVncSessionBackend)
#include "test_vnc_session_backend.moc"
Reference in New Issue View Git Blame Copy Permalink