DarkSingularity/orbithub
Internal
Public Access
0
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases 7 Wiki Activity

Add VNC Hextile decoding

Browse Source 
Implements RFC 6143 SS7.7.4: rectangles announced as Hextile (type 5)
tile the update into 16x16 blocks, each either raw pixels or a
background fill plus an optional list of foreground/individually-
colored subrects, with background/foreground persisting across tiles
within one rectangle when not re-specified.

The pure byte-decode logic (tile metadata, subrect list) lives in new
src/vnc_pixel_codecs.h/.cpp, kept separate from
VncSessionBackend's wire-sequencing state machine so it's unit-testable
without a socket -- the pattern the plan calls for continuing into the
ZRLE/Tight work still ahead. Adds 5 fake-server tests covering a raw
tile, a background-only solid fill, uncoloured and individually-colored
subrects, and a 4-tile rectangle proving background persistence and
correct tile-cursor wraparound.

Verified against the live TightVNC test server (connect, frame,
cursor, clipboard all still work); that particular server always
chose Raw for the actual framebuffer content during this session, so
Hextile's real-world path isn't independently confirmed live -- the
unit tests are the primary correctness evidence for this phase.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmith
2026-09-15 20:49:08 -06:00
co-authored by Claude Sonnet 5
parent 9dd1af21d6
commit e49fa0cf26
7 changed files with 598 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/CMakeLists.txt
+1
View File
@@ -29,6 +29,7 @@ add_test(NAME test_ssh_session_backend COMMAND test_ssh_session_backend)
add_executable(test_vnc_session_backend
test_vnc_session_backend.cpp
${CMAKE_SOURCE_DIR}/src/vnc_session_backend.cpp
${CMAKE_SOURCE_DIR}/src/vnc_pixel_codecs.cpp
${CMAKE_SOURCE_DIR}/src/session_backend.h
)
target_include_directories(test_vnc_session_backend PRIVATE ${CMAKE_SOURCE_DIR}/src)
tests/test_vnc_session_backend.cpp
+281
View File
@@ -79,6 +79,52 @@ SessionConnectOptions makeOptions(const QString& password = QString())
options.password = password;
return options;
}
void appendU16(QByteArray& buf, quint16 value)
{
buf.append(static_cast<char>((value >> 8) & 0xFF));
buf.append(static_cast<char>(value & 0xFF));
}
// Pixel bytes as expected under our negotiated SetPixelFormat: 32bpp
// little-endian, byte order B,G,R,pad.
QByteArray pixelBytes(int r, int g, int b)
{
QByteArray bytes;
bytes.append(static_cast<char>(b));
bytes.append(static_cast<char>(g));
bytes.append(static_cast<char>(r));
bytes.append(char(0));
return bytes;
}
QByteArray serverInitBytes(int width, int height)
{
QByteArray serverInit;
appendU16(serverInit, static_cast<quint16>(width));
appendU16(serverInit, static_cast<quint16>(height));
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
return serverInit;
}
// A FramebufferUpdate message with exactly one Hextile-encoded rectangle
// (RFC 6143 encoding type 5), given the already fully-formed tile byte
// stream to follow the rectangle header.
QByteArray hextileFramebufferUpdate(int x, int y, int width, int height, const QByteArray& tileData)
{
QByteArray update;
update.append(char(0)); update.append(char(0));
update.append(char(0)); update.append(char(1)); // 1 rectangle
appendU16(update, static_cast<quint16>(x));
appendU16(update, static_cast<quint16>(y));
appendU16(update, static_cast<quint16>(width));
appendU16(update, static_cast<quint16>(height));
update.append(char(0)); update.append(char(0)); update.append(char(0));
update.append(char(5)); // encoding = Hextile
update.append(tileData);
return update;
}
}
class TestVncSessionBackend : public QObject
@@ -114,6 +160,11 @@ private slots:
void setClipboardTextSendsClientCutText();
void cursorPseudoEncodingProducesExpectedImageAndHotspot();
void zeroSizeCursorPseudoEncodingHidesCursor();
void hextileRawTileProducesExpectedPixels();
void hextileBackgroundOnlyTileFillsSolidColor();
void hextileUncolouredSubrectUsesForeground();
void hextileColouredSubrectUsesOwnColor();
void hextileMultiTilePersistsBackgroundAcrossTiles();
private:
std::unique_ptr<FakeVncServer> m_server;
@@ -843,5 +894,235 @@ void TestVncSessionBackend::zeroSizeCursorPseudoEncodingHidesCursor()
QTRY_VERIFY(gotHidden);
}
void TestVncSessionBackend::hextileRawTileProducesExpectedPixels()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(2, 2));
break;
case 3: { // a single Raw-subencoding Hextile tile covering the whole 2x2 rectangle
QByteArray tile;
tile.append(char(0x01)); // HextileFlags::kRaw
tile += pixelBytes(255, 0, 0);
tile += pixelBytes(0, 255, 0);
tile += pixelBytes(0, 0, 255);
tile += pixelBytes(255, 255, 255);
m_server->sendWhenConnected(hextileFramebufferUpdate(0, 0, 2, 2, tile));
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(2, 2));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(255, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(0, 255, 0));
QCOMPARE(m_lastFrame.pixelColor(0, 1), QColor(0, 0, 255));
QCOMPARE(m_lastFrame.pixelColor(1, 1), QColor(255, 255, 255));
}
void TestVncSessionBackend::hextileBackgroundOnlyTileFillsSolidColor()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(3, 3));
break;
case 3: { // BackgroundSpecified only, no subrects -- solid fill
QByteArray tile;
tile.append(char(0x02)); // HextileFlags::kBackgroundSpecified
tile += pixelBytes(0, 128, 255);
m_server->sendWhenConnected(hextileFramebufferUpdate(0, 0, 3, 3, tile));
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(3, 3));
for (int y = 0; y < 3; ++y) {
for (int x = 0; x < 3; ++x) {
QCOMPARE(m_lastFrame.pixelColor(x, y), QColor(0, 128, 255));
}
}
}
void TestVncSessionBackend::hextileUncolouredSubrectUsesForeground()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(4, 4));
break;
case 3: {
// Background(black) + Foreground(red) + one uncoloured 2x2
// subrect at tile-local (1,1) -- must render in the foreground
// color, everywhere else in the background color.
QByteArray tile;
tile.append(char(0x02 | 0x04 | 0x08)); // Background|Foreground|AnySubrects
tile += pixelBytes(0, 0, 0); // background
tile += pixelBytes(255, 0, 0); // foreground
tile.append(char(1)); // subrect count
tile.append(char(0x11)); // xy: x=1, y=1
tile.append(char(0x11)); // wh: width=2, height=2
m_server->sendWhenConnected(hextileFramebufferUpdate(0, 0, 4, 4, tile));
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(4, 4));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(0, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(1, 1), QColor(255, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(2, 2), QColor(255, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(3, 3), QColor(0, 0, 0));
}
void TestVncSessionBackend::hextileColouredSubrectUsesOwnColor()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(3, 3));
break;
case 3: {
// No background/foreground specified this tile -- both stay at
// the rectangle-start default (black). One individually-
// coloured 1x1 subrect at tile-local (0,0).
QByteArray tile;
tile.append(char(0x08 | 0x10)); // AnySubrects|SubrectsColoured
tile.append(char(1)); // subrect count
tile += pixelBytes(0, 255, 0); // this subrect's own color
tile.append(char(0x00)); // xy: x=0, y=0
tile.append(char(0x00)); // wh: width=1, height=1
m_server->sendWhenConnected(hextileFramebufferUpdate(0, 0, 3, 3, tile));
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(3, 3));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(0, 255, 0));
QCOMPARE(m_lastFrame.pixelColor(1, 1), QColor(0, 0, 0));
QCOMPARE(m_lastFrame.pixelColor(2, 2), QColor(0, 0, 0));
}
void TestVncSessionBackend::hextileMultiTilePersistsBackgroundAcrossTiles()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2:
m_server->sendWhenConnected(serverInitBytes(20, 20));
break;
case 3: {
// A 20x20 rectangle tiles as (0,0) 16x16, (16,0) 4x16,
// (0,16) 16x4, (16,16) 4x4 -- only the first tile specifies a
// background color; the other three specify nothing at all
// (subencoding 0x00) and must inherit it.
QByteArray tile1;
tile1.append(char(0x02)); // BackgroundSpecified
tile1 += pixelBytes(0, 0, 255);
QByteArray tile2(1, char(0x00));
QByteArray tile3(1, char(0x00));
QByteArray tile4(1, char(0x00));
m_server->sendWhenConnected(
hextileFramebufferUpdate(0, 0, 20, 20, tile1 + tile2 + tile3 + tile4));
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(m_gotFrame);
QCOMPARE(m_lastFrame.size(), QSize(20, 20));
QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(0, 0, 255));
QCOMPARE(m_lastFrame.pixelColor(19, 0), QColor(0, 0, 255));
QCOMPARE(m_lastFrame.pixelColor(0, 19), QColor(0, 0, 255));
QCOMPARE(m_lastFrame.pixelColor(19, 19), QColor(0, 0, 255));
QCOMPARE(m_lastFrame.pixelColor(10, 10), QColor(0, 0, 255));
}
QTEST_GUILESS_MAIN(TestVncSessionBackend)
#include "test_vnc_session_backend.moc"