From bb022edcf233bd284cdc26b2971c04330d07b5c2 Mon Sep 17 00:00:00 2001 From: Keith Smith Date: Tue, 15 Sep 2026 21:02:26 -0600 Subject: [PATCH] Add VNC ZRLE decoding Implements RFC 6143 SS7.7.6: a ZRLE rectangle is a 4-byte compressed length followed by that many zlib-compressed bytes, decompressing to 64x64 tiles each using one of five subencodings (Raw, Solid, packed palette, Plain RLE, Palette RLE). The zlib stream persists for the whole connection rather than being reset per-rectangle or per-update, so VncSessionBackend now owns a lazily-initialized, persistent z_stream torn down only in resetProtocolState() on a fresh connect/reconnect. Since the entire rectangle's compressed data decompresses into memory in one shot, tile parsing is a plain synchronous loop rather than needing its own RfbState values -- only the compressed-length and compressed-data reads are actual protocol states. Tile decoding (the five subencodings, including the continuation-byte run-length encoding shared by two of them) lives in vnc_pixel_codecs.h/.cpp alongside the Hextile decoder, unit-tested with 6 new tests covering each subencoding plus a persistence test that splits one continuous deflate stream across two separate FramebufferUpdate messages -- it only decodes correctly if the connection's inflate stream is retained between them. Adds a top-level find_package(ZLIB REQUIRED) + ZLIB::ZLIB link (previously only pulled in transitively via vendored FreeRDP's own smartcard-emulation feature, which happened to have it enabled but shouldn't be relied on for that). Live-verified against the TightVNC test server that nothing regressed (connect, cursor, clipboard); that server consistently sends Raw for actual framebuffer content regardless of announced encodings, so Hextile/ZRLE's live decode path isn't independently confirmed against a real server -- the unit tests are the primary evidence here. Co-Authored-By: Claude Sonnet 5 --- CMakeLists.txt | 2 + docs/PROGRESS.md | 42 +++- src/vnc_pixel_codecs.cpp | 174 +++++++++++++ src/vnc_pixel_codecs.h | 14 ++ src/vnc_session_backend.cpp | 154 +++++++++++- src/vnc_session_backend.h | 21 +- tests/CMakeLists.txt | 2 +- tests/test_vnc_session_backend.cpp | 382 +++++++++++++++++++++++++++++ 8 files changed, 772 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b1f5f9..0411fee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,7 @@ include(GNUInstallDirs) find_package(Qt6 6.2 REQUIRED COMPONENTS Widgets Sql Network) find_package(OpenSSL REQUIRED) +find_package(ZLIB REQUIRED) qt_standard_project_setup() @@ -155,6 +156,7 @@ add_executable(orbithub WIN32 MACOSX_BUNDLE ${ORBITHUB_SOURCES}) target_link_libraries(orbithub PRIVATE Qt6::Widgets Qt6::Sql Qt6::Network) target_link_libraries(orbithub PRIVATE KodoTerm::KodoTerm) target_link_libraries(orbithub PRIVATE OpenSSL::Crypto) +target_link_libraries(orbithub PRIVATE ZLIB::ZLIB) target_compile_definitions(orbithub PRIVATE ORBITHUB_VERSION_STRING="${PROJECT_VERSION}") if(TARGET freerdp AND TARGET winpr) target_compile_definitions(orbithub PRIVATE ORBITHUB_HAS_FREERDP) diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index ef2be33..52c4be5 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -121,18 +121,25 @@ Delivered: keyboard (Qt key -> X11 keysym mapping) and mouse/wheel input forwarding - `VncDisplayWidget` mirroring `RdpDisplayWidget`'s scale-to-fit rendering and input-forwarding shape -- 19 unit tests (`tests/test_vnc_session_backend.cpp`): pure-function +- 36 unit tests (`tests/test_vnc_session_backend.cpp`): pure-function coverage (DES key prep verified against an independently documented test vector, keysym mapping, socket-error mapping) plus state-machine coverage against a scripted in-process fake RFB server (all three protocol-version handshake shapes, auth success/failure, unsupported - security types, pixel-accurate Raw decoding) -- caught and fixed a real - re-entrancy bug (`abort()` synchronously re-firing `disconnected()` - mid-`failConnection()`, silently overwriting a specific error with a - generic one) + security types, pixel-accurate Raw/Hextile/ZRLE decoding including a + ZRLE zlib-stream-persistence test across two separate + `FramebufferUpdate` messages, cursor/clipboard round trips) -- caught + and fixed a real re-entrancy bug (`abort()` synchronously re-firing + `disconnected()` mid-`failConnection()`, silently overwriting a specific + error with a generic one) - Verified live against a real, independently implemented VNC server (TightVNC on Windows): connect with VNC Authentication, correct - framebuffer dimensions and pixel data, clean disconnect, reconnect + framebuffer dimensions and pixel data, clean disconnect, reconnect, + live cursor-shape and clipboard-send checks. That particular server + consistently chose Raw for actual framebuffer content regardless of + which compression encodings were announced, so Hextile/ZRLE's + real-world decode path isn't independently confirmed live -- the unit + tests are the primary correctness evidence for those two - Per-tab VNC-only display mode toggle (tab-bar right-click -> `Display Mode`): `Scale to Fit` (default, matches RDP's behavior) or `Actual Size (Scrollbars)` -- renders the remote framebuffer at its @@ -142,20 +149,35 @@ Delivered: it degenerates to an exact 1:1 mapping once the widget's own bounds are fixed to the remote's size, so no separate rendering path was needed. Persisted across sessions like the terminal theme preference. +- Robustness fix: an unrecognized `FramebufferUpdate` rectangle encoding + used to abort the connection generically; `kAnnouncedEncodings` is now + the single source of truth for what `SetEncodings` announces and what + the rectangle-dispatch `switch` can decode, with a regression test + pinning that every announced encoding has a working case +- Clipboard sync (`ServerCutText`/`ClientCutText`, Latin-1 only -- no + Unicode extension) in both directions +- Remote cursor shape sync via RFB's Cursor pseudo-encoding, mirroring + `RdpDisplayWidget`'s cursor handling in `VncDisplayWidget` +- Hextile and ZRLE compression encodings, in addition to Raw + CopyRect -- + meaningfully reduces bandwidth over slower links versus Raw alone. Pure + tile/pixel decode logic lives in `src/vnc_pixel_codecs.h/.cpp`, kept + separate from the wire-sequencing state machine so it's unit-testable + without a socket. ZRLE decoding links `ZLIB::ZLIB` (found via a fresh + top-level `find_package(ZLIB REQUIRED)`, independent of whether + vendored FreeRDP's own internal zlib usage stays enabled) Known gaps (explicit scope decisions, not oversights -- see issue #3 for follow-up tracking): - Apple's Screen Sharing authentication (Diffie-Hellman + AES, security type 30) isn't implemented, so this can't yet reach macOS's built-in VNC server -- only standard VNC Authentication (type 2) and no-auth (type 1) -- Raw + CopyRect encodings only -- no Hextile/ZRLE/Tight compression, so - bandwidth usage is higher over slow links than a full VNC client +- Tight compression isn't implemented yet (Raw + CopyRect + Hextile + ZRLE + are); Tight typically compresses best of all of these, so bandwidth over + very slow links is still not as good as a full VNC client would achieve - No dynamic resize (connects at the server's native resolution; the `Scale to Fit`/`Actual Size` toggle changes how that fixed resolution is displayed locally, not what resolution is requested from the guest -- VNC has no equivalent of RDP's MS-RDPEDISP for that) -- No remote cursor shape sync (local default cursor only) -- No clipboard sync ## Milestone 7 - Cross-Platform Protocol Hardening diff --git a/src/vnc_pixel_codecs.cpp b/src/vnc_pixel_codecs.cpp index d5004f3..ecb9e04 100644 --- a/src/vnc_pixel_codecs.cpp +++ b/src/vnc_pixel_codecs.cpp @@ -77,4 +77,178 @@ QVector decodeHextileSubrects(bool coloured, int subrectCount, return subrects; } +int decodeZrleTile(const QByteArray& data, int offset, int tileWidth, int tileHeight, + QVector& pixels) +{ + const int pixelCount = tileWidth * tileHeight; + const auto* bytes = reinterpret_cast(data.constData()); + const int size = data.size(); + if (offset < 0 || offset >= size || pixelCount <= 0) { + return -1; + } + + int pos = offset; + const quint8 subencoding = bytes[pos]; + ++pos; + + // CPIXEL is 3 bytes for our negotiated 32bpp/24-depth true-color + // format -- the padding byte a full pixel would have is simply + // omitted. rgbFromPixelBytes() already only reads the first 3 bytes it + // is given (B,G,R order), so it doubles as the CPIXEL reader. + auto readCpixel = [&](QRgb* out) -> bool { + if (pos + 3 > size) { + return false; + } + *out = rgbFromPixelBytes(bytes + pos); + pos += 3; + return true; + }; + + // Shared continuation-byte run-length reader for both RLE subencoding + // families: keep summing bytes while they equal 255, add the final + // (non-255) byte, and the true run length is that sum plus one. + auto readRunLength = [&](int* out) -> bool { + int total = 0; + for (;;) { + if (pos >= size) { + return false; + } + const quint8 b = bytes[pos]; + ++pos; + total += b; + if (b != 255) { + break; + } + } + *out = total + 1; + return true; + }; + + if (subencoding == 0) { // Raw + pixels.reserve(pixels.size() + pixelCount); + for (int i = 0; i < pixelCount; ++i) { + QRgb color = 0; + if (!readCpixel(&color)) { + return -1; + } + pixels.append(color); + } + return pos - offset; + } + + if (subencoding == 1) { // Solid + QRgb color = 0; + if (!readCpixel(&color)) { + return -1; + } + pixels.reserve(pixels.size() + pixelCount); + for (int i = 0; i < pixelCount; ++i) { + pixels.append(color); + } + return pos - offset; + } + + if (subencoding >= 2 && subencoding <= 16) { // Packed palette + const int paletteSize = subencoding; + QVector palette; + palette.reserve(paletteSize); + for (int i = 0; i < paletteSize; ++i) { + QRgb color = 0; + if (!readCpixel(&color)) { + return -1; + } + palette.append(color); + } + + int bitsPerPixel = 4; + if (paletteSize == 2) { + bitsPerPixel = 1; + } else if (paletteSize <= 4) { + bitsPerPixel = 2; + } + const int rowBytes = (tileWidth * bitsPerPixel + 7) / 8; + + pixels.reserve(pixels.size() + pixelCount); + for (int y = 0; y < tileHeight; ++y) { + if (pos + rowBytes > size) { + return -1; + } + int bitPos = 0; + for (int x = 0; x < tileWidth; ++x) { + const int byteIndex = pos + (bitPos / 8); + const int shift = 8 - (bitPos % 8) - bitsPerPixel; + const int mask = (1 << bitsPerPixel) - 1; + const int index = (bytes[byteIndex] >> shift) & mask; + if (index >= palette.size()) { + return -1; + } + pixels.append(palette.at(index)); + bitPos += bitsPerPixel; + } + pos += rowBytes; + } + return pos - offset; + } + + if (subencoding == 128) { // Plain RLE + int produced = 0; + while (produced < pixelCount) { + QRgb color = 0; + if (!readCpixel(&color)) { + return -1; + } + int runLength = 0; + if (!readRunLength(&runLength)) { + return -1; + } + for (int i = 0; i < runLength && produced < pixelCount; ++i, ++produced) { + pixels.append(color); + } + } + return pos - offset; + } + + if (subencoding >= 130) { // Palette RLE (129 is unused/invalid, falls through below) + const int paletteSize = subencoding - 128; + QVector palette; + palette.reserve(paletteSize); + for (int i = 0; i < paletteSize; ++i) { + QRgb color = 0; + if (!readCpixel(&color)) { + return -1; + } + palette.append(color); + } + + int produced = 0; + while (produced < pixelCount) { + if (pos >= size) { + return -1; + } + const quint8 indexByte = bytes[pos]; + ++pos; + + int index = indexByte; + int runLength = 1; + if (indexByte >= 128) { + index = indexByte - 128; + if (!readRunLength(&runLength)) { + return -1; + } + } + if (index >= palette.size()) { + return -1; + } + const QRgb color = palette.at(index); + for (int i = 0; i < runLength && produced < pixelCount; ++i, ++produced) { + pixels.append(color); + } + } + return pos - offset; + } + + // Subencodings 17-127 and 129 are not defined by RFC 6143. + return -1; +} + } diff --git a/src/vnc_pixel_codecs.h b/src/vnc_pixel_codecs.h index c521e06..dbd9690 100644 --- a/src/vnc_pixel_codecs.h +++ b/src/vnc_pixel_codecs.h @@ -60,6 +60,20 @@ int hextileSubrectByteCount(bool coloured, int subrectCount); QVector decodeHextileSubrects(bool coloured, int subrectCount, const QByteArray& data, QRgb foreground); +// Decodes one ZRLE tile (RFC 6143 SS7.7.6) from `data`, starting at +// `offset` (which must point at the tile's own 1-byte subencoding). +// `data` holds an entire rectangle's worth of already-zlib-decompressed +// bytes (possibly several tiles' worth) -- this reads only as much as the +// one tile needs and never looks past `data.size()`. On success, appends +// exactly tileWidth*tileHeight pixels (row-major) to `pixels` (which is +// NOT cleared first, so callers can accumulate across tiles if desired -- +// VncSessionBackend clears/reuses a fresh vector per tile) and returns the +// number of bytes consumed. Returns -1 for a malformed/truncated tile +// (should never happen against a spec-compliant server, but must not read +// out of bounds against an adversarial or buggy one). +int decodeZrleTile(const QByteArray& data, int offset, int tileWidth, int tileHeight, + QVector& pixels); + } #endif diff --git a/src/vnc_session_backend.cpp b/src/vnc_session_backend.cpp index ada9aef..207ff3d 100644 --- a/src/vnc_session_backend.cpp +++ b/src/vnc_session_backend.cpp @@ -10,6 +10,8 @@ #include +#include + #include #include #include @@ -35,14 +37,23 @@ constexpr quint8 kServerMsgServerCutText = 3; constexpr qint32 kEncRaw = 0; constexpr qint32 kEncCopyRect = 1; constexpr qint32 kEncHextile = 5; +constexpr qint32 kEncZRLE = 16; // RFC 6143 SS7.8.2 "Cursor pseudo-encoding": not a real screen-content // encoding -- a rectangle with this type carries a cursor shape update // instead (hotspot in x/y, image dims in width/height), never painted into // the framebuffer. constexpr qint32 kEncCursor = -239; -constexpr std::array kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile, - kEncCursor }; +constexpr std::array kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile, + kEncZRLE, kEncCursor }; + +constexpr int kZrleTileSize = 64; + +// Sanity cap on a single ZRLE rectangle's compressed-length field, purely +// to reject an obviously-hostile/corrupt claim before attempting to +// buffer/allocate that many bytes -- well beyond anything a real screen +// update should ever compress to. +constexpr quint32 kMaxZrleCompressedLength = 64u * 1024u * 1024u; quint16 readU16BE(const QByteArray& buf, int offset) { @@ -93,8 +104,11 @@ VncSessionBackend::VncSessionBackend(const Profile& profile, QObject* parent) m_hextileForeground(qRgb(0, 0, 0)), m_hextileSubencoding(0), m_hextileSubrectsRemaining(0), - m_hextileSubrectsColoured(false) + m_hextileSubrectsColoured(false), + m_zrleInflateStream(new z_stream_s()), + m_zrleInflateInitialized(false) { + std::memset(m_zrleInflateStream, 0, sizeof(z_stream_s)); connect(m_socket, &QTcpSocket::connected, this, &VncSessionBackend::onSocketConnected); connect(m_socket, &QTcpSocket::readyRead, this, &VncSessionBackend::onSocketReadyRead); connect(m_socket, &QTcpSocket::disconnected, this, &VncSessionBackend::onSocketDisconnected); @@ -106,6 +120,10 @@ VncSessionBackend::~VncSessionBackend() if (m_socket->state() != QAbstractSocket::UnconnectedState) { m_socket->abort(); } + if (m_zrleInflateInitialized) { + inflateEnd(m_zrleInflateStream); + } + delete m_zrleInflateStream; } void VncSessionBackend::connectSession(const SessionConnectOptions& options) @@ -341,6 +359,8 @@ void VncSessionBackend::onSocketDisconnected() && m_rfbState != RfbState::WaitingHextileTileMeta && m_rfbState != RfbState::WaitingHextileSubrectData && m_rfbState != RfbState::WaitingHextileRawTileData + && m_rfbState != RfbState::WaitingZrleCompressedLength + && m_rfbState != RfbState::WaitingZrleCompressedData && m_rfbState != RfbState::WaitingSetColourMapHeader && m_rfbState != RfbState::WaitingSetColourMapData && m_rfbState != RfbState::WaitingServerCutTextHeader @@ -389,6 +409,16 @@ void VncSessionBackend::resetProtocolState() m_hextileTileY = 0; m_hextileBackground = qRgb(0, 0, 0); m_hextileForeground = qRgb(0, 0, 0); + + // The ZRLE zlib stream is scoped to one TCP connection's lifetime, not + // to individual rectangles or updates -- tear it down here so a fresh + // connect/reconnect starts a clean stream instead of feeding new + // compressed data through decoder state left over from a previous + // session. + if (m_zrleInflateInitialized) { + inflateEnd(m_zrleInflateStream); + m_zrleInflateInitialized = false; + } } bool VncSessionBackend::haveBytes(int count) const @@ -873,6 +903,9 @@ void VncSessionBackend::processReceiveBuffer() m_rfbState = RfbState::WaitingHextileTileSubencoding; } break; + case kEncZRLE: + m_rfbState = RfbState::WaitingZrleCompressedLength; + break; default: { // SetEncodings (see kAnnouncedEncodings) is entirely // client-controlled, so a spec-compliant server will never @@ -1083,6 +1116,121 @@ void VncSessionBackend::processReceiveBuffer() break; } + case RfbState::WaitingZrleCompressedLength: { + if (!haveBytes(4)) { + return; + } + const quint32 length = readU32BE(m_recvBuffer, 0); + m_recvBuffer.remove(0, 4); + if (length > kMaxZrleCompressedLength) { + failConnection(QStringLiteral("The VNC server sent an implausibly large ZRLE update."), + QStringLiteral("ZRLE compressed length %1").arg(length)); + return; + } + m_pendingLength = length; + m_rfbState = RfbState::WaitingZrleCompressedData; + break; + } + + case RfbState::WaitingZrleCompressedData: { + if (!haveBytes(static_cast(m_pendingLength))) { + return; + } + const QByteArray compressed = m_recvBuffer.left(static_cast(m_pendingLength)); + m_recvBuffer.remove(0, static_cast(m_pendingLength)); + + if (!m_zrleInflateInitialized) { + std::memset(m_zrleInflateStream, 0, sizeof(z_stream_s)); + if (inflateInit(m_zrleInflateStream) != Z_OK) { + failConnection(QStringLiteral("Failed to initialize ZRLE decompression."), + QStringLiteral("inflateInit() failed")); + return; + } + m_zrleInflateInitialized = true; + } + + // The zlib stream is persistent across the whole connection + // (RFC 6143 SS7.7.6) -- Z_NO_FLUSH, never Z_FINISH, and the + // stream is only ever torn down in resetProtocolState() on a + // fresh connect/reconnect. + QByteArray decompressed; + constexpr int kChunkSize = 65536; + char outBuffer[kChunkSize]; + m_zrleInflateStream->next_in = + reinterpret_cast(const_cast(compressed.constData())); + m_zrleInflateStream->avail_in = static_cast(compressed.size()); + int inflateResult = Z_OK; + while (m_zrleInflateStream->avail_in > 0) { + m_zrleInflateStream->next_out = reinterpret_cast(outBuffer); + m_zrleInflateStream->avail_out = kChunkSize; + inflateResult = inflate(m_zrleInflateStream, Z_NO_FLUSH); + const int produced = kChunkSize - static_cast(m_zrleInflateStream->avail_out); + if (produced > 0) { + decompressed.append(outBuffer, produced); + } + if (inflateResult != Z_OK) { + break; + } + } + if (inflateResult != Z_OK && inflateResult != Z_STREAM_END) { + failConnection(QStringLiteral("The VNC server sent malformed ZRLE-compressed data."), + QStringLiteral("inflate() returned %1").arg(inflateResult)); + return; + } + + const int rectWidth = m_currentRectangle.width; + const int rectHeight = m_currentRectangle.height; + int decodeOffset = 0; + bool malformedTile = false; + QString malformedTileDetail; + { + // Scoped so the QPainter is destroyed (ending the paint + // session) before onRectangleFinished() below, which may + // emit frameUpdated(m_framebuffer) -- matching how + // WaitingRawPixelData/WaitingCopyRectSource already end + // their own painters before finishing a rectangle. + QPainter painter(&m_framebuffer); + for (int tileY = 0; tileY < rectHeight && !malformedTile; tileY += kZrleTileSize) { + const int tileHeight = qMin(kZrleTileSize, rectHeight - tileY); + for (int tileX = 0; tileX < rectWidth; tileX += kZrleTileSize) { + const int tileWidth = qMin(kZrleTileSize, rectWidth - tileX); + + QVector pixels; + const int consumed = VncPixelCodecs::decodeZrleTile( + decompressed, decodeOffset, tileWidth, tileHeight, pixels); + if (consumed < 0 || pixels.size() != tileWidth * tileHeight) { + malformedTile = true; + malformedTileDetail = QStringLiteral("Tile at (%1,%2) size %3x%4") + .arg(tileX) + .arg(tileY) + .arg(tileWidth) + .arg(tileHeight); + break; + } + decodeOffset += consumed; + + QImage tileImage(tileWidth, tileHeight, QImage::Format_RGB32); + for (int y = 0; y < tileHeight; ++y) { + for (int x = 0; x < tileWidth; ++x) { + tileImage.setPixel(x, y, pixels.at((y * tileWidth) + x)); + } + } + painter.drawImage(m_currentRectangle.x + tileX, + m_currentRectangle.y + tileY, tileImage); + } + } + } + + if (malformedTile) { + failConnection(QStringLiteral("The VNC server sent a malformed ZRLE tile."), + malformedTileDetail); + return; + } + + onRectangleFinished(); + break; + } + case RfbState::WaitingSetColourMapHeader: { if (!haveBytes(5)) { return; diff --git a/src/vnc_session_backend.h b/src/vnc_session_backend.h index c9ad570..4e36555 100644 --- a/src/vnc_session_backend.h +++ b/src/vnc_session_backend.h @@ -10,6 +10,7 @@ #include class QTcpSocket; +struct z_stream_s; // Implements RFB (RFC 6143) directly against QTcpSocket -- there is no // permissively licensed VNC client library to vendor the way FreeRDP was @@ -22,11 +23,10 @@ class QTcpSocket; // // Scope (see plan / issue #3 for the full rationale): standard VNC // Authentication (security type 2) and no-auth (type 1) only -- not -// Apple's Screen Sharing scheme (type 30). Raw + CopyRect + Hextile -// encodings (ZRLE/Tight compression not yet implemented). No dynamic -// resize. Clipboard sync (Latin-1 only, per RFB's ServerCutText/ -// ClientCutText) and remote cursor shape sync (the Cursor pseudo-encoding) -// are supported. +// Apple's Screen Sharing scheme (type 30). Raw + CopyRect + Hextile + ZRLE +// encodings (Tight compression not yet implemented). No dynamic resize. +// Clipboard sync (Latin-1 only, per RFB's ServerCutText/ClientCutText) and +// remote cursor shape sync (the Cursor pseudo-encoding) are supported. class VncSessionBackend : public SessionBackend { Q_OBJECT @@ -90,6 +90,8 @@ private: WaitingHextileTileMeta, WaitingHextileSubrectData, WaitingHextileRawTileData, + WaitingZrleCompressedLength, + WaitingZrleCompressedData, WaitingSetColourMapHeader, WaitingSetColourMapData, WaitingServerCutTextHeader, @@ -139,6 +141,15 @@ private: int m_hextileSubrectsRemaining; bool m_hextileSubrectsColoured; + // ZRLE's zlib stream (RFC 6143 SS7.7.6) persists for the whole + // connection, not per-rectangle or per-update -- lazily initialized on + // the first ZRLE rectangle, torn down and reset on every fresh + // connect/reconnect via resetProtocolState(). z_stream_s is only + // forward-declared here so doesn't leak into every includer of + // this header; the full type is only needed in the .cpp. + z_stream_s* m_zrleInflateStream; + bool m_zrleInflateInitialized; + void setState(SessionState state, const QString& message); void resetProtocolState(); void processReceiveBuffer(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2768db7..fbb7757 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -34,7 +34,7 @@ add_executable(test_vnc_session_backend ) target_include_directories(test_vnc_session_backend PRIVATE ${CMAKE_SOURCE_DIR}/src) target_link_libraries(test_vnc_session_backend PRIVATE - Qt6::Core Qt6::Gui Qt6::Network Qt6::Test OpenSSL::Crypto + Qt6::Core Qt6::Gui Qt6::Network Qt6::Test OpenSSL::Crypto ZLIB::ZLIB ) add_test(NAME test_vnc_session_backend COMMAND test_vnc_session_backend) diff --git a/tests/test_vnc_session_backend.cpp b/tests/test_vnc_session_backend.cpp index b039ae5..0eb497d 100644 --- a/tests/test_vnc_session_backend.cpp +++ b/tests/test_vnc_session_backend.cpp @@ -4,6 +4,8 @@ #include #include +#include + namespace { // Independently documented bit-reversal example for VNC Authentication's // DES key prep (password "COW"): 'C'=0x43, 'O'=0x4F, 'W'=0x57, each @@ -125,6 +127,88 @@ QByteArray hextileFramebufferUpdate(int x, int y, int width, int height, const Q update.append(tileData); return update; } + +void appendU32(QByteArray& buf, quint32 value) +{ + buf.append(static_cast((value >> 24) & 0xFF)); + buf.append(static_cast((value >> 16) & 0xFF)); + buf.append(static_cast((value >> 8) & 0xFF)); + buf.append(static_cast(value & 0xFF)); +} + +// ZRLE's CPIXEL: 3 bytes, B,G,R order (the pad byte a full 32bpp pixel +// would have is simply omitted) -- matches VncPixelCodecs::rgbFromPixelBytes. +QByteArray cpixelBytes(int r, int g, int b) +{ + QByteArray bytes; + bytes.append(static_cast(b)); + bytes.append(static_cast(g)); + bytes.append(static_cast(r)); + return bytes; +} + +// A FramebufferUpdate message with exactly one ZRLE-encoded rectangle +// (RFC 6143 encoding type 16): a 4-byte compressed length followed by that +// many already-zlib-compressed bytes. +QByteArray zrleFramebufferUpdate(int x, int y, int width, int height, const QByteArray& compressed) +{ + QByteArray update; + update.append(char(0)); update.append(char(0)); + update.append(char(0)); update.append(char(1)); // 1 rectangle + appendU16(update, static_cast(x)); + appendU16(update, static_cast(y)); + appendU16(update, static_cast(width)); + appendU16(update, static_cast(height)); + update.append(char(0)); update.append(char(0)); update.append(char(0)); + update.append(char(16)); // encoding = ZRLE + appendU32(update, static_cast(compressed.size())); + update.append(compressed); + return update; +} + +// Compresses `input` as a single, complete, self-contained zlib stream +// (its own header + Z_FINISH) -- correct for tests where each +// VncSessionBackend under test lazily starts its own fresh inflate stream +// on the first ZRLE rectangle it ever sees. +QByteArray zlibCompressWhole(const QByteArray& input) +{ + z_stream stream{}; + deflateInit(&stream, Z_DEFAULT_COMPRESSION); + QByteArray output(static_cast(deflateBound(&stream, static_cast(input.size()))), + char(0)); + stream.next_in = reinterpret_cast(const_cast(input.constData())); + stream.avail_in = static_cast(input.size()); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + deflate(&stream, Z_FINISH); + output.resize(output.size() - static_cast(stream.avail_out)); + deflateEnd(&stream); + return output; +} + +// Compresses `input` as one chunk of an ongoing (not yet finished, unless +// `finish` is true) zlib stream carried in `stream` across calls -- used to +// build two separately-decodable compressed chunks that only both decode +// correctly against a single persistent inflate stream, for testing that +// ZRLE's zlib stream is retained across rectangles/updates rather than +// reset each time. +QByteArray zlibCompressChunk(z_stream& stream, const QByteArray& input, bool finish) +{ + QByteArray output(static_cast(deflateBound(&stream, static_cast(input.size()))) + 64, + char(0)); + stream.next_in = reinterpret_cast(const_cast(input.constData())); + stream.avail_in = static_cast(input.size()); + const int flushMode = finish ? Z_FINISH : Z_SYNC_FLUSH; + int totalOut = 0; + do { + stream.next_out = reinterpret_cast(output.data() + totalOut); + stream.avail_out = static_cast(output.size() - totalOut); + deflate(&stream, flushMode); + totalOut = output.size() - static_cast(stream.avail_out); + } while (stream.avail_out == 0); + output.resize(totalOut); + return output; +} } class TestVncSessionBackend : public QObject @@ -165,6 +249,12 @@ private slots: void hextileUncolouredSubrectUsesForeground(); void hextileColouredSubrectUsesOwnColor(); void hextileMultiTilePersistsBackgroundAcrossTiles(); + void zrleRawTileProducesExpectedPixels(); + void zrleSolidTileFillsColor(); + void zrlePackedPaletteTileProducesExpectedPixels(); + void zrlePlainRleTileProducesExpectedPixels(); + void zrlePaletteRleTileProducesExpectedPixels(); + void zrleStreamPersistsAcrossTwoFramebufferUpdates(); private: std::unique_ptr m_server; @@ -1124,5 +1214,297 @@ void TestVncSessionBackend::hextileMultiTilePersistsBackgroundAcrossTiles() QCOMPARE(m_lastFrame.pixelColor(10, 10), QColor(0, 0, 255)); } +void TestVncSessionBackend::zrleRawTileProducesExpectedPixels() +{ + QByteArray tile; + tile.append(char(0)); // Raw + tile += cpixelBytes(255, 0, 0); + tile += cpixelBytes(0, 255, 0); + const QByteArray compressed = zlibCompressWhole(tile); + + connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() { + m_server->sendWhenConnected(QByteArray("RFB 003.008\n")); + }); + connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() { + switch (m_server->nextStep()) { + case 0: { + QByteArray securityTypes; + securityTypes.append(char(1)); + securityTypes.append(char(1)); + m_server->sendWhenConnected(securityTypes); + break; + } + case 1: + m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK + break; + case 2: + m_server->sendWhenConnected(serverInitBytes(2, 1)); + break; + case 3: + m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 2, 1, compressed)); + break; + default: + break; + } + }); + + m_backend->connectSession(makeOptions()); + QTRY_VERIFY(m_gotFrame); + QCOMPARE(m_lastFrame.size(), QSize(2, 1)); + QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(255, 0, 0)); + QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(0, 255, 0)); +} + +void TestVncSessionBackend::zrleSolidTileFillsColor() +{ + QByteArray tile; + tile.append(char(1)); // Solid + tile += cpixelBytes(0, 128, 255); + const QByteArray compressed = zlibCompressWhole(tile); + + connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() { + m_server->sendWhenConnected(QByteArray("RFB 003.008\n")); + }); + connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() { + switch (m_server->nextStep()) { + case 0: { + QByteArray securityTypes; + securityTypes.append(char(1)); + securityTypes.append(char(1)); + m_server->sendWhenConnected(securityTypes); + break; + } + case 1: + m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK + break; + case 2: + m_server->sendWhenConnected(serverInitBytes(2, 2)); + break; + case 3: + m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 2, 2, compressed)); + break; + default: + break; + } + }); + + m_backend->connectSession(makeOptions()); + QTRY_VERIFY(m_gotFrame); + QCOMPARE(m_lastFrame.size(), QSize(2, 2)); + for (int y = 0; y < 2; ++y) { + for (int x = 0; x < 2; ++x) { + QCOMPARE(m_lastFrame.pixelColor(x, y), QColor(0, 128, 255)); + } + } +} + +void TestVncSessionBackend::zrlePackedPaletteTileProducesExpectedPixels() +{ + // 4 pixels, 2-color palette, 1 bit/pixel, packed MSB-first: indices + // [0,1,1,0] -> byte 0b0110_0000. + QByteArray tile; + tile.append(char(2)); // packed palette, paletteSize = 2 + tile += cpixelBytes(10, 20, 30); // palette[0] + tile += cpixelBytes(200, 210, 220); // palette[1] + tile.append(char(0x60)); + const QByteArray compressed = zlibCompressWhole(tile); + + connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() { + m_server->sendWhenConnected(QByteArray("RFB 003.008\n")); + }); + connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() { + switch (m_server->nextStep()) { + case 0: { + QByteArray securityTypes; + securityTypes.append(char(1)); + securityTypes.append(char(1)); + m_server->sendWhenConnected(securityTypes); + break; + } + case 1: + m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK + break; + case 2: + m_server->sendWhenConnected(serverInitBytes(4, 1)); + break; + case 3: + m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 4, 1, compressed)); + break; + default: + break; + } + }); + + m_backend->connectSession(makeOptions()); + QTRY_VERIFY(m_gotFrame); + QCOMPARE(m_lastFrame.size(), QSize(4, 1)); + QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(10, 20, 30)); + QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(200, 210, 220)); + QCOMPARE(m_lastFrame.pixelColor(2, 0), QColor(200, 210, 220)); + QCOMPARE(m_lastFrame.pixelColor(3, 0), QColor(10, 20, 30)); +} + +void TestVncSessionBackend::zrlePlainRleTileProducesExpectedPixels() +{ + // One run of length 3 (encoded as total=2, single continuation byte + // since 2 != 255) covering the whole 3x1 tile. + QByteArray tile; + tile.append(char(128)); // Plain RLE + tile += cpixelBytes(50, 60, 70); + tile.append(char(2)); + const QByteArray compressed = zlibCompressWhole(tile); + + connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() { + m_server->sendWhenConnected(QByteArray("RFB 003.008\n")); + }); + connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() { + switch (m_server->nextStep()) { + case 0: { + QByteArray securityTypes; + securityTypes.append(char(1)); + securityTypes.append(char(1)); + m_server->sendWhenConnected(securityTypes); + break; + } + case 1: + m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK + break; + case 2: + m_server->sendWhenConnected(serverInitBytes(3, 1)); + break; + case 3: + m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 3, 1, compressed)); + break; + default: + break; + } + }); + + m_backend->connectSession(makeOptions()); + QTRY_VERIFY(m_gotFrame); + QCOMPARE(m_lastFrame.size(), QSize(3, 1)); + for (int x = 0; x < 3; ++x) { + QCOMPARE(m_lastFrame.pixelColor(x, 0), QColor(50, 60, 70)); + } +} + +void TestVncSessionBackend::zrlePaletteRleTileProducesExpectedPixels() +{ + // palette = [colorA, colorB]. Entries: index 0, run length 1 (index + // byte < 128, no run-length byte follows); then index 1, run length 2 + // (index byte 0x81 = 128 + 1, followed by a run-length byte encoding + // total=1 -> actual length 2). + QByteArray tile; + tile.append(char(130)); // Palette RLE, paletteSize = 2 + tile += cpixelBytes(1, 2, 3); // palette[0] = colorA + tile += cpixelBytes(4, 5, 6); // palette[1] = colorB + tile.append(char(0x00)); + tile.append(char(0x81)); + tile.append(char(0x01)); + const QByteArray compressed = zlibCompressWhole(tile); + + connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() { + m_server->sendWhenConnected(QByteArray("RFB 003.008\n")); + }); + connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() { + switch (m_server->nextStep()) { + case 0: { + QByteArray securityTypes; + securityTypes.append(char(1)); + securityTypes.append(char(1)); + m_server->sendWhenConnected(securityTypes); + break; + } + case 1: + m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK + break; + case 2: + m_server->sendWhenConnected(serverInitBytes(3, 1)); + break; + case 3: + m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 3, 1, compressed)); + break; + default: + break; + } + }); + + m_backend->connectSession(makeOptions()); + QTRY_VERIFY(m_gotFrame); + QCOMPARE(m_lastFrame.size(), QSize(3, 1)); + QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(1, 2, 3)); + QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(4, 5, 6)); + QCOMPARE(m_lastFrame.pixelColor(2, 0), QColor(4, 5, 6)); +} + +void TestVncSessionBackend::zrleStreamPersistsAcrossTwoFramebufferUpdates() +{ + QByteArray tile1; + tile1.append(char(0)); // Raw + tile1 += cpixelBytes(255, 0, 0); + tile1 += cpixelBytes(0, 255, 0); + + QByteArray tile2; + tile2.append(char(1)); // Solid + tile2 += cpixelBytes(10, 20, 30); + + // One continuous deflate stream split into two chunks -- the second + // chunk is only decodable by an inflate stream that already consumed + // the first, which is exactly what should happen if + // VncSessionBackend's ZRLE stream persists across separate + // FramebufferUpdate messages instead of being reset each time. + z_stream deflateStream{}; + deflateInit(&deflateStream, Z_DEFAULT_COMPRESSION); + const QByteArray compressed1 = zlibCompressChunk(deflateStream, tile1, false); + const QByteArray compressed2 = zlibCompressChunk(deflateStream, tile2, true); + deflateEnd(&deflateStream); + + // Capture each frame as it arrives, rather than polling m_lastFrame + // between separately-timed QTRY_VERIFY checks -- the two updates can + // both land before the first check ever gets scheduled to run, so only + // recording state exactly when each signal fires is race-free here. + QVector frames; + connect(m_backend.get(), &SessionBackend::frameUpdated, this, + [&frames](const QImage& frame) { frames.append(frame.copy()); }); + + connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() { + m_server->sendWhenConnected(QByteArray("RFB 003.008\n")); + }); + connect(m_server.get(), &FakeVncServer::dataReceived, this, + [this, compressed1, compressed2]() { + switch (m_server->nextStep()) { + case 0: { + QByteArray securityTypes; + securityTypes.append(char(1)); + securityTypes.append(char(1)); + m_server->sendWhenConnected(securityTypes); + break; + } + case 1: + m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK + break; + case 2: + m_server->sendWhenConnected(serverInitBytes(2, 1)); + break; + case 3: // first (non-incremental) FramebufferUpdateRequest + m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 2, 1, compressed1)); + break; + case 4: // next (incremental) FramebufferUpdateRequest + m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 2, 1, compressed2)); + break; + default: + break; + } + }); + + m_backend->connectSession(makeOptions()); + QTRY_VERIFY(frames.size() >= 2); + QCOMPARE(m_lastState, SessionState::Connected); + QCOMPARE(frames.at(0).pixelColor(0, 0), QColor(255, 0, 0)); + QCOMPARE(frames.at(0).pixelColor(1, 0), QColor(0, 255, 0)); + QCOMPARE(frames.at(1).pixelColor(0, 0), QColor(10, 20, 30)); + QCOMPARE(frames.at(1).pixelColor(1, 0), QColor(10, 20, 30)); +} + QTEST_GUILESS_MAIN(TestVncSessionBackend) #include "test_vnc_session_backend.moc"