Add VNC ZRLE decoding

Implements RFC 6143 SS7.7.6: a ZRLE rectangle is a 4-byte compressed length followed by that many zlib-compressed bytes, decompressing to 64x64 tiles each using one of five subencodings (Raw, Solid, packed palette, Plain RLE, Palette RLE). The zlib stream persists for the whole connection rather than being reset per-rectangle or per-update, so VncSessionBackend now owns a lazily-initialized, persistent z_stream torn down only in resetProtocolState() on a fresh connect/reconnect. Since the entire rectangle's compressed data decompresses into memory in one shot, tile parsing is a plain synchronous loop rather than needing its own RfbState values -- only the compressed-length and compressed-data reads are actual protocol states. Tile decoding (the five subencodings, including the continuation-byte run-length encoding shared by two of them) lives in vnc_pixel_codecs.h/.cpp alongside the Hextile decoder, unit-tested with 6 new tests covering each subencoding plus a persistence test that splits one continuous deflate stream across two separate FramebufferUpdate messages -- it only decodes correctly if the connection's inflate stream is retained between them. Adds a top-level find_package(ZLIB REQUIRED) + ZLIB::ZLIB link (previously only pulled in transitively via vendored FreeRDP's own smartcard-emulation feature, which happened to have it enabled but shouldn't be relied on for that). Live-verified against the TightVNC test server that nothing regressed (connect, cursor, clipboard); that server consistently sends Raw for actual framebuffer content regardless of announced encodings, so Hextile/ZRLE's live decode path isn't independently confirmed against a real server -- the unit tests are the primary evidence here. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-15 21:02:26 -06:00
parent e49fa0cf26
commit bb022edcf2
8 changed files with 772 additions and 19 deletions
@@ -34,7 +34,7 @@ add_executable(test_vnc_session_backend
 )
 target_include_directories(test_vnc_session_backend PRIVATE ${CMAKE_SOURCE_DIR}/src)
 target_link_libraries(test_vnc_session_backend PRIVATE
-    Qt6::Core Qt6::Gui Qt6::Network Qt6::Test OpenSSL::Crypto
+    Qt6::Core Qt6::Gui Qt6::Network Qt6::Test OpenSSL::Crypto ZLIB::ZLIB
 )
 add_test(NAME test_vnc_session_backend COMMAND test_vnc_session_backend)

@@ -4,6 +4,8 @@
 #include <QTcpSocket>
 #include <QTest>

+#include <zlib.h>
+
 namespace {
 // Independently documented bit-reversal example for VNC Authentication's
 // DES key prep (password "COW"): 'C'=0x43, 'O'=0x4F, 'W'=0x57, each
@@ -125,6 +127,88 @@ QByteArray hextileFramebufferUpdate(int x, int y, int width, int height, const Q
    update.append(tileData);
    return update;
 }
+
+void appendU32(QByteArray& buf, quint32 value)
+{
+    buf.append(static_cast<char>((value >> 24) & 0xFF));
+    buf.append(static_cast<char>((value >> 16) & 0xFF));
+    buf.append(static_cast<char>((value >> 8) & 0xFF));
+    buf.append(static_cast<char>(value & 0xFF));
+}
+
+// ZRLE's CPIXEL: 3 bytes, B,G,R order (the pad byte a full 32bpp pixel
+// would have is simply omitted) -- matches VncPixelCodecs::rgbFromPixelBytes.
+QByteArray cpixelBytes(int r, int g, int b)
+{
+    QByteArray bytes;
+    bytes.append(static_cast<char>(b));
+    bytes.append(static_cast<char>(g));
+    bytes.append(static_cast<char>(r));
+    return bytes;
+}
+
+// A FramebufferUpdate message with exactly one ZRLE-encoded rectangle
+// (RFC 6143 encoding type 16): a 4-byte compressed length followed by that
+// many already-zlib-compressed bytes.
+QByteArray zrleFramebufferUpdate(int x, int y, int width, int height, const QByteArray& compressed)
+{
+    QByteArray update;
+    update.append(char(0)); update.append(char(0));
+    update.append(char(0)); update.append(char(1)); // 1 rectangle
+    appendU16(update, static_cast<quint16>(x));
+    appendU16(update, static_cast<quint16>(y));
+    appendU16(update, static_cast<quint16>(width));
+    appendU16(update, static_cast<quint16>(height));
+    update.append(char(0)); update.append(char(0)); update.append(char(0));
+    update.append(char(16)); // encoding = ZRLE
+    appendU32(update, static_cast<quint32>(compressed.size()));
+    update.append(compressed);
+    return update;
+}
+
+// Compresses `input` as a single, complete, self-contained zlib stream
+// (its own header + Z_FINISH) -- correct for tests where each
+// VncSessionBackend under test lazily starts its own fresh inflate stream
+// on the first ZRLE rectangle it ever sees.
+QByteArray zlibCompressWhole(const QByteArray& input)
+{
+    z_stream stream{};
+    deflateInit(&stream, Z_DEFAULT_COMPRESSION);
+    QByteArray output(static_cast<int>(deflateBound(&stream, static_cast<uLong>(input.size()))),
+                      char(0));
+    stream.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(input.constData()));
+    stream.avail_in = static_cast<uInt>(input.size());
+    stream.next_out = reinterpret_cast<Bytef*>(output.data());
+    stream.avail_out = static_cast<uInt>(output.size());
+    deflate(&stream, Z_FINISH);
+    output.resize(output.size() - static_cast<int>(stream.avail_out));
+    deflateEnd(&stream);
+    return output;
+}
+
+// Compresses `input` as one chunk of an ongoing (not yet finished, unless
+// `finish` is true) zlib stream carried in `stream` across calls -- used to
+// build two separately-decodable compressed chunks that only both decode
+// correctly against a single persistent inflate stream, for testing that
+// ZRLE's zlib stream is retained across rectangles/updates rather than
+// reset each time.
+QByteArray zlibCompressChunk(z_stream& stream, const QByteArray& input, bool finish)
+{
+    QByteArray output(static_cast<int>(deflateBound(&stream, static_cast<uLong>(input.size()))) + 64,
+                      char(0));
+    stream.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(input.constData()));
+    stream.avail_in = static_cast<uInt>(input.size());
+    const int flushMode = finish ? Z_FINISH : Z_SYNC_FLUSH;
+    int totalOut = 0;
+    do {
+        stream.next_out = reinterpret_cast<Bytef*>(output.data() + totalOut);
+        stream.avail_out = static_cast<uInt>(output.size() - totalOut);
+        deflate(&stream, flushMode);
+        totalOut = output.size() - static_cast<int>(stream.avail_out);
+    } while (stream.avail_out == 0);
+    output.resize(totalOut);
+    return output;
+}
 }

 class TestVncSessionBackend : public QObject
@@ -165,6 +249,12 @@ private slots:
    void hextileUncolouredSubrectUsesForeground();
    void hextileColouredSubrectUsesOwnColor();
    void hextileMultiTilePersistsBackgroundAcrossTiles();
+    void zrleRawTileProducesExpectedPixels();
+    void zrleSolidTileFillsColor();
+    void zrlePackedPaletteTileProducesExpectedPixels();
+    void zrlePlainRleTileProducesExpectedPixels();
+    void zrlePaletteRleTileProducesExpectedPixels();
+    void zrleStreamPersistsAcrossTwoFramebufferUpdates();

 private:
    std::unique_ptr<FakeVncServer> m_server;
@@ -1124,5 +1214,297 @@ void TestVncSessionBackend::hextileMultiTilePersistsBackgroundAcrossTiles()
    QCOMPARE(m_lastFrame.pixelColor(10, 10), QColor(0, 0, 255));
 }

+void TestVncSessionBackend::zrleRawTileProducesExpectedPixels()
+{
+    QByteArray tile;
+    tile.append(char(0)); // Raw
+    tile += cpixelBytes(255, 0, 0);
+    tile += cpixelBytes(0, 255, 0);
+    const QByteArray compressed = zlibCompressWhole(tile);
+
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() {
+        switch (m_server->nextStep()) {
+        case 0: {
+            QByteArray securityTypes;
+            securityTypes.append(char(1));
+            securityTypes.append(char(1));
+            m_server->sendWhenConnected(securityTypes);
+            break;
+        }
+        case 1:
+            m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
+            break;
+        case 2:
+            m_server->sendWhenConnected(serverInitBytes(2, 1));
+            break;
+        case 3:
+            m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 2, 1, compressed));
+            break;
+        default:
+            break;
+        }
+    });
+
+    m_backend->connectSession(makeOptions());
+    QTRY_VERIFY(m_gotFrame);
+    QCOMPARE(m_lastFrame.size(), QSize(2, 1));
+    QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(255, 0, 0));
+    QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(0, 255, 0));
+}
+
+void TestVncSessionBackend::zrleSolidTileFillsColor()
+{
+    QByteArray tile;
+    tile.append(char(1)); // Solid
+    tile += cpixelBytes(0, 128, 255);
+    const QByteArray compressed = zlibCompressWhole(tile);
+
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() {
+        switch (m_server->nextStep()) {
+        case 0: {
+            QByteArray securityTypes;
+            securityTypes.append(char(1));
+            securityTypes.append(char(1));
+            m_server->sendWhenConnected(securityTypes);
+            break;
+        }
+        case 1:
+            m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
+            break;
+        case 2:
+            m_server->sendWhenConnected(serverInitBytes(2, 2));
+            break;
+        case 3:
+            m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 2, 2, compressed));
+            break;
+        default:
+            break;
+        }
+    });
+
+    m_backend->connectSession(makeOptions());
+    QTRY_VERIFY(m_gotFrame);
+    QCOMPARE(m_lastFrame.size(), QSize(2, 2));
+    for (int y = 0; y < 2; ++y) {
+        for (int x = 0; x < 2; ++x) {
+            QCOMPARE(m_lastFrame.pixelColor(x, y), QColor(0, 128, 255));
+        }
+    }
+}
+
+void TestVncSessionBackend::zrlePackedPaletteTileProducesExpectedPixels()
+{
+    // 4 pixels, 2-color palette, 1 bit/pixel, packed MSB-first: indices
+    // [0,1,1,0] -> byte 0b0110_0000.
+    QByteArray tile;
+    tile.append(char(2)); // packed palette, paletteSize = 2
+    tile += cpixelBytes(10, 20, 30); // palette[0]
+    tile += cpixelBytes(200, 210, 220); // palette[1]
+    tile.append(char(0x60));
+    const QByteArray compressed = zlibCompressWhole(tile);
+
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() {
+        switch (m_server->nextStep()) {
+        case 0: {
+            QByteArray securityTypes;
+            securityTypes.append(char(1));
+            securityTypes.append(char(1));
+            m_server->sendWhenConnected(securityTypes);
+            break;
+        }
+        case 1:
+            m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
+            break;
+        case 2:
+            m_server->sendWhenConnected(serverInitBytes(4, 1));
+            break;
+        case 3:
+            m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 4, 1, compressed));
+            break;
+        default:
+            break;
+        }
+    });
+
+    m_backend->connectSession(makeOptions());
+    QTRY_VERIFY(m_gotFrame);
+    QCOMPARE(m_lastFrame.size(), QSize(4, 1));
+    QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(10, 20, 30));
+    QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(200, 210, 220));
+    QCOMPARE(m_lastFrame.pixelColor(2, 0), QColor(200, 210, 220));
+    QCOMPARE(m_lastFrame.pixelColor(3, 0), QColor(10, 20, 30));
+}
+
+void TestVncSessionBackend::zrlePlainRleTileProducesExpectedPixels()
+{
+    // One run of length 3 (encoded as total=2, single continuation byte
+    // since 2 != 255) covering the whole 3x1 tile.
+    QByteArray tile;
+    tile.append(char(128)); // Plain RLE
+    tile += cpixelBytes(50, 60, 70);
+    tile.append(char(2));
+    const QByteArray compressed = zlibCompressWhole(tile);
+
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() {
+        switch (m_server->nextStep()) {
+        case 0: {
+            QByteArray securityTypes;
+            securityTypes.append(char(1));
+            securityTypes.append(char(1));
+            m_server->sendWhenConnected(securityTypes);
+            break;
+        }
+        case 1:
+            m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
+            break;
+        case 2:
+            m_server->sendWhenConnected(serverInitBytes(3, 1));
+            break;
+        case 3:
+            m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 3, 1, compressed));
+            break;
+        default:
+            break;
+        }
+    });
+
+    m_backend->connectSession(makeOptions());
+    QTRY_VERIFY(m_gotFrame);
+    QCOMPARE(m_lastFrame.size(), QSize(3, 1));
+    for (int x = 0; x < 3; ++x) {
+        QCOMPARE(m_lastFrame.pixelColor(x, 0), QColor(50, 60, 70));
+    }
+}
+
+void TestVncSessionBackend::zrlePaletteRleTileProducesExpectedPixels()
+{
+    // palette = [colorA, colorB]. Entries: index 0, run length 1 (index
+    // byte < 128, no run-length byte follows); then index 1, run length 2
+    // (index byte 0x81 = 128 + 1, followed by a run-length byte encoding
+    // total=1 -> actual length 2).
+    QByteArray tile;
+    tile.append(char(130)); // Palette RLE, paletteSize = 2
+    tile += cpixelBytes(1, 2, 3); // palette[0] = colorA
+    tile += cpixelBytes(4, 5, 6); // palette[1] = colorB
+    tile.append(char(0x00));
+    tile.append(char(0x81));
+    tile.append(char(0x01));
+    const QByteArray compressed = zlibCompressWhole(tile);
+
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, compressed]() {
+        switch (m_server->nextStep()) {
+        case 0: {
+            QByteArray securityTypes;
+            securityTypes.append(char(1));
+            securityTypes.append(char(1));
+            m_server->sendWhenConnected(securityTypes);
+            break;
+        }
+        case 1:
+            m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
+            break;
+        case 2:
+            m_server->sendWhenConnected(serverInitBytes(3, 1));
+            break;
+        case 3:
+            m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 3, 1, compressed));
+            break;
+        default:
+            break;
+        }
+    });
+
+    m_backend->connectSession(makeOptions());
+    QTRY_VERIFY(m_gotFrame);
+    QCOMPARE(m_lastFrame.size(), QSize(3, 1));
+    QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(1, 2, 3));
+    QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(4, 5, 6));
+    QCOMPARE(m_lastFrame.pixelColor(2, 0), QColor(4, 5, 6));
+}
+
+void TestVncSessionBackend::zrleStreamPersistsAcrossTwoFramebufferUpdates()
+{
+    QByteArray tile1;
+    tile1.append(char(0)); // Raw
+    tile1 += cpixelBytes(255, 0, 0);
+    tile1 += cpixelBytes(0, 255, 0);
+
+    QByteArray tile2;
+    tile2.append(char(1)); // Solid
+    tile2 += cpixelBytes(10, 20, 30);
+
+    // One continuous deflate stream split into two chunks -- the second
+    // chunk is only decodable by an inflate stream that already consumed
+    // the first, which is exactly what should happen if
+    // VncSessionBackend's ZRLE stream persists across separate
+    // FramebufferUpdate messages instead of being reset each time.
+    z_stream deflateStream{};
+    deflateInit(&deflateStream, Z_DEFAULT_COMPRESSION);
+    const QByteArray compressed1 = zlibCompressChunk(deflateStream, tile1, false);
+    const QByteArray compressed2 = zlibCompressChunk(deflateStream, tile2, true);
+    deflateEnd(&deflateStream);
+
+    // Capture each frame as it arrives, rather than polling m_lastFrame
+    // between separately-timed QTRY_VERIFY checks -- the two updates can
+    // both land before the first check ever gets scheduled to run, so only
+    // recording state exactly when each signal fires is race-free here.
+    QVector<QImage> frames;
+    connect(m_backend.get(), &SessionBackend::frameUpdated, this,
+            [&frames](const QImage& frame) { frames.append(frame.copy()); });
+
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this,
+            [this, compressed1, compressed2]() {
+                switch (m_server->nextStep()) {
+                case 0: {
+                    QByteArray securityTypes;
+                    securityTypes.append(char(1));
+                    securityTypes.append(char(1));
+                    m_server->sendWhenConnected(securityTypes);
+                    break;
+                }
+                case 1:
+                    m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
+                    break;
+                case 2:
+                    m_server->sendWhenConnected(serverInitBytes(2, 1));
+                    break;
+                case 3: // first (non-incremental) FramebufferUpdateRequest
+                    m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 2, 1, compressed1));
+                    break;
+                case 4: // next (incremental) FramebufferUpdateRequest
+                    m_server->sendWhenConnected(zrleFramebufferUpdate(0, 0, 2, 1, compressed2));
+                    break;
+                default:
+                    break;
+                }
+            });
+
+    m_backend->connectSession(makeOptions());
+    QTRY_VERIFY(frames.size() >= 2);
+    QCOMPARE(m_lastState, SessionState::Connected);
+    QCOMPARE(frames.at(0).pixelColor(0, 0), QColor(255, 0, 0));
+    QCOMPARE(frames.at(0).pixelColor(1, 0), QColor(0, 255, 0));
+    QCOMPARE(frames.at(1).pixelColor(0, 0), QColor(10, 20, 30));
+    QCOMPARE(frames.at(1).pixelColor(1, 0), QColor(10, 20, 30));
+}
+
 QTEST_GUILESS_MAIN(TestVncSessionBackend)
 #include "test_vnc_session_backend.moc"