Add VNC Tight decoding

Implements RFC 6143's Tight encoding: a compression-control byte (low 4 bits reset one of 4 persistent zlib streams; high nibble selects Fill/JPEG/Basic mode) followed by Fill's 3-byte solid color, JPEG's compact-length-prefixed baseline JPEG covering the whole rectangle (decoded via libjpeg-turbo directly, not QImage's plugin, to avoid a packaging-dependent runtime failure mode), or Basic mode's compact-length-prefixed zlib payload plus a filter (Copy, Palette, or Gradient) applied after decompression. Unlike Hextile/ZRLE, Tight has no internal tiling -- one rectangle is one filtered/compressed unit. The three filters live in vnc_pixel_codecs.h/.cpp alongside the Hextile/ZRLE decoders. Adds find_package(JPEG REQUIRED) + JPEG::JPEG as a new build dependency (confirmed available via libjpeg-turbo on this dev machine). 5 new tests cover Fill, Basic+Copy, Basic+Palette, JPEG (round-tripped through a real libjpeg-turbo-encoded fixture, compared with tolerance since JPEG is lossy), and the stream-reset flag correctly tearing down and reinitializing a targeted stream rather than erroring on stale state. Known, documented gap: this decoder always treats Basic-mode payloads as zlib-compressed; the real protocol allows very small payloads to skip compression, which couldn't be verified with confidence against the RFC text alone and is narrow enough in practice (tiny solid areas are virtually always sent as Fill instead) to leave unhandled for now -- it fails that one rectangle's decode cleanly rather than misinterpreting it silently. The Gradient filter is implemented from the spec description but is the least exercised of the three in this pass. Live-verified against the TightVNC test server that nothing regressed; that server still consistently chose Raw for actual framebuffer content regardless of announced encodings, so Tight's live decode path isn't independently confirmed against a real server here either -- the unit tests are the primary evidence. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-15 21:13:17 -06:00
parent bb022edcf2
commit 4fca8fce41
8 changed files with 901 additions and 26 deletions
@@ -34,7 +34,7 @@ add_executable(test_vnc_session_backend
 )
 target_include_directories(test_vnc_session_backend PRIVATE ${CMAKE_SOURCE_DIR}/src)
 target_link_libraries(test_vnc_session_backend PRIVATE
-    Qt6::Core Qt6::Gui Qt6::Network Qt6::Test OpenSSL::Crypto ZLIB::ZLIB
+    Qt6::Core Qt6::Gui Qt6::Network Qt6::Test OpenSSL::Crypto ZLIB::ZLIB JPEG::JPEG
 )
 add_test(NAME test_vnc_session_backend COMMAND test_vnc_session_backend)

@@ -6,6 +6,12 @@

 #include <zlib.h>

+extern "C" {
+#include <jpeglib.h>
+}
+
+#include <cstdlib>
+
 namespace {
 // Independently documented bit-reversal example for VNC Authentication's
 // DES key prep (password "COW"): 'C'=0x43, 'O'=0x4F, 'W'=0x57, each
@@ -209,6 +215,88 @@ QByteArray zlibCompressChunk(z_stream& stream, const QByteArray& input, bool fin
    output.resize(totalOut);
    return output;
 }
+
+// Mirrors VncSessionBackend's compact-length decode in reverse, for
+// building test fixtures.
+void appendTightCompactLength(QByteArray& buf, quint32 length)
+{
+    buf.append(static_cast<char>((length & 0x7F) | ((length > 0x7F) ? 0x80 : 0)));
+    if (length <= 0x7F) {
+        return;
+    }
+    length >>= 7;
+    buf.append(static_cast<char>((length & 0x7F) | ((length > 0x7F) ? 0x80 : 0)));
+    if (length <= 0x7F) {
+        return;
+    }
+    length >>= 7;
+    buf.append(static_cast<char>(length & 0xFF));
+}
+
+// A FramebufferUpdate message with exactly one Tight-encoded rectangle
+// (RFC 6143 encoding type 7), given the already fully-formed byte stream
+// to follow the rectangle header (starting with the compression-control
+// byte).
+QByteArray tightFramebufferUpdate(int x, int y, int width, int height, const QByteArray& tightData)
+{
+    QByteArray update;
+    update.append(char(0)); update.append(char(0));
+    update.append(char(0)); update.append(char(1)); // 1 rectangle
+    appendU16(update, static_cast<quint16>(x));
+    appendU16(update, static_cast<quint16>(y));
+    appendU16(update, static_cast<quint16>(width));
+    appendU16(update, static_cast<quint16>(height));
+    update.append(char(0)); update.append(char(0)); update.append(char(0));
+    update.append(char(7)); // encoding = Tight
+    update.append(tightData);
+    return update;
+}
+
+// Encodes a solid-colored width x height baseline JPEG using libjpeg-turbo
+// directly (mirroring the decode side's use of the same library), for
+// exercising Tight's JPEG sub-mode against real, valid JPEG bytes.
+QByteArray encodeJpegForTest(int width, int height, QRgb color)
+{
+    jpeg_compress_struct cinfo;
+    jpeg_error_mgr jerr;
+    cinfo.err = jpeg_std_error(&jerr);
+    jpeg_create_compress(&cinfo);
+
+    unsigned char* buffer = nullptr;
+    unsigned long bufferSize = 0;
+    jpeg_mem_dest(&cinfo, &buffer, &bufferSize);
+
+    cinfo.image_width = static_cast<JDIMENSION>(width);
+    cinfo.image_height = static_cast<JDIMENSION>(height);
+    cinfo.input_components = 3;
+    cinfo.in_color_space = JCS_RGB;
+    jpeg_set_defaults(&cinfo);
+    jpeg_set_quality(&cinfo, 100, TRUE);
+    // Disable chroma subsampling so a solid color round-trips as exactly
+    // as JPEG's DCT quantization allows, for a tighter test tolerance.
+    for (int i = 0; i < cinfo.num_components; ++i) {
+        cinfo.comp_info[i].h_samp_factor = 1;
+        cinfo.comp_info[i].v_samp_factor = 1;
+    }
+
+    jpeg_start_compress(&cinfo, TRUE);
+    QVector<unsigned char> row(width * 3);
+    for (int x = 0; x < width; ++x) {
+        row[(x * 3) + 0] = static_cast<unsigned char>(qRed(color));
+        row[(x * 3) + 1] = static_cast<unsigned char>(qGreen(color));
+        row[(x * 3) + 2] = static_cast<unsigned char>(qBlue(color));
+    }
+    JSAMPROW rowPointer[1] = { row.data() };
+    for (int y = 0; y < height; ++y) {
+        jpeg_write_scanlines(&cinfo, rowPointer, 1);
+    }
+    jpeg_finish_compress(&cinfo);
+
+    const QByteArray result(reinterpret_cast<const char*>(buffer), static_cast<int>(bufferSize));
+    jpeg_destroy_compress(&cinfo);
+    std::free(buffer);
+    return result;
+}
 }

 class TestVncSessionBackend : public QObject
@@ -255,6 +343,11 @@ private slots:
    void zrlePlainRleTileProducesExpectedPixels();
    void zrlePaletteRleTileProducesExpectedPixels();
    void zrleStreamPersistsAcrossTwoFramebufferUpdates();
+    void tightFillProducesExpectedPixels();
+    void tightBasicCopyFilterProducesExpectedPixels();
+    void tightBasicPaletteFilterProducesExpectedPixels();
+    void tightJpegProducesExpectedPixels();
+    void tightStreamResetFlagAllowsIndependentDecoding();

 private:
    std::unique_ptr<FakeVncServer> m_server;
@@ -1506,5 +1599,268 @@ void TestVncSessionBackend::zrleStreamPersistsAcrossTwoFramebufferUpdates()
    QCOMPARE(frames.at(1).pixelColor(1, 0), QColor(10, 20, 30));
 }

+void TestVncSessionBackend::tightFillProducesExpectedPixels()
+{
+    QByteArray tight;
+    tight.append(char(0x80)); // mode = Fill, no stream-reset bits
+    tight += cpixelBytes(10, 200, 30);
+
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, tight]() {
+        switch (m_server->nextStep()) {
+        case 0: {
+            QByteArray securityTypes;
+            securityTypes.append(char(1));
+            securityTypes.append(char(1));
+            m_server->sendWhenConnected(securityTypes);
+            break;
+        }
+        case 1:
+            m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
+            break;
+        case 2:
+            m_server->sendWhenConnected(serverInitBytes(3, 3));
+            break;
+        case 3:
+            m_server->sendWhenConnected(tightFramebufferUpdate(0, 0, 3, 3, tight));
+            break;
+        default:
+            break;
+        }
+    });
+
+    m_backend->connectSession(makeOptions());
+    QTRY_VERIFY(m_gotFrame);
+    QCOMPARE(m_lastFrame.size(), QSize(3, 3));
+    for (int y = 0; y < 3; ++y) {
+        for (int x = 0; x < 3; ++x) {
+            QCOMPARE(m_lastFrame.pixelColor(x, y), QColor(10, 200, 30));
+        }
+    }
+}
+
+void TestVncSessionBackend::tightBasicCopyFilterProducesExpectedPixels()
+{
+    QByteArray filtered;
+    filtered += cpixelBytes(255, 0, 0);
+    filtered += cpixelBytes(0, 255, 0);
+    filtered += cpixelBytes(0, 0, 255);
+    filtered += cpixelBytes(255, 255, 255);
+    const QByteArray compressed = zlibCompressWhole(filtered);
+
+    QByteArray tight;
+    tight.append(char(0x00)); // mode = Basic, stream 0, Copy implied
+    appendTightCompactLength(tight, static_cast<quint32>(compressed.size()));
+    tight += compressed;
+
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, tight]() {
+        switch (m_server->nextStep()) {
+        case 0: {
+            QByteArray securityTypes;
+            securityTypes.append(char(1));
+            securityTypes.append(char(1));
+            m_server->sendWhenConnected(securityTypes);
+            break;
+        }
+        case 1:
+            m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
+            break;
+        case 2:
+            m_server->sendWhenConnected(serverInitBytes(2, 2));
+            break;
+        case 3:
+            m_server->sendWhenConnected(tightFramebufferUpdate(0, 0, 2, 2, tight));
+            break;
+        default:
+            break;
+        }
+    });
+
+    m_backend->connectSession(makeOptions());
+    QTRY_VERIFY(m_gotFrame);
+    QCOMPARE(m_lastFrame.size(), QSize(2, 2));
+    QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(255, 0, 0));
+    QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(0, 255, 0));
+    QCOMPARE(m_lastFrame.pixelColor(0, 1), QColor(0, 0, 255));
+    QCOMPARE(m_lastFrame.pixelColor(1, 1), QColor(255, 255, 255));
+}
+
+void TestVncSessionBackend::tightBasicPaletteFilterProducesExpectedPixels()
+{
+    // 4 pixels, 2-color palette, 1 bit/pixel, packed MSB-first: indices
+    // [0,1,1,0] -> byte 0b0110_0000 (same packing convention as ZRLE's
+    // packed palette; with only one row here, continuous vs row-padded
+    // bit-packing happen to coincide).
+    QByteArray filtered;
+    filtered.append(char(0x01)); // paletteSize - 1 = 1 -> paletteSize = 2
+    filtered += cpixelBytes(10, 20, 30); // palette[0]
+    filtered += cpixelBytes(200, 210, 220); // palette[1]
+    filtered.append(char(0x60));
+    const QByteArray compressed = zlibCompressWhole(filtered);
+
+    QByteArray tight;
+    tight.append(char(0x40)); // mode = Basic, stream 0, explicit filter follows
+    tight.append(char(1)); // filter id = Palette
+    appendTightCompactLength(tight, static_cast<quint32>(compressed.size()));
+    tight += compressed;
+
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, tight]() {
+        switch (m_server->nextStep()) {
+        case 0: {
+            QByteArray securityTypes;
+            securityTypes.append(char(1));
+            securityTypes.append(char(1));
+            m_server->sendWhenConnected(securityTypes);
+            break;
+        }
+        case 1:
+            m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
+            break;
+        case 2:
+            m_server->sendWhenConnected(serverInitBytes(4, 1));
+            break;
+        case 3:
+            m_server->sendWhenConnected(tightFramebufferUpdate(0, 0, 4, 1, tight));
+            break;
+        default:
+            break;
+        }
+    });
+
+    m_backend->connectSession(makeOptions());
+    QTRY_VERIFY(m_gotFrame);
+    QCOMPARE(m_lastFrame.size(), QSize(4, 1));
+    QCOMPARE(m_lastFrame.pixelColor(0, 0), QColor(10, 20, 30));
+    QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(200, 210, 220));
+    QCOMPARE(m_lastFrame.pixelColor(2, 0), QColor(200, 210, 220));
+    QCOMPARE(m_lastFrame.pixelColor(3, 0), QColor(10, 20, 30));
+}
+
+void TestVncSessionBackend::tightJpegProducesExpectedPixels()
+{
+    // 8x8 (one full JPEG MCU at 1x1 chroma sampling) solid red.
+    const QByteArray jpegBytes = encodeJpegForTest(8, 8, qRgb(220, 20, 20));
+
+    QByteArray tight;
+    tight.append(char(0x90)); // mode = JPEG
+    appendTightCompactLength(tight, static_cast<quint32>(jpegBytes.size()));
+    tight += jpegBytes;
+
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, tight]() {
+        switch (m_server->nextStep()) {
+        case 0: {
+            QByteArray securityTypes;
+            securityTypes.append(char(1));
+            securityTypes.append(char(1));
+            m_server->sendWhenConnected(securityTypes);
+            break;
+        }
+        case 1:
+            m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
+            break;
+        case 2:
+            m_server->sendWhenConnected(serverInitBytes(8, 8));
+            break;
+        case 3:
+            m_server->sendWhenConnected(tightFramebufferUpdate(0, 0, 8, 8, tight));
+            break;
+        default:
+            break;
+        }
+    });
+
+    m_backend->connectSession(makeOptions());
+    QTRY_VERIFY(m_gotFrame);
+    QCOMPARE(m_lastFrame.size(), QSize(8, 8));
+    // JPEG is lossy even at quality 100 -- assert closeness, not equality.
+    const QColor decoded = m_lastFrame.pixelColor(4, 4);
+    QVERIFY(qAbs(decoded.red() - 220) <= 8);
+    QVERIFY(qAbs(decoded.green() - 20) <= 8);
+    QVERIFY(qAbs(decoded.blue() - 20) <= 8);
+}
+
+void TestVncSessionBackend::tightStreamResetFlagAllowsIndependentDecoding()
+{
+    QByteArray filteredA;
+    filteredA += cpixelBytes(10, 20, 30);
+    filteredA += cpixelBytes(10, 20, 30);
+    filteredA += cpixelBytes(10, 20, 30);
+    filteredA += cpixelBytes(10, 20, 30);
+    QByteArray filteredB;
+    filteredB += cpixelBytes(200, 210, 220);
+    filteredB += cpixelBytes(200, 210, 220);
+    filteredB += cpixelBytes(200, 210, 220);
+    filteredB += cpixelBytes(200, 210, 220);
+
+    // Each independently a complete, self-contained zlib stream (its own
+    // deflateInit + Z_FINISH) -- decoding the second one correctly after
+    // the first, on the same persistent stream slot, requires that
+    // stream-reset bit 0 in rectangle B's control byte actually tore down
+    // and re-initialized stream 0 rather than trying to keep feeding an
+    // already-finished stream more data.
+    const QByteArray compressedA = zlibCompressWhole(filteredA);
+    const QByteArray compressedB = zlibCompressWhole(filteredB);
+
+    QByteArray tightA;
+    tightA.append(char(0x00)); // mode = Basic, stream 0, Copy, no reset
+    appendTightCompactLength(tightA, static_cast<quint32>(compressedA.size()));
+    tightA += compressedA;
+
+    QByteArray tightB;
+    tightB.append(char(0x01)); // mode = Basic, stream 0, Copy, reset stream 0
+    appendTightCompactLength(tightB, static_cast<quint32>(compressedB.size()));
+    tightB += compressedB;
+
+    QVector<QImage> frames;
+    connect(m_backend.get(), &SessionBackend::frameUpdated, this,
+            [&frames](const QImage& frame) { frames.append(frame.copy()); });
+
+    connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
+        m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
+    });
+    connect(m_server.get(), &FakeVncServer::dataReceived, this, [this, tightA, tightB]() {
+        switch (m_server->nextStep()) {
+        case 0: {
+            QByteArray securityTypes;
+            securityTypes.append(char(1));
+            securityTypes.append(char(1));
+            m_server->sendWhenConnected(securityTypes);
+            break;
+        }
+        case 1:
+            m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
+            break;
+        case 2:
+            m_server->sendWhenConnected(serverInitBytes(2, 2));
+            break;
+        case 3: // first (non-incremental) FramebufferUpdateRequest
+            m_server->sendWhenConnected(tightFramebufferUpdate(0, 0, 2, 2, tightA));
+            break;
+        case 4: // next (incremental) FramebufferUpdateRequest
+            m_server->sendWhenConnected(tightFramebufferUpdate(0, 0, 2, 2, tightB));
+            break;
+        default:
+            break;
+        }
+    });
+
+    m_backend->connectSession(makeOptions());
+    QTRY_VERIFY(frames.size() >= 2);
+    QCOMPARE(m_lastState, SessionState::Connected);
+    QCOMPARE(frames.at(0).pixelColor(0, 0), QColor(10, 20, 30));
+    QCOMPARE(frames.at(1).pixelColor(0, 0), QColor(200, 210, 220));
+}
+
 QTEST_GUILESS_MAIN(TestVncSessionBackend)
 #include "test_vnc_session_backend.moc"