Add VNC ZRLE decoding

Implements RFC 6143 SS7.7.6: a ZRLE rectangle is a 4-byte compressed
length followed by that many zlib-compressed bytes, decompressing to
64x64 tiles each using one of five subencodings (Raw, Solid, packed
palette, Plain RLE, Palette RLE). The zlib stream persists for the
whole connection rather than being reset per-rectangle or per-update,
so VncSessionBackend now owns a lazily-initialized, persistent
z_stream torn down only in resetProtocolState() on a fresh
connect/reconnect.

Since the entire rectangle's compressed data decompresses into memory
in one shot, tile parsing is a plain synchronous loop rather than
needing its own RfbState values -- only the compressed-length and
compressed-data reads are actual protocol states. Tile decoding (the
five subencodings, including the continuation-byte run-length
encoding shared by two of them) lives in vnc_pixel_codecs.h/.cpp
alongside the Hextile decoder, unit-tested with 6 new tests covering
each subencoding plus a persistence test that splits one continuous
deflate stream across two separate FramebufferUpdate messages -- it
only decodes correctly if the connection's inflate stream is retained
between them.

Adds a top-level find_package(ZLIB REQUIRED) + ZLIB::ZLIB link
(previously only pulled in transitively via vendored FreeRDP's own
smartcard-emulation feature, which happened to have it enabled but
shouldn't be relied on for that).

Live-verified against the TightVNC test server that nothing regressed
(connect, cursor, clipboard); that server consistently sends Raw for
actual framebuffer content regardless of announced encodings, so
Hextile/ZRLE's live decode path isn't independently confirmed against
a real server -- the unit tests are the primary evidence here.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/vnc_pixel_codecs.cpp

@@ -77,4 +77,178 @@ QVector<HextileSubrect> decodeHextileSubrects(bool coloured, int subrectCount,
     return subrects;
 }
 
+int decodeZrleTile(const QByteArray& data, int offset, int tileWidth, int tileHeight,
+                   QVector<QRgb>& pixels)
+{
+    const int pixelCount = tileWidth * tileHeight;
+    const auto* bytes = reinterpret_cast<const uchar*>(data.constData());
+    const int size = data.size();
+    if (offset < 0 || offset >= size || pixelCount <= 0) {
+        return -1;
+    }
+
+    int pos = offset;
+    const quint8 subencoding = bytes[pos];
+    ++pos;
+
+    // CPIXEL is 3 bytes for our negotiated 32bpp/24-depth true-color
+    // format -- the padding byte a full pixel would have is simply
+    // omitted. rgbFromPixelBytes() already only reads the first 3 bytes it
+    // is given (B,G,R order), so it doubles as the CPIXEL reader.
+    auto readCpixel = [&](QRgb* out) -> bool {
+        if (pos + 3 > size) {
+            return false;
+        }
+        *out = rgbFromPixelBytes(bytes + pos);
+        pos += 3;
+        return true;
+    };
+
+    // Shared continuation-byte run-length reader for both RLE subencoding
+    // families: keep summing bytes while they equal 255, add the final
+    // (non-255) byte, and the true run length is that sum plus one.
+    auto readRunLength = [&](int* out) -> bool {
+        int total = 0;
+        for (;;) {
+            if (pos >= size) {
+                return false;
+            }
+            const quint8 b = bytes[pos];
+            ++pos;
+            total += b;
+            if (b != 255) {
+                break;
+            }
+        }
+        *out = total + 1;
+        return true;
+    };
+
+    if (subencoding == 0) { // Raw
+        pixels.reserve(pixels.size() + pixelCount);
+        for (int i = 0; i < pixelCount; ++i) {
+            QRgb color = 0;
+            if (!readCpixel(&color)) {
+                return -1;
+            }
+            pixels.append(color);
+        }
+        return pos - offset;
+    }
+
+    if (subencoding == 1) { // Solid
+        QRgb color = 0;
+        if (!readCpixel(&color)) {
+            return -1;
+        }
+        pixels.reserve(pixels.size() + pixelCount);
+        for (int i = 0; i < pixelCount; ++i) {
+            pixels.append(color);
+        }
+        return pos - offset;
+    }
+
+    if (subencoding >= 2 && subencoding <= 16) { // Packed palette
+        const int paletteSize = subencoding;
+        QVector<QRgb> palette;
+        palette.reserve(paletteSize);
+        for (int i = 0; i < paletteSize; ++i) {
+            QRgb color = 0;
+            if (!readCpixel(&color)) {
+                return -1;
+            }
+            palette.append(color);
+        }
+
+        int bitsPerPixel = 4;
+        if (paletteSize == 2) {
+            bitsPerPixel = 1;
+        } else if (paletteSize <= 4) {
+            bitsPerPixel = 2;
+        }
+        const int rowBytes = (tileWidth * bitsPerPixel + 7) / 8;
+
+        pixels.reserve(pixels.size() + pixelCount);
+        for (int y = 0; y < tileHeight; ++y) {
+            if (pos + rowBytes > size) {
+                return -1;
+            }
+            int bitPos = 0;
+            for (int x = 0; x < tileWidth; ++x) {
+                const int byteIndex = pos + (bitPos / 8);
+                const int shift = 8 - (bitPos % 8) - bitsPerPixel;
+                const int mask = (1 << bitsPerPixel) - 1;
+                const int index = (bytes[byteIndex] >> shift) & mask;
+                if (index >= palette.size()) {
+                    return -1;
+                }
+                pixels.append(palette.at(index));
+                bitPos += bitsPerPixel;
+            }
+            pos += rowBytes;
+        }
+        return pos - offset;
+    }
+
+    if (subencoding == 128) { // Plain RLE
+        int produced = 0;
+        while (produced < pixelCount) {
+            QRgb color = 0;
+            if (!readCpixel(&color)) {
+                return -1;
+            }
+            int runLength = 0;
+            if (!readRunLength(&runLength)) {
+                return -1;
+            }
+            for (int i = 0; i < runLength && produced < pixelCount; ++i, ++produced) {
+                pixels.append(color);
+            }
+        }
+        return pos - offset;
+    }
+
+    if (subencoding >= 130) { // Palette RLE (129 is unused/invalid, falls through below)
+        const int paletteSize = subencoding - 128;
+        QVector<QRgb> palette;
+        palette.reserve(paletteSize);
+        for (int i = 0; i < paletteSize; ++i) {
+            QRgb color = 0;
+            if (!readCpixel(&color)) {
+                return -1;
+            }
+            palette.append(color);
+        }
+
+        int produced = 0;
+        while (produced < pixelCount) {
+            if (pos >= size) {
+                return -1;
+            }
+            const quint8 indexByte = bytes[pos];
+            ++pos;
+
+            int index = indexByte;
+            int runLength = 1;
+            if (indexByte >= 128) {
+                index = indexByte - 128;
+                if (!readRunLength(&runLength)) {
+                    return -1;
+                }
+            }
+            if (index >= palette.size()) {
+                return -1;
+            }
+            const QRgb color = palette.at(index);
+            for (int i = 0; i < runLength && produced < pixelCount; ++i, ++produced) {
+                pixels.append(color);
+            }
+        }
+        return pos - offset;
+    }
+
+    // Subencodings 17-127 and 129 are not defined by RFC 6143.
+    return -1;
+}
+
 }

src/vnc_pixel_codecs.h

@@ -60,6 +60,20 @@ int hextileSubrectByteCount(bool coloured, int subrectCount);
 QVector<HextileSubrect> decodeHextileSubrects(bool coloured, int subrectCount,
                                               const QByteArray& data, QRgb foreground);
 
+// Decodes one ZRLE tile (RFC 6143 SS7.7.6) from `data`, starting at
+// `offset` (which must point at the tile's own 1-byte subencoding).
+// `data` holds an entire rectangle's worth of already-zlib-decompressed
+// bytes (possibly several tiles' worth) -- this reads only as much as the
+// one tile needs and never looks past `data.size()`. On success, appends
+// exactly tileWidth*tileHeight pixels (row-major) to `pixels` (which is
+// NOT cleared first, so callers can accumulate across tiles if desired --
+// VncSessionBackend clears/reuses a fresh vector per tile) and returns the
+// number of bytes consumed. Returns -1 for a malformed/truncated tile
+// (should never happen against a spec-compliant server, but must not read
+// out of bounds against an adversarial or buggy one).
+int decodeZrleTile(const QByteArray& data, int offset, int tileWidth, int tileHeight,
+                   QVector<QRgb>& pixels);
+
 }
 
 #endif

src/vnc_session_backend.cpp

@@ -10,6 +10,8 @@
 
 #include <openssl/des.h>
 
+#include <zlib.h>
+
 #include <array>
 #include <cstring>
 #include <limits>
@@ -35,14 +37,23 @@ constexpr quint8 kServerMsgServerCutText = 3;
 constexpr qint32 kEncRaw = 0;
 constexpr qint32 kEncCopyRect = 1;
 constexpr qint32 kEncHextile = 5;
+constexpr qint32 kEncZRLE = 16;
 // RFC 6143 SS7.8.2 "Cursor pseudo-encoding": not a real screen-content
 // encoding -- a rectangle with this type carries a cursor shape update
 // instead (hotspot in x/y, image dims in width/height), never painted into
 // the framebuffer.
 constexpr qint32 kEncCursor = -239;
 
-constexpr std::array<qint32, 4> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile,
-                                                        kEncCursor };
+constexpr std::array<qint32, 5> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile,
+                                                        kEncZRLE, kEncCursor };
+
+constexpr int kZrleTileSize = 64;
+
+// Sanity cap on a single ZRLE rectangle's compressed-length field, purely
+// to reject an obviously-hostile/corrupt claim before attempting to
+// buffer/allocate that many bytes -- well beyond anything a real screen
+// update should ever compress to.
+constexpr quint32 kMaxZrleCompressedLength = 64u * 1024u * 1024u;
 
 quint16 readU16BE(const QByteArray& buf, int offset)
 {
@@ -93,8 +104,11 @@ VncSessionBackend::VncSessionBackend(const Profile& profile, QObject* parent)
       m_hextileForeground(qRgb(0, 0, 0)),
       m_hextileSubencoding(0),
       m_hextileSubrectsRemaining(0),
-      m_hextileSubrectsColoured(false)
+      m_hextileSubrectsColoured(false),
+      m_zrleInflateStream(new z_stream_s()),
+      m_zrleInflateInitialized(false)
 {
+    std::memset(m_zrleInflateStream, 0, sizeof(z_stream_s));
     connect(m_socket, &QTcpSocket::connected, this, &VncSessionBackend::onSocketConnected);
     connect(m_socket, &QTcpSocket::readyRead, this, &VncSessionBackend::onSocketReadyRead);
     connect(m_socket, &QTcpSocket::disconnected, this, &VncSessionBackend::onSocketDisconnected);
@@ -106,6 +120,10 @@ VncSessionBackend::~VncSessionBackend()
     if (m_socket->state() != QAbstractSocket::UnconnectedState) {
         m_socket->abort();
     }
+    if (m_zrleInflateInitialized) {
+        inflateEnd(m_zrleInflateStream);
+    }
+    delete m_zrleInflateStream;
 }
 
 void VncSessionBackend::connectSession(const SessionConnectOptions& options)
@@ -341,6 +359,8 @@ void VncSessionBackend::onSocketDisconnected()
             && m_rfbState != RfbState::WaitingHextileTileMeta
             && m_rfbState != RfbState::WaitingHextileSubrectData
             && m_rfbState != RfbState::WaitingHextileRawTileData
+            && m_rfbState != RfbState::WaitingZrleCompressedLength
+            && m_rfbState != RfbState::WaitingZrleCompressedData
             && m_rfbState != RfbState::WaitingSetColourMapHeader
             && m_rfbState != RfbState::WaitingSetColourMapData
             && m_rfbState != RfbState::WaitingServerCutTextHeader
@@ -389,6 +409,16 @@ void VncSessionBackend::resetProtocolState()
     m_hextileTileY = 0;
     m_hextileBackground = qRgb(0, 0, 0);
     m_hextileForeground = qRgb(0, 0, 0);
+
+    // The ZRLE zlib stream is scoped to one TCP connection's lifetime, not
+    // to individual rectangles or updates -- tear it down here so a fresh
+    // connect/reconnect starts a clean stream instead of feeding new
+    // compressed data through decoder state left over from a previous
+    // session.
+    if (m_zrleInflateInitialized) {
+        inflateEnd(m_zrleInflateStream);
+        m_zrleInflateInitialized = false;
+    }
 }
 
 bool VncSessionBackend::haveBytes(int count) const
@@ -873,6 +903,9 @@ void VncSessionBackend::processReceiveBuffer()
                     m_rfbState = RfbState::WaitingHextileTileSubencoding;
                 }
                 break;
+            case kEncZRLE:
+                m_rfbState = RfbState::WaitingZrleCompressedLength;
+                break;
             default: {
                 // SetEncodings (see kAnnouncedEncodings) is entirely
                 // client-controlled, so a spec-compliant server will never
@@ -1083,6 +1116,121 @@ void VncSessionBackend::processReceiveBuffer()
             break;
         }
 
+        case RfbState::WaitingZrleCompressedLength: {
+            if (!haveBytes(4)) {
+                return;
+            }
+            const quint32 length = readU32BE(m_recvBuffer, 0);
+            m_recvBuffer.remove(0, 4);
+            if (length > kMaxZrleCompressedLength) {
+                failConnection(QStringLiteral("The VNC server sent an implausibly large ZRLE update."),
+                               QStringLiteral("ZRLE compressed length %1").arg(length));
+                return;
+            }
+            m_pendingLength = length;
+            m_rfbState = RfbState::WaitingZrleCompressedData;
+            break;
+        }
+
+        case RfbState::WaitingZrleCompressedData: {
+            if (!haveBytes(static_cast<int>(m_pendingLength))) {
+                return;
+            }
+            const QByteArray compressed = m_recvBuffer.left(static_cast<int>(m_pendingLength));
+            m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
+
+            if (!m_zrleInflateInitialized) {
+                std::memset(m_zrleInflateStream, 0, sizeof(z_stream_s));
+                if (inflateInit(m_zrleInflateStream) != Z_OK) {
+                    failConnection(QStringLiteral("Failed to initialize ZRLE decompression."),
+                                   QStringLiteral("inflateInit() failed"));
+                    return;
+                }
+                m_zrleInflateInitialized = true;
+            }
+
+            // The zlib stream is persistent across the whole connection
+            // (RFC 6143 SS7.7.6) -- Z_NO_FLUSH, never Z_FINISH, and the
+            // stream is only ever torn down in resetProtocolState() on a
+            // fresh connect/reconnect.
+            QByteArray decompressed;
+            constexpr int kChunkSize = 65536;
+            char outBuffer[kChunkSize];
+            m_zrleInflateStream->next_in =
+                reinterpret_cast<Bytef*>(const_cast<char*>(compressed.constData()));
+            m_zrleInflateStream->avail_in = static_cast<uInt>(compressed.size());
+            int inflateResult = Z_OK;
+            while (m_zrleInflateStream->avail_in > 0) {
+                m_zrleInflateStream->next_out = reinterpret_cast<Bytef*>(outBuffer);
+                m_zrleInflateStream->avail_out = kChunkSize;
+                inflateResult = inflate(m_zrleInflateStream, Z_NO_FLUSH);
+                const int produced = kChunkSize - static_cast<int>(m_zrleInflateStream->avail_out);
+                if (produced > 0) {
+                    decompressed.append(outBuffer, produced);
+                }
+                if (inflateResult != Z_OK) {
+                    break;
+                }
+            }
+            if (inflateResult != Z_OK && inflateResult != Z_STREAM_END) {
+                failConnection(QStringLiteral("The VNC server sent malformed ZRLE-compressed data."),
+                               QStringLiteral("inflate() returned %1").arg(inflateResult));
+                return;
+            }
+
+            const int rectWidth = m_currentRectangle.width;
+            const int rectHeight = m_currentRectangle.height;
+            int decodeOffset = 0;
+            bool malformedTile = false;
+            QString malformedTileDetail;
+            {
+                // Scoped so the QPainter is destroyed (ending the paint
+                // session) before onRectangleFinished() below, which may
+                // emit frameUpdated(m_framebuffer) -- matching how
+                // WaitingRawPixelData/WaitingCopyRectSource already end
+                // their own painters before finishing a rectangle.
+                QPainter painter(&m_framebuffer);
+                for (int tileY = 0; tileY < rectHeight && !malformedTile; tileY += kZrleTileSize) {
+                    const int tileHeight = qMin(kZrleTileSize, rectHeight - tileY);
+                    for (int tileX = 0; tileX < rectWidth; tileX += kZrleTileSize) {
+                        const int tileWidth = qMin(kZrleTileSize, rectWidth - tileX);
+
+                        QVector<QRgb> pixels;
+                        const int consumed = VncPixelCodecs::decodeZrleTile(
+                            decompressed, decodeOffset, tileWidth, tileHeight, pixels);
+                        if (consumed < 0 || pixels.size() != tileWidth * tileHeight) {
+                            malformedTile = true;
+                            malformedTileDetail = QStringLiteral("Tile at (%1,%2) size %3x%4")
+                                                      .arg(tileX)
+                                                      .arg(tileY)
+                                                      .arg(tileWidth)
+                                                      .arg(tileHeight);
+                            break;
+                        }
+                        decodeOffset += consumed;
+
+                        QImage tileImage(tileWidth, tileHeight, QImage::Format_RGB32);
+                        for (int y = 0; y < tileHeight; ++y) {
+                            for (int x = 0; x < tileWidth; ++x) {
+                                tileImage.setPixel(x, y, pixels.at((y * tileWidth) + x));
+                            }
+                        }
+                        painter.drawImage(m_currentRectangle.x + tileX,
+                                          m_currentRectangle.y + tileY, tileImage);
+                    }
+                }
+            }
+
+            if (malformedTile) {
+                failConnection(QStringLiteral("The VNC server sent a malformed ZRLE tile."),
+                               malformedTileDetail);
+                return;
+            }
+
+            onRectangleFinished();
+            break;
+        }
+
         case RfbState::WaitingSetColourMapHeader: {
             if (!haveBytes(5)) {
                 return;

src/vnc_session_backend.h

@@ -10,6 +10,7 @@
 #include <QRgb>
 
 class QTcpSocket;
+struct z_stream_s;
 
 // Implements RFB (RFC 6143) directly against QTcpSocket -- there is no
 // permissively licensed VNC client library to vendor the way FreeRDP was
@@ -22,11 +23,10 @@ class QTcpSocket;
 //
 // Scope (see plan / issue #3 for the full rationale): standard VNC
 // Authentication (security type 2) and no-auth (type 1) only -- not
-// Apple's Screen Sharing scheme (type 30). Raw + CopyRect + Hextile
-// encodings (ZRLE/Tight compression not yet implemented). No dynamic
-// resize. Clipboard sync (Latin-1 only, per RFB's ServerCutText/
-// ClientCutText) and remote cursor shape sync (the Cursor pseudo-encoding)
-// are supported.
+// Apple's Screen Sharing scheme (type 30). Raw + CopyRect + Hextile + ZRLE
+// encodings (Tight compression not yet implemented). No dynamic resize.
+// Clipboard sync (Latin-1 only, per RFB's ServerCutText/ClientCutText) and
+// remote cursor shape sync (the Cursor pseudo-encoding) are supported.
 class VncSessionBackend : public SessionBackend
 {
     Q_OBJECT
@@ -90,6 +90,8 @@ private:
         WaitingHextileTileMeta,
         WaitingHextileSubrectData,
         WaitingHextileRawTileData,
+        WaitingZrleCompressedLength,
+        WaitingZrleCompressedData,
         WaitingSetColourMapHeader,
         WaitingSetColourMapData,
         WaitingServerCutTextHeader,
@@ -139,6 +141,15 @@ private:
     int m_hextileSubrectsRemaining;
     bool m_hextileSubrectsColoured;
 
+    // ZRLE's zlib stream (RFC 6143 SS7.7.6) persists for the whole
+    // connection, not per-rectangle or per-update -- lazily initialized on
+    // the first ZRLE rectangle, torn down and reset on every fresh
+    // connect/reconnect via resetProtocolState(). z_stream_s is only
+    // forward-declared here so <zlib.h> doesn't leak into every includer of
+    // this header; the full type is only needed in the .cpp.
+    z_stream_s* m_zrleInflateStream;
+    bool m_zrleInflateInitialized;
+
     void setState(SessionState state, const QString& message);
     void resetProtocolState();
     void processReceiveBuffer();