Add VNC ZRLE decoding

Implements RFC 6143 SS7.7.6: a ZRLE rectangle is a 4-byte compressed
length followed by that many zlib-compressed bytes, decompressing to
64x64 tiles each using one of five subencodings (Raw, Solid, packed
palette, Plain RLE, Palette RLE). The zlib stream persists for the
whole connection rather than being reset per-rectangle or per-update,
so VncSessionBackend now owns a lazily-initialized, persistent
z_stream torn down only in resetProtocolState() on a fresh
connect/reconnect.

Since the entire rectangle's compressed data decompresses into memory
in one shot, tile parsing is a plain synchronous loop rather than
needing its own RfbState values -- only the compressed-length and
compressed-data reads are actual protocol states. Tile decoding (the
five subencodings, including the continuation-byte run-length
encoding shared by two of them) lives in vnc_pixel_codecs.h/.cpp
alongside the Hextile decoder, unit-tested with 6 new tests covering
each subencoding plus a persistence test that splits one continuous
deflate stream across two separate FramebufferUpdate messages -- it
only decodes correctly if the connection's inflate stream is retained
between them.

Adds a top-level find_package(ZLIB REQUIRED) + ZLIB::ZLIB link
(previously only pulled in transitively via vendored FreeRDP's own
smartcard-emulation feature, which happened to have it enabled but
shouldn't be relied on for that).

Live-verified against the TightVNC test server that nothing regressed
(connect, cursor, clipboard); that server consistently sends Raw for
actual framebuffer content regardless of announced encodings, so
Hextile/ZRLE's live decode path isn't independently confirmed against
a real server -- the unit tests are the primary evidence here.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-09-15 21:02:26 -06:00
1 parent e49fa0cf26
commit bb022edcf2
8 files changed
+772 -19

No files matched your search

+174
View File
@@ -77,4 +77,178 @@ QVector<HextileSubrect> decodeHextileSubrects(bool coloured, int subrectCount,
return subrects;
}
int decodeZrleTile(const QByteArray& data, int offset, int tileWidth, int tileHeight,
QVector<QRgb>& pixels)
{
const int pixelCount = tileWidth * tileHeight;
const auto* bytes = reinterpret_cast<const uchar*>(data.constData());
const int size = data.size();
if (offset < 0 || offset >= size || pixelCount <= 0) {
return -1;
}
int pos = offset;
const quint8 subencoding = bytes[pos];
++pos;
// CPIXEL is 3 bytes for our negotiated 32bpp/24-depth true-color
// format -- the padding byte a full pixel would have is simply
// omitted. rgbFromPixelBytes() already only reads the first 3 bytes it
// is given (B,G,R order), so it doubles as the CPIXEL reader.
auto readCpixel = [&](QRgb* out) -> bool {
if (pos + 3 > size) {
return false;
}
*out = rgbFromPixelBytes(bytes + pos);
pos += 3;
return true;
};
// Shared continuation-byte run-length reader for both RLE subencoding
// families: keep summing bytes while they equal 255, add the final
// (non-255) byte, and the true run length is that sum plus one.
auto readRunLength = [&](int* out) -> bool {
int total = 0;
for (;;) {
if (pos >= size) {
return false;
}
const quint8 b = bytes[pos];
++pos;
total += b;
if (b != 255) {
break;
}
}
*out = total + 1;
return true;
};
if (subencoding == 0) { // Raw
pixels.reserve(pixels.size() + pixelCount);
for (int i = 0; i < pixelCount; ++i) {
QRgb color = 0;
if (!readCpixel(&color)) {
return -1;
}
pixels.append(color);
}
return pos - offset;
}
if (subencoding == 1) { // Solid
QRgb color = 0;
if (!readCpixel(&color)) {
return -1;
}
pixels.reserve(pixels.size() + pixelCount);
for (int i = 0; i < pixelCount; ++i) {
pixels.append(color);
}
return pos - offset;
}
if (subencoding >= 2 && subencoding <= 16) { // Packed palette
const int paletteSize = subencoding;
QVector<QRgb> palette;
palette.reserve(paletteSize);
for (int i = 0; i < paletteSize; ++i) {
QRgb color = 0;
if (!readCpixel(&color)) {
return -1;
}
palette.append(color);
}
int bitsPerPixel = 4;
if (paletteSize == 2) {
bitsPerPixel = 1;
} else if (paletteSize <= 4) {
bitsPerPixel = 2;
}
const int rowBytes = (tileWidth * bitsPerPixel + 7) / 8;
pixels.reserve(pixels.size() + pixelCount);
for (int y = 0; y < tileHeight; ++y) {
if (pos + rowBytes > size) {
return -1;
}
int bitPos = 0;
for (int x = 0; x < tileWidth; ++x) {
const int byteIndex = pos + (bitPos / 8);
const int shift = 8 - (bitPos % 8) - bitsPerPixel;
const int mask = (1 << bitsPerPixel) - 1;
const int index = (bytes[byteIndex] >> shift) & mask;
if (index >= palette.size()) {
return -1;
}
pixels.append(palette.at(index));
bitPos += bitsPerPixel;
}
pos += rowBytes;
}
return pos - offset;
}
if (subencoding == 128) { // Plain RLE
int produced = 0;
while (produced < pixelCount) {
QRgb color = 0;
if (!readCpixel(&color)) {
return -1;
}
int runLength = 0;
if (!readRunLength(&runLength)) {
return -1;
}
for (int i = 0; i < runLength && produced < pixelCount; ++i, ++produced) {
pixels.append(color);
}
}
return pos - offset;
}
if (subencoding >= 130) { // Palette RLE (129 is unused/invalid, falls through below)
const int paletteSize = subencoding - 128;
QVector<QRgb> palette;
palette.reserve(paletteSize);
for (int i = 0; i < paletteSize; ++i) {
QRgb color = 0;
if (!readCpixel(&color)) {
return -1;
}
palette.append(color);
}
int produced = 0;
while (produced < pixelCount) {
if (pos >= size) {
return -1;
}
const quint8 indexByte = bytes[pos];
++pos;
int index = indexByte;
int runLength = 1;
if (indexByte >= 128) {
index = indexByte - 128;
if (!readRunLength(&runLength)) {
return -1;
}
}
if (index >= palette.size()) {
return -1;
}
const QRgb color = palette.at(index);
for (int i = 0; i < runLength && produced < pixelCount; ++i, ++produced) {
pixels.append(color);
}
}
return pos - offset;
}
// Subencodings 17-127 and 129 are not defined by RFC 6143.
return -1;
}
}