Internal
Public Access
Add VNC ZRLE decoding
Implements RFC 6143 SS7.7.6: a ZRLE rectangle is a 4-byte compressed length followed by that many zlib-compressed bytes, decompressing to 64x64 tiles each using one of five subencodings (Raw, Solid, packed palette, Plain RLE, Palette RLE). The zlib stream persists for the whole connection rather than being reset per-rectangle or per-update, so VncSessionBackend now owns a lazily-initialized, persistent z_stream torn down only in resetProtocolState() on a fresh connect/reconnect. Since the entire rectangle's compressed data decompresses into memory in one shot, tile parsing is a plain synchronous loop rather than needing its own RfbState values -- only the compressed-length and compressed-data reads are actual protocol states. Tile decoding (the five subencodings, including the continuation-byte run-length encoding shared by two of them) lives in vnc_pixel_codecs.h/.cpp alongside the Hextile decoder, unit-tested with 6 new tests covering each subencoding plus a persistence test that splits one continuous deflate stream across two separate FramebufferUpdate messages -- it only decodes correctly if the connection's inflate stream is retained between them. Adds a top-level find_package(ZLIB REQUIRED) + ZLIB::ZLIB link (previously only pulled in transitively via vendored FreeRDP's own smartcard-emulation feature, which happened to have it enabled but shouldn't be relied on for that). Live-verified against the TightVNC test server that nothing regressed (connect, cursor, clipboard); that server consistently sends Raw for actual framebuffer content regardless of announced encodings, so Hextile/ZRLE's live decode path isn't independently confirmed against a real server -- the unit tests are the primary evidence here. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e49fa0cf26
commit
bb022edcf2
8 files changed
+772
-19
No files matched your search
@@ -77,4 +77,178 @@ QVector<HextileSubrect> decodeHextileSubrects(bool coloured, int subrectCount,
|
||||
return subrects;
|
||||
}
|
||||
|
||||
int decodeZrleTile(const QByteArray& data, int offset, int tileWidth, int tileHeight,
|
||||
QVector<QRgb>& pixels)
|
||||
{
|
||||
const int pixelCount = tileWidth * tileHeight;
|
||||
const auto* bytes = reinterpret_cast<const uchar*>(data.constData());
|
||||
const int size = data.size();
|
||||
if (offset < 0 || offset >= size || pixelCount <= 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
int pos = offset;
|
||||
const quint8 subencoding = bytes[pos];
|
||||
++pos;
|
||||
|
||||
// CPIXEL is 3 bytes for our negotiated 32bpp/24-depth true-color
|
||||
// format -- the padding byte a full pixel would have is simply
|
||||
// omitted. rgbFromPixelBytes() already only reads the first 3 bytes it
|
||||
// is given (B,G,R order), so it doubles as the CPIXEL reader.
|
||||
auto readCpixel = [&](QRgb* out) -> bool {
|
||||
if (pos + 3 > size) {
|
||||
return false;
|
||||
}
|
||||
*out = rgbFromPixelBytes(bytes + pos);
|
||||
pos += 3;
|
||||
return true;
|
||||
};
|
||||
|
||||
// Shared continuation-byte run-length reader for both RLE subencoding
|
||||
// families: keep summing bytes while they equal 255, add the final
|
||||
// (non-255) byte, and the true run length is that sum plus one.
|
||||
auto readRunLength = [&](int* out) -> bool {
|
||||
int total = 0;
|
||||
for (;;) {
|
||||
if (pos >= size) {
|
||||
return false;
|
||||
}
|
||||
const quint8 b = bytes[pos];
|
||||
++pos;
|
||||
total += b;
|
||||
if (b != 255) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
*out = total + 1;
|
||||
return true;
|
||||
};
|
||||
|
||||
if (subencoding == 0) { // Raw
|
||||
pixels.reserve(pixels.size() + pixelCount);
|
||||
for (int i = 0; i < pixelCount; ++i) {
|
||||
QRgb color = 0;
|
||||
if (!readCpixel(&color)) {
|
||||
return -1;
|
||||
}
|
||||
pixels.append(color);
|
||||
}
|
||||
return pos - offset;
|
||||
}
|
||||
|
||||
if (subencoding == 1) { // Solid
|
||||
QRgb color = 0;
|
||||
if (!readCpixel(&color)) {
|
||||
return -1;
|
||||
}
|
||||
pixels.reserve(pixels.size() + pixelCount);
|
||||
for (int i = 0; i < pixelCount; ++i) {
|
||||
pixels.append(color);
|
||||
}
|
||||
return pos - offset;
|
||||
}
|
||||
|
||||
if (subencoding >= 2 && subencoding <= 16) { // Packed palette
|
||||
const int paletteSize = subencoding;
|
||||
QVector<QRgb> palette;
|
||||
palette.reserve(paletteSize);
|
||||
for (int i = 0; i < paletteSize; ++i) {
|
||||
QRgb color = 0;
|
||||
if (!readCpixel(&color)) {
|
||||
return -1;
|
||||
}
|
||||
palette.append(color);
|
||||
}
|
||||
|
||||
int bitsPerPixel = 4;
|
||||
if (paletteSize == 2) {
|
||||
bitsPerPixel = 1;
|
||||
} else if (paletteSize <= 4) {
|
||||
bitsPerPixel = 2;
|
||||
}
|
||||
const int rowBytes = (tileWidth * bitsPerPixel + 7) / 8;
|
||||
|
||||
pixels.reserve(pixels.size() + pixelCount);
|
||||
for (int y = 0; y < tileHeight; ++y) {
|
||||
if (pos + rowBytes > size) {
|
||||
return -1;
|
||||
}
|
||||
int bitPos = 0;
|
||||
for (int x = 0; x < tileWidth; ++x) {
|
||||
const int byteIndex = pos + (bitPos / 8);
|
||||
const int shift = 8 - (bitPos % 8) - bitsPerPixel;
|
||||
const int mask = (1 << bitsPerPixel) - 1;
|
||||
const int index = (bytes[byteIndex] >> shift) & mask;
|
||||
if (index >= palette.size()) {
|
||||
return -1;
|
||||
}
|
||||
pixels.append(palette.at(index));
|
||||
bitPos += bitsPerPixel;
|
||||
}
|
||||
pos += rowBytes;
|
||||
}
|
||||
return pos - offset;
|
||||
}
|
||||
|
||||
if (subencoding == 128) { // Plain RLE
|
||||
int produced = 0;
|
||||
while (produced < pixelCount) {
|
||||
QRgb color = 0;
|
||||
if (!readCpixel(&color)) {
|
||||
return -1;
|
||||
}
|
||||
int runLength = 0;
|
||||
if (!readRunLength(&runLength)) {
|
||||
return -1;
|
||||
}
|
||||
for (int i = 0; i < runLength && produced < pixelCount; ++i, ++produced) {
|
||||
pixels.append(color);
|
||||
}
|
||||
}
|
||||
return pos - offset;
|
||||
}
|
||||
|
||||
if (subencoding >= 130) { // Palette RLE (129 is unused/invalid, falls through below)
|
||||
const int paletteSize = subencoding - 128;
|
||||
QVector<QRgb> palette;
|
||||
palette.reserve(paletteSize);
|
||||
for (int i = 0; i < paletteSize; ++i) {
|
||||
QRgb color = 0;
|
||||
if (!readCpixel(&color)) {
|
||||
return -1;
|
||||
}
|
||||
palette.append(color);
|
||||
}
|
||||
|
||||
int produced = 0;
|
||||
while (produced < pixelCount) {
|
||||
if (pos >= size) {
|
||||
return -1;
|
||||
}
|
||||
const quint8 indexByte = bytes[pos];
|
||||
++pos;
|
||||
|
||||
int index = indexByte;
|
||||
int runLength = 1;
|
||||
if (indexByte >= 128) {
|
||||
index = indexByte - 128;
|
||||
if (!readRunLength(&runLength)) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
if (index >= palette.size()) {
|
||||
return -1;
|
||||
}
|
||||
const QRgb color = palette.at(index);
|
||||
for (int i = 0; i < runLength && produced < pixelCount; ++i, ++produced) {
|
||||
pixels.append(color);
|
||||
}
|
||||
}
|
||||
return pos - offset;
|
||||
}
|
||||
|
||||
// Subencodings 17-127 and 129 are not defined by RFC 6143.
|
||||
return -1;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in new issue
Block a user