DarkSingularity/orbithub
Internal
Public Access
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 7 Wiki Activity

Add VNC ZRLE decoding

Browse Source 
Implements RFC 6143 SS7.7.6: a ZRLE rectangle is a 4-byte compressed
length followed by that many zlib-compressed bytes, decompressing to
64x64 tiles each using one of five subencodings (Raw, Solid, packed
palette, Plain RLE, Palette RLE). The zlib stream persists for the
whole connection rather than being reset per-rectangle or per-update,
so VncSessionBackend now owns a lazily-initialized, persistent
z_stream torn down only in resetProtocolState() on a fresh
connect/reconnect.

Since the entire rectangle's compressed data decompresses into memory
in one shot, tile parsing is a plain synchronous loop rather than
needing its own RfbState values -- only the compressed-length and
compressed-data reads are actual protocol states. Tile decoding (the
five subencodings, including the continuation-byte run-length
encoding shared by two of them) lives in vnc_pixel_codecs.h/.cpp
alongside the Hextile decoder, unit-tested with 6 new tests covering
each subencoding plus a persistence test that splits one continuous
deflate stream across two separate FramebufferUpdate messages -- it
only decodes correctly if the connection's inflate stream is retained
between them.

Adds a top-level find_package(ZLIB REQUIRED) + ZLIB::ZLIB link
(previously only pulled in transitively via vendored FreeRDP's own
smartcard-emulation feature, which happened to have it enabled but
shouldn't be relied on for that).

Live-verified against the TightVNC test server that nothing regressed
(connect, cursor, clipboard); that server consistently sends Raw for
actual framebuffer content regardless of announced encodings, so
Hextile/ZRLE's live decode path isn't independently confirmed against
a real server -- the unit tests are the primary evidence here.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmith
2026-09-15 21:02:26 -06:00
co-authored by Claude Sonnet 5
parent e49fa0cf26
commit bb022edcf2
8 changed files with 772 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vnc_session_backend.cpp
+151 -3
View File
@@ -10,6 +10,8 @@
#include <openssl/des.h>
#include <zlib.h>
#include <array>
#include <cstring>
#include <limits>
@@ -35,14 +37,23 @@ constexpr quint8 kServerMsgServerCutText = 3;
constexpr qint32 kEncRaw = 0;
constexpr qint32 kEncCopyRect = 1;
constexpr qint32 kEncHextile = 5;
constexpr qint32 kEncZRLE = 16;
// RFC 6143 SS7.8.2 "Cursor pseudo-encoding": not a real screen-content
// encoding -- a rectangle with this type carries a cursor shape update
// instead (hotspot in x/y, image dims in width/height), never painted into
// the framebuffer.
constexpr qint32 kEncCursor = -239;
constexpr std::array<qint32, 4> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile,
kEncCursor };
constexpr std::array<qint32, 5> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile,
kEncZRLE, kEncCursor };
constexpr int kZrleTileSize = 64;
// Sanity cap on a single ZRLE rectangle's compressed-length field, purely
// to reject an obviously-hostile/corrupt claim before attempting to
// buffer/allocate that many bytes -- well beyond anything a real screen
// update should ever compress to.
constexpr quint32 kMaxZrleCompressedLength = 64u * 1024u * 1024u;
quint16 readU16BE(const QByteArray& buf, int offset)
{
@@ -93,8 +104,11 @@ VncSessionBackend::VncSessionBackend(const Profile& profile, QObject* parent)
m_hextileForeground(qRgb(0, 0, 0)),
m_hextileSubencoding(0),
m_hextileSubrectsRemaining(0),
m_hextileSubrectsColoured(false)
m_hextileSubrectsColoured(false),
m_zrleInflateStream(new z_stream_s()),
m_zrleInflateInitialized(false)
{
std::memset(m_zrleInflateStream, 0, sizeof(z_stream_s));
connect(m_socket, &QTcpSocket::connected, this, &VncSessionBackend::onSocketConnected);
connect(m_socket, &QTcpSocket::readyRead, this, &VncSessionBackend::onSocketReadyRead);
connect(m_socket, &QTcpSocket::disconnected, this, &VncSessionBackend::onSocketDisconnected);
@@ -106,6 +120,10 @@ VncSessionBackend::~VncSessionBackend()
if (m_socket->state() != QAbstractSocket::UnconnectedState) {
m_socket->abort();
}
if (m_zrleInflateInitialized) {
inflateEnd(m_zrleInflateStream);
}
delete m_zrleInflateStream;
}
void VncSessionBackend::connectSession(const SessionConnectOptions& options)
@@ -341,6 +359,8 @@ void VncSessionBackend::onSocketDisconnected()
&& m_rfbState != RfbState::WaitingHextileTileMeta
&& m_rfbState != RfbState::WaitingHextileSubrectData
&& m_rfbState != RfbState::WaitingHextileRawTileData
&& m_rfbState != RfbState::WaitingZrleCompressedLength
&& m_rfbState != RfbState::WaitingZrleCompressedData
&& m_rfbState != RfbState::WaitingSetColourMapHeader
&& m_rfbState != RfbState::WaitingSetColourMapData
&& m_rfbState != RfbState::WaitingServerCutTextHeader
@@ -389,6 +409,16 @@ void VncSessionBackend::resetProtocolState()
m_hextileTileY = 0;
m_hextileBackground = qRgb(0, 0, 0);
m_hextileForeground = qRgb(0, 0, 0);
// The ZRLE zlib stream is scoped to one TCP connection's lifetime, not
// to individual rectangles or updates -- tear it down here so a fresh
// connect/reconnect starts a clean stream instead of feeding new
// compressed data through decoder state left over from a previous
// session.
if (m_zrleInflateInitialized) {
inflateEnd(m_zrleInflateStream);
m_zrleInflateInitialized = false;
}
}
bool VncSessionBackend::haveBytes(int count) const
@@ -873,6 +903,9 @@ void VncSessionBackend::processReceiveBuffer()
m_rfbState = RfbState::WaitingHextileTileSubencoding;
}
break;
case kEncZRLE:
m_rfbState = RfbState::WaitingZrleCompressedLength;
break;
default: {
// SetEncodings (see kAnnouncedEncodings) is entirely
// client-controlled, so a spec-compliant server will never
@@ -1083,6 +1116,121 @@ void VncSessionBackend::processReceiveBuffer()
break;
}
case RfbState::WaitingZrleCompressedLength: {
if (!haveBytes(4)) {
return;
}
const quint32 length = readU32BE(m_recvBuffer, 0);
m_recvBuffer.remove(0, 4);
if (length > kMaxZrleCompressedLength) {
failConnection(QStringLiteral("The VNC server sent an implausibly large ZRLE update."),
QStringLiteral("ZRLE compressed length %1").arg(length));
return;
}
m_pendingLength = length;
m_rfbState = RfbState::WaitingZrleCompressedData;
break;
}
case RfbState::WaitingZrleCompressedData: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const QByteArray compressed = m_recvBuffer.left(static_cast<int>(m_pendingLength));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
if (!m_zrleInflateInitialized) {
std::memset(m_zrleInflateStream, 0, sizeof(z_stream_s));
if (inflateInit(m_zrleInflateStream) != Z_OK) {
failConnection(QStringLiteral("Failed to initialize ZRLE decompression."),
QStringLiteral("inflateInit() failed"));
return;
}
m_zrleInflateInitialized = true;
}
// The zlib stream is persistent across the whole connection
// (RFC 6143 SS7.7.6) -- Z_NO_FLUSH, never Z_FINISH, and the
// stream is only ever torn down in resetProtocolState() on a
// fresh connect/reconnect.
QByteArray decompressed;
constexpr int kChunkSize = 65536;
char outBuffer[kChunkSize];
m_zrleInflateStream->next_in =
reinterpret_cast<Bytef*>(const_cast<char*>(compressed.constData()));
m_zrleInflateStream->avail_in = static_cast<uInt>(compressed.size());
int inflateResult = Z_OK;
while (m_zrleInflateStream->avail_in > 0) {
m_zrleInflateStream->next_out = reinterpret_cast<Bytef*>(outBuffer);
m_zrleInflateStream->avail_out = kChunkSize;
inflateResult = inflate(m_zrleInflateStream, Z_NO_FLUSH);
const int produced = kChunkSize - static_cast<int>(m_zrleInflateStream->avail_out);
if (produced > 0) {
decompressed.append(outBuffer, produced);
}
if (inflateResult != Z_OK) {
break;
}
}
if (inflateResult != Z_OK && inflateResult != Z_STREAM_END) {
failConnection(QStringLiteral("The VNC server sent malformed ZRLE-compressed data."),
QStringLiteral("inflate() returned %1").arg(inflateResult));
return;
}
const int rectWidth = m_currentRectangle.width;
const int rectHeight = m_currentRectangle.height;
int decodeOffset = 0;
bool malformedTile = false;
QString malformedTileDetail;
{
// Scoped so the QPainter is destroyed (ending the paint
// session) before onRectangleFinished() below, which may
// emit frameUpdated(m_framebuffer) -- matching how
// WaitingRawPixelData/WaitingCopyRectSource already end
// their own painters before finishing a rectangle.
QPainter painter(&m_framebuffer);
for (int tileY = 0; tileY < rectHeight && !malformedTile; tileY += kZrleTileSize) {
const int tileHeight = qMin(kZrleTileSize, rectHeight - tileY);
for (int tileX = 0; tileX < rectWidth; tileX += kZrleTileSize) {
const int tileWidth = qMin(kZrleTileSize, rectWidth - tileX);
QVector<QRgb> pixels;
const int consumed = VncPixelCodecs::decodeZrleTile(
decompressed, decodeOffset, tileWidth, tileHeight, pixels);
if (consumed < 0 || pixels.size() != tileWidth * tileHeight) {
malformedTile = true;
malformedTileDetail = QStringLiteral("Tile at (%1,%2) size %3x%4")
.arg(tileX)
.arg(tileY)
.arg(tileWidth)
.arg(tileHeight);
break;
}
decodeOffset += consumed;
QImage tileImage(tileWidth, tileHeight, QImage::Format_RGB32);
for (int y = 0; y < tileHeight; ++y) {
for (int x = 0; x < tileWidth; ++x) {
tileImage.setPixel(x, y, pixels.at((y * tileWidth) + x));
}
}
painter.drawImage(m_currentRectangle.x + tileX,
m_currentRectangle.y + tileY, tileImage);
}
}
}
if (malformedTile) {
failConnection(QStringLiteral("The VNC server sent a malformed ZRLE tile."),
malformedTileDetail);
return;
}
onRectangleFinished();
break;
}
case RfbState::WaitingSetColourMapHeader: {
if (!haveBytes(5)) {
return;