DarkSingularity/orbithub
Internal
Public Access
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 7 Wiki Activity

Add VNC Tight decoding

Browse Source 
Implements RFC 6143's Tight encoding: a compression-control byte (low
4 bits reset one of 4 persistent zlib streams; high nibble selects
Fill/JPEG/Basic mode) followed by Fill's 3-byte solid color, JPEG's
compact-length-prefixed baseline JPEG covering the whole rectangle
(decoded via libjpeg-turbo directly, not QImage's plugin, to avoid a
packaging-dependent runtime failure mode), or Basic mode's
compact-length-prefixed zlib payload plus a filter (Copy, Palette, or
Gradient) applied after decompression. Unlike Hextile/ZRLE, Tight has
no internal tiling -- one rectangle is one filtered/compressed unit.

The three filters live in vnc_pixel_codecs.h/.cpp alongside the
Hextile/ZRLE decoders. Adds find_package(JPEG REQUIRED) + JPEG::JPEG
as a new build dependency (confirmed available via libjpeg-turbo on
this dev machine). 5 new tests cover Fill, Basic+Copy, Basic+Palette,
JPEG (round-tripped through a real libjpeg-turbo-encoded fixture,
compared with tolerance since JPEG is lossy), and the stream-reset
flag correctly tearing down and reinitializing a targeted stream
rather than erroring on stale state.

Known, documented gap: this decoder always treats Basic-mode payloads
as zlib-compressed; the real protocol allows very small payloads to
skip compression, which couldn't be verified with confidence against
the RFC text alone and is narrow enough in practice (tiny solid areas
are virtually always sent as Fill instead) to leave unhandled for now
-- it fails that one rectangle's decode cleanly rather than
misinterpreting it silently. The Gradient filter is implemented from
the spec description but is the least exercised of the three in this
pass.

Live-verified against the TightVNC test server that nothing
regressed; that server still consistently chose Raw for actual
framebuffer content regardless of announced encodings, so Tight's live
decode path isn't independently confirmed against a real server here
either -- the unit tests are the primary evidence.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmith
2026-09-15 21:13:17 -06:00
co-authored by Claude Sonnet 5
parent bb022edcf2
commit 4fca8fce41
8 changed files with 901 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vnc_session_backend.cpp
+321 -3
View File
@@ -12,6 +12,10 @@
#include <zlib.h>
extern "C" {
#include <jpeglib.h>
}
#include <array>
#include <cstring>
#include <limits>
@@ -38,14 +42,26 @@ constexpr qint32 kEncRaw = 0;
constexpr qint32 kEncCopyRect = 1;
constexpr qint32 kEncHextile = 5;
constexpr qint32 kEncZRLE = 16;
constexpr qint32 kEncTight = 7;
// RFC 6143 SS7.8.2 "Cursor pseudo-encoding": not a real screen-content
// encoding -- a rectangle with this type carries a cursor shape update
// instead (hotspot in x/y, image dims in width/height), never painted into
// the framebuffer.
constexpr qint32 kEncCursor = -239;
constexpr std::array<qint32, 5> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile,
kEncZRLE, kEncCursor };
constexpr std::array<qint32, 6> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile,
kEncZRLE, kEncTight, kEncCursor };
// Tight compression-control byte, high nibble (after the low 4 bits'
// stream-reset flags are stripped off).
constexpr quint8 kTightModeFill = 0x08;
constexpr quint8 kTightModeJpeg = 0x09;
// Modes 0-7 are "Basic" compression; bits 0-1 of the mode select which of
// the 4 persistent zlib streams to use, bit 2 flags that an explicit
// filter-id byte follows (Copy is implied when it's clear).
constexpr quint8 kTightBasicModeMax = 0x07;
constexpr quint8 kTightBasicExplicitFilterFlag = 0x04;
constexpr quint8 kTightBasicStreamIndexMask = 0x03;
constexpr int kZrleTileSize = 64;
@@ -81,6 +97,62 @@ void appendU32BE(QByteArray& buf, quint32 value)
buf.append(static_cast<char>((value >> 8) & 0xFF));
buf.append(static_cast<char>(value & 0xFF));
}
// Decodes a Tight "JPEG" rectangle (a full baseline JPEG image covering
// the whole rectangle, no tiling) via libjpeg-turbo directly -- chosen
// over QImage::fromData(..., "JPEG") specifically to avoid depending on
// Qt's own JPEG plugin being present at runtime across every packaging
// target. Returns false on any decode failure or dimension mismatch
// rather than risk painting a misinterpreted image.
bool decodeJpegRectangle(const QByteArray& jpegBytes, int width, int height, QImage* outImage)
{
if (jpegBytes.isEmpty() || width <= 0 || height <= 0) {
return false;
}
jpeg_decompress_struct cinfo;
jpeg_error_mgr jerr;
cinfo.err = jpeg_std_error(&jerr);
jpeg_create_decompress(&cinfo);
jpeg_mem_src(&cinfo, reinterpret_cast<const unsigned char*>(jpegBytes.constData()),
static_cast<unsigned long>(jpegBytes.size()));
if (jpeg_read_header(&cinfo, TRUE) != JPEG_HEADER_OK) {
jpeg_destroy_decompress(&cinfo);
return false;
}
cinfo.out_color_space = JCS_RGB;
jpeg_start_decompress(&cinfo);
if (static_cast<int>(cinfo.output_width) != width
|| static_cast<int>(cinfo.output_height) != height) {
jpeg_finish_decompress(&cinfo);
jpeg_destroy_decompress(&cinfo);
return false;
}
QImage image(width, height, QImage::Format_RGB32);
const int rowStride = static_cast<int>(cinfo.output_width) * cinfo.output_components;
QVector<unsigned char> rowBuffer(rowStride);
unsigned char* rowPointer[1];
while (cinfo.output_scanline < cinfo.output_height) {
rowPointer[0] = rowBuffer.data();
jpeg_read_scanlines(&cinfo, rowPointer, 1);
const int y = static_cast<int>(cinfo.output_scanline) - 1;
auto* dst = reinterpret_cast<QRgb*>(image.scanLine(y));
for (int x = 0; x < width; ++x) {
const unsigned char* p = rowBuffer.constData() + (x * 3);
dst[x] = qRgb(p[0], p[1], p[2]);
}
}
jpeg_finish_decompress(&cinfo);
jpeg_destroy_decompress(&cinfo);
*outImage = image;
return true;
}
}
VncSessionBackend::VncSessionBackend(const Profile& profile, QObject* parent)
@@ -106,9 +178,18 @@ VncSessionBackend::VncSessionBackend(const Profile& profile, QObject* parent)
m_hextileSubrectsRemaining(0),
m_hextileSubrectsColoured(false),
m_zrleInflateStream(new z_stream_s()),
m_zrleInflateInitialized(false)
m_zrleInflateInitialized(false),
m_tightInflateStreams{ new z_stream_s(), new z_stream_s(), new z_stream_s(),
new z_stream_s() },
m_tightInflateInitialized{ false, false, false, false },
m_tightCompressionMode(0),
m_tightFilterId(0),
m_tightLengthByteIndex(0)
{
std::memset(m_zrleInflateStream, 0, sizeof(z_stream_s));
for (z_stream_s* stream : m_tightInflateStreams) {
std::memset(stream, 0, sizeof(z_stream_s));
}
connect(m_socket, &QTcpSocket::connected, this, &VncSessionBackend::onSocketConnected);
connect(m_socket, &QTcpSocket::readyRead, this, &VncSessionBackend::onSocketReadyRead);
connect(m_socket, &QTcpSocket::disconnected, this, &VncSessionBackend::onSocketDisconnected);
@@ -124,6 +205,12 @@ VncSessionBackend::~VncSessionBackend()
inflateEnd(m_zrleInflateStream);
}
delete m_zrleInflateStream;
for (int i = 0; i < 4; ++i) {
if (m_tightInflateInitialized[i]) {
inflateEnd(m_tightInflateStreams[i]);
}
delete m_tightInflateStreams[i];
}
}
void VncSessionBackend::connectSession(const SessionConnectOptions& options)
@@ -361,6 +448,11 @@ void VncSessionBackend::onSocketDisconnected()
&& m_rfbState != RfbState::WaitingHextileRawTileData
&& m_rfbState != RfbState::WaitingZrleCompressedLength
&& m_rfbState != RfbState::WaitingZrleCompressedData
&& m_rfbState != RfbState::WaitingTightCompressionControl
&& m_rfbState != RfbState::WaitingTightFillColor
&& m_rfbState != RfbState::WaitingTightFilterId
&& m_rfbState != RfbState::WaitingTightLengthByte
&& m_rfbState != RfbState::WaitingTightPayload
&& m_rfbState != RfbState::WaitingSetColourMapHeader
&& m_rfbState != RfbState::WaitingSetColourMapData
&& m_rfbState != RfbState::WaitingServerCutTextHeader
@@ -419,6 +511,12 @@ void VncSessionBackend::resetProtocolState()
inflateEnd(m_zrleInflateStream);
m_zrleInflateInitialized = false;
}
for (int i = 0; i < 4; ++i) {
if (m_tightInflateInitialized[i]) {
inflateEnd(m_tightInflateStreams[i]);
m_tightInflateInitialized[i] = false;
}
}
}
bool VncSessionBackend::haveBytes(int count) const
@@ -588,6 +686,38 @@ void VncSessionBackend::advanceHextileTile()
}
}
bool VncSessionBackend::inflateTightStream(int streamIndex, const QByteArray& compressed,
QByteArray* decompressed)
{
z_stream_s* stream = m_tightInflateStreams[streamIndex];
if (!m_tightInflateInitialized[streamIndex]) {
std::memset(stream, 0, sizeof(z_stream_s));
if (inflateInit(stream) != Z_OK) {
return false;
}
m_tightInflateInitialized[streamIndex] = true;
}
constexpr int kChunkSize = 65536;
char outBuffer[kChunkSize];
stream->next_in = reinterpret_cast<Bytef*>(const_cast<char*>(compressed.constData()));
stream->avail_in = static_cast<uInt>(compressed.size());
int result = Z_OK;
while (stream->avail_in > 0) {
stream->next_out = reinterpret_cast<Bytef*>(outBuffer);
stream->avail_out = kChunkSize;
result = inflate(stream, Z_NO_FLUSH);
const int produced = kChunkSize - static_cast<int>(stream->avail_out);
if (produced > 0) {
decompressed->append(outBuffer, produced);
}
if (result != Z_OK) {
break;
}
}
return result == Z_OK || result == Z_STREAM_END;
}
void VncSessionBackend::processReceiveBuffer()
{
for (;;) {
@@ -906,6 +1036,9 @@ void VncSessionBackend::processReceiveBuffer()
case kEncZRLE:
m_rfbState = RfbState::WaitingZrleCompressedLength;
break;
case kEncTight:
m_rfbState = RfbState::WaitingTightCompressionControl;
break;
default: {
// SetEncodings (see kAnnouncedEncodings) is entirely
// client-controlled, so a spec-compliant server will never
@@ -1231,6 +1364,191 @@ void VncSessionBackend::processReceiveBuffer()
break;
}
case RfbState::WaitingTightCompressionControl: {
if (!haveBytes(1)) {
return;
}
const quint8 controlByte = static_cast<quint8>(m_recvBuffer.at(0));
m_recvBuffer.remove(0, 1);
// Low 4 bits: reset flags for the 4 persistent Basic-mode zlib
// streams, independent of whatever mode this particular
// rectangle itself uses.
for (int i = 0; i < 4; ++i) {
if ((controlByte & (1 << i)) != 0 && m_tightInflateInitialized[i]) {
inflateEnd(m_tightInflateStreams[i]);
m_tightInflateInitialized[i] = false;
}
}
const quint8 mode = controlByte >> 4;
m_tightCompressionMode = mode;
if (mode == kTightModeFill) {
m_rfbState = RfbState::WaitingTightFillColor;
} else if (mode == kTightModeJpeg) {
m_tightLengthByteIndex = 0;
m_pendingLength = 0;
m_rfbState = RfbState::WaitingTightLengthByte;
} else if (mode <= kTightBasicModeMax) {
if ((mode & kTightBasicExplicitFilterFlag) != 0) {
m_rfbState = RfbState::WaitingTightFilterId;
} else {
m_tightFilterId = 0; // Copy implied
m_tightLengthByteIndex = 0;
m_pendingLength = 0;
m_rfbState = RfbState::WaitingTightLengthByte;
}
} else {
failConnection(
QStringLiteral("The VNC server sent an invalid Tight compression-control byte."),
QStringLiteral("Control byte 0x%1").arg(controlByte, 2, 16, QChar('0')));
return;
}
break;
}
case RfbState::WaitingTightFillColor: {
if (!haveBytes(3)) {
return;
}
const QRgb color = VncPixelCodecs::rgbFromPixelBytes(
reinterpret_cast<const uchar*>(m_recvBuffer.constData()));
m_recvBuffer.remove(0, 3);
if (m_currentRectangle.width > 0 && m_currentRectangle.height > 0) {
QPainter painter(&m_framebuffer);
painter.fillRect(QRect(m_currentRectangle.x, m_currentRectangle.y,
m_currentRectangle.width, m_currentRectangle.height),
QColor::fromRgb(color));
}
onRectangleFinished();
break;
}
case RfbState::WaitingTightFilterId: {
if (!haveBytes(1)) {
return;
}
m_tightFilterId = static_cast<quint8>(m_recvBuffer.at(0));
m_recvBuffer.remove(0, 1);
if (m_tightFilterId > 2) {
failConnection(QStringLiteral("The VNC server sent an unrecognized Tight filter id."),
QStringLiteral("Filter id %1").arg(m_tightFilterId));
return;
}
m_tightLengthByteIndex = 0;
m_pendingLength = 0;
m_rfbState = RfbState::WaitingTightLengthByte;
break;
}
case RfbState::WaitingTightLengthByte: {
if (!haveBytes(1)) {
return;
}
const quint8 b = static_cast<quint8>(m_recvBuffer.at(0));
m_recvBuffer.remove(0, 1);
if (m_tightLengthByteIndex == 0) {
m_pendingLength = (b & 0x7F);
} else if (m_tightLengthByteIndex == 1) {
m_pendingLength |= static_cast<quint32>(b & 0x7F) << 7;
} else {
m_pendingLength |= static_cast<quint32>(b) << 14;
}
// Compact-length encoding: up to 3 bytes, continuation-bit
// style (only the first two bytes' high bit can request
// another byte; the third byte's full 8 bits are always used
// as the final contribution, capping the value at ~4MB).
const bool continuation = (m_tightLengthByteIndex < 2) && ((b & 0x80) != 0);
++m_tightLengthByteIndex;
if (continuation) {
break;
}
m_rfbState = RfbState::WaitingTightPayload;
break;
}
case RfbState::WaitingTightPayload: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const QByteArray payload = m_recvBuffer.left(static_cast<int>(m_pendingLength));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
const int rectWidth = m_currentRectangle.width;
const int rectHeight = m_currentRectangle.height;
if (rectWidth <= 0 || rectHeight <= 0) {
onRectangleFinished();
break;
}
if (m_tightCompressionMode == kTightModeJpeg) {
QImage jpegImage;
if (!decodeJpegRectangle(payload, rectWidth, rectHeight, &jpegImage)) {
failConnection(QStringLiteral("The VNC server sent malformed Tight JPEG data."),
QStringLiteral("JPEG payload length %1").arg(payload.size()));
return;
}
{
QPainter painter(&m_framebuffer);
painter.drawImage(m_currentRectangle.x, m_currentRectangle.y, jpegImage);
}
onRectangleFinished();
break;
}
// Basic compression: always zlib-compressed by this decoder
// (the real protocol allows very small payloads to skip
// compression entirely; not handled here -- see class comment).
const int streamIndex = m_tightCompressionMode & kTightBasicStreamIndexMask;
QByteArray filtered;
if (!inflateTightStream(streamIndex, payload, &filtered)) {
failConnection(
QStringLiteral("The VNC server sent malformed Tight-compressed data."),
QStringLiteral("Stream %1, payload length %2").arg(streamIndex).arg(payload.size()));
return;
}
QVector<QRgb> pixels;
switch (m_tightFilterId) {
case 0:
pixels = VncPixelCodecs::decodeTightCopyFilter(filtered, rectWidth, rectHeight);
break;
case 1:
pixels = VncPixelCodecs::decodeTightPaletteFilter(filtered, rectWidth, rectHeight);
break;
case 2:
pixels = VncPixelCodecs::decodeTightGradientFilter(filtered, rectWidth, rectHeight);
break;
default:
break;
}
if (pixels.size() != rectWidth * rectHeight) {
failConnection(
QStringLiteral("The VNC server sent a malformed Tight rectangle."),
QStringLiteral("Filter %1 produced %2 pixels, expected %3")
.arg(m_tightFilterId)
.arg(pixels.size())
.arg(rectWidth * rectHeight));
return;
}
{
QImage rectImage(rectWidth, rectHeight, QImage::Format_RGB32);
for (int y = 0; y < rectHeight; ++y) {
for (int x = 0; x < rectWidth; ++x) {
rectImage.setPixel(x, y, pixels.at((y * rectWidth) + x));
}
}
QPainter painter(&m_framebuffer);
painter.drawImage(m_currentRectangle.x, m_currentRectangle.y, rectImage);
}
onRectangleFinished();
break;
}
case RfbState::WaitingSetColourMapHeader: {
if (!haveBytes(5)) {
return;