DarkSingularity/orbithub
Internal
Public Access
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 8 Wiki Activity

Add VNC Hextile decoding

Browse Source 
Implements RFC 6143 SS7.7.4: rectangles announced as Hextile (type 5)
tile the update into 16x16 blocks, each either raw pixels or a
background fill plus an optional list of foreground/individually-
colored subrects, with background/foreground persisting across tiles
within one rectangle when not re-specified.

The pure byte-decode logic (tile metadata, subrect list) lives in new
src/vnc_pixel_codecs.h/.cpp, kept separate from
VncSessionBackend's wire-sequencing state machine so it's unit-testable
without a socket -- the pattern the plan calls for continuing into the
ZRLE/Tight work still ahead. Adds 5 fake-server tests covering a raw
tile, a background-only solid fill, uncoloured and individually-colored
subrects, and a 4-tile rectangle proving background persistence and
correct tile-cursor wraparound.

Verified against the live TightVNC test server (connect, frame,
cursor, clipboard all still work); that particular server always
chose Raw for the actual framebuffer content during this session, so
Hextile's real-world path isn't independently confirmed live -- the
unit tests are the primary correctness evidence for this phase.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmith
2026-09-15 20:49:08 -06:00
co-authored by Claude Sonnet 5
parent 9dd1af21d6
commit e49fa0cf26
7 changed files with 598 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vnc_pixel_codecs.cpp
+80
View File
@@ -0,0 +1,80 @@
#include "vnc_pixel_codecs.h"
namespace VncPixelCodecs {
QRgb rgbFromPixelBytes(const uchar* bytes)
{
// Byte order B,G,R,pad -- matches VncSessionBackend's negotiated
// SetPixelFormat (32bpp little-endian, R at shift 16 / G at 8 / B at 0).
return qRgb(bytes[2], bytes[1], bytes[0]);
}
int hextileFixedMetaByteCount(quint8 subencoding)
{
int count = 0;
if ((subencoding & HextileFlags::kBackgroundSpecified) != 0) {
count += 4;
}
if ((subencoding & HextileFlags::kForegroundSpecified) != 0) {
count += 4;
}
if ((subencoding & HextileFlags::kAnySubrects) != 0) {
count += 1;
}
return count;
}
int decodeHextileFixedMeta(quint8 subencoding, const QByteArray& data, QRgb* background,
QRgb* foreground)
{
int offset = 0;
const auto* bytes = reinterpret_cast<const uchar*>(data.constData());
if ((subencoding & HextileFlags::kBackgroundSpecified) != 0) {
*background = rgbFromPixelBytes(bytes + offset);
offset += 4;
}
if ((subencoding & HextileFlags::kForegroundSpecified) != 0) {
*foreground = rgbFromPixelBytes(bytes + offset);
offset += 4;
}
if ((subencoding & HextileFlags::kAnySubrects) != 0) {
return static_cast<int>(static_cast<quint8>(data.at(offset)));
}
return 0;
}
int hextileSubrectByteCount(bool coloured, int subrectCount)
{
return subrectCount * (coloured ? 6 : 2);
}
QVector<HextileSubrect> decodeHextileSubrects(bool coloured, int subrectCount,
const QByteArray& data, QRgb foreground)
{
QVector<HextileSubrect> subrects;
subrects.reserve(subrectCount);
const auto* bytes = reinterpret_cast<const uchar*>(data.constData());
const int stride = coloured ? 6 : 2;
for (int i = 0; i < subrectCount; ++i) {
const uchar* entry = bytes + (i * stride);
QRgb color = foreground;
int fieldOffset = 0;
if (coloured) {
color = rgbFromPixelBytes(entry);
fieldOffset = 4;
}
const uchar xy = entry[fieldOffset];
const uchar wh = entry[fieldOffset + 1];
// High nibble = x (or width-1), low nibble = y (or height-1).
const int x = (xy >> 4) & 0x0F;
const int y = xy & 0x0F;
const int width = ((wh >> 4) & 0x0F) + 1;
const int height = (wh & 0x0F) + 1;
subrects.append(HextileSubrect{QRect(x, y, width, height), color});
}
return subrects;
}
}
src/vnc_pixel_codecs.h
+65
View File
@@ -0,0 +1,65 @@
#ifndef ORBITHUB_VNC_PIXEL_CODECS_H
#define ORBITHUB_VNC_PIXEL_CODECS_H
#include <QByteArray>
#include <QRect>
#include <QRgb>
#include <QVector>
// Pure, state-free decode helpers for VNC/RFB pixel encodings, kept out of
// VncSessionBackend so the wire-sequencing (when to read what off the
// socket) and wire-decoding (how to interpret already-buffered bytes) stay
// separate and the latter is unit-testable without a live connection.
namespace VncPixelCodecs {
// Converts one pixel's worth of raw bytes as sent under
// VncSessionBackend's negotiated SetPixelFormat (32bpp, little-endian,
// byte order B,G,R,pad) into a QRgb.
QRgb rgbFromPixelBytes(const uchar* bytes);
// A single filled rectangle, in tile-local coordinates (0,0 = the tile's
// own top-left corner, not the enclosing FramebufferUpdate rectangle's).
struct HextileSubrect {
QRect rect;
QRgb color = 0;
};
// RFC 6143 SS7.7.4 Hextile tile subencoding byte flags.
namespace HextileFlags {
constexpr quint8 kRaw = 0x01;
constexpr quint8 kBackgroundSpecified = 0x02;
constexpr quint8 kForegroundSpecified = 0x04;
constexpr quint8 kAnySubrects = 0x08;
constexpr quint8 kSubrectsColoured = 0x10;
}
// Byte length of a Hextile tile's "fixed" metadata -- the optional
// background/foreground color updates plus the optional subrect count --
// derivable from the subencoding byte alone, before any of those bytes are
// available. Only meaningful when HextileFlags::kRaw is *not* set (a Raw
// tile has no metadata at all, just tileWidth*tileHeight raw pixels).
int hextileFixedMetaByteCount(quint8 subencoding);
// Parses the `hextileFixedMetaByteCount(subencoding)` bytes described
// above. Updates *background/*foreground in place only when the
// corresponding flag is set in `subencoding` -- callers should persist
// their previous values across tiles in the same rectangle and pass them
// in here unchanged when a color isn't re-specified, since RFC 6143 has
// each tile inherit the last-specified colors. Returns the subrect count
// (0 if HextileFlags::kAnySubrects isn't set).
int decodeHextileFixedMeta(quint8 subencoding, const QByteArray& data, QRgb* background,
QRgb* foreground);
// Byte length of `subrectCount` subrects' worth of data, given whether
// they're individually colored (HextileFlags::kSubrectsColoured).
int hextileSubrectByteCount(bool coloured, int subrectCount);
// Parses `subrectCount` subrects (xy + wh bytes, plus a per-subrect color
// when `coloured`) out of `data`, substituting `foreground` in for any
// that aren't individually colored.
QVector<HextileSubrect> decodeHextileSubrects(bool coloured, int subrectCount,
const QByteArray& data, QRgb foreground);
}
#endif
src/vnc_session_backend.cpp
+144 -2
View File
@@ -1,5 +1,7 @@
#include "vnc_session_backend.h"
#include "vnc_pixel_codecs.h"
#include <QPainter>
#include <QStringList>
#include <QTcpSocket>
@@ -32,13 +34,15 @@ constexpr quint8 kServerMsgServerCutText = 3;
// of truth for what we tell the server we can decode via SetEncodings.
constexpr qint32 kEncRaw = 0;
constexpr qint32 kEncCopyRect = 1;
constexpr qint32 kEncHextile = 5;
// RFC 6143 SS7.8.2 "Cursor pseudo-encoding": not a real screen-content
// encoding -- a rectangle with this type carries a cursor shape update
// instead (hotspot in x/y, image dims in width/height), never painted into
// the framebuffer.
constexpr qint32 kEncCursor = -239;
constexpr std::array<qint32, 3> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncCursor };
constexpr std::array<qint32, 4> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile,
kEncCursor };
quint16 readU16BE(const QByteArray& buf, int offset)
{
@@ -82,7 +86,14 @@ VncSessionBackend::VncSessionBackend(const Profile& profile, QObject* parent)
m_pendingRectanglesRemaining(0),
m_pointerButtonMask(0),
m_lastPointerX(0),
m_lastPointerY(0)
m_lastPointerY(0),
m_hextileTileX(0),
m_hextileTileY(0),
m_hextileBackground(qRgb(0, 0, 0)),
m_hextileForeground(qRgb(0, 0, 0)),
m_hextileSubencoding(0),
m_hextileSubrectsRemaining(0),
m_hextileSubrectsColoured(false)
{
connect(m_socket, &QTcpSocket::connected, this, &VncSessionBackend::onSocketConnected);
connect(m_socket, &QTcpSocket::readyRead, this, &VncSessionBackend::onSocketReadyRead);
@@ -326,6 +337,10 @@ void VncSessionBackend::onSocketDisconnected()
&& m_rfbState != RfbState::WaitingRawPixelData
&& m_rfbState != RfbState::WaitingCopyRectSource
&& m_rfbState != RfbState::WaitingCursorPixelData
&& m_rfbState != RfbState::WaitingHextileTileSubencoding
&& m_rfbState != RfbState::WaitingHextileTileMeta
&& m_rfbState != RfbState::WaitingHextileSubrectData
&& m_rfbState != RfbState::WaitingHextileRawTileData
&& m_rfbState != RfbState::WaitingSetColourMapHeader
&& m_rfbState != RfbState::WaitingSetColourMapData
&& m_rfbState != RfbState::WaitingServerCutTextHeader
@@ -370,6 +385,10 @@ void VncSessionBackend::resetProtocolState()
m_framebuffer = QImage();
m_pendingRectanglesRemaining = 0;
m_pointerButtonMask = 0;
m_hextileTileX = 0;
m_hextileTileY = 0;
m_hextileBackground = qRgb(0, 0, 0);
m_hextileForeground = qRgb(0, 0, 0);
}
bool VncSessionBackend::haveBytes(int count) const
@@ -516,6 +535,29 @@ void VncSessionBackend::onRectangleFinished()
requestFramebufferUpdate(true);
}
QRect VncSessionBackend::currentHextileTileRect() const
{
const int tileWidth = qMin(16, m_currentRectangle.width - m_hextileTileX);
const int tileHeight = qMin(16, m_currentRectangle.height - m_hextileTileY);
return QRect(m_currentRectangle.x + m_hextileTileX, m_currentRectangle.y + m_hextileTileY,
tileWidth, tileHeight);
}
void VncSessionBackend::advanceHextileTile()
{
m_hextileTileX += 16;
if (m_hextileTileX >= m_currentRectangle.width) {
m_hextileTileX = 0;
m_hextileTileY += 16;
}
if (m_hextileTileY >= m_currentRectangle.height) {
onRectangleFinished();
} else {
m_rfbState = RfbState::WaitingHextileTileSubencoding;
}
}
void VncSessionBackend::processReceiveBuffer()
{
for (;;) {
@@ -819,6 +861,18 @@ void VncSessionBackend::processReceiveBuffer()
case kEncCursor:
m_rfbState = RfbState::WaitingCursorPixelData;
break;
case kEncHextile:
m_hextileTileX = 0;
m_hextileTileY = 0;
m_hextileBackground = qRgb(0, 0, 0);
m_hextileForeground = qRgb(0, 0, 0);
if (m_currentRectangle.width <= 0 || m_currentRectangle.height <= 0) {
// No tiles to decode at all.
onRectangleFinished();
} else {
m_rfbState = RfbState::WaitingHextileTileSubencoding;
}
break;
default: {
// SetEncodings (see kAnnouncedEncodings) is entirely
// client-controlled, so a spec-compliant server will never
@@ -941,6 +995,94 @@ void VncSessionBackend::processReceiveBuffer()
break;
}
case RfbState::WaitingHextileTileSubencoding: {
if (!haveBytes(1)) {
return;
}
m_hextileSubencoding = static_cast<quint8>(m_recvBuffer.at(0));
m_recvBuffer.remove(0, 1);
if ((m_hextileSubencoding & VncPixelCodecs::HextileFlags::kRaw) != 0) {
m_rfbState = RfbState::WaitingHextileRawTileData;
} else {
m_pendingLength = static_cast<quint32>(
VncPixelCodecs::hextileFixedMetaByteCount(m_hextileSubencoding));
m_rfbState = RfbState::WaitingHextileTileMeta;
}
break;
}
case RfbState::WaitingHextileTileMeta: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const QByteArray data = m_recvBuffer.left(static_cast<int>(m_pendingLength));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
const int subrectCount = VncPixelCodecs::decodeHextileFixedMeta(
m_hextileSubencoding, data, &m_hextileBackground, &m_hextileForeground);
// Every tile is filled with the (possibly just-updated,
// possibly inherited) background colour first, regardless of
// whether this tile re-specified it.
QPainter painter(&m_framebuffer);
painter.fillRect(currentHextileTileRect(), QColor::fromRgb(m_hextileBackground));
m_hextileSubrectsColoured =
(m_hextileSubencoding & VncPixelCodecs::HextileFlags::kSubrectsColoured) != 0;
m_hextileSubrectsRemaining = subrectCount;
if (subrectCount == 0) {
advanceHextileTile();
} else {
m_pendingLength = static_cast<quint32>(VncPixelCodecs::hextileSubrectByteCount(
m_hextileSubrectsColoured, subrectCount));
m_rfbState = RfbState::WaitingHextileSubrectData;
}
break;
}
case RfbState::WaitingHextileSubrectData: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const QByteArray data = m_recvBuffer.left(static_cast<int>(m_pendingLength));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
const QVector<VncPixelCodecs::HextileSubrect> subrects =
VncPixelCodecs::decodeHextileSubrects(m_hextileSubrectsColoured,
m_hextileSubrectsRemaining, data,
m_hextileForeground);
const QRect tileRect = currentHextileTileRect();
QPainter painter(&m_framebuffer);
for (const VncPixelCodecs::HextileSubrect& subrect : subrects) {
painter.fillRect(subrect.rect.translated(tileRect.topLeft()),
QColor::fromRgb(subrect.color));
}
advanceHextileTile();
break;
}
case RfbState::WaitingHextileRawTileData: {
const QRect tileRect = currentHextileTileRect();
const qint64 byteCount =
static_cast<qint64>(tileRect.width()) * tileRect.height() * 4;
if (!haveBytes(static_cast<int>(byteCount))) {
return;
}
if (byteCount > 0) {
const QImage tileImage(reinterpret_cast<const uchar*>(m_recvBuffer.constData()),
tileRect.width(), tileRect.height(),
tileRect.width() * 4, QImage::Format_RGB32);
QPainter painter(&m_framebuffer);
painter.drawImage(tileRect.topLeft(), tileImage);
}
m_recvBuffer.remove(0, static_cast<int>(byteCount));
advanceHextileTile();
break;
}
case RfbState::WaitingSetColourMapHeader: {
if (!haveBytes(5)) {
return;
src/vnc_session_backend.h
+25 -4
View File
@@ -6,6 +6,8 @@
#include <QAbstractSocket>
#include <QByteArray>
#include <QImage>
#include <QRect>
#include <QRgb>
class QTcpSocket;
@@ -20,10 +22,11 @@ class QTcpSocket;
//
// Scope (see plan / issue #3 for the full rationale): standard VNC
// Authentication (security type 2) and no-auth (type 1) only -- not
// Apple's Screen Sharing scheme (type 30). Raw + CopyRect encodings only.
// No dynamic resize. Clipboard sync (Latin-1 only, per RFB's
// ServerCutText/ClientCutText) and remote cursor shape sync (the Cursor
// pseudo-encoding) are supported.
// Apple's Screen Sharing scheme (type 30). Raw + CopyRect + Hextile
// encodings (ZRLE/Tight compression not yet implemented). No dynamic
// resize. Clipboard sync (Latin-1 only, per RFB's ServerCutText/
// ClientCutText) and remote cursor shape sync (the Cursor pseudo-encoding)
// are supported.
class VncSessionBackend : public SessionBackend
{
Q_OBJECT
@@ -83,6 +86,10 @@ private:
WaitingRawPixelData,
WaitingCopyRectSource,
WaitingCursorPixelData,
WaitingHextileTileSubencoding,
WaitingHextileTileMeta,
WaitingHextileSubrectData,
WaitingHextileRawTileData,
WaitingSetColourMapHeader,
WaitingSetColourMapData,
WaitingServerCutTextHeader,
@@ -120,6 +127,18 @@ private:
int m_lastPointerX;
int m_lastPointerY;
// Hextile decode state (RFC 6143 SS7.7.4): tile-cursor position relative
// to the current rectangle's origin, plus the background/foreground
// colors, which persist across tiles within one rectangle whenever a
// tile doesn't re-specify them.
int m_hextileTileX;
int m_hextileTileY;
QRgb m_hextileBackground;
QRgb m_hextileForeground;
quint8 m_hextileSubencoding;
int m_hextileSubrectsRemaining;
bool m_hextileSubrectsColoured;
void setState(SessionState state, const QString& message);
void resetProtocolState();
void processReceiveBuffer();
@@ -134,6 +153,8 @@ private:
void sendPointerEvent();
void sendWheelClick(quint8 wheelBit);
void sendClientCutText(const QString& text);
QRect currentHextileTileRect() const;
void advanceHextileTile();
};
#endif