DarkSingularity/orbithub
Internal
Public Access
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 7 Wiki Activity

Add VNC Hextile decoding

Browse Source 
Implements RFC 6143 SS7.7.4: rectangles announced as Hextile (type 5)
tile the update into 16x16 blocks, each either raw pixels or a
background fill plus an optional list of foreground/individually-
colored subrects, with background/foreground persisting across tiles
within one rectangle when not re-specified.

The pure byte-decode logic (tile metadata, subrect list) lives in new
src/vnc_pixel_codecs.h/.cpp, kept separate from
VncSessionBackend's wire-sequencing state machine so it's unit-testable
without a socket -- the pattern the plan calls for continuing into the
ZRLE/Tight work still ahead. Adds 5 fake-server tests covering a raw
tile, a background-only solid fill, uncoloured and individually-colored
subrects, and a 4-tile rectangle proving background persistence and
correct tile-cursor wraparound.

Verified against the live TightVNC test server (connect, frame,
cursor, clipboard all still work); that particular server always
chose Raw for the actual framebuffer content during this session, so
Hextile's real-world path isn't independently confirmed live -- the
unit tests are the primary correctness evidence for this phase.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmith
2026-09-15 20:49:08 -06:00
co-authored by Claude Sonnet 5
parent 9dd1af21d6
commit e49fa0cf26
7 changed files with 598 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vnc_session_backend.cpp
+144 -2
View File
@@ -1,5 +1,7 @@
#include "vnc_session_backend.h"
#include "vnc_pixel_codecs.h"
#include <QPainter>
#include <QStringList>
#include <QTcpSocket>
@@ -32,13 +34,15 @@ constexpr quint8 kServerMsgServerCutText = 3;
// of truth for what we tell the server we can decode via SetEncodings.
constexpr qint32 kEncRaw = 0;
constexpr qint32 kEncCopyRect = 1;
constexpr qint32 kEncHextile = 5;
// RFC 6143 SS7.8.2 "Cursor pseudo-encoding": not a real screen-content
// encoding -- a rectangle with this type carries a cursor shape update
// instead (hotspot in x/y, image dims in width/height), never painted into
// the framebuffer.
constexpr qint32 kEncCursor = -239;
constexpr std::array<qint32, 3> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncCursor };
constexpr std::array<qint32, 4> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile,
kEncCursor };
quint16 readU16BE(const QByteArray& buf, int offset)
{
@@ -82,7 +86,14 @@ VncSessionBackend::VncSessionBackend(const Profile& profile, QObject* parent)
m_pendingRectanglesRemaining(0),
m_pointerButtonMask(0),
m_lastPointerX(0),
m_lastPointerY(0)
m_lastPointerY(0),
m_hextileTileX(0),
m_hextileTileY(0),
m_hextileBackground(qRgb(0, 0, 0)),
m_hextileForeground(qRgb(0, 0, 0)),
m_hextileSubencoding(0),
m_hextileSubrectsRemaining(0),
m_hextileSubrectsColoured(false)
{
connect(m_socket, &QTcpSocket::connected, this, &VncSessionBackend::onSocketConnected);
connect(m_socket, &QTcpSocket::readyRead, this, &VncSessionBackend::onSocketReadyRead);
@@ -326,6 +337,10 @@ void VncSessionBackend::onSocketDisconnected()
&& m_rfbState != RfbState::WaitingRawPixelData
&& m_rfbState != RfbState::WaitingCopyRectSource
&& m_rfbState != RfbState::WaitingCursorPixelData
&& m_rfbState != RfbState::WaitingHextileTileSubencoding
&& m_rfbState != RfbState::WaitingHextileTileMeta
&& m_rfbState != RfbState::WaitingHextileSubrectData
&& m_rfbState != RfbState::WaitingHextileRawTileData
&& m_rfbState != RfbState::WaitingSetColourMapHeader
&& m_rfbState != RfbState::WaitingSetColourMapData
&& m_rfbState != RfbState::WaitingServerCutTextHeader
@@ -370,6 +385,10 @@ void VncSessionBackend::resetProtocolState()
m_framebuffer = QImage();
m_pendingRectanglesRemaining = 0;
m_pointerButtonMask = 0;
m_hextileTileX = 0;
m_hextileTileY = 0;
m_hextileBackground = qRgb(0, 0, 0);
m_hextileForeground = qRgb(0, 0, 0);
}
bool VncSessionBackend::haveBytes(int count) const
@@ -516,6 +535,29 @@ void VncSessionBackend::onRectangleFinished()
requestFramebufferUpdate(true);
}
QRect VncSessionBackend::currentHextileTileRect() const
{
const int tileWidth = qMin(16, m_currentRectangle.width - m_hextileTileX);
const int tileHeight = qMin(16, m_currentRectangle.height - m_hextileTileY);
return QRect(m_currentRectangle.x + m_hextileTileX, m_currentRectangle.y + m_hextileTileY,
tileWidth, tileHeight);
}
void VncSessionBackend::advanceHextileTile()
{
m_hextileTileX += 16;
if (m_hextileTileX >= m_currentRectangle.width) {
m_hextileTileX = 0;
m_hextileTileY += 16;
}
if (m_hextileTileY >= m_currentRectangle.height) {
onRectangleFinished();
} else {
m_rfbState = RfbState::WaitingHextileTileSubencoding;
}
}
void VncSessionBackend::processReceiveBuffer()
{
for (;;) {
@@ -819,6 +861,18 @@ void VncSessionBackend::processReceiveBuffer()
case kEncCursor:
m_rfbState = RfbState::WaitingCursorPixelData;
break;
case kEncHextile:
m_hextileTileX = 0;
m_hextileTileY = 0;
m_hextileBackground = qRgb(0, 0, 0);
m_hextileForeground = qRgb(0, 0, 0);
if (m_currentRectangle.width <= 0 || m_currentRectangle.height <= 0) {
// No tiles to decode at all.
onRectangleFinished();
} else {
m_rfbState = RfbState::WaitingHextileTileSubencoding;
}
break;
default: {
// SetEncodings (see kAnnouncedEncodings) is entirely
// client-controlled, so a spec-compliant server will never
@@ -941,6 +995,94 @@ void VncSessionBackend::processReceiveBuffer()
break;
}
case RfbState::WaitingHextileTileSubencoding: {
if (!haveBytes(1)) {
return;
}
m_hextileSubencoding = static_cast<quint8>(m_recvBuffer.at(0));
m_recvBuffer.remove(0, 1);
if ((m_hextileSubencoding & VncPixelCodecs::HextileFlags::kRaw) != 0) {
m_rfbState = RfbState::WaitingHextileRawTileData;
} else {
m_pendingLength = static_cast<quint32>(
VncPixelCodecs::hextileFixedMetaByteCount(m_hextileSubencoding));
m_rfbState = RfbState::WaitingHextileTileMeta;
}
break;
}
case RfbState::WaitingHextileTileMeta: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const QByteArray data = m_recvBuffer.left(static_cast<int>(m_pendingLength));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
const int subrectCount = VncPixelCodecs::decodeHextileFixedMeta(
m_hextileSubencoding, data, &m_hextileBackground, &m_hextileForeground);
// Every tile is filled with the (possibly just-updated,
// possibly inherited) background colour first, regardless of
// whether this tile re-specified it.
QPainter painter(&m_framebuffer);
painter.fillRect(currentHextileTileRect(), QColor::fromRgb(m_hextileBackground));
m_hextileSubrectsColoured =
(m_hextileSubencoding & VncPixelCodecs::HextileFlags::kSubrectsColoured) != 0;
m_hextileSubrectsRemaining = subrectCount;
if (subrectCount == 0) {
advanceHextileTile();
} else {
m_pendingLength = static_cast<quint32>(VncPixelCodecs::hextileSubrectByteCount(
m_hextileSubrectsColoured, subrectCount));
m_rfbState = RfbState::WaitingHextileSubrectData;
}
break;
}
case RfbState::WaitingHextileSubrectData: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const QByteArray data = m_recvBuffer.left(static_cast<int>(m_pendingLength));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
const QVector<VncPixelCodecs::HextileSubrect> subrects =
VncPixelCodecs::decodeHextileSubrects(m_hextileSubrectsColoured,
m_hextileSubrectsRemaining, data,
m_hextileForeground);
const QRect tileRect = currentHextileTileRect();
QPainter painter(&m_framebuffer);
for (const VncPixelCodecs::HextileSubrect& subrect : subrects) {
painter.fillRect(subrect.rect.translated(tileRect.topLeft()),
QColor::fromRgb(subrect.color));
}
advanceHextileTile();
break;
}
case RfbState::WaitingHextileRawTileData: {
const QRect tileRect = currentHextileTileRect();
const qint64 byteCount =
static_cast<qint64>(tileRect.width()) * tileRect.height() * 4;
if (!haveBytes(static_cast<int>(byteCount))) {
return;
}
if (byteCount > 0) {
const QImage tileImage(reinterpret_cast<const uchar*>(m_recvBuffer.constData()),
tileRect.width(), tileRect.height(),
tileRect.width() * 4, QImage::Format_RGB32);
QPainter painter(&m_framebuffer);
painter.drawImage(tileRect.topLeft(), tileImage);
}
m_recvBuffer.remove(0, static_cast<int>(byteCount));
advanceHextileTile();
break;
}
case RfbState::WaitingSetColourMapHeader: {
if (!haveBytes(5)) {
return;