DarkSingularity/orbithub
Internal
Public Access
0
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases 7 Wiki Activity

Fix VNC robustness gap: unannounced encodings now fail clearly, not silently

Browse Source 
Previously an unrecognized rectangle encoding in a FramebufferUpdate
aborted the connection with a generic message, and there was nothing
tying the set of encodings we announce via SetEncodings to the set we
actually know how to decode. Introduces kAnnouncedEncodings as the
single source of truth for both, converts the rectangle dispatch to a
switch keyed off it, and gives the (still intentionally fatal --
there's no safe way to skip an unknown-length payload) fallback a
message that identifies it as a protocol violation rather than "not
supported". Adds a regression test asserting every announced encoding
has a working dispatch case, so future encodings (Hextile/ZRLE/Tight/
Cursor) can't be added to the announced list without matching decode
support. Also replaces scattered inline magic numbers for RFB
message-type constants with named constants, in prep for the
clipboard/cursor/compression work that follows.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmith
2026-09-15 20:27:18 -06:00
co-authored by Claude Sonnet 5
parent 0b5454197c
commit 35d4daec7f
2 changed files with 192 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_vnc_session_backend.cpp
+129
View File
@@ -108,6 +108,8 @@ private slots:
void unsupportedSecurityTypeReachesFailedState();
void rfb33ServerWithNoAuthConnectsWithoutSecurityResult();
void rawFramebufferUpdateProducesExpectedPixels();
void copyRectEncodingDoesNotFailConnection();
void unannouncedEncodingFailsConnectionWithClearMessage();
private:
std::unique_ptr<FakeVncServer> m_server;
@@ -491,5 +493,132 @@ void TestVncSessionBackend::rawFramebufferUpdateProducesExpectedPixels()
QCOMPARE(m_lastFrame.pixelColor(1, 0), QColor(0, 255, 0));
}
// Regression guard for the WaitingRectangleHeader dispatch: every encoding
// this backend announces via SetEncodings must have a working decode path.
// Raw is already exercised by rawFramebufferUpdateProducesExpectedPixels();
// this covers CopyRect. Whenever a new encoding is added to
// kAnnouncedEncodings in vnc_session_backend.cpp, a matching test belongs
// here (or nearby) so the announced list and the dispatch switch can never
// silently drift apart.
void TestVncSessionBackend::copyRectEncodingDoesNotFailConnection()
{
int frameCount = 0;
connect(m_backend.get(), &SessionBackend::frameUpdated, this,
[&frameCount](const QImage&) { ++frameCount; });
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
m_server->received.clear();
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2: { // ClientInit
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(2)); // width = 2
serverInit.append(char(0)); serverInit.append(char(2)); // height = 2
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
m_server->sendWhenConnected(serverInit);
break;
}
case 3: { // initial non-incremental request -> seed the framebuffer with Raw
QByteArray update;
update.append(char(0)); update.append(char(0));
update.append(char(0)); update.append(char(1)); // 1 rectangle
update.append(char(0)); update.append(char(0)); // x
update.append(char(0)); update.append(char(0)); // y
update.append(char(0)); update.append(char(2)); // width
update.append(char(0)); update.append(char(2)); // height
update.append(char(0)); update.append(char(0)); update.append(char(0));
update.append(char(0)); // encoding = Raw
update.append(QByteArray(2 * 2 * 4, char(0x11)));
m_server->sendWhenConnected(update);
break;
}
case 4: { // next incremental request -> a CopyRect rectangle
QByteArray update;
update.append(char(0)); update.append(char(0));
update.append(char(0)); update.append(char(1)); // 1 rectangle
update.append(char(0)); update.append(char(0)); // x
update.append(char(0)); update.append(char(0)); // y
update.append(char(0)); update.append(char(2)); // width
update.append(char(0)); update.append(char(2)); // height
update.append(char(0)); update.append(char(0)); update.append(char(0));
update.append(char(1)); // encoding = CopyRect
update.append(char(0)); update.append(char(0)); // src x = 0
update.append(char(0)); update.append(char(0)); // src y = 0
m_server->sendWhenConnected(update);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_VERIFY(frameCount >= 2);
QCOMPARE(m_lastState, SessionState::Connected);
}
void TestVncSessionBackend::unannouncedEncodingFailsConnectionWithClearMessage()
{
connect(m_server.get(), &FakeVncServer::clientConnected, this, [this]() {
m_server->sendWhenConnected(QByteArray("RFB 003.008\n"));
});
connect(m_server.get(), &FakeVncServer::dataReceived, this, [this]() {
m_server->received.clear();
switch (m_server->nextStep()) {
case 0: {
QByteArray securityTypes;
securityTypes.append(char(1));
securityTypes.append(char(1));
m_server->sendWhenConnected(securityTypes);
break;
}
case 1:
m_server->sendWhenConnected(QByteArray(4, char(0))); // SecurityResult: OK
break;
case 2: { // ClientInit
QByteArray serverInit;
serverInit.append(char(0)); serverInit.append(char(1)); // width = 1
serverInit.append(char(0)); serverInit.append(char(1)); // height = 1
serverInit.append(QByteArray(16, char(0)));
serverInit.append(QByteArray(4, char(0)));
m_server->sendWhenConnected(serverInit);
break;
}
case 3: { // a rectangle whose encoding was never announced via SetEncodings
QByteArray update;
update.append(char(0)); update.append(char(0));
update.append(char(0)); update.append(char(1)); // 1 rectangle
update.append(char(0)); update.append(char(0)); // x
update.append(char(0)); update.append(char(0)); // y
update.append(char(0)); update.append(char(1)); // width
update.append(char(0)); update.append(char(1)); // height
update.append(char(0)); update.append(char(0)); update.append(char(0));
update.append(char(99)); // encoding = 99, never announced
m_server->sendWhenConnected(update);
break;
}
default:
break;
}
});
m_backend->connectSession(makeOptions());
QTRY_COMPARE(m_lastState, SessionState::Failed);
QVERIFY(m_lastErrorDisplay.contains(QStringLiteral("non-compliant")));
}
QTEST_GUILESS_MAIN(TestVncSessionBackend)
#include "test_vnc_session_backend.moc"