Files
orbithub/src/vnc_session_backend.cpp
T
ksmithandClaude Sonnet 5 776db5ec04 Allow blank usernames, asking for one at connect time instead
Closes #21. SSH and RDP profiles previously hard-required a username
to even save the profile; that validation is dropped, and
SessionTab::requestConnectOptions() now prompts for it at connect
time when blank, reusing the existing password-prompt bar in
unmasked mode -- the same pattern already used for a blank password.

VNC's username is trickier: most VNC servers never use one (plain VNC
Authentication and no-auth don't), only the two Apple auth schemes
(security types 30/33) do, and which auth method gets used isn't known
until mid-connection, after the server's security-type list has been
negotiated -- too late for the pre-connect prompt SSH/RDP uses. Adds a
new async request/response pair to SessionBackend, usernameRequested()
signal / provideUsername() slot, mirroring the existing SSH host-key-
confirmation pattern. VncSessionBackend pauses its state machine right
before computing an Apple-auth response if no username is available --
without consuming the already-buffered prime/host-key bytes, so
resuming re-parses them identically -- emits the request, and resumes
via provideUsername(). Cancelling (or submitting blank) fails the
connection cleanly instead of sending Apple auth an empty username.

The username is kept on the tab's in-memory profile copy for its
lifetime, not written back to the saved profile, matching how
passwords are already handled.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-16 08:01:57 -06:00

1991 lines
77 KiB
C++

#include "vnc_session_backend.h"
#include "vnc_apple_dh_auth.h"
#include "vnc_apple_rsa_auth.h"
#include "vnc_pixel_codecs.h"
#include <QPainter>
#include <QStringList>
#include <QTcpSocket>
#include <QTimer>
#include <QtGlobal>
#include <openssl/des.h>
#include <zlib.h>
extern "C" {
#include <jpeglib.h>
}
#include <array>
#include <cstring>
#include <limits>
namespace {
// RFB (RFC 6143) message-type constants, client -> server.
constexpr char kMsgSetPixelFormat = 0;
constexpr char kMsgSetEncodings = 2;
constexpr char kMsgFramebufferUpdateRequest = 3;
constexpr char kMsgKeyEvent = 4;
constexpr char kMsgPointerEvent = 5;
constexpr char kMsgClientCutText = 6;
// RFB message-type constants, server -> client.
constexpr quint8 kServerMsgFramebufferUpdate = 0;
constexpr quint8 kServerMsgSetColourMapEntries = 1;
constexpr quint8 kServerMsgBell = 2;
constexpr quint8 kServerMsgServerCutText = 3;
// Encoding types. This list grows as new FramebufferUpdate rectangle
// encodings are supported; kAnnouncedEncodings below is the single source
// of truth for what we tell the server we can decode via SetEncodings.
constexpr qint32 kEncRaw = 0;
constexpr qint32 kEncCopyRect = 1;
constexpr qint32 kEncHextile = 5;
constexpr qint32 kEncZRLE = 16;
constexpr qint32 kEncTight = 7;
// RFC 6143 SS7.8.2 "Cursor pseudo-encoding": not a real screen-content
// encoding -- a rectangle with this type carries a cursor shape update
// instead (hotspot in x/y, image dims in width/height), never painted into
// the framebuffer.
constexpr qint32 kEncCursor = -239;
// Apple's Screen Sharing authentication (Diffie-Hellman + AES). Not part
// of RFC 6143 -- see vnc_apple_dh_auth.h.
constexpr quint8 kSecurityTypeNone = 1;
constexpr quint8 kSecurityTypeVncAuth = 2;
constexpr quint8 kSecurityTypeAppleDh = 30;
// Apple's RSA-based scheme -- see vnc_apple_rsa_auth.h. Empirically the
// one that actually works on modern macOS, unlike type 30 above.
constexpr quint8 kSecurityTypeAppleRsa = 33;
constexpr std::array<qint32, 6> kAnnouncedEncodings = { kEncRaw, kEncCopyRect, kEncHextile,
kEncZRLE, kEncTight, kEncCursor };
// Tight compression-control byte, high nibble (after the low 4 bits'
// stream-reset flags are stripped off).
constexpr quint8 kTightModeFill = 0x08;
constexpr quint8 kTightModeJpeg = 0x09;
// Modes 0-7 are "Basic" compression; bits 0-1 of the mode select which of
// the 4 persistent zlib streams to use, bit 2 flags that an explicit
// filter-id byte follows (Copy is implied when it's clear).
constexpr quint8 kTightBasicModeMax = 0x07;
constexpr quint8 kTightBasicExplicitFilterFlag = 0x04;
constexpr quint8 kTightBasicStreamIndexMask = 0x03;
constexpr int kZrleTileSize = 64;
// Sanity cap on a single ZRLE rectangle's compressed-length field, purely
// to reject an obviously-hostile/corrupt claim before attempting to
// buffer/allocate that many bytes -- well beyond anything a real screen
// update should ever compress to.
constexpr quint32 kMaxZrleCompressedLength = 64u * 1024u * 1024u;
quint16 readU16BE(const QByteArray& buf, int offset)
{
const auto* p = reinterpret_cast<const uchar*>(buf.constData()) + offset;
return static_cast<quint16>((p[0] << 8) | p[1]);
}
quint32 readU32BE(const QByteArray& buf, int offset)
{
const auto* p = reinterpret_cast<const uchar*>(buf.constData()) + offset;
return (static_cast<quint32>(p[0]) << 24) | (static_cast<quint32>(p[1]) << 16)
| (static_cast<quint32>(p[2]) << 8) | static_cast<quint32>(p[3]);
}
void appendU16BE(QByteArray& buf, quint16 value)
{
buf.append(static_cast<char>((value >> 8) & 0xFF));
buf.append(static_cast<char>(value & 0xFF));
}
void appendU32BE(QByteArray& buf, quint32 value)
{
buf.append(static_cast<char>((value >> 24) & 0xFF));
buf.append(static_cast<char>((value >> 16) & 0xFF));
buf.append(static_cast<char>((value >> 8) & 0xFF));
buf.append(static_cast<char>(value & 0xFF));
}
// Decodes a Tight "JPEG" rectangle (a full baseline JPEG image covering
// the whole rectangle, no tiling) via libjpeg-turbo directly -- chosen
// over QImage::fromData(..., "JPEG") specifically to avoid depending on
// Qt's own JPEG plugin being present at runtime across every packaging
// target. Returns false on any decode failure or dimension mismatch
// rather than risk painting a misinterpreted image.
bool decodeJpegRectangle(const QByteArray& jpegBytes, int width, int height, QImage* outImage)
{
if (jpegBytes.isEmpty() || width <= 0 || height <= 0) {
return false;
}
jpeg_decompress_struct cinfo;
jpeg_error_mgr jerr;
cinfo.err = jpeg_std_error(&jerr);
jpeg_create_decompress(&cinfo);
jpeg_mem_src(&cinfo, reinterpret_cast<const unsigned char*>(jpegBytes.constData()),
static_cast<unsigned long>(jpegBytes.size()));
if (jpeg_read_header(&cinfo, TRUE) != JPEG_HEADER_OK) {
jpeg_destroy_decompress(&cinfo);
return false;
}
cinfo.out_color_space = JCS_RGB;
jpeg_start_decompress(&cinfo);
if (static_cast<int>(cinfo.output_width) != width
|| static_cast<int>(cinfo.output_height) != height) {
jpeg_finish_decompress(&cinfo);
jpeg_destroy_decompress(&cinfo);
return false;
}
QImage image(width, height, QImage::Format_RGB32);
const int rowStride = static_cast<int>(cinfo.output_width) * cinfo.output_components;
QVector<unsigned char> rowBuffer(rowStride);
unsigned char* rowPointer[1];
while (cinfo.output_scanline < cinfo.output_height) {
rowPointer[0] = rowBuffer.data();
jpeg_read_scanlines(&cinfo, rowPointer, 1);
const int y = static_cast<int>(cinfo.output_scanline) - 1;
auto* dst = reinterpret_cast<QRgb*>(image.scanLine(y));
for (int x = 0; x < width; ++x) {
const unsigned char* p = rowBuffer.constData() + (x * 3);
dst[x] = qRgb(p[0], p[1], p[2]);
}
}
jpeg_finish_decompress(&cinfo);
jpeg_destroy_decompress(&cinfo);
*outImage = image;
return true;
}
}
VncSessionBackend::VncSessionBackend(const Profile& profile, QObject* parent)
: SessionBackend(profile, parent),
m_socket(new QTcpSocket(this)),
m_state(SessionState::Disconnected),
m_rfbState(RfbState::Idle),
m_userInitiatedDisconnect(false),
m_reconnectPending(false),
m_negotiatedMinorVersion(8),
m_securityTypeCount(0),
m_chosenSecurityType(0),
m_pendingLength(0),
m_pendingRectanglesRemaining(0),
m_pointerButtonMask(0),
m_lastPointerX(0),
m_lastPointerY(0),
m_hextileTileX(0),
m_hextileTileY(0),
m_hextileBackground(qRgb(0, 0, 0)),
m_hextileForeground(qRgb(0, 0, 0)),
m_hextileSubencoding(0),
m_hextileSubrectsRemaining(0),
m_hextileSubrectsColoured(false),
m_zrleInflateStream(new z_stream_s()),
m_zrleInflateInitialized(false),
m_tightInflateStreams{ new z_stream_s(), new z_stream_s(), new z_stream_s(),
new z_stream_s() },
m_tightInflateInitialized{ false, false, false, false },
m_tightCompressionMode(0),
m_tightFilterId(0),
m_tightLengthByteIndex(0),
m_appleAuthKeyLength(0),
m_waitingForUsername(false)
{
std::memset(m_zrleInflateStream, 0, sizeof(z_stream_s));
for (z_stream_s* stream : m_tightInflateStreams) {
std::memset(stream, 0, sizeof(z_stream_s));
}
connect(m_socket, &QTcpSocket::connected, this, &VncSessionBackend::onSocketConnected);
connect(m_socket, &QTcpSocket::readyRead, this, &VncSessionBackend::onSocketReadyRead);
connect(m_socket, &QTcpSocket::disconnected, this, &VncSessionBackend::onSocketDisconnected);
connect(m_socket, &QTcpSocket::errorOccurred, this, &VncSessionBackend::onSocketErrorOccurred);
}
VncSessionBackend::~VncSessionBackend()
{
if (m_socket->state() != QAbstractSocket::UnconnectedState) {
m_socket->abort();
}
if (m_zrleInflateInitialized) {
inflateEnd(m_zrleInflateStream);
}
delete m_zrleInflateStream;
for (int i = 0; i < 4; ++i) {
if (m_tightInflateInitialized[i]) {
inflateEnd(m_tightInflateStreams[i]);
}
delete m_tightInflateStreams[i];
}
}
void VncSessionBackend::connectSession(const SessionConnectOptions& options)
{
if (m_state == SessionState::Connected || m_state == SessionState::Connecting) {
emit eventLogged(QStringLiteral("Connect skipped: session is already active."));
return;
}
const Profile& p = profile();
if (p.host.trimmed().isEmpty()) {
const QString message = QStringLiteral("Host is required for VNC connections.");
setState(SessionState::Failed, message);
emit connectionError(message, message);
return;
}
if (p.port < 1 || p.port > 65535) {
const QString message = QStringLiteral("Port must be between 1 and 65535.");
setState(SessionState::Failed, message);
emit connectionError(message, message);
return;
}
m_userInitiatedDisconnect = false;
m_reconnectPending = false;
m_activeOptions = options;
resetProtocolState();
setState(SessionState::Connecting, QStringLiteral("Connecting to VNC endpoint..."));
emit eventLogged(
QStringLiteral("Opening TCP connection to %1:%2.").arg(p.host.trimmed()).arg(p.port));
m_socket->connectToHost(p.host.trimmed(), static_cast<quint16>(p.port));
}
void VncSessionBackend::disconnectSession()
{
if (m_socket->state() == QAbstractSocket::UnconnectedState) {
if (m_state != SessionState::Disconnected) {
setState(SessionState::Disconnected, QStringLiteral("Session is disconnected."));
}
return;
}
m_userInitiatedDisconnect = true;
emit eventLogged(QStringLiteral("Disconnect requested."));
m_socket->disconnectFromHost();
QTimer::singleShot(1500, this, [this]() {
if (m_socket->state() != QAbstractSocket::UnconnectedState) {
emit eventLogged(QStringLiteral("Force-closing VNC connection."));
m_socket->abort();
}
});
}
void VncSessionBackend::reconnectSession(const SessionConnectOptions& options)
{
emit eventLogged(QStringLiteral("Reconnect requested."));
if (m_socket->state() == QAbstractSocket::UnconnectedState) {
connectSession(options);
return;
}
m_reconnectPending = true;
m_reconnectOptions = options;
m_userInitiatedDisconnect = true;
m_socket->disconnectFromHost();
}
void VncSessionBackend::sendInput(const QString&)
{
emit eventLogged(QStringLiteral("Input ignored: VNC backend uses direct keyboard/mouse events."));
}
void VncSessionBackend::confirmHostKey(bool)
{
// RFB has no host-key-verification concept (unlike SSH); nothing to do.
}
void VncSessionBackend::updateTerminalSize(int, int)
{
// No dynamic resize support for VNC in this pass (see class comment).
// The display widget still reports viewport geometry for its own local
// scale-to-fit bookkeeping; there's just nothing to act on server-side
// here yet.
}
void VncSessionBackend::sendKeyEvent(int key,
quint32 nativeScanCode,
const QString& text,
bool pressed,
int modifiers)
{
Q_UNUSED(nativeScanCode);
Q_UNUSED(modifiers);
if (m_socket->state() != QAbstractSocket::ConnectedState) {
return;
}
const quint32 keysym = keysymForQtKey(key, text);
if (keysym == 0) {
return;
}
QByteArray msg;
msg.append(kMsgKeyEvent);
msg.append(pressed ? char(1) : char(0));
appendU16BE(msg, 0); // padding
appendU32BE(msg, keysym);
m_socket->write(msg);
}
void VncSessionBackend::sendMouseMoveEvent(int x, int y)
{
m_lastPointerX = x;
m_lastPointerY = y;
sendPointerEvent();
}
void VncSessionBackend::sendMouseButtonEvent(int x, int y, int button, bool pressed)
{
m_lastPointerX = x;
m_lastPointerY = y;
quint8 bit = 0;
switch (static_cast<Qt::MouseButton>(button)) {
case Qt::LeftButton:
bit = 0x01;
break;
case Qt::MiddleButton:
bit = 0x02;
break;
case Qt::RightButton:
bit = 0x04;
break;
default:
break;
}
if (bit != 0) {
m_pointerButtonMask = pressed ? static_cast<quint8>(m_pointerButtonMask | bit)
: static_cast<quint8>(m_pointerButtonMask & ~bit);
}
sendPointerEvent();
}
void VncSessionBackend::sendMouseWheelEvent(int x, int y, int deltaX, int deltaY)
{
m_lastPointerX = x;
m_lastPointerY = y;
// De facto convention (not part of the base RFC, but honored by
// essentially every real VNC server): wheel motion is represented as a
// momentary press+release of buttons 4/5 (vertical) or 6/7 (horizontal,
// less universally supported but common enough to include).
constexpr int kDegreesPerClick = 120;
if (deltaY != 0) {
const int clicks = qMax(1, qAbs(deltaY) / kDegreesPerClick);
const quint8 bit = deltaY > 0 ? 0x08 : 0x10;
for (int i = 0; i < clicks; ++i) {
sendWheelClick(bit);
}
}
if (deltaX != 0) {
const int clicks = qMax(1, qAbs(deltaX) / kDegreesPerClick);
const quint8 bit = deltaX > 0 ? 0x40 : 0x20;
for (int i = 0; i < clicks; ++i) {
sendWheelClick(bit);
}
}
}
void VncSessionBackend::setClipboardText(const QString& text)
{
if (m_socket->state() != QAbstractSocket::ConnectedState) {
return;
}
// Unlike RDP's CLIPRDR (list -> server request -> response), RFB's
// ClientCutText has no negotiation: send it immediately.
sendClientCutText(text);
}
void VncSessionBackend::onSocketConnected()
{
emit eventLogged(QStringLiteral("TCP connection established; waiting for VNC handshake."));
m_rfbState = RfbState::WaitingProtocolVersion;
}
void VncSessionBackend::onSocketReadyRead()
{
m_recvBuffer.append(m_socket->readAll());
processReceiveBuffer();
}
void VncSessionBackend::onSocketDisconnected()
{
if (m_reconnectPending) {
m_reconnectPending = false;
const SessionConnectOptions options = m_reconnectOptions;
setState(SessionState::Disconnected, QStringLiteral("Reconnecting..."));
QTimer::singleShot(0, this, [this, options]() { connectSession(options); });
return;
}
if (m_userInitiatedDisconnect) {
m_userInitiatedDisconnect = false;
setState(SessionState::Disconnected, QStringLiteral("Session disconnected."));
emit eventLogged(QStringLiteral("VNC connection closed after disconnect request."));
return;
}
// Guard against re-entrancy: failConnection() below calls
// m_socket->abort(), which synchronously re-emits disconnected() (Qt's
// socket engine fires it immediately, not via the event loop) before
// abort() itself returns -- so this slot can run again, mid-call,
// while m_state is already Failed from the explicit failure that
// triggered the abort. Without this guard that would silently
// overwrite an already-correct, specific error with this function's
// generic one.
if (m_state == SessionState::Connecting || m_state == SessionState::Connected) {
// Pre-3.8 servers have no explicit SecurityResult message, so a
// closed socket mid-handshake is how they signal an auth failure
// (or any other rejection) -- call that out specifically rather
// than a generic "connection closed" message.
const bool stillHandshaking = m_rfbState != RfbState::WaitingServerMessageType
&& m_rfbState != RfbState::WaitingFramebufferUpdateHeader
&& m_rfbState != RfbState::WaitingRectangleHeader
&& m_rfbState != RfbState::WaitingRawPixelData
&& m_rfbState != RfbState::WaitingCopyRectSource
&& m_rfbState != RfbState::WaitingCursorPixelData
&& m_rfbState != RfbState::WaitingHextileTileSubencoding
&& m_rfbState != RfbState::WaitingHextileTileMeta
&& m_rfbState != RfbState::WaitingHextileSubrectData
&& m_rfbState != RfbState::WaitingHextileRawTileData
&& m_rfbState != RfbState::WaitingZrleCompressedLength
&& m_rfbState != RfbState::WaitingZrleCompressedData
&& m_rfbState != RfbState::WaitingTightCompressionControl
&& m_rfbState != RfbState::WaitingTightFillColor
&& m_rfbState != RfbState::WaitingTightFilterId
&& m_rfbState != RfbState::WaitingTightLengthByte
&& m_rfbState != RfbState::WaitingTightPayload
&& m_rfbState != RfbState::WaitingSetColourMapHeader
&& m_rfbState != RfbState::WaitingSetColourMapData
&& m_rfbState != RfbState::WaitingServerCutTextHeader
&& m_rfbState != RfbState::WaitingServerCutTextData;
const QString message = stillHandshaking
? QStringLiteral("The VNC server closed the connection during the handshake -- "
"this often means the password was incorrect.")
: QStringLiteral("The VNC server closed the connection.");
setState(SessionState::Failed, message);
emit connectionError(message,
QStringLiteral("Socket disconnected unexpectedly (protocol state %1)")
.arg(static_cast<int>(m_rfbState)));
}
}
void VncSessionBackend::onSocketErrorOccurred(QAbstractSocket::SocketError error)
{
const QString raw = m_socket->errorString();
if (m_state == SessionState::Connecting || m_state == SessionState::Connected) {
const QString display = mapSocketError(error, raw);
setState(SessionState::Failed, display);
emit connectionError(display, raw);
}
}
void VncSessionBackend::setState(SessionState state, const QString& message)
{
m_state = state;
emit stateChanged(state, message);
emit eventLogged(message);
}
void VncSessionBackend::resetProtocolState()
{
m_rfbState = RfbState::Idle;
m_recvBuffer.clear();
m_negotiatedMinorVersion = 8;
m_securityTypeCount = 0;
m_offeredSecurityTypes.clear();
m_chosenSecurityType = 0;
m_pendingLength = 0;
m_framebuffer = QImage();
m_pendingRectanglesRemaining = 0;
m_pointerButtonMask = 0;
m_hextileTileX = 0;
m_hextileTileY = 0;
m_hextileBackground = qRgb(0, 0, 0);
m_hextileForeground = qRgb(0, 0, 0);
// The ZRLE zlib stream is scoped to one TCP connection's lifetime, not
// to individual rectangles or updates -- tear it down here so a fresh
// connect/reconnect starts a clean stream instead of feeding new
// compressed data through decoder state left over from a previous
// session.
if (m_zrleInflateInitialized) {
inflateEnd(m_zrleInflateStream);
m_zrleInflateInitialized = false;
}
for (int i = 0; i < 4; ++i) {
if (m_tightInflateInitialized[i]) {
inflateEnd(m_tightInflateStreams[i]);
m_tightInflateInitialized[i] = false;
}
}
m_appleAuthGenerator.clear();
m_appleAuthKeyLength = 0;
m_waitingForUsername = false;
m_promptedUsername.clear();
}
bool VncSessionBackend::haveBytes(int count) const
{
return m_recvBuffer.size() >= count;
}
void VncSessionBackend::failConnection(const QString& displayMessage, const QString& rawMessage)
{
setState(SessionState::Failed, displayMessage);
emit connectionError(displayMessage, rawMessage);
m_rfbState = RfbState::Idle;
m_socket->abort();
}
void VncSessionBackend::sendVersionReply()
{
// m_negotiatedMinorVersion is always a single digit (3, 7, or 8).
const QByteArray reply =
QStringLiteral("RFB 003.00%1\n").arg(m_negotiatedMinorVersion).toLatin1();
m_socket->write(reply);
}
void VncSessionBackend::sendClientInit()
{
const char shared = 1; // always share the desktop with other viewers
m_socket->write(&shared, 1);
}
void VncSessionBackend::sendSetPixelFormatAndEncodings()
{
// Request a fixed 32bpp true-color format whose in-memory byte layout
// (little-endian word, R at shift 16 / G at 8 / B at 0) matches
// QImage::Format_RGB32 exactly, so incoming Raw pixel data can be
// wrapped with zero conversion -- the same trick
// RdpSessionBackend::orbitEndPaint uses for FreeRDP's GDI buffer.
QByteArray setPixelFormat;
setPixelFormat.append(kMsgSetPixelFormat);
setPixelFormat.append(3, char(0)); // padding
setPixelFormat.append(char(32)); // bits-per-pixel
setPixelFormat.append(char(24)); // depth
setPixelFormat.append(char(0)); // big-endian-flag: false
setPixelFormat.append(char(1)); // true-color-flag: true
appendU16BE(setPixelFormat, 255); // red-max
appendU16BE(setPixelFormat, 255); // green-max
appendU16BE(setPixelFormat, 255); // blue-max
setPixelFormat.append(char(16)); // red-shift
setPixelFormat.append(char(8)); // green-shift
setPixelFormat.append(char(0)); // blue-shift
setPixelFormat.append(3, char(0)); // padding
m_socket->write(setPixelFormat);
QByteArray setEncodings;
setEncodings.append(kMsgSetEncodings);
setEncodings.append(char(0)); // padding
appendU16BE(setEncodings, static_cast<quint16>(kAnnouncedEncodings.size()));
for (qint32 encoding : kAnnouncedEncodings) {
appendU32BE(setEncodings, static_cast<quint32>(encoding));
}
m_socket->write(setEncodings);
}
void VncSessionBackend::requestFramebufferUpdate(bool incremental)
{
QByteArray msg;
msg.append(kMsgFramebufferUpdateRequest);
msg.append(incremental ? char(1) : char(0));
appendU16BE(msg, 0); // x
appendU16BE(msg, 0); // y
appendU16BE(msg, static_cast<quint16>(qBound(0, m_framebuffer.width(), 65535)));
appendU16BE(msg, static_cast<quint16>(qBound(0, m_framebuffer.height(), 65535)));
m_socket->write(msg);
}
void VncSessionBackend::sendPointerEvent()
{
if (m_socket->state() != QAbstractSocket::ConnectedState) {
return;
}
QByteArray msg;
msg.append(kMsgPointerEvent);
msg.append(static_cast<char>(m_pointerButtonMask));
appendU16BE(msg, static_cast<quint16>(qBound(0, m_lastPointerX, 65535)));
appendU16BE(msg, static_cast<quint16>(qBound(0, m_lastPointerY, 65535)));
m_socket->write(msg);
}
void VncSessionBackend::sendWheelClick(quint8 wheelBit)
{
if (m_socket->state() != QAbstractSocket::ConnectedState) {
return;
}
const quint16 x = static_cast<quint16>(qBound(0, m_lastPointerX, 65535));
const quint16 y = static_cast<quint16>(qBound(0, m_lastPointerY, 65535));
QByteArray press;
press.append(kMsgPointerEvent);
press.append(static_cast<char>(m_pointerButtonMask | wheelBit));
appendU16BE(press, x);
appendU16BE(press, y);
m_socket->write(press);
QByteArray release;
release.append(kMsgPointerEvent);
release.append(static_cast<char>(m_pointerButtonMask));
appendU16BE(release, x);
appendU16BE(release, y);
m_socket->write(release);
}
void VncSessionBackend::sendClientCutText(const QString& text)
{
// Latin-1 only, matching ServerCutText's decode above -- RFB has no
// Unicode clipboard extension in scope here.
const QByteArray latin1 = text.toLatin1();
QByteArray msg;
msg.append(kMsgClientCutText);
msg.append(3, char(0)); // padding
appendU32BE(msg, static_cast<quint32>(latin1.size()));
msg.append(latin1);
m_socket->write(msg);
}
void VncSessionBackend::sendAppleRsaHostKeyRequest()
{
// Fixed request packet asking the server for its RSA host key, per
// security type 33's sub-protocol (confirmed against the `asyncvnc`
// reference -- see vnc_apple_rsa_auth.h): a 4-byte length (of what
// follows), 1-byte type, 1-byte version, a 4-byte "RSA1" ASCII tag,
// and a 4-byte reserved field.
QByteArray msg;
appendU32BE(msg, 10);
msg.append(char(1)); // type: request
msg.append(char(0)); // version
msg += QByteArray("RSA1");
appendU32BE(msg, 0); // reserved
m_socket->write(msg);
}
QString VncSessionBackend::effectiveUsername() const
{
const QString profileUsername = profile().username.trimmed();
return profileUsername.isEmpty() ? m_promptedUsername : profileUsername;
}
bool VncSessionBackend::ensureUsernameAvailable()
{
if (!effectiveUsername().isEmpty()) {
return true;
}
if (!m_waitingForUsername) {
m_waitingForUsername = true;
emit usernameRequested(
QStringLiteral("A username is required for %1's authentication method:")
.arg(profile().host));
}
return false;
}
void VncSessionBackend::provideUsername(const QString& username)
{
if (!m_waitingForUsername) {
return;
}
m_waitingForUsername = false;
if (m_state != SessionState::Connecting) {
// The connection already failed or was torn down while the
// prompt was pending -- nothing left to resume.
return;
}
m_promptedUsername = username.trimmed();
if (m_promptedUsername.isEmpty()) {
failConnection(
QStringLiteral("A username is required for this VNC server's authentication method."),
QStringLiteral("Username prompt was cancelled or left empty"));
return;
}
processReceiveBuffer();
}
void VncSessionBackend::finishHandshakeIntoRunningState()
{
emit remoteDesktopSizeChanged(m_framebuffer.width(), m_framebuffer.height());
sendSetPixelFormatAndEncodings();
requestFramebufferUpdate(false);
m_rfbState = RfbState::WaitingServerMessageType;
setState(SessionState::Connected, QStringLiteral("VNC session established."));
}
void VncSessionBackend::onRectangleFinished()
{
--m_pendingRectanglesRemaining;
if (m_pendingRectanglesRemaining > 0) {
m_rfbState = RfbState::WaitingRectangleHeader;
return;
}
emit frameUpdated(m_framebuffer);
m_rfbState = RfbState::WaitingServerMessageType;
requestFramebufferUpdate(true);
}
QRect VncSessionBackend::currentHextileTileRect() const
{
const int tileWidth = qMin(16, m_currentRectangle.width - m_hextileTileX);
const int tileHeight = qMin(16, m_currentRectangle.height - m_hextileTileY);
return QRect(m_currentRectangle.x + m_hextileTileX, m_currentRectangle.y + m_hextileTileY,
tileWidth, tileHeight);
}
void VncSessionBackend::advanceHextileTile()
{
m_hextileTileX += 16;
if (m_hextileTileX >= m_currentRectangle.width) {
m_hextileTileX = 0;
m_hextileTileY += 16;
}
if (m_hextileTileY >= m_currentRectangle.height) {
onRectangleFinished();
} else {
m_rfbState = RfbState::WaitingHextileTileSubencoding;
}
}
bool VncSessionBackend::inflateTightStream(int streamIndex, const QByteArray& compressed,
QByteArray* decompressed)
{
z_stream_s* stream = m_tightInflateStreams[streamIndex];
if (!m_tightInflateInitialized[streamIndex]) {
std::memset(stream, 0, sizeof(z_stream_s));
if (inflateInit(stream) != Z_OK) {
return false;
}
m_tightInflateInitialized[streamIndex] = true;
}
constexpr int kChunkSize = 65536;
char outBuffer[kChunkSize];
stream->next_in = reinterpret_cast<Bytef*>(const_cast<char*>(compressed.constData()));
stream->avail_in = static_cast<uInt>(compressed.size());
int result = Z_OK;
while (stream->avail_in > 0) {
stream->next_out = reinterpret_cast<Bytef*>(outBuffer);
stream->avail_out = kChunkSize;
result = inflate(stream, Z_NO_FLUSH);
const int produced = kChunkSize - static_cast<int>(stream->avail_out);
if (produced > 0) {
decompressed->append(outBuffer, produced);
}
if (result != Z_OK) {
break;
}
}
return result == Z_OK || result == Z_STREAM_END;
}
void VncSessionBackend::processReceiveBuffer()
{
for (;;) {
switch (m_rfbState) {
case RfbState::Idle:
return;
case RfbState::WaitingProtocolVersion: {
if (!haveBytes(12)) {
return;
}
const QByteArray versionLine = m_recvBuffer.left(12);
m_recvBuffer.remove(0, 12);
bool ok = versionLine.size() == 12 && versionLine.startsWith("RFB ")
&& versionLine.at(11) == '\n';
int major = 0;
int minor = 0;
if (ok) {
bool majorOk = false;
bool minorOk = false;
major = versionLine.mid(4, 3).toInt(&majorOk);
minor = versionLine.mid(8, 3).toInt(&minorOk);
ok = majorOk && minorOk;
}
if (!ok || major < 3) {
failConnection(QStringLiteral("Server did not send a valid RFB protocol version."),
QString::fromLatin1(versionLine));
return;
}
if (major > 3 || minor >= 8) {
m_negotiatedMinorVersion = 8;
} else if (minor == 7) {
m_negotiatedMinorVersion = 7;
} else {
m_negotiatedMinorVersion = 3;
}
sendVersionReply();
m_rfbState = (m_negotiatedMinorVersion >= 7) ? RfbState::WaitingSecurityTypeCount
: RfbState::WaitingSecurityTypeV33;
break;
}
case RfbState::WaitingSecurityTypeCount: {
if (!haveBytes(1)) {
return;
}
m_securityTypeCount = static_cast<quint8>(m_recvBuffer.at(0));
m_recvBuffer.remove(0, 1);
m_rfbState = (m_securityTypeCount == 0) ? RfbState::WaitingSecurityFailureReasonLength
: RfbState::WaitingSecurityTypeList;
break;
}
case RfbState::WaitingSecurityTypeList: {
if (!haveBytes(m_securityTypeCount)) {
return;
}
m_offeredSecurityTypes = m_recvBuffer.left(m_securityTypeCount);
m_recvBuffer.remove(0, m_securityTypeCount);
// Preference order when multiple are offered: None needs no
// credentials at all so it's preferred outright; between
// Apple's two schemes, type 30 (DH+AES)'s wire format is
// confirmed against an independent, authoritative source
// (neatvnc's rfb-proto.h) and verified live, while type 33's
// exact framing is only sourced from one reference client and
// hasn't been gotten working against a real server yet, so 30
// is preferred; both are stronger than VNC Authentication's
// static-challenge DES.
quint8 chosen = 0;
if (m_offeredSecurityTypes.contains(static_cast<char>(kSecurityTypeNone))) {
chosen = kSecurityTypeNone;
} else if (m_offeredSecurityTypes.contains(static_cast<char>(kSecurityTypeAppleDh))) {
chosen = kSecurityTypeAppleDh;
} else if (m_offeredSecurityTypes.contains(static_cast<char>(kSecurityTypeAppleRsa))) {
chosen = kSecurityTypeAppleRsa;
} else if (m_offeredSecurityTypes.contains(static_cast<char>(kSecurityTypeVncAuth))) {
chosen = kSecurityTypeVncAuth;
}
if (chosen == 0) {
QStringList offered;
for (char type : m_offeredSecurityTypes) {
offered.push_back(QString::number(static_cast<quint8>(type)));
}
failConnection(
QStringLiteral(
"The VNC server requires an authentication method OrbitHub doesn't "
"support yet."),
QStringLiteral("Offered security types: %1").arg(offered.join(QStringLiteral(", "))));
return;
}
m_chosenSecurityType = chosen;
m_socket->write(QByteArray(1, static_cast<char>(chosen)));
if (chosen == kSecurityTypeVncAuth) {
m_rfbState = RfbState::WaitingVncAuthChallenge;
} else if (chosen == kSecurityTypeAppleDh) {
m_rfbState = RfbState::WaitingAppleAuthParams;
} else if (chosen == kSecurityTypeAppleRsa) {
sendAppleRsaHostKeyRequest();
m_rfbState = RfbState::WaitingAppleRsaHostKeyHeader;
} else if (m_negotiatedMinorVersion >= 8) {
m_rfbState = RfbState::WaitingSecurityResult;
} else {
sendClientInit();
m_rfbState = RfbState::WaitingServerInitHeader;
}
break;
}
case RfbState::WaitingSecurityTypeV33: {
if (!haveBytes(4)) {
return;
}
const quint32 type = readU32BE(m_recvBuffer, 0);
m_recvBuffer.remove(0, 4);
if (type == 0) {
m_rfbState = RfbState::WaitingSecurityFailureReasonLength;
} else if (type == kSecurityTypeNone) {
m_chosenSecurityType = kSecurityTypeNone;
sendClientInit();
m_rfbState = RfbState::WaitingServerInitHeader;
} else if (type == kSecurityTypeVncAuth) {
m_chosenSecurityType = kSecurityTypeVncAuth;
m_rfbState = RfbState::WaitingVncAuthChallenge;
} else if (type == kSecurityTypeAppleDh) {
m_chosenSecurityType = kSecurityTypeAppleDh;
m_rfbState = RfbState::WaitingAppleAuthParams;
} else if (type == kSecurityTypeAppleRsa) {
m_chosenSecurityType = kSecurityTypeAppleRsa;
sendAppleRsaHostKeyRequest();
m_rfbState = RfbState::WaitingAppleRsaHostKeyHeader;
} else {
failConnection(
QStringLiteral(
"The VNC server requires an authentication method OrbitHub doesn't "
"support yet."),
QStringLiteral("Security type %1").arg(type));
return;
}
break;
}
case RfbState::WaitingSecurityFailureReasonLength: {
if (!haveBytes(4)) {
return;
}
m_pendingLength = readU32BE(m_recvBuffer, 0);
m_recvBuffer.remove(0, 4);
m_rfbState = RfbState::WaitingSecurityFailureReason;
break;
}
case RfbState::WaitingSecurityFailureReason: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const QString reason = QString::fromUtf8(m_recvBuffer.left(static_cast<int>(m_pendingLength)));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
failConnection(reason.isEmpty()
? QStringLiteral("The VNC server refused the connection.")
: QStringLiteral("The VNC server refused the connection: %1").arg(reason),
reason);
return;
}
case RfbState::WaitingVncAuthChallenge: {
if (!haveBytes(16)) {
return;
}
const QByteArray challenge = m_recvBuffer.left(16);
m_recvBuffer.remove(0, 16);
if (m_activeOptions.password.isEmpty()) {
failConnection(QStringLiteral("Password is required for this VNC server."),
QStringLiteral(
"VNC Authentication requested but no password was provided."));
return;
}
m_socket->write(vncAuthResponse(challenge, m_activeOptions.password));
if (m_negotiatedMinorVersion >= 8) {
m_rfbState = RfbState::WaitingSecurityResult;
} else {
sendClientInit();
m_rfbState = RfbState::WaitingServerInitHeader;
}
break;
}
case RfbState::WaitingAppleAuthParams: {
// Confirmed against a real macOS Screen Sharing server: a
// literal 2-byte generator (not length-prefixed), then a
// 2-byte key length applying to both the prime and the
// server's public key that follow.
if (!haveBytes(4)) {
return;
}
m_appleAuthGenerator = m_recvBuffer.left(2);
m_appleAuthKeyLength = readU16BE(m_recvBuffer, 2);
m_recvBuffer.remove(0, 4);
m_pendingLength = m_appleAuthKeyLength * 2;
m_rfbState = RfbState::WaitingAppleAuthPrimeAndServerKey;
break;
}
case RfbState::WaitingAppleAuthPrimeAndServerKey: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
// Checked before consuming any bytes: if a username is needed
// and not yet available, pause here (leaving m_recvBuffer
// untouched) until provideUsername() resumes us and this same
// case re-parses identically.
if (!ensureUsernameAvailable()) {
return;
}
const int keyLength = static_cast<int>(m_appleAuthKeyLength);
const QByteArray prime = m_recvBuffer.left(keyLength);
const QByteArray serverPublicKey = m_recvBuffer.mid(keyLength, keyLength);
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
const VncAppleDhAuth::Response response = VncAppleDhAuth::computeResponse(
m_appleAuthGenerator, prime, serverPublicKey, effectiveUsername(),
m_activeOptions.password);
if (response.clientPublicKey.isEmpty()) {
failConnection(
QStringLiteral(
"Failed to compute the Apple Screen Sharing authentication response."),
QStringLiteral("VncAppleDhAuth::computeResponse() failed"));
return;
}
// Confirmed via an independent, authoritative source (neatvnc's
// rfb-proto.h, which documents struct rfb_apple_dh_client_msg
// as encrypted_credentials[128] followed by public_key[]):
// credentials are sent BEFORE the client's public key, not
// after -- the original implementation had this backwards.
m_socket->write(response.encryptedCredentials);
m_socket->write(response.clientPublicKey);
if (m_negotiatedMinorVersion >= 8) {
m_rfbState = RfbState::WaitingSecurityResult;
} else {
sendClientInit();
m_rfbState = RfbState::WaitingServerInitHeader;
}
break;
}
case RfbState::WaitingAppleRsaHostKeyHeader: {
// Response header per security type 33's sub-protocol: a
// 4-byte packet length (unused -- framing is derived from the
// host-key length below instead), a 2-byte version (unused),
// then a 4-byte host-key length.
if (!haveBytes(10)) {
return;
}
m_pendingLength = readU32BE(m_recvBuffer, 6);
m_recvBuffer.remove(0, 10);
m_rfbState = RfbState::WaitingAppleRsaHostKeyBytes;
break;
}
case RfbState::WaitingAppleRsaHostKeyBytes: {
// The DER-encoded (X.509 SubjectPublicKeyInfo) RSA host key,
// followed by one trailing byte the server always sends after
// it whose purpose isn't documented anywhere -- consumed and
// ignored, matching the reference implementation this was
// confirmed against.
const int totalBytes = static_cast<int>(m_pendingLength) + 1;
if (!haveBytes(totalBytes)) {
return;
}
// See the matching comment in WaitingAppleAuthPrimeAndServerKey.
if (!ensureUsernameAvailable()) {
return;
}
const QByteArray hostKeyDer = m_recvBuffer.left(static_cast<int>(m_pendingLength));
m_recvBuffer.remove(0, totalBytes);
const VncAppleRsaAuth::Response response = VncAppleRsaAuth::computeResponse(
hostKeyDer, effectiveUsername(), m_activeOptions.password);
if (response.encryptedCredentials.isEmpty() || response.encryptedAesKey.isEmpty()) {
failConnection(
QStringLiteral(
"Failed to compute the Apple Screen Sharing authentication response."),
QStringLiteral("VncAppleRsaAuth::computeResponse() failed"));
return;
}
QByteArray responseMsg;
appendU32BE(responseMsg,
static_cast<quint32>(6 + 2 + response.encryptedCredentials.size() + 2
+ response.encryptedAesKey.size()));
responseMsg.append(char(1)); // type: response
responseMsg.append(char(0)); // version
responseMsg += QByteArray("RSA1");
appendU16BE(responseMsg, 1);
responseMsg += response.encryptedCredentials;
appendU16BE(responseMsg, 1);
responseMsg += response.encryptedAesKey;
m_socket->write(responseMsg);
if (m_negotiatedMinorVersion >= 8) {
m_rfbState = RfbState::WaitingSecurityResult;
} else {
sendClientInit();
m_rfbState = RfbState::WaitingServerInitHeader;
}
break;
}
case RfbState::WaitingSecurityResult: {
if (!haveBytes(4)) {
return;
}
const quint32 result = readU32BE(m_recvBuffer, 0);
m_recvBuffer.remove(0, 4);
if (result == 0) {
sendClientInit();
m_rfbState = RfbState::WaitingServerInitHeader;
} else {
m_rfbState = RfbState::WaitingSecurityResultReasonLength;
}
break;
}
case RfbState::WaitingSecurityResultReasonLength: {
if (!haveBytes(4)) {
return;
}
m_pendingLength = readU32BE(m_recvBuffer, 0);
m_recvBuffer.remove(0, 4);
m_rfbState = RfbState::WaitingSecurityResultReason;
break;
}
case RfbState::WaitingSecurityResultReason: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const QString reason = QString::fromUtf8(m_recvBuffer.left(static_cast<int>(m_pendingLength)));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
failConnection(QStringLiteral("Authentication failed: %1")
.arg(reason.isEmpty() ? QStringLiteral("incorrect password") : reason),
reason);
return;
}
case RfbState::WaitingServerInitHeader: {
if (!haveBytes(24)) {
return;
}
const quint16 width = readU16BE(m_recvBuffer, 0);
const quint16 height = readU16BE(m_recvBuffer, 2);
const quint32 nameLength = readU32BE(m_recvBuffer, 20);
m_recvBuffer.remove(0, 24);
m_framebuffer =
QImage(qMax(1, static_cast<int>(width)), qMax(1, static_cast<int>(height)),
QImage::Format_RGB32);
m_framebuffer.fill(Qt::black);
m_pendingLength = nameLength;
m_rfbState = RfbState::WaitingServerName;
break;
}
case RfbState::WaitingServerName: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
// The desktop name isn't surfaced anywhere yet; just consume it.
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
finishHandshakeIntoRunningState();
break;
}
case RfbState::WaitingServerMessageType: {
if (!haveBytes(1)) {
return;
}
const quint8 messageType = static_cast<quint8>(m_recvBuffer.at(0));
m_recvBuffer.remove(0, 1);
switch (messageType) {
case kServerMsgFramebufferUpdate:
m_rfbState = RfbState::WaitingFramebufferUpdateHeader;
break;
case kServerMsgSetColourMapEntries:
m_rfbState = RfbState::WaitingSetColourMapHeader;
break;
case kServerMsgBell:
// Bell -- nothing to render; stay in the same state.
emit eventLogged(QStringLiteral("Remote bell."));
break;
case kServerMsgServerCutText:
m_rfbState = RfbState::WaitingServerCutTextHeader;
break;
default:
failConnection(
QStringLiteral("The VNC server sent a message OrbitHub doesn't understand."),
QStringLiteral("Unrecognized server message type %1").arg(messageType));
return;
}
break;
}
case RfbState::WaitingFramebufferUpdateHeader: {
if (!haveBytes(3)) {
return;
}
const quint16 count = readU16BE(m_recvBuffer, 1);
m_recvBuffer.remove(0, 3);
m_pendingRectanglesRemaining = count;
if (count == 0) {
emit frameUpdated(m_framebuffer);
m_rfbState = RfbState::WaitingServerMessageType;
requestFramebufferUpdate(true);
} else {
m_rfbState = RfbState::WaitingRectangleHeader;
}
break;
}
case RfbState::WaitingRectangleHeader: {
if (!haveBytes(12)) {
return;
}
m_currentRectangle.x = readU16BE(m_recvBuffer, 0);
m_currentRectangle.y = readU16BE(m_recvBuffer, 2);
m_currentRectangle.width = readU16BE(m_recvBuffer, 4);
m_currentRectangle.height = readU16BE(m_recvBuffer, 6);
m_currentRectangle.encoding = static_cast<qint32>(readU32BE(m_recvBuffer, 8));
m_recvBuffer.remove(0, 12);
switch (m_currentRectangle.encoding) {
case kEncRaw:
m_rfbState = RfbState::WaitingRawPixelData;
break;
case kEncCopyRect:
m_rfbState = RfbState::WaitingCopyRectSource;
break;
case kEncCursor:
m_rfbState = RfbState::WaitingCursorPixelData;
break;
case kEncHextile:
m_hextileTileX = 0;
m_hextileTileY = 0;
m_hextileBackground = qRgb(0, 0, 0);
m_hextileForeground = qRgb(0, 0, 0);
if (m_currentRectangle.width <= 0 || m_currentRectangle.height <= 0) {
// No tiles to decode at all.
onRectangleFinished();
} else {
m_rfbState = RfbState::WaitingHextileTileSubencoding;
}
break;
case kEncZRLE:
m_rfbState = RfbState::WaitingZrleCompressedLength;
break;
case kEncTight:
m_rfbState = RfbState::WaitingTightCompressionControl;
break;
default: {
// SetEncodings (see kAnnouncedEncodings) is entirely
// client-controlled, so a spec-compliant server will never
// send an encoding we didn't announce. Reaching here means
// either a non-compliant server, or -- more likely in
// practice -- an OrbitHub bug where an encoding was added to
// kAnnouncedEncodings without a matching case above. There's
// no safe way to skip an unrecognized rectangle's payload
// (its length depends on decoding it), so this must stay
// fatal rather than attempt to guess and resync.
QStringList announced;
for (qint32 encoding : kAnnouncedEncodings) {
announced.push_back(QString::number(encoding));
}
failConnection(
QStringLiteral("The VNC server sent an encoding it was never offered -- "
"this indicates a non-compliant server."),
QStringLiteral("Encoding type %1 (announced: %2)")
.arg(m_currentRectangle.encoding)
.arg(announced.join(QStringLiteral(", "))));
return;
}
}
break;
}
case RfbState::WaitingRawPixelData: {
const qint64 byteCount =
static_cast<qint64>(m_currentRectangle.width) * m_currentRectangle.height * 4;
if (byteCount < 0 || byteCount > std::numeric_limits<int>::max()) {
failConnection(QStringLiteral("The VNC server sent an implausibly large update."),
QStringLiteral("Raw rectangle %1x%2")
.arg(m_currentRectangle.width)
.arg(m_currentRectangle.height));
return;
}
if (!haveBytes(static_cast<int>(byteCount))) {
return;
}
if (byteCount > 0) {
const QImage rectImage(reinterpret_cast<const uchar*>(m_recvBuffer.constData()),
m_currentRectangle.width, m_currentRectangle.height,
m_currentRectangle.width * 4, QImage::Format_RGB32);
QPainter painter(&m_framebuffer);
painter.drawImage(m_currentRectangle.x, m_currentRectangle.y, rectImage);
}
m_recvBuffer.remove(0, static_cast<int>(byteCount));
onRectangleFinished();
break;
}
case RfbState::WaitingCopyRectSource: {
if (!haveBytes(4)) {
return;
}
const int srcX = readU16BE(m_recvBuffer, 0);
const int srcY = readU16BE(m_recvBuffer, 2);
m_recvBuffer.remove(0, 4);
const QImage srcCopy =
m_framebuffer.copy(srcX, srcY, m_currentRectangle.width, m_currentRectangle.height);
QPainter painter(&m_framebuffer);
painter.drawImage(m_currentRectangle.x, m_currentRectangle.y, srcCopy);
onRectangleFinished();
break;
}
case RfbState::WaitingCursorPixelData: {
// Cursor pseudo-encoding (RFC 6143 SS7.8.2): x/y in the already-
// parsed rectangle header are the hotspot, not screen position;
// width/height are the cursor image's own dimensions. Never
// painted into m_framebuffer. Payload is width*height pixels in
// our negotiated 32bpp format, followed by a row-padded,
// MSB-first-per-byte opacity bitmask.
const int width = m_currentRectangle.width;
const int height = m_currentRectangle.height;
const qint64 maskRowBytes = (static_cast<qint64>(width) + 7) / 8;
const qint64 pixelBytes = static_cast<qint64>(width) * height * 4;
const qint64 maskBytes = maskRowBytes * height;
const qint64 totalBytes = pixelBytes + maskBytes;
if (totalBytes < 0 || totalBytes > std::numeric_limits<int>::max()) {
failConnection(QStringLiteral("The VNC server sent an implausibly large cursor image."),
QStringLiteral("Cursor rectangle %1x%2").arg(width).arg(height));
return;
}
if (!haveBytes(static_cast<int>(totalBytes))) {
return;
}
// A 0x0 cursor rectangle is the spec's way of saying "hide the
// cursor"; treat any other degenerate (zero-area) size the same
// way rather than trying to build an empty QImage.
if (width <= 0 || height <= 0) {
emit cursorHidden();
} else {
QImage cursorImage(width, height, QImage::Format_ARGB32);
const auto* pixelData = reinterpret_cast<const uchar*>(m_recvBuffer.constData());
const uchar* maskData = pixelData + pixelBytes;
for (int y = 0; y < height; ++y) {
for (int x = 0; x < width; ++x) {
const uchar* px = pixelData + ((static_cast<qint64>(y) * width + x) * 4);
// Matches our negotiated SetPixelFormat: little-
// endian 32bpp, R at shift 16 / G at 8 / B at 0 --
// byte order B,G,R,pad.
const uchar b = px[0];
const uchar g = px[1];
const uchar r = px[2];
const uchar maskByte = maskData[y * maskRowBytes + (x / 8)];
const bool opaque = (maskByte & (0x80 >> (x % 8))) != 0;
cursorImage.setPixel(x, y, qRgba(r, g, b, opaque ? 255 : 0));
}
}
emit cursorImageChanged(cursorImage,
QPoint(m_currentRectangle.x, m_currentRectangle.y));
}
m_recvBuffer.remove(0, static_cast<int>(totalBytes));
onRectangleFinished();
break;
}
case RfbState::WaitingHextileTileSubencoding: {
if (!haveBytes(1)) {
return;
}
m_hextileSubencoding = static_cast<quint8>(m_recvBuffer.at(0));
m_recvBuffer.remove(0, 1);
if ((m_hextileSubencoding & VncPixelCodecs::HextileFlags::kRaw) != 0) {
m_rfbState = RfbState::WaitingHextileRawTileData;
} else {
m_pendingLength = static_cast<quint32>(
VncPixelCodecs::hextileFixedMetaByteCount(m_hextileSubencoding));
m_rfbState = RfbState::WaitingHextileTileMeta;
}
break;
}
case RfbState::WaitingHextileTileMeta: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const QByteArray data = m_recvBuffer.left(static_cast<int>(m_pendingLength));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
const int subrectCount = VncPixelCodecs::decodeHextileFixedMeta(
m_hextileSubencoding, data, &m_hextileBackground, &m_hextileForeground);
// Every tile is filled with the (possibly just-updated,
// possibly inherited) background colour first, regardless of
// whether this tile re-specified it.
QPainter painter(&m_framebuffer);
painter.fillRect(currentHextileTileRect(), QColor::fromRgb(m_hextileBackground));
m_hextileSubrectsColoured =
(m_hextileSubencoding & VncPixelCodecs::HextileFlags::kSubrectsColoured) != 0;
m_hextileSubrectsRemaining = subrectCount;
if (subrectCount == 0) {
advanceHextileTile();
} else {
m_pendingLength = static_cast<quint32>(VncPixelCodecs::hextileSubrectByteCount(
m_hextileSubrectsColoured, subrectCount));
m_rfbState = RfbState::WaitingHextileSubrectData;
}
break;
}
case RfbState::WaitingHextileSubrectData: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const QByteArray data = m_recvBuffer.left(static_cast<int>(m_pendingLength));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
const QVector<VncPixelCodecs::HextileSubrect> subrects =
VncPixelCodecs::decodeHextileSubrects(m_hextileSubrectsColoured,
m_hextileSubrectsRemaining, data,
m_hextileForeground);
const QRect tileRect = currentHextileTileRect();
QPainter painter(&m_framebuffer);
for (const VncPixelCodecs::HextileSubrect& subrect : subrects) {
painter.fillRect(subrect.rect.translated(tileRect.topLeft()),
QColor::fromRgb(subrect.color));
}
advanceHextileTile();
break;
}
case RfbState::WaitingHextileRawTileData: {
const QRect tileRect = currentHextileTileRect();
const qint64 byteCount =
static_cast<qint64>(tileRect.width()) * tileRect.height() * 4;
if (!haveBytes(static_cast<int>(byteCount))) {
return;
}
if (byteCount > 0) {
const QImage tileImage(reinterpret_cast<const uchar*>(m_recvBuffer.constData()),
tileRect.width(), tileRect.height(),
tileRect.width() * 4, QImage::Format_RGB32);
QPainter painter(&m_framebuffer);
painter.drawImage(tileRect.topLeft(), tileImage);
}
m_recvBuffer.remove(0, static_cast<int>(byteCount));
advanceHextileTile();
break;
}
case RfbState::WaitingZrleCompressedLength: {
if (!haveBytes(4)) {
return;
}
const quint32 length = readU32BE(m_recvBuffer, 0);
m_recvBuffer.remove(0, 4);
if (length > kMaxZrleCompressedLength) {
failConnection(QStringLiteral("The VNC server sent an implausibly large ZRLE update."),
QStringLiteral("ZRLE compressed length %1").arg(length));
return;
}
m_pendingLength = length;
m_rfbState = RfbState::WaitingZrleCompressedData;
break;
}
case RfbState::WaitingZrleCompressedData: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const QByteArray compressed = m_recvBuffer.left(static_cast<int>(m_pendingLength));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
if (!m_zrleInflateInitialized) {
std::memset(m_zrleInflateStream, 0, sizeof(z_stream_s));
if (inflateInit(m_zrleInflateStream) != Z_OK) {
failConnection(QStringLiteral("Failed to initialize ZRLE decompression."),
QStringLiteral("inflateInit() failed"));
return;
}
m_zrleInflateInitialized = true;
}
// The zlib stream is persistent across the whole connection
// (RFC 6143 SS7.7.6) -- Z_NO_FLUSH, never Z_FINISH, and the
// stream is only ever torn down in resetProtocolState() on a
// fresh connect/reconnect.
QByteArray decompressed;
constexpr int kChunkSize = 65536;
char outBuffer[kChunkSize];
m_zrleInflateStream->next_in =
reinterpret_cast<Bytef*>(const_cast<char*>(compressed.constData()));
m_zrleInflateStream->avail_in = static_cast<uInt>(compressed.size());
int inflateResult = Z_OK;
while (m_zrleInflateStream->avail_in > 0) {
m_zrleInflateStream->next_out = reinterpret_cast<Bytef*>(outBuffer);
m_zrleInflateStream->avail_out = kChunkSize;
inflateResult = inflate(m_zrleInflateStream, Z_NO_FLUSH);
const int produced = kChunkSize - static_cast<int>(m_zrleInflateStream->avail_out);
if (produced > 0) {
decompressed.append(outBuffer, produced);
}
if (inflateResult != Z_OK) {
break;
}
}
if (inflateResult != Z_OK && inflateResult != Z_STREAM_END) {
failConnection(QStringLiteral("The VNC server sent malformed ZRLE-compressed data."),
QStringLiteral("inflate() returned %1").arg(inflateResult));
return;
}
const int rectWidth = m_currentRectangle.width;
const int rectHeight = m_currentRectangle.height;
int decodeOffset = 0;
bool malformedTile = false;
QString malformedTileDetail;
{
// Scoped so the QPainter is destroyed (ending the paint
// session) before onRectangleFinished() below, which may
// emit frameUpdated(m_framebuffer) -- matching how
// WaitingRawPixelData/WaitingCopyRectSource already end
// their own painters before finishing a rectangle.
QPainter painter(&m_framebuffer);
for (int tileY = 0; tileY < rectHeight && !malformedTile; tileY += kZrleTileSize) {
const int tileHeight = qMin(kZrleTileSize, rectHeight - tileY);
for (int tileX = 0; tileX < rectWidth; tileX += kZrleTileSize) {
const int tileWidth = qMin(kZrleTileSize, rectWidth - tileX);
QVector<QRgb> pixels;
const int consumed = VncPixelCodecs::decodeZrleTile(
decompressed, decodeOffset, tileWidth, tileHeight, pixels);
if (consumed < 0 || pixels.size() != tileWidth * tileHeight) {
malformedTile = true;
malformedTileDetail = QStringLiteral("Tile at (%1,%2) size %3x%4")
.arg(tileX)
.arg(tileY)
.arg(tileWidth)
.arg(tileHeight);
break;
}
decodeOffset += consumed;
QImage tileImage(tileWidth, tileHeight, QImage::Format_RGB32);
for (int y = 0; y < tileHeight; ++y) {
for (int x = 0; x < tileWidth; ++x) {
tileImage.setPixel(x, y, pixels.at((y * tileWidth) + x));
}
}
painter.drawImage(m_currentRectangle.x + tileX,
m_currentRectangle.y + tileY, tileImage);
}
}
}
if (malformedTile) {
failConnection(QStringLiteral("The VNC server sent a malformed ZRLE tile."),
malformedTileDetail);
return;
}
onRectangleFinished();
break;
}
case RfbState::WaitingTightCompressionControl: {
if (!haveBytes(1)) {
return;
}
const quint8 controlByte = static_cast<quint8>(m_recvBuffer.at(0));
m_recvBuffer.remove(0, 1);
// Low 4 bits: reset flags for the 4 persistent Basic-mode zlib
// streams, independent of whatever mode this particular
// rectangle itself uses.
for (int i = 0; i < 4; ++i) {
if ((controlByte & (1 << i)) != 0 && m_tightInflateInitialized[i]) {
inflateEnd(m_tightInflateStreams[i]);
m_tightInflateInitialized[i] = false;
}
}
const quint8 mode = controlByte >> 4;
m_tightCompressionMode = mode;
if (mode == kTightModeFill) {
m_rfbState = RfbState::WaitingTightFillColor;
} else if (mode == kTightModeJpeg) {
m_tightLengthByteIndex = 0;
m_pendingLength = 0;
m_rfbState = RfbState::WaitingTightLengthByte;
} else if (mode <= kTightBasicModeMax) {
if ((mode & kTightBasicExplicitFilterFlag) != 0) {
m_rfbState = RfbState::WaitingTightFilterId;
} else {
m_tightFilterId = 0; // Copy implied
m_tightLengthByteIndex = 0;
m_pendingLength = 0;
m_rfbState = RfbState::WaitingTightLengthByte;
}
} else {
failConnection(
QStringLiteral("The VNC server sent an invalid Tight compression-control byte."),
QStringLiteral("Control byte 0x%1").arg(controlByte, 2, 16, QChar('0')));
return;
}
break;
}
case RfbState::WaitingTightFillColor: {
if (!haveBytes(3)) {
return;
}
const QRgb color = VncPixelCodecs::rgbFromPixelBytes(
reinterpret_cast<const uchar*>(m_recvBuffer.constData()));
m_recvBuffer.remove(0, 3);
if (m_currentRectangle.width > 0 && m_currentRectangle.height > 0) {
QPainter painter(&m_framebuffer);
painter.fillRect(QRect(m_currentRectangle.x, m_currentRectangle.y,
m_currentRectangle.width, m_currentRectangle.height),
QColor::fromRgb(color));
}
onRectangleFinished();
break;
}
case RfbState::WaitingTightFilterId: {
if (!haveBytes(1)) {
return;
}
m_tightFilterId = static_cast<quint8>(m_recvBuffer.at(0));
m_recvBuffer.remove(0, 1);
if (m_tightFilterId > 2) {
failConnection(QStringLiteral("The VNC server sent an unrecognized Tight filter id."),
QStringLiteral("Filter id %1").arg(m_tightFilterId));
return;
}
m_tightLengthByteIndex = 0;
m_pendingLength = 0;
m_rfbState = RfbState::WaitingTightLengthByte;
break;
}
case RfbState::WaitingTightLengthByte: {
if (!haveBytes(1)) {
return;
}
const quint8 b = static_cast<quint8>(m_recvBuffer.at(0));
m_recvBuffer.remove(0, 1);
if (m_tightLengthByteIndex == 0) {
m_pendingLength = (b & 0x7F);
} else if (m_tightLengthByteIndex == 1) {
m_pendingLength |= static_cast<quint32>(b & 0x7F) << 7;
} else {
m_pendingLength |= static_cast<quint32>(b) << 14;
}
// Compact-length encoding: up to 3 bytes, continuation-bit
// style (only the first two bytes' high bit can request
// another byte; the third byte's full 8 bits are always used
// as the final contribution, capping the value at ~4MB).
const bool continuation = (m_tightLengthByteIndex < 2) && ((b & 0x80) != 0);
++m_tightLengthByteIndex;
if (continuation) {
break;
}
m_rfbState = RfbState::WaitingTightPayload;
break;
}
case RfbState::WaitingTightPayload: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
const QByteArray payload = m_recvBuffer.left(static_cast<int>(m_pendingLength));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
const int rectWidth = m_currentRectangle.width;
const int rectHeight = m_currentRectangle.height;
if (rectWidth <= 0 || rectHeight <= 0) {
onRectangleFinished();
break;
}
if (m_tightCompressionMode == kTightModeJpeg) {
QImage jpegImage;
if (!decodeJpegRectangle(payload, rectWidth, rectHeight, &jpegImage)) {
failConnection(QStringLiteral("The VNC server sent malformed Tight JPEG data."),
QStringLiteral("JPEG payload length %1").arg(payload.size()));
return;
}
{
QPainter painter(&m_framebuffer);
painter.drawImage(m_currentRectangle.x, m_currentRectangle.y, jpegImage);
}
onRectangleFinished();
break;
}
// Basic compression: always zlib-compressed by this decoder
// (the real protocol allows very small payloads to skip
// compression entirely; not handled here -- see class comment).
const int streamIndex = m_tightCompressionMode & kTightBasicStreamIndexMask;
QByteArray filtered;
if (!inflateTightStream(streamIndex, payload, &filtered)) {
failConnection(
QStringLiteral("The VNC server sent malformed Tight-compressed data."),
QStringLiteral("Stream %1, payload length %2").arg(streamIndex).arg(payload.size()));
return;
}
QVector<QRgb> pixels;
switch (m_tightFilterId) {
case 0:
pixels = VncPixelCodecs::decodeTightCopyFilter(filtered, rectWidth, rectHeight);
break;
case 1:
pixels = VncPixelCodecs::decodeTightPaletteFilter(filtered, rectWidth, rectHeight);
break;
case 2:
pixels = VncPixelCodecs::decodeTightGradientFilter(filtered, rectWidth, rectHeight);
break;
default:
break;
}
if (pixels.size() != rectWidth * rectHeight) {
failConnection(
QStringLiteral("The VNC server sent a malformed Tight rectangle."),
QStringLiteral("Filter %1 produced %2 pixels, expected %3")
.arg(m_tightFilterId)
.arg(pixels.size())
.arg(rectWidth * rectHeight));
return;
}
{
QImage rectImage(rectWidth, rectHeight, QImage::Format_RGB32);
for (int y = 0; y < rectHeight; ++y) {
for (int x = 0; x < rectWidth; ++x) {
rectImage.setPixel(x, y, pixels.at((y * rectWidth) + x));
}
}
QPainter painter(&m_framebuffer);
painter.drawImage(m_currentRectangle.x, m_currentRectangle.y, rectImage);
}
onRectangleFinished();
break;
}
case RfbState::WaitingSetColourMapHeader: {
if (!haveBytes(5)) {
return;
}
const quint16 numColors = readU16BE(m_recvBuffer, 3);
m_recvBuffer.remove(0, 5);
m_pendingLength = static_cast<quint32>(numColors) * 6;
m_rfbState = (m_pendingLength == 0) ? RfbState::WaitingServerMessageType
: RfbState::WaitingSetColourMapData;
break;
}
case RfbState::WaitingSetColourMapData: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
// We always request a true-color pixel format, so a well
// behaved server should never actually send this; consume it
// to stay in sync in case one does anyway.
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
m_rfbState = RfbState::WaitingServerMessageType;
break;
}
case RfbState::WaitingServerCutTextHeader: {
if (!haveBytes(7)) {
return;
}
const quint32 length = readU32BE(m_recvBuffer, 3);
m_recvBuffer.remove(0, 7);
m_pendingLength = length;
m_rfbState =
(length == 0) ? RfbState::WaitingServerMessageType : RfbState::WaitingServerCutTextData;
break;
}
case RfbState::WaitingServerCutTextData: {
if (!haveBytes(static_cast<int>(m_pendingLength))) {
return;
}
// RFB's ServerCutText is Latin-1 only (RFC 6143 SS7.5.4) -- no
// Unicode extension is implemented, unlike RDP's CLIPRDR.
const QString text =
QString::fromLatin1(m_recvBuffer.left(static_cast<int>(m_pendingLength)));
m_recvBuffer.remove(0, static_cast<int>(m_pendingLength));
m_rfbState = RfbState::WaitingServerMessageType;
emit remoteClipboardTextChanged(text);
break;
}
}
}
}
QByteArray VncSessionBackend::desKeyFromPassword(const QString& password)
{
QByteArray key(8, char(0));
const QByteArray latin1 = password.toLatin1();
for (int i = 0; i < 8 && i < latin1.size(); ++i) {
key[i] = latin1.at(i);
}
// RFB's VNC Authentication builds the DES key from the password with
// each byte's bits reversed -- a historical quirk of the original
// RealVNC implementation (it read the password into the DES key
// registers MSB-first where DES itself expects LSB-first), preserved
// here purely for protocol compatibility.
for (int i = 0; i < key.size(); ++i) {
quint8 byte = static_cast<quint8>(key.at(i));
quint8 reversed = 0;
for (int bit = 0; bit < 8; ++bit) {
reversed = static_cast<quint8>((reversed << 1) | (byte & 1));
byte = static_cast<quint8>(byte >> 1);
}
key[i] = static_cast<char>(reversed);
}
return key;
}
QByteArray VncSessionBackend::vncAuthResponse(const QByteArray& challenge, const QString& password)
{
if (challenge.size() != 16) {
return QByteArray();
}
const QByteArray key = desKeyFromPassword(password);
// The classic DES_* API is deprecated in OpenSSL 3.0 in favor of EVP,
// but VNC Authentication is inherently single-block-DES-ECB (a RealVNC
// protocol quirk, not a real security mechanism), and the classic API
// remains present and correct for exactly this narrow use.
#if defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
DES_cblock desKey;
std::memcpy(desKey, key.constData(), 8);
DES_key_schedule schedule;
DES_set_key_unchecked(&desKey, &schedule);
QByteArray response(16, char(0));
for (int block = 0; block < 2; ++block) {
DES_cblock input;
DES_cblock output;
std::memcpy(input, challenge.constData() + (block * 8), 8);
DES_ecb_encrypt(&input, &output, &schedule, DES_ENCRYPT);
std::memcpy(response.data() + (block * 8), output, 8);
}
#if defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic pop
#endif
return response;
}
QString VncSessionBackend::mapSocketError(QAbstractSocket::SocketError error, const QString& rawDetail)
{
switch (error) {
case QAbstractSocket::ConnectionRefusedError:
return QStringLiteral("Connection refused by remote host.");
case QAbstractSocket::RemoteHostClosedError:
return QStringLiteral("The VNC server closed the connection.");
case QAbstractSocket::HostNotFoundError:
return QStringLiteral("Host could not be resolved.");
case QAbstractSocket::SocketTimeoutError:
return QStringLiteral("Connection timed out.");
case QAbstractSocket::NetworkError:
return QStringLiteral("No route to host.");
case QAbstractSocket::SslHandshakeFailedError:
return QStringLiteral("A secure connection could not be established.");
default:
break;
}
return rawDetail.isEmpty() ? QStringLiteral("VNC connection failed.") : rawDetail;
}
quint32 VncSessionBackend::keysymForQtKey(int key, const QString& text)
{
switch (key) {
case Qt::Key_Backspace:
return 0xff08;
case Qt::Key_Tab:
case Qt::Key_Backtab:
return 0xff09;
case Qt::Key_Return:
case Qt::Key_Enter:
return 0xff0d;
case Qt::Key_Escape:
return 0xff1b;
case Qt::Key_Delete:
return 0xffff;
case Qt::Key_Home:
return 0xff50;
case Qt::Key_Left:
return 0xff51;
case Qt::Key_Up:
return 0xff52;
case Qt::Key_Right:
return 0xff53;
case Qt::Key_Down:
return 0xff54;
case Qt::Key_PageUp:
return 0xff55;
case Qt::Key_PageDown:
return 0xff56;
case Qt::Key_End:
return 0xff57;
case Qt::Key_Insert:
return 0xff63;
case Qt::Key_Menu:
return 0xff67;
case Qt::Key_NumLock:
return 0xff7f;
case Qt::Key_Pause:
return 0xff13;
case Qt::Key_ScrollLock:
return 0xff14;
case Qt::Key_Print:
return 0xff61;
// Ambiguous left/right modifier keys default to their left variant --
// RFB's KeyEvent has no notion of "native scancode" to disambiguate
// sides the way RDP's scancode table does.
case Qt::Key_Shift:
return 0xffe1;
case Qt::Key_Control:
return 0xffe3;
case Qt::Key_CapsLock:
return 0xffe5;
case Qt::Key_Meta:
return 0xffe7;
case Qt::Key_Alt:
return 0xffe9;
case Qt::Key_AltGr:
return 0xffea;
case Qt::Key_Super_L:
case Qt::Key_Super_R:
return 0xffeb;
default:
break;
}
if (key >= Qt::Key_F1 && key <= Qt::Key_F24) {
return 0xffbeu + static_cast<quint32>(key - Qt::Key_F1);
}
if (!text.isEmpty()) {
const uint codepoint = text.at(0).unicode();
if (codepoint >= 0x20 && codepoint <= 0xff) {
return codepoint;
}
if (codepoint > 0xff) {
// X11's convention for representing a Unicode codepoint beyond
// Latin-1 as a keysym.
return 0x01000000u + codepoint;
}
}
return 0;
}