orbithub/src/vnc_session_backend.h

#ifndef ORBITHUB_VNC_SESSION_BACKEND_H
#define ORBITHUB_VNC_SESSION_BACKEND_H

#include "session_backend.h"

#include <QAbstractSocket>
#include <QByteArray>
#include <QImage>
#include <QRect>
#include <QRgb>

#include <array>

class QTcpSocket;
struct z_stream_s;

// Implements RFB (RFC 6143) directly against QTcpSocket -- there is no
// permissively licensed VNC client library to vendor the way FreeRDP was
// for RDP (LibVNCClient is GPLv2, gtk-vnc is LGPL but GTK-tied), so this is
// an original implementation. Threading follows SshSessionBackend's model
// (a QObject moved to its own QThread, driven by Qt's own async socket
// signals) rather than RdpSessionBackend's manual worker-thread/blocking
// loop, since QTcpSocket is already async -- there's no legacy synchronous
// C API to wrangle here.
//
// Scope (see plan / issue #3 for the full rationale): standard VNC
// Authentication (security type 2), no-auth (type 1), and Apple's Screen
// Sharing scheme (type 30, Diffie-Hellman + AES -- see
// vnc_apple_dh_auth.h; not part of RFC 6143, implemented against the
// well-established reverse-engineered wire format). Raw + CopyRect +
// Hextile + ZRLE + Tight encodings. No dynamic resize. Clipboard sync
// (Latin-1 only, per RFB's ServerCutText/ClientCutText) and remote cursor
// shape sync (the Cursor pseudo-encoding) are supported.
//
// Tight decoding gap: the real protocol allows the server to skip zlib
// compression entirely for very small Basic-mode payloads; this decoder
// always attempts to zlib-inflate them, so a server that takes that
// shortcut on a given rectangle would have that one rectangle fail rather
// than decode. This is intentionally not special-cased (the exact trigger
// condition/wire signaling for it could not be verified with confidence
// against the RFC text alone, and it only affects rare, tiny rectangles --
// solid or near-solid tiny areas are virtually always sent as Fill instead
// in practice) -- see docs/PROGRESS.md.
class VncSessionBackend : public SessionBackend
{
    Q_OBJECT

public:
    explicit VncSessionBackend(const Profile& profile, QObject* parent = nullptr);
    ~VncSessionBackend() override;

    // Pure, state-free helpers exposed as public statics purely so tests
    // can exercise them without a live connection.
    static QByteArray vncAuthResponse(const QByteArray& challenge, const QString& password);
    static QByteArray desKeyFromPassword(const QString& password);
    static QString mapSocketError(QAbstractSocket::SocketError error, const QString& rawDetail);
    static quint32 keysymForQtKey(int key, const QString& text);

public slots:
    void connectSession(const SessionConnectOptions& options) override;
    void disconnectSession() override;
    void reconnectSession(const SessionConnectOptions& options) override;
    void sendInput(const QString& input) override;
    void confirmHostKey(bool trustHost) override;
    void updateTerminalSize(int columns, int rows) override;
    void sendKeyEvent(int key,
                      quint32 nativeScanCode,
                      const QString& text,
                      bool pressed,
                      int modifiers) override;
    void sendMouseMoveEvent(int x, int y) override;
    void sendMouseButtonEvent(int x, int y, int button, bool pressed) override;
    void sendMouseWheelEvent(int x, int y, int deltaX, int deltaY) override;
    void setClipboardText(const QString& text) override;

private slots:
    void onSocketConnected();
    void onSocketReadyRead();
    void onSocketDisconnected();
    void onSocketErrorOccurred(QAbstractSocket::SocketError error);

private:
    enum class RfbState {
        Idle,
        WaitingProtocolVersion,
        WaitingSecurityTypeCount,
        WaitingSecurityTypeList,
        WaitingSecurityTypeV33,
        WaitingSecurityFailureReasonLength,
        WaitingSecurityFailureReason,
        WaitingVncAuthChallenge,
        WaitingSecurityResult,
        WaitingSecurityResultReasonLength,
        WaitingSecurityResultReason,
        WaitingServerInitHeader,
        WaitingServerName,
        WaitingServerMessageType,
        WaitingFramebufferUpdateHeader,
        WaitingRectangleHeader,
        WaitingRawPixelData,
        WaitingCopyRectSource,
        WaitingCursorPixelData,
        WaitingHextileTileSubencoding,
        WaitingHextileTileMeta,
        WaitingHextileSubrectData,
        WaitingHextileRawTileData,
        WaitingZrleCompressedLength,
        WaitingZrleCompressedData,
        WaitingTightCompressionControl,
        WaitingTightFillColor,
        WaitingTightFilterId,
        WaitingTightLengthByte,
        WaitingTightPayload,
        WaitingAppleAuthGeneratorLength,
        WaitingAppleAuthGeneratorBytes,
        WaitingAppleAuthPrimeLength,
        WaitingAppleAuthPrimeAndServerKey,
        WaitingSetColourMapHeader,
        WaitingSetColourMapData,
        WaitingServerCutTextHeader,
        WaitingServerCutTextData,
    };

    struct PendingRectangle {
        int x = 0;
        int y = 0;
        int width = 0;
        int height = 0;
        qint32 encoding = 0;
    };

    QTcpSocket* m_socket;
    SessionState m_state;
    RfbState m_rfbState;
    QByteArray m_recvBuffer;
    bool m_userInitiatedDisconnect;
    bool m_reconnectPending;
    SessionConnectOptions m_reconnectOptions;
    SessionConnectOptions m_activeOptions;

    int m_negotiatedMinorVersion;
    int m_securityTypeCount;
    QByteArray m_offeredSecurityTypes;
    quint8 m_chosenSecurityType;
    quint32 m_pendingLength;

    QImage m_framebuffer;
    int m_pendingRectanglesRemaining;
    PendingRectangle m_currentRectangle;

    quint8 m_pointerButtonMask;
    int m_lastPointerX;
    int m_lastPointerY;

    // Hextile decode state (RFC 6143 SS7.7.4): tile-cursor position relative
    // to the current rectangle's origin, plus the background/foreground
    // colors, which persist across tiles within one rectangle whenever a
    // tile doesn't re-specify them.
    int m_hextileTileX;
    int m_hextileTileY;
    QRgb m_hextileBackground;
    QRgb m_hextileForeground;
    quint8 m_hextileSubencoding;
    int m_hextileSubrectsRemaining;
    bool m_hextileSubrectsColoured;

    // ZRLE's zlib stream (RFC 6143 SS7.7.6) persists for the whole
    // connection, not per-rectangle or per-update -- lazily initialized on
    // the first ZRLE rectangle, torn down and reset on every fresh
    // connect/reconnect via resetProtocolState(). z_stream_s is only
    // forward-declared here so <zlib.h> doesn't leak into every includer of
    // this header; the full type is only needed in the .cpp.
    z_stream_s* m_zrleInflateStream;
    bool m_zrleInflateInitialized;

    // Tight decode state (RFC 6143 SS7.7.4). Unlike ZRLE, Tight's "Basic"
    // compression mode has 4 independent persistent zlib streams (chosen
    // per-rectangle by 2 bits of the compression-control byte), each with
    // its own lifecycle -- reset individually via the control byte's low 4
    // bits, otherwise persisting like ZRLE's single stream.
    std::array<z_stream_s*, 4> m_tightInflateStreams;
    std::array<bool, 4> m_tightInflateInitialized;
    quint8 m_tightCompressionMode; // compression-control byte >> 4
    quint8 m_tightFilterId;
    int m_tightLengthByteIndex;

    // Apple Screen Sharing authentication state (security type 30, not
    // part of RFC 6143 -- see vnc_apple_dh_auth.h). m_appleAuthGenerator
    // must persist across the generator-length/generator-bytes state
    // hop; m_appleAuthPrimeLength must persist from when it's first read
    // until the combined prime+server-public-key buffer (2x that length)
    // has fully arrived, since m_pendingLength gets reused to track that
    // combined byte count in the meantime.
    QByteArray m_appleAuthGenerator;
    quint32 m_appleAuthPrimeLength;

    void setState(SessionState state, const QString& message);
    void resetProtocolState();
    void processReceiveBuffer();
    bool haveBytes(int count) const;
    void sendVersionReply();
    void sendClientInit();
    void sendSetPixelFormatAndEncodings();
    void requestFramebufferUpdate(bool incremental);
    void failConnection(const QString& displayMessage, const QString& rawMessage);
    void finishHandshakeIntoRunningState();
    void onRectangleFinished();
    void sendPointerEvent();
    void sendWheelClick(quint8 wheelBit);
    void sendClientCutText(const QString& text);
    QRect currentHextileTileRect() const;
    void advanceHextileTile();
    bool inflateTightStream(int streamIndex, const QByteArray& compressed, QByteArray* decompressed);
};

#endif