Fix Apple DH auth wire-order bug, add type 33 fallback, fix missing VNC password prompt

Live testing against a real macOS Screen Sharing server surfaced two
real bugs, independent of each other:

1. SessionTab::requestConnectOptions() never prompted for a password
   on VNC profiles (only SSH/RDP) -- every VNC connection went out
   with an empty password regardless of what the server needed. VNC
   now gets its own prompt; an empty password is allowed through
   (unlike RDP's hard requirement) since no-auth VNC servers exist and
   there's no way to know client-side before the security-type
   negotiation happens.

2. VncSessionBackend's Apple DH (type 30) response sent the client's
   public key before the encrypted credentials. Cross-checking against
   neatvnc's rfb-proto.h (an independent, authoritative reference: both
   the wire struct definitions and the full server-side verification
   code, matched field-by-field against this implementation) showed
   the correct order is credentials first, then public key -- exactly
   backwards from what was implemented. Fixed, with a new regression
   test that decrypts the credentials back out using the trailing
   public-key bytes to derive the shared secret, which would fail if
   the fields were swapped again.

Also adds security type 33 (RSA + AES, src/vnc_apple_rsa_auth.h) as a
fallback Apple auth scheme, sourced from the `asyncvnc` PyPI package.
Preference when multiple are offered: None > AppleDH(30) >
AppleRSA(33) > VNCAuth(2).

Neither scheme has been gotten working live yet against the specific
macOS Tahoe (26.6.2) server available for testing -- type 30's wire
format is now verified correct byte-for-byte against the independent
reference above, but the server still rejects it with a generic
"Authentication or authorization failure"; type 33 is rejected even
earlier, right after the initial host-key request. macOS Tahoe was
released after this assistant's knowledge cutoff, so there may be a
protocol or permission-model change specific to it that isn't
reflected in either reference. Documented as an open issue in
docs/PROGRESS.md rather than claimed as working.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/vnc_apple_rsa_auth.cpp

@@ -0,0 +1,118 @@
+#include "vnc_apple_rsa_auth.h"
+
+#include <openssl/evp.h>
+#include <openssl/rand.h>
+#include <openssl/rsa.h>
+#include <openssl/x509.h>
+
+#include <memory>
+
+namespace VncAppleRsaAuth {
+
+namespace {
+
+struct EvpPkeyDeleter {
+    void operator()(EVP_PKEY* key) const { EVP_PKEY_free(key); }
+};
+struct EvpPkeyCtxDeleter {
+    void operator()(EVP_PKEY_CTX* ctx) const { EVP_PKEY_CTX_free(ctx); }
+};
+struct CipherCtxDeleter {
+    void operator()(EVP_CIPHER_CTX* ctx) const { EVP_CIPHER_CTX_free(ctx); }
+};
+
+using EvpPkeyPtr = std::unique_ptr<EVP_PKEY, EvpPkeyDeleter>;
+using EvpPkeyCtxPtr = std::unique_ptr<EVP_PKEY_CTX, EvpPkeyCtxDeleter>;
+using CipherCtxPtr = std::unique_ptr<EVP_CIPHER_CTX, CipherCtxDeleter>;
+
+}
+
+QByteArray packCredential(const QString& text)
+{
+    QByteArray data = text.toUtf8();
+    data.append(char(0));
+    if (data.size() < 64) {
+        QByteArray padding(64 - data.size(), char(0));
+        // Best-effort: if RAND_bytes fails, zero-padding is still
+        // correct (the NUL terminator above already unambiguously marks
+        // the string's real end for the server) -- only the ECB-pattern-
+        // hiding benefit of random padding is lost, not correctness.
+        RAND_bytes(reinterpret_cast<unsigned char*>(padding.data()), padding.size());
+        data += padding;
+    } else {
+        data = data.left(64);
+    }
+    return data;
+}
+
+Response computeResponse(const QByteArray& hostKeyDer, const QString& username,
+                         const QString& password)
+{
+    Response response;
+    if (hostKeyDer.isEmpty()) {
+        return response;
+    }
+
+    const auto* derPtr = reinterpret_cast<const unsigned char*>(hostKeyDer.constData());
+    EvpPkeyPtr hostKey(d2i_PUBKEY(nullptr, &derPtr, hostKeyDer.size()));
+    if (!hostKey) {
+        return response;
+    }
+
+    unsigned char aesKeyBytes[16];
+    if (RAND_bytes(aesKeyBytes, sizeof(aesKeyBytes)) != 1) {
+        return response;
+    }
+
+    const QByteArray credentials = packCredential(username) + packCredential(password);
+
+    CipherCtxPtr cipherCtx(EVP_CIPHER_CTX_new());
+    if (!cipherCtx
+        || EVP_EncryptInit_ex(cipherCtx.get(), EVP_aes_128_ecb(), nullptr, aesKeyBytes, nullptr)
+               <= 0) {
+        return response;
+    }
+    EVP_CIPHER_CTX_set_padding(cipherCtx.get(), 0);
+
+    QByteArray encryptedCredentials(credentials.size() + EVP_MAX_BLOCK_LENGTH, char(0));
+    int outLen1 = 0;
+    if (EVP_EncryptUpdate(cipherCtx.get(),
+                          reinterpret_cast<unsigned char*>(encryptedCredentials.data()), &outLen1,
+                          reinterpret_cast<const unsigned char*>(credentials.constData()),
+                          credentials.size())
+        <= 0) {
+        return response;
+    }
+    int outLen2 = 0;
+    if (EVP_EncryptFinal_ex(cipherCtx.get(),
+                            reinterpret_cast<unsigned char*>(encryptedCredentials.data()) + outLen1,
+                            &outLen2)
+        <= 0) {
+        return response;
+    }
+    encryptedCredentials.resize(outLen1 + outLen2);
+
+    EvpPkeyCtxPtr rsaCtx(EVP_PKEY_CTX_new(hostKey.get(), nullptr));
+    if (!rsaCtx || EVP_PKEY_encrypt_init(rsaCtx.get()) <= 0
+        || EVP_PKEY_CTX_set_rsa_padding(rsaCtx.get(), RSA_PKCS1_PADDING) <= 0) {
+        return response;
+    }
+    size_t encryptedKeyLen = 0;
+    if (EVP_PKEY_encrypt(rsaCtx.get(), nullptr, &encryptedKeyLen, aesKeyBytes, sizeof(aesKeyBytes))
+        <= 0) {
+        return response;
+    }
+    QByteArray encryptedAesKey(static_cast<int>(encryptedKeyLen), char(0));
+    if (EVP_PKEY_encrypt(rsaCtx.get(), reinterpret_cast<unsigned char*>(encryptedAesKey.data()),
+                         &encryptedKeyLen, aesKeyBytes, sizeof(aesKeyBytes))
+        <= 0) {
+        return response;
+    }
+    encryptedAesKey.resize(static_cast<int>(encryptedKeyLen));
+
+    response.encryptedCredentials = encryptedCredentials;
+    response.encryptedAesKey = encryptedAesKey;
+    return response;
+}
+
+}