Support protocol version 3.8

This makes it possible to support additional security types, but
none are implemented yet.

src/vncserver/qvncclient.cpp

@@ -262,10 +262,10 @@ void QVncClient::initializeConnection()
     m_clientSocket->setSocketDescriptor(m_socketHandle);
     emit socketCreated(m_clientSocket);
 
-    // send protocol version
-    const char *proto = "RFB 003.003\n";
+    // send maximum supported protocol version
+    const char *proto = "RFB 003.008\n";
     m_clientSocket->write(proto, 12);
-    m_state = Protocol;
+    m_state = ClientState::Protocol;
 
     if (lcVnc().isDebugEnabled())
         connect(m_clientSocket, &QIODevice::bytesWritten, this, [this](qint64 bytes){
@@ -292,7 +292,7 @@ QTcpSocket *QVncClient::clientSocket() const
 void QVncClient::setDirty(const QRegion &region)
 {
     m_dirtyRegion += region;
-    if (m_state == Connected)
+    if (m_state == ClientState::Connected)
         scheduleUpdate();
 }
 
@@ -523,41 +523,66 @@ int QVncClient::dirtyMapCount() const
 
 void QVncClient::readClient()
 {
-    qCDebug(lcVnc) << "readClient" << m_state;
+    qCDebug(lcVnc) << "readClient" << int(m_state);
     switch (m_state) {
-        case Disconnected:
+        case ClientState::Disconnected:
 
             break;
-        case Protocol:
+        case ClientState::Protocol:
             if (m_clientSocket->bytesAvailable() >= 12) {
                 char proto[13];
                 m_clientSocket->read(proto, 12);
                 proto[12] = '\0';
                 qCDebug(lcVnc, "Client protocol version %s", proto);
                 if (!strcmp(proto, "RFB 003.008\n")) {
-                    m_protocolVersion = V3_8;
+                    m_protocolVersion = ProtocolVersion::V3_8;
                 } else if (!strcmp(proto, "RFB 003.007\n")) {
-                    m_protocolVersion = V3_7;
+                    m_protocolVersion = ProtocolVersion::V3_7;
                 } else {
-                    m_protocolVersion = V3_3;
+                    m_protocolVersion = ProtocolVersion::V3_3;
                 }
 
-                if (m_protocolVersion == V3_3) {
+                if (m_protocolVersion == ProtocolVersion::V3_3) {
                     // No authentication
                     quint32 auth = htonl(1);
                     m_clientSocket->write((char *) &auth, sizeof(auth));
-                    m_state = Init;
+                    m_state = ClientState::Init;
+                } else {
+                    // Authentication negotiation
+                    // No security for now
+                    const char supportedSecurity[] {1, SecurityNone}; // count, list
+                    m_clientSocket->write(supportedSecurity, sizeof(supportedSecurity));
+                    m_state = ClientState::Security;
+
+                    // Version 3.8+: This is where we would send a failure message and terminate the
+                    // connection e.g. because we require a stronger security:
+                    // U32 numchars; n*U8 reason-string
                 }
             }
             break;
-        case Authentication:
+        case ClientState::Authentication:
 
             break;
-        case Init:
+
+        case ClientState::Security:
+            if (m_clientSocket->bytesAvailable() >= 1) {
+                m_clientSocket->read(reinterpret_cast<char *>(&m_securityType), 1);
+                qCDebug(lcVnc) << "Security type:" << m_securityType;
+                if (m_protocolVersion >= ProtocolVersion::V3_8) {
+                    // SecurityResult
+                    // OK = 0, Failed = 1 (TooManyAttempts = 2)
+                    quint32 result = htonl(0);
+                    m_clientSocket->write(reinterpret_cast<char*>(&result), sizeof(result));
+                }
+                m_state = ClientState::Init;
+            }
+        break;
+        case ClientState::Init:
             if (m_clientSocket->bytesAvailable() >= 1) {
                 quint8 shared;
                 m_clientSocket->read((char *) &shared, 1);
 
+                qCDebug(lcVnc) << "Client init, shared:" << shared;
                 // Server Init msg
                 QRfbServerInit sim;
                 QRfbPixelFormat &format = sim.format;
@@ -664,11 +689,11 @@ void QVncClient::readClient()
                 sim.height = m_screenSize.height();
                 sim.setName("Qt VNC Server");
                 sim.write(m_clientSocket);
-                m_state = Connected;
+                m_state = ClientState::Connected;
             }
             break;
 
-        case Connected:
+        case ClientState::Connected:
             do {
                 if (!m_handleMsg) {
                     m_clientSocket->read((char *)&m_msgType, 1);
@@ -705,14 +730,12 @@ void QVncClient::readClient()
                 }
             } while (!m_handleMsg && m_clientSocket->bytesAvailable());
             break;
-    default:
-        break;
     }
 }
 
 void QVncClient::discardClient()
 {
-    m_state = Disconnected;
+    m_state = ClientState::Disconnected;
     QMetaObject::invokeMethod(m_server, "discardClient", Q_ARG(QObject*, this));
 }
 
@@ -797,97 +820,6 @@ void QVncClient::setPixelFormat()
     }
 }
 
-
-/*
-
-
-0,Raw,[RFC6143]
-1,CopyRect,[RFC6143]
-2,RRE,[RFC6143]
-5,Hextile,[RFC6143]
-9-10,Possibly used in UltraVNC,historic assignment
-16,ZRLE,[RFC6143]
--239,Cursor pseudo-encoding,[RFC6143]
--223,DesktopSize pseudo-encoding,[RFC6143]
-4,CoRRE,historic assignment
-6,zlib,historic assignment
-7,tight,historic assignment
-8,zlibhex,historic assignment
-15,TRLE,[RFC6143]
-17,Hitachi ZYWRLE,historic assignment
-20,H.264,[Tristan_Richardson]
-21,JPEG,historic assignment [Tristan_Richardson]
-22,JRLE,historic assignment [Tristan_Richardson]
-23,VA H.264,[Tristan_Richardson]
-24,ZRLE2,[Tristan_Richardson]
-50,Open H.264 Encoding,[https://github.com/rfbproto/rfbproto/blob/master/rfbproto.rst][Maxim_Devaev]
-1000-1002,Apple Inc.,[Michael_Stein]
-1011,Apple Inc.,[Michael_Stein]
-1024 to 1099,RealVNC,historic assignment [Tristan_Richardson]
-1100-1105,Apple Inc.,[Michael_Stein]
--1 to -222,tight options,historic assignment
--224 to -238,tight options,historic assignment
--240 to -256,tight options,historic assignment
--257 to -272,Anthony Liguori,historic assignment
--273 to -304,VMWare,historic assignment
--305,gii,historic assignment
--306,popa,historic assignment
--307,Peter Astrand DesktopName,historic assignment
--308,Pierre Ossman ExtendedDesktopSize,historic assignment
--309,Colin Dean xvp,historic assignment
--310,OLIVE Call Control,historic assignment
--311,ClientRedirect,[Brian_Hinz]
--312,Fence,TigerVNC
--313,ContinuousUpdates,TigerVNC
--314,CursorWithAlpha,[Tristan_Richardson]
--315,ColorMap,[Tristan_Richardson]
--412 to -512,TurboVNC fine-grained quality level,historic assignment
--523 to -528,Car Connectivity,"[Jörg_Brakensiek]
-        [Terminal Mode v1.0]"
--763 to -768,TurboVNC subsampling level,historic assignment
-0x48323634,VA H.264,[David_Verbeiren]
-
----
-
-
-Number	Name
--23 to -32	JPEG Quality Level Pseudo-encoding
--223	DesktopSize Pseudo-encoding
--224	LastRect Pseudo-encoding
--239	Cursor Pseudo-encoding
--240	X Cursor Pseudo-encoding
--247 to -256	Compression Level Pseudo-encoding
--257	QEMU Pointer Motion Change Pseudo-encoding
--258	QEMU Extended Key Event Pseudo-encoding
--259	QEMU Audio Pseudo-encoding
--261	QEMU LED State Pseudo-encoding
--305	gii Pseudo-encoding
--307	DesktopName Pseudo-encoding
--308	ExtendedDesktopSize Pseudo-encoding
--309	xvp Pseudo-encoding
--312	Fence Pseudo-encoding
--313	ContinuousUpdates Pseudo-encoding
--314	Cursor With Alpha Pseudo-encoding
--412 to -512	JPEG Fine-Grained Quality Level Pseudo-encoding
--763 to -768	JPEG Subsampling Level Pseudo-encoding
-0x574d5664	VMware Cursor Pseudo-encoding
-0x574d5665	VMware Cursor State Pseudo-encoding
-0x574d5666	VMware Cursor Position Pseudo-encoding
-0x574d5667	VMware Key Repeat Pseudo-encoding
-0x574d5668	VMware LED state Pseudo-encoding
-0x574d5669	VMware Display Mode Change Pseudo-encoding
-0x574d566a	VMware Virtual Machine State Pseudo-encoding
-0xc0a1e5ce	Extended Clipboard Pseudo-encoding
-
-
-
-
-  */
-
-
-
-
-
 static const char *encodingName(qint32 enc)
 {
     switch (enc) {

src/vncserver/qvncclient_p.h

@@ -157,19 +157,36 @@ private:
     QRfbEncoder *createEncoder(Encodings encoding);
     QRegion dirtyRegion() const { return m_dirtyRegion; }
 
-    enum ClientState {
+    enum class ClientState {
         Disconnected,
         Protocol,
         Authentication,
+        Security,
         Init,
         Connected
     };
-    enum ProtocolVersion {
+    enum class ProtocolVersion {
         V3_3,
         V3_7,
         V3_8
     };
 
+    enum SecurityType : uchar {
+        SecurityInvalid               = 0,
+        SecurityNone                  = 1,
+        SecurityVncAuthentication     = 2,
+        SecurityRsaAes                = 5,
+        SecurityRsaAesUnencrypted     = 6,
+        SecurityRsaAesTwoStep         = 13,
+        SecurityTight                 = 16,
+        SecurityVeNCrypt              = 19,
+        SecurityXvp                   = 22,
+        SecurityDiffieHellman         = 30,
+        SecurityRsaAes256             = 129,
+        SecurityRsaAes256Unencrypted  = 130,
+        SecurityRsaAes256TwoStep      = 133
+    };
+
     void setPixelFormat();
     void setEncodings();
     void frameBufferUpdateRequest();
@@ -213,6 +230,7 @@ private:
     QImage m_previousImage;
     DirtyMap *m_dirtyMap = nullptr;
     ProtocolVersion m_protocolVersion = ProtocolVersion::V3_3;
+    SecurityType m_securityType = SecurityInvalid;
     z_stream_s *m_zlibStream = nullptr;
     QImage m_currentImage;
     bool m_currentImageIsFlipped = false;