WIP: Add VNC Authentication

Enabled automatically when LibTomCrypt is available. Password
hardcoded to 'ABCD' for testing.

src/quickvnc/qvncitem.h

 #ifndef VNCCONNECTOR_H
 #define VNCCONNECTOR_H
 
+#include <QElapsedTimer>
+#include <QImage>
 #include <QQuickItem>
-#include <QtQuick/private/qquickshadereffectsource_p.h>
 
 QT_BEGIN_NAMESPACE
 

src/vncserver/CMakeLists.txt

@@ -2,6 +2,8 @@
 ## VNC Server Module:
 #####################################################################
 
+include("FindLibTomCrypt.cmake")
+
 qt_internal_add_module(VncServer
     SOURCES
         qrfbprotocol.cpp qrfbprotocol_p.h
@@ -26,3 +28,10 @@ qt_internal_add_module(VncServer
     GENERATE_CPP_EXPORTS
     GENERATE_PRIVATE_CPP_EXPORTS
 )
+
+qt_internal_extend_target(VncServer CONDITION LibTomCrypt_FOUND
+    LIBRARIES
+        LibTomCrypt::LibTomCrypt
+    DEFINES
+        LIBTOMCRYPT_FOUND
+)

src/vncserver/FindLibTomCrypt.cmake

+#.rst:
+# FindLibTomCrypt
+# ---------
+#
+# Try to locate the libtomcrypt client library.
+# If found, this will define the following variables:
+#
+# ``LibTomCrypt_FOUND``
+#     True if the libtomcrypt library is available
+# ``LibTomCrypt_INCLUDE_DIRS``
+#     The libtomcrypt include directories
+# ``LibTomCrypt_LIBRARIES``
+#     The libtomcrypt libraries for linking
+#
+# If ``LibTomCrypt_FOUND`` is TRUE, it will also define the following
+# imported target:
+#
+# ``LibTomCrypt::LibTomCrypt``
+#     The libtomcrypt client library
+
+find_package(PkgConfig QUIET)
+pkg_check_modules(PC_LibTomCrypt libtomcrypt)
+
+find_path(LibTomCrypt_INCLUDE_DIR
+          NAMES tomcrypt.h
+          HINTS ${PC_LibTomCrypt_INCLUDEDIR}
+          )
+
+find_library(LibTomCrypt_LIBRARY
+             NAMES libtomcrypt tomcrypt
+             HINTS ${PC_LibTomCrypt_LIBDIR})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LibTomCrypt DEFAULT_MSG LibTomCrypt_LIBRARY LibTomCrypt_INCLUDE_DIR)
+
+if(LibTomCrypt_FOUND)
+  set(LibTomCrypt_INCLUDE_DIRS "${LibTomCrypt_INCLUDE_DIR}")
+  set(LibTomCrypt_LIBRARIES "${LibTomCrypt_LIBRARY}")
+  if(NOT TARGET LibTomCrypt::LibTomCrypt)
+    add_library(LibTomCrypt::LibTomCrypt UNKNOWN IMPORTED)
+    set_target_properties(LibTomCrypt::LibTomCrypt PROPERTIES
+                          IMPORTED_LOCATION "${LibTomCrypt_LIBRARIES}"
+                          INTERFACE_INCLUDE_DIRECTORIES "${LibTomCrypt_INCLUDE_DIRS}")
+  endif()
+endif()
+
+mark_as_advanced(LibTomCrypt_INCLUDE_DIR LibTomCrypt_LIBRARY)
+
+include(FeatureSummary)
+set_package_properties(LibTomCrypt PROPERTIES
+  URL "https://libtom.net"
+  DESCRIPTION "LibTomCrypt library")
+

src/vncserver/qvncclient.cpp

@@ -39,6 +39,12 @@
 #include <QtGui/qguiapplication.h>
 #include <QtGui/private/qimage_p.h>
 
+
+#ifdef LIBTOMCRYPT_FOUND
+#include <tomcrypt.h>
+#define QT_VNC_AUTH // todo configure feature
+#endif
+
 #ifdef Q_OS_WIN
 #include <winsock2.h>
 #else
@@ -46,12 +52,12 @@
 #endif
 
 #if !defined(QT_VNCSERVER_NO_ZLIB)
+#include <qrandom.h>
 #  include <zlib.h>
 #endif
 
 QT_BEGIN_NAMESPACE
 
-
 //TODO: support different stride/subimage
 
 class QVNCDirtyMap
@@ -232,15 +238,32 @@ public:
     QVNCDirtyMapOptimized<quint32> map;
 };
 
+int QVncClient::sharedButtonState = Qt::NoButton;
 
 
-int QVncClient::sharedButtonState = Qt::NoButton;
+static inline QByteArray reverseBits(const char *bytes, int len)
+{
+    auto rev = [](const uchar b) -> uchar {
+     return ((b * 0x0802LU & 0x22110LU) | (b * 0x8020LU & 0x88440LU)) * 0x10101LU >> 16;
+    };
+    QByteArray result;
+    result.reserve(len);
+
+    for (int i = 0; i < len; ++i)
+        result += rev(bytes[i]);
+
+    qDebug() << "Twiddle:" << result;
+
+    return result;
+}
 
 QVncClient::QVncClient(QVncServer *server)
     : QObject(nullptr)
     , m_server(server)
     , m_supportDesktopSize(false)
 {
+    const char pwd[8]{'A', 'B', 'C', 'D', 0};  //Just testing...
+    m_passwordTest = reverseBits(pwd, sizeof(pwd));
 }
 
 void QVncClient::initialize(qintptr handle)
@@ -521,6 +544,26 @@ int QVncClient::dirtyMapCount() const
     return m_dirtyMap ? m_dirtyMap->map.numDirty : 0;
 }
 
+#ifdef LIBTOMCRYPT_FOUND
+static QByteArray desHash(const QByteArray &data, const QByteArray &passwd)
+{
+    Q_ASSERT(passwd.length() == 8 && data.length() == 16);
+
+    auto getData = [](const QByteArray &ba) -> const unsigned char * {return reinterpret_cast<const unsigned char*>(ba.constData()); };
+
+    Symmetric_key des_key;
+    des_setup(getData(passwd), passwd.length(), 0, &des_key);
+
+    uchar output[16];
+    auto *input = getData(data);
+
+    des_ecb_encrypt(input, output, &des_key);
+    des_ecb_encrypt(input + 8, output+8, &des_key);
+
+    return QByteArray{reinterpret_cast<char*>(output), 16};
+}
+#endif
+
 void QVncClient::readClient()
 {
     qCDebug(lcVnc) << "readClient" << int(m_state);
@@ -549,25 +592,73 @@ void QVncClient::readClient()
                     m_state = ClientState::Init;
                 } else {
                     // Authentication negotiation
-                    // No security for now
-                    const char supportedSecurity[] {1, SecurityNone}; // count, list
+                    const char supportedSecurity[] {
+#ifdef QT_VNC_AUTH
+                        SecurityVncAuthentication,
+#endif
+                                SecurityNone};
+                    const char size = sizeof(supportedSecurity);
+                    m_clientSocket->write(&size, 1);
                     m_clientSocket->write(supportedSecurity, sizeof(supportedSecurity));
                     m_state = ClientState::Security;
 
-                    // Version 3.8+: This is where we would send a failure message and terminate the
+                    // Version 3.8+: This is where we can send a failure message and terminate the
                     // connection e.g. because we require a stronger security:
                     // U32 numchars; n*U8 reason-string
                 }
             }
             break;
         case ClientState::Authentication:
+#ifdef QT_VNC_AUTH
+            if (m_securityType == SecurityVncAuthentication && m_clientSocket->bytesAvailable() >= 16) {
+                char response[16];
+                m_clientSocket->read(response, 16);
+                auto expected = desHash(m_randomChallenge, m_passwordTest);
+                bool passwordMatch = (expected == QByteArray(response, 16));
+                qDebug() << "Password test" << passwordMatch;
+
+                if (!passwordMatch) {
+                    auto dbg = qDebug();
+                    dbg << "Expected:" << Qt::hex;
+                    for (int i = 0; i < expected.length(); ++i)
+                        dbg << int(uchar(expected[i]));
+                    dbg << "Response:" << Qt::hex;
+                    for (int i = 0; i < 16; ++i)
+                        dbg << int(uchar(response[i]));
+                }
 
+                quint32 result = htonl(passwordMatch ? 0 : 1);
+                m_clientSocket->write(reinterpret_cast<char*>(&result), sizeof(result));
+                if (passwordMatch) {
+                    m_state = ClientState::Init;
+                } else {
+                    const QByteArray errorString {"Authentication failure."};
+                    int strLen = htonl(errorString.length());
+                    m_clientSocket->write(reinterpret_cast<char*>(&strLen), sizeof(strLen));
+                    m_clientSocket->write(errorString);
+//                    discardClient();
+                    m_state = ClientState::Disconnected;
+                }
+            }
+#endif // QT_VNC_AUTH
             break;
 
         case ClientState::Security:
             if (m_clientSocket->bytesAvailable() >= 1) {
                 m_clientSocket->read(reinterpret_cast<char *>(&m_securityType), 1);
                 qCDebug(lcVnc) << "Security type:" << m_securityType;
+
+#ifdef QT_VNC_AUTH
+                if (m_securityType == SecurityVncAuthentication) {
+                    auto *secureGenerator = QRandomGenerator::system();
+                    char challengeData[16];
+                    secureGenerator->fillRange(reinterpret_cast<quint32*>(&challengeData), sizeof(challengeData) / sizeof(quint32));
+                    m_clientSocket->write(challengeData, sizeof(challengeData));
+                    m_randomChallenge = {challengeData, sizeof(challengeData)};
+                    m_state = ClientState::Authentication;
+                    break;
+                }
+#endif
                 if (m_protocolVersion >= ProtocolVersion::V3_8) {
                     // SecurityResult
                     // OK = 0, Failed = 1 (TooManyAttempts = 2)

src/vncserver/qvncclient_p.h

@@ -234,6 +234,8 @@ private:
     z_stream_s *m_zlibStream = nullptr;
     QImage m_currentImage;
     bool m_currentImageIsFlipped = false;
+    QByteArray m_passwordTest;
+    QByteArray m_randomChallenge;
 
     // ### Not nice
     static int sharedButtonState;