diff --git a/App.qml b/App.qml
new file mode 100644
index 0000000000000000000000000000000000000000..ab4337c57ac4bf58fed241ac86227dd461f61210
--- /dev/null
+++ b/App.qml
@@ -0,0 +1,25 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+import QtQuick 2.15
+import QtQuick.Controls 2.15
+import QtQuick.Window 2.15
+
+ApplicationWindow {
+ id: mainWindow
+ visible: true
+ width: Screen.width / 3
+ height: Screen.height / 3
+ //width: Screen.width / 2
+ //height: Screen.height / 2
+ //flags: Qt.FramelessWindowHint | Qt.Window
+ //visibility: Window.FullScreen
+ color: "black" // Optional background color for the main window
+
+ // @disable-check M300
+ Screen01 {
+ anchors.fill: parent
+ }
+
+}
+
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a6ba292889269b74864eab563ac9330d296ca4cd
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,48 @@
+
+cmake_minimum_required(VERSION 3.21.1)
+
+add_subdirectory(aimodel)
+
+project(QtAiInferenceApi LANGUAGES CXX)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+
+set(CMAKE_AUTOMOC ON)
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+set(QT_QML_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/qml)
+set(QML_IMPORT_PATH ${QT_QML_OUTPUT_DIRECTORY}
+ CACHE STRING "Import paths for Qt Creator's code model"
+ FORCE
+)
+
+find_package(Qt6 6.8 REQUIRED COMPONENTS Core Gui Qml Quick Multimedia)
+qt_standard_project_setup(REQUIRES 6.8)
+
+qt_add_executable(${CMAKE_PROJECT_NAME}
+ main.cpp
+)
+
+qt_add_qml_module(${CMAKE_PROJECT_NAME}
+ URI qtaiinferenceapi
+ VERSION 1.0
+ RESOURCES
+ qtquickcontrols2.conf
+ QML_FILES
+ App.qml
+ Screen01.ui.qml
+ )
+
+target_link_libraries(${CMAKE_PROJECT_NAME}
+ PRIVATE
+ Qt6::Quick
+ Qt6::Multimedia
+ QtAiModelApi
+)
+
+
+include(GNUInstallDirs)
+install(TARGETS ${CMAKE_PROJECT_NAME}
+ BUNDLE DESTINATION .
+ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+)
diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt
new file mode 100644
index 0000000000000000000000000000000000000000..136d90045663ab0acaeff2a3ecf0013c968de8c9
--- /dev/null
+++ b/LICENSES/Apache-2.0.txt
@@ -0,0 +1,61 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
+ 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
+ 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
+ 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
+ (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and
+ (b) You must cause any modified files to carry prominent notices stating that You changed the files; and
+ (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
+ (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
+ 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
+ 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
+ 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
+ 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+ 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/LICENSES/BSD-3-Clause.txt b/LICENSES/BSD-3-Clause.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b91bbd89468099ae5c3028f176783976ca538187
--- /dev/null
+++ b/LICENSES/BSD-3-Clause.txt
@@ -0,0 +1,9 @@
+Copyright (c) <year> <owner>.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+ 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/LICENSES/LGPL-3.0-only.txt b/LICENSES/LGPL-3.0-only.txt
new file mode 100644
index 0000000000000000000000000000000000000000..65c5ca88a67c30becee01c5a8816d964b03862f9
--- /dev/null
+++ b/LICENSES/LGPL-3.0-only.txt
@@ -0,0 +1,165 @@
+ GNU LESSER GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+ This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+ 0. Additional Definitions.
+
+ As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+ "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+ An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+ A "Combined Work" is a work produced by combining or linking an
+Application with the Library. The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+ The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+ The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+ 1. Exception to Section 3 of the GNU GPL.
+
+ You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+ 2. Conveying Modified Versions.
+
+ If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+ a) under this License, provided that you make a good faith effort to
+ ensure that, in the event an Application does not supply the
+ function or data, the facility still operates, and performs
+ whatever part of its purpose remains meaningful, or
+
+ b) under the GNU GPL, with none of the additional permissions of
+ this License applicable to that copy.
+
+ 3. Object Code Incorporating Material from Library Header Files.
+
+ The object code form of an Application may incorporate material from
+a header file that is part of the Library. You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+ a) Give prominent notice with each copy of the object code that the
+ Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the object code with a copy of the GNU GPL and this license
+ document.
+
+ 4. Combined Works.
+
+ You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+ a) Give prominent notice with each copy of the Combined Work that
+ the Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the Combined Work with a copy of the GNU GPL and this license
+ document.
+
+ c) For a Combined Work that displays copyright notices during
+ execution, include the copyright notice for the Library among
+ these notices, as well as a reference directing the user to the
+ copies of the GNU GPL and this license document.
+
+ d) Do one of the following:
+
+ 0) Convey the Minimal Corresponding Source under the terms of this
+ License, and the Corresponding Application Code in a form
+ suitable for, and under terms that permit, the user to
+ recombine or relink the Application with a modified version of
+ the Linked Version to produce a modified Combined Work, in the
+ manner specified by section 6 of the GNU GPL for conveying
+ Corresponding Source.
+
+ 1) Use a suitable shared library mechanism for linking with the
+ Library. A suitable mechanism is one that (a) uses at run time
+ a copy of the Library already present on the user's computer
+ system, and (b) will operate properly with a modified version
+ of the Library that is interface-compatible with the Linked
+ Version.
+
+ e) Provide Installation Information, but only if you would otherwise
+ be required to provide such information under section 6 of the
+ GNU GPL, and only to the extent that such information is
+ necessary to install and execute a modified version of the
+ Combined Work produced by recombining or relinking the
+ Application with a modified version of the Linked Version. (If
+ you use option 4d0, the Installation Information must accompany
+ the Minimal Corresponding Source and Corresponding Application
+ Code. If you use option 4d1, you must provide the Installation
+ Information in the manner specified by section 6 of the GNU GPL
+ for conveying Corresponding Source.)
+
+ 5. Combined Libraries.
+
+ You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+ a) Accompany the combined library with a copy of the same work based
+ on the Library, uncombined with any other library facilities,
+ conveyed under the terms of this License.
+
+ b) Give prominent notice with the combined library that part of it
+ is a work based on the Library, and explaining where to find the
+ accompanying uncombined form of the same work.
+
+ 6. Revised Versions of the GNU Lesser General Public License.
+
+ The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+ If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/LICENSES/LicenseRef-Qt-Commercial.txt b/LICENSES/LicenseRef-Qt-Commercial.txt
new file mode 100644
index 0000000000000000000000000000000000000000..825b1f358b1f42d762112defbf0a193c9d968488
--- /dev/null
+++ b/LICENSES/LicenseRef-Qt-Commercial.txt
@@ -0,0 +1,8 @@
+Licensees holding valid commercial Qt licenses may use this software in
+accordance with the the terms contained in a written agreement between
+you and The Qt Company. Alternatively, the terms and conditions that were
+accepted by the licensee when buying and/or downloading the
+software do apply.
+
+For the latest licensing terms and conditions, see https://www.qt.io/terms-conditions.
+For further information use the contact form at https://www.qt.io/contact-us.
diff --git a/QtAiInferenceApi.qmlproject b/QtAiInferenceApi.qmlproject
new file mode 100644
index 0000000000000000000000000000000000000000..df4f7cb5746a3e738324487cbd906d0b9f11fd04
--- /dev/null
+++ b/QtAiInferenceApi.qmlproject
@@ -0,0 +1,79 @@
+// prop: json-converted
+// prop: auto-generated
+
+import QmlProject
+
+Project {
+ mainFile: "App.qml"
+ mainUiFile: "Screen.ui.qml"
+ targetDirectory: "/opt/QtAiInferenecApi"
+ enableCMakeGeneration: true
+ enablePythonGeneration: false
+ widgetApp: true
+ importPaths: [ "." ]
+
+ qdsVersion: "4.6"
+ quickVersion: "6.8"
+ qt6Project: true
+ qtForMCUs: false
+
+ multilanguageSupport: true
+ primaryLanguage: "en"
+ supportedLanguages: [ "en" ]
+
+ Environment {
+ QML_COMPAT_RESOLVE_URLS_ON_ASSIGNMENT: "1"
+ QT_AUTO_SCREEN_SCALE_FACTOR: "1"
+ QT_ENABLE_HIGHDPI_SCALING: "0"
+ QT_LOGGING_RULES: "qt.qml.connections=false"
+ QT_QUICK_CONTROLS_CONF: "qtquickcontrols2.conf"
+ }
+
+ QmlFiles {
+ directory: "."
+ }
+
+ JavaScriptFiles {
+ directory: "."
+ }
+
+ ImageFiles {
+ directory: "."
+ }
+
+ Files {
+ filter: "*.conf"
+ files: [
+ "qtquickcontrols2.conf"
+ ]
+ }
+
+ Files {
+ directory: "."
+ filter: "qmldir"
+ }
+
+ Files {
+ filter: "*.ttf;*.otf"
+ }
+
+ Files {
+ filter: "*.wav;*.mp3"
+ }
+
+ Files {
+ filter: "*.mp4"
+ }
+
+ Files {
+ filter: "*.glsl;*.glslv;*.glslf;*.vsh;*.fsh;*.vert;*.frag"
+ }
+
+ Files {
+ filter: "*.qsb"
+ }
+
+ Files {
+ filter: "*.json"
+ }
+}
diff --git a/README.md b/README.md
index 737a487642e223c350dd32a486f885fd7d78d47a..cf1cf320c8a46b2206fb4aa3c03448e0f15727be 100644
--- a/README.md
+++ b/README.md
@@ -1,93 +1,268 @@
# Qt AI Inference API
+This projects contains the proof-of-concept for a new Qt AI Inference API. The purpose of the API is to let you easily use different types of AI models for inference from your Qt code, either from C++ or directly from QML! The API abstracts the details of the underlying model and framework implementations, allowing you to just tell what type of input and output you would like to use, and Qt will set things up for you! You can also chain different models together for pipelines.
+## How it works
-## Getting started
+When you declare a model in your code, Qt will infer from the given input and output type what backend it will set up for the model. The backends are implemented as QPlugins. Currently, the backends are:
-To make it easy for you to get started with GitLab, here's a list of recommended next steps.
+| Input type | Output type | Qt backend | Description |
+|------------|-------------|---------------|-------------------------------------------------------------------------------|
+| Text\|Image| Text | QtOllamaModel | Uses ollama to load LLM models and communicate to them with ollama's REST API |
+| Speech | Text | QtAsrModel | Uses Whisper for Automatic Speech Recognition (ASR), or speech-to-text |
+| Image | Json | QtTritonModel | Uses Triton to load a model for object detection from images |
+| Image | Json | QtYoloModel | Uses a YOLO model for object detection from images |
+| Text | Speech | QtPiperModel | Uses Piper TTS model to convert text into speech |
-Already a pro? Just edit this README.md and make it your own. Want to make it easy? [Use the template at the bottom](#editing-this-readme)!
+Note, the Qt backends expect the underlying backend implementation (ollama, Whisper...) to be running, and will not take care of starting them up for you. You need to start them yourself, e.g. in the case of QtOllamaModel, loading the intended model to ollama's memory by running:
+```
+ollama run <model>
+```
+
+## Building the library
+
+To build the API library, you need to have a Qt kit (6.7 or newer). Additional dependencies for specific plugins are:
+* QtSpeech additional library for QtTtsModel
+* OpenCV for QtTritonModel
+* QtMultimedia for QtYoloModel
-## Add your files
+In Qt Creator, open the library project by choosing the CMakeLists.txt file under qt-ai-inference-api/aimodel/, configure with your Qt kit and build the library. You can also choose the qt-ai-inference-api/CMakeLists.txt to build the whole project, including the example app that ships with the API. For an example on how to include the library project into your Qt app project, see the [Qt AI App example app](https://git.qt.io/ai-ui/qt-ai-app)
-- [ ] [Create](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#create-a-file) or [upload](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#upload-a-file) files
-- [ ] [Add files using the command line](https://docs.gitlab.com/ee/gitlab-basics/add-file.html#add-a-file-using-the-command-line) or push an existing Git repository with the following command:
+## The API
+
+Currently, the API consists of one class, QAiModel being the C++ implementation, and MultiModal its QML counterpart.
+QAiModel inherits [QObject](https://doc.qt.io/qt-6/qobject.html). To use the QML API, use the following import statement in your QML code:
```
-cd existing_repo
-git remote add origin https://git.qt.io/edge-ai/qt-ai-inference-api.git
-git branch -M main
-git push -uf origin main
+import qtaimodel
```
-## Integrate with your tools
+### Properties
-- [ ] [Set up project integrations](https://git.qt.io/edge-ai/qt-ai-inference-api/-/settings/integrations)
+**AiModelPrivateInterface::AiModelTypes type**
-## Collaborate with your team
+A combination of AiModelType flags to tell what type of model to instantiate. Possible values are:
-- [ ] [Invite team members and collaborators](https://docs.gitlab.com/ee/user/project/members/)
-- [ ] [Create a new merge request](https://docs.gitlab.com/ee/user/project/merge_requests/creating_merge_requests.html)
-- [ ] [Automatically close issues from merge requests](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#closing-issues-automatically)
-- [ ] [Enable merge request approvals](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/)
-- [ ] [Set auto-merge](https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html)
+| Name | Value | Description |
+|-------------|---------|--------------------------------------------------------------------------|
+| InputText | 0x00001 | The model takes text as input |
+| InputAudio | 0x00002 | The model takes speech as input |
+| InputVideo | 0x00004 | The model takes video as input |
+| InputImage | 0x00008 | The model takes image as input |
+| InputJson | 0x00010 | The model takes JSON as input |
+| OutputText | 0x00100 | The model outputs text |
+| OutputAudio | 0x00200 |The model outputs speech |
+| OutputVideo | 0x00400 |The model outputs video |
+| OutputImage | 0x00800 |The model outputs image |
+| OutputJson | 0x01000 |The model outputs JSON |
-## Test and Deploy
+For supported input-output combinations, see the table under "How it works" section.
-Use the built-in continuous integration in GitLab.
+Example:
+```
+import qtaimodel
-- [ ] [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/quick_start/index.html)
-- [ ] [Analyze your code for known vulnerabilities with Static Application Security Testing (SAST)](https://docs.gitlab.com/ee/user/application_security/sast/)
-- [ ] [Deploy to Kubernetes, Amazon EC2, or Amazon ECS using Auto Deploy](https://docs.gitlab.com/ee/topics/autodevops/requirements.html)
-- [ ] [Use pull-based deployments for improved Kubernetes management](https://docs.gitlab.com/ee/user/clusters/agent/)
-- [ ] [Set up protected environments](https://docs.gitlab.com/ee/ci/environments/protected_environments.html)
+MultiModal {
+ // initiates a LLM model which takes text input, and produces text output
+ type: MultiModal.InputText | MultiModal.OutputText
+}
+```
-***
+| | |
+|------------------|-----------------------------------------------------|
+| Write method: | void setType(AiModelPrivateInterface::AiModelTypes) |
+| Read method: | AiModelPrivateInterface::AiModelTypes type() |
+| Notifier signal: | void typeChanged() |
-# Editing this README
+**QString prompt**
-When you're ready to make this README your own, just edit this file and use the handy template below (or feel free to structure it however you want - this is just a starting point!). Thanks to [makeareadme.com](https://www.makeareadme.com/) for this template.
+The prompt for the model. This can be used to provide a more persistent prompt that will be combined with anything provided to the model with pushData(). The prompt will be prepended to any data provided with pushData(). Note, only setting the prompt will not send anything to the underlying model; you need to use pushData() to trigger that.
-## Suggestions for a good README
+Example:
+```
+import qtaimodel
+
+MultiModal {
+ id: model
+ type: MultiModal.InputText | MultiModal.OutputText
+ prompt: "Summarize the following text:"
+}
+
+function summarizeText() {
+ model.pushData("Lorem ipsum")
+ // The actual prompt sent to the underlying model will be "Summarize the following text: Lorem ipsum"
+}
+```
+| | |
+|------------------|-----------------------------------------------------|
+| Write method: | void setPrompt(QString) |
+| Read method: | QString prompt() |
+| Notifier signal: | void promptChanged() |
-Every project is different, so consider which of these sections apply to yours. The sections used in the template are suggestions for most open source projects. Also keep in mind that while a README can be too long and detailed, too long is better than too short. If you think your README is too long, consider utilizing another form of documentation rather than cutting out information.
+**QString model**
-## Name
-Choose a self-explaining name for your project.
+Use to tell the underlying framework what specific model to use.
-## Description
-Let people know what your project can do specifically. Provide context and add a link to any reference visitors might be unfamiliar with. A list of Features or a Background subsection can also be added here. If there are alternatives to your project, this is a good place to list differentiating factors.
+Example:
+```
+import qtaimodel
+
+MultiModal {
+ id: model
+ // Runs a QtOllamaModel, where ollama uses deepseek-r1 model
+ type: MultiModal.InputText | MultiModal.OutputText
+ model: "deepseek-r1"
+}
+```
-## Badges
-On some READMEs, you may see small images that convey metadata, such as whether or not all the tests are passing for the project. You can use Shields to add some to your README. Many services also have instructions for adding a badge.
+| | |
+|------------------|-----------------------------------------------------|
+| Write method: | void setModel(QString) |
+| Read method: | QString model() |
+| Notifier signal: | void modelChanged() |
-## Visuals
-Depending on what you are making, it can be a good idea to include screenshots or even a video (you'll frequently see GIFs rather than actual videos). Tools like ttygif can help, but check out Asciinema for a more sophisticated method.
+**QVariantList rag**
-## Installation
-Within a particular ecosystem, there may be a common way of installing things, such as using Yarn, NuGet, or Homebrew. However, consider the possibility that whoever is reading your README is a novice and would like more guidance. Listing specific steps helps remove ambiguity and gets people to using your project as quickly as possible. If it only runs in a specific context like a particular programming language version or operating system or has dependencies that have to be installed manually, also add a Requirements subsection.
+Retrieval-Augmented Generation data to use for the model, if it supports it. RAG supports currently only chromadb, which should be running on background.
-## Usage
-Use examples liberally, and show the expected output if you can. It's helpful to have inline the smallest example of usage that you can demonstrate, while providing links to more sophisticated examples if they are too long to reasonably include in the README.
+Example:
+```
+import qtaimodel
+
+ MultiModal {
+ id: llamaModel
+ type: (MultiModal.InputText | MultiModal.OutputText)
+ model: "llama3.2"
+ prompt: "Which item has best armor bonus?"
+ rag: ["Cloth of Authority | Armour Class +1",
+ "Drunken Cloth | Constitution +2 (up to 20)",
+ "Icebite Robe | Resistance to Damage Types: Cold damage.",
+ "Obsidian Laced Robe | Grants Resistance to Damage Types: Fire damage.",
+ "Moon Devotion Robe | Advantage on Constitution Saving throws.",
+ ]
+ }
+```
-## Support
-Tell people where they can go to for help. It can be any combination of an issue tracker, a chat room, an email address, etc.
+| | |
+|------------------|-----------------------------------------------------|
+| Write method: | void setRag(QByteArray) |
+| Read method: | QByteArray rag() |
+| Notifier signal: | void ragChanged() |
-## Roadmap
-If you have ideas for releases in the future, it is a good idea to list them in the README.
+**QVector<QAiModel*> inputs**
-## Contributing
-State if you are open to contributions and what your requirements are for accepting them.
+A list of models this model will use as its inputs. This allows for chaining models together to create pipelines. You can use the Optional flag with the model's type to tell whether it's an optional or mandatory input. For mandatory inputs, this model will not process any other inputs before the mandatory one has something to offer. For optional ones, other inputs will be processed regardless if that input has data available or not.
+
+Example:
+```
+import qtaimodel
+
+// The ASR model will convert speech to text and pass it to the LLM model. Its "optional" is set as true,
+// so if the LLM model has other ways of receiving input, such as typing, it will not block processing
+// those while waiting for the output of the ASR model.
+MultiModal {
+ id: asrModel
+ type: MultiModal.InputAudio | MultiModal.OutputText
+ optional: true
+}
+
+MultiModal {
+ id: llmModel
+ type: MultiModal.InputText | MultiModal.OutputText
+ inputs: [asrModel]
+}
+
+```
+
+| | |
+|------------------|-----------------------------------------------------|
+| Write method: | void setInputs(QVector<QAiModel*>) |
+| Read method: | QVector<QAiModel*> inputs() |
+| Notifier signal: | void inputsChanged() |
+
+**bool processing**
+
+Whether the model is currently processing a request.
+
+Example:
+```
+import qtaimodel
+
+MultiModal {
+ id: model
+ ...
+}
+
+BusyIndicator {
+ running: model.processing
+}
+```
-For people who want to make changes to your project, it's helpful to have some documentation on how to get started. Perhaps there is a script that they should run or some environment variables that they need to set. Make these steps explicit. These instructions could also be useful to your future self.
+| | |
+|------------------|-----------------------------------------------------|
+| Write method: | void setProcessing(bool) |
+| Read method: | bool processing() |
+| Notifier signal: | void processingChanged() |
+
+
+**bool buffered**
+
+Whether the model should buffer the latest result for later use.
+
+
+| | |
+|------------------|-----------------------------------------------------|
+| Write method: | void setBuffered(bool) |
+| Read method: | bool buffered() |
+| Notifier signal: | void bufferedChanged() |
+
+
+**bool optional**
+
+This boolean can be used when using the model as an input for another model, telling the other model(s) to treat this one as optional input. If this is set to true, the other model(s) will not wait for this model to produce output before processing its other inputs. If this is set to false, the other model(s) will wait for this model to produce output before processing its other inputs. Default is false. Has no effect if this model is not being used as input by any other model.
+
+
+| | |
+|------------------|-----------------------------------------------------|
+| Write method: | void setOptional(bool) |
+| Read method: | bool optional() |
+| Notifier signal: | void optionalChanged() |
+
+### Methods
+
+
+**Q_INVOKABLE void pushData(QVariant data)**
+
+Push data to the model. The argument can be of any type supported by QVariant. The underlying QAiModel implementation will convert it based on its expected input type.
+
+
+QVariant data - The data to push to the model. If the prompt property has been set, it will be prepended to the data before sending the final request to the underlying model.
+
+### Signals
+
+**void gotResult(QVariant result)**
+
+Emitted when the underlying model has finished processing and returns a result. The result is passed as QVariant, converted from the output type the underlying model provides.
+
+Example:
+```
+import qtaimodel
+
+MultiModal {
+ id: model
+ types: MultiModal.InputText | MultiModal.OutputText
+ onGotResult: (result) => {
+ someLabel.text = result
+ }
+}
+
+```
-You can also document commands to lint the code or run tests. These steps help to ensure high code quality and reduce the likelihood that the changes inadvertently break something. Having instructions for running tests is especially helpful if it requires external setup, such as starting a Selenium server for testing in a browser.
+## Known issues
-## Authors and acknowledgment
-Show your appreciation to those who have contributed to the project.
+* Currently the C++ API is not public, meaning the API is only usable from QML. This will change in a future patch.
-## License
-For open source projects, say how it is licensed.
+## Additional links
-## Project status
-If you have run out of energy or time for your project, put a note at the top of the README saying that development has slowed down or stopped completely. Someone may choose to fork your project or volunteer to step in as a maintainer or owner, allowing your project to keep going. You can also make an explicit request for maintainers.
+- [The example app](https://git.qt.io/ai-ui/qt-ai-app)
+- [Qt Project JIRA](https://bugreports.qt.io) - If you would like to leave ideas, suggestions or bug reports, please use the "QtAiApi" label so we can easily gather them!
diff --git a/Screen01.ui.qml b/Screen01.ui.qml
new file mode 100644
index 0000000000000000000000000000000000000000..ebd7d30520b865b85d0b2b38d79108808efdb26e
--- /dev/null
+++ b/Screen01.ui.qml
@@ -0,0 +1,125 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+// TODO: check if all is really needed here
+import QtQuick 2.15
+import QtQuick.Layouts
+import QtQuick.Controls
+import Qt.labs.platform // TODO: Is this used here?
+import QtMultimedia
+
+import qtaimodel
+
+Rectangle {
+ id: rectangle
+ anchors.fill: parent
+ color: "#ffffff" // TODO: Use some better color?
+
+ property string llamaPrompt: "You are an assistant.\n"
+ property string imageFile: ""
+
+ ColumnLayout {
+ RowLayout {
+ id: buttonRow
+ Button {
+ text: "Record audio"
+ onClicked: {
+ recorder.record()
+ }
+ }
+ Button {
+ text: "Stop audio recording"
+ onClicked: {
+ recorder.stop()
+ if (recorder.actualLocation != "") {
+ speechToText.pushData(recorder.actualLocation)
+ }
+ if (imageFile != "") {
+ imageToText.pushData(imageFile)
+ }
+ }
+ }
+ }
+
+ RowLayout {
+ Button {
+ text: qsTr("Open image")
+ onClicked: fileDialog.open()
+ }
+ Text {
+ id: result
+ text: rectangle.imageFile
+ }
+ }
+
+ TextField {
+ text: llamaPrompt
+ implicitWidth: 300
+ onEditingFinished: llamaModel.prompt = text
+ }
+
+ TextArea {
+ placeholderText: "Enter context"
+ background: Rectangle {
+ color: "lightgreen"
+ }
+
+ implicitWidth: 300
+ implicitHeight: 200
+ onEditingFinished: llamaModel.rag = [text]
+ }
+
+ Image {
+ source: imageFile
+ }
+ }
+
+ FileDialog {
+ id: fileDialog
+ folder: StandardPaths.standardLocations(StandardPaths.PicturesLocation)[0]
+ nameFilters: ["*.*"]
+ onAccepted: {
+ imageFile = fileDialog.file
+ }
+ onRejected: {}
+ }
+
+ CaptureSession {
+ audioInput: AudioInput {}
+ recorder: MediaRecorder {
+ id: recorder
+ mediaFormat {
+ fileFormat: MediaFormat.Wave
+ }
+ }
+ }
+
+ MultiModal {
+ id: imageToText
+ type: (MultiModal.InputImage | MultiModal.OutputText)
+ model: "llava-phi3" // TODO: replace with Janus model from DeepSeek
+ prompt: "What is in the picture?"
+ optional: true
+ buffered: true
+ }
+
+ MultiModal {
+ id: speechToText
+ type: (MultiModal.InputAudio | MultiModal.OutputText)
+ model: "turbo"
+ }
+
+ MultiModal {
+ id: llamaModel
+ type: (MultiModal.InputText | MultiModal.OutputText)
+ model: "gemma3:4b"
+ prompt: llamaPrompt
+ inputs: [ imageToText, speechToText ]
+ }
+
+ MultiModal {
+ id: text2speech
+ type: (MultiModal.InputText | MultiModal.OutputAudio)
+ inputs: [ llamaModel ]
+ }
+}
diff --git a/aimodel/CMakeLists.txt b/aimodel/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..605edc97337e3623adcad9dc524c549e207f7f77
--- /dev/null
+++ b/aimodel/CMakeLists.txt
@@ -0,0 +1,46 @@
+
+cmake_minimum_required(VERSION 3.21.1)
+set(QT_QML_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/qml)
+set(QML_IMPORT_PATH ${QT_QML_OUTPUT_DIRECTORY}
+ CACHE STRING "Import paths for Qt Creator's code model"
+ FORCE
+)
+
+project(QtAiModelApi LANGUAGES CXX)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+
+set(CMAKE_AUTOMOC ON)
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+
+
+add_subdirectory(plugins)
+
+find_package(Qt6 6.8 REQUIRED COMPONENTS Core Qml Quick Network)
+qt_standard_project_setup(REQUIRES 6.8)
+
+qt_add_library(QtAiModelPluginInterface
+ qaimodelinterface_p.h
+ chromadb.h chromadb.cpp
+)
+target_link_libraries(QtAiModelPluginInterface
+ PRIVATE
+ Qt6::Core
+ Qt6::Network
+)
+
+qt_add_qml_module(QtAiModelApi
+ URI qtaimodel
+ VERSION 1.0
+ SHARED
+ SOURCES
+ qaimodel.h qaimodel.cpp
+ )
+
+qt_import_qml_plugins(QtAiModelApi)
+
+target_link_libraries(QtAiModelApi
+ PRIVATE
+ Qt6::Quick
+ QtAiModelPluginInterface
+)
diff --git a/aimodel/asr_server/asr_server.py b/aimodel/asr_server/asr_server.py
new file mode 100755
index 0000000000000000000000000000000000000000..d064ed1da77b77229caf205076900692b386b327
--- /dev/null
+++ b/aimodel/asr_server/asr_server.py
@@ -0,0 +1,54 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2025 The Qt Company Ltd.
+# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+from http.server import BaseHTTPRequestHandler,HTTPServer
+from os import curdir, sep
+import cgi
+import whisper
+import simplejson
+
+PORT_NUMBER = 8003
+
+#This class will handles any incoming request from
+#the browser
+class myHandler(BaseHTTPRequestHandler):
+
+ #Handler for the POST requests
+ def do_POST(self):
+ print("do_POST");
+ if self.path=="/send":
+ self.data_string = self.rfile.read(int(self.headers['Content-Length']))
+ print("data_string: " + self.data_string.decode())
+ json_data = simplejson.loads(self.data_string)
+ print("json_data: " + simplejson.dumps(json_data))
+ print("MODEL: " + json_data["model"])
+ print("FILE: " + json_data["file"])
+ result = model.transcribe(json_data["file"], language="en")
+ print("Sending response")
+ self.send_response(200)
+ self.end_headers()
+ json_response = { "response" : result["text"] }
+ json_response_text = simplejson.dumps(json_response)
+ self.wfile.write(json_response_text.encode("utf-8"))
+ return
+
+
+try:
+ #Create a web server and define the handler to manage the
+ #incoming request
+ server = HTTPServer(('', PORT_NUMBER), myHandler)
+ print('Started httpserver on port ' , PORT_NUMBER)
+
+ #Load the AI model for the Whisker ASR
+ model = whisper.load_model("turbo")
+
+ #Wait forever for incoming htto requests
+ server.serve_forever()
+
+except KeyboardInterrupt:
+ print('^C received, shutting down the web server')
+ server.socket.close()
+
+
diff --git a/aimodel/chromadb.cpp b/aimodel/chromadb.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..22ee94f136145cb27352b3884549ee8131d1a273
--- /dev/null
+++ b/aimodel/chromadb.cpp
@@ -0,0 +1,140 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#include "chromadb.h"
+#include <QJsonArray>
+#include <QJsonDocument>
+#include <QJsonObject>
+#include <QRestReply>
+#include <QCryptographicHash>
+#include <QTimer>
+
+using namespace Qt::StringLiterals;
+
+
+void ChromaDb::sendRequest(
+ QString url,
+ QJsonObject object,
+ std::function<void(QJsonDocument, int)> lambda)
+{
+ QNetworkRequest request(url);
+ request.setRawHeader("Content-Type", "application/json");
+ qDebug() << object["ids"] << object["documents"];
+ m_restApi.post(request, QJsonDocument(object).toJson(), this, [=](QRestReply &reply) {
+ std::optional<QJsonDocument> json = reply.readJson();
+ if (reply.isHttpStatusSuccess()) {
+ lambda(json ? json.value() : QJsonDocument(), reply.httpStatus());
+ } else {
+ qDebug() << "JSON decode error:" << request.url() << "HTTP status:" << reply.httpStatus();
+ setError(true);
+ }
+ });
+}
+
+
+ChromaDb::ChromaDb(QObject *parent)
+ : QObject{parent},
+ m_manager(this)
+ , m_restApi(&m_manager)
+{
+
+}
+
+bool ChromaDb::connected() const
+{
+ return m_connected;
+}
+
+void ChromaDb::connect(bool c)
+{
+ if (connected()) {
+ qDebug() << "Already connected";
+ } else {
+ sendRequest(m_chromadb_url_base + "collections",
+ QJsonObject({{"name", "embeddings_db"}, {"get_or_create", true}}),
+ [this](auto json, int httpCode) {
+ qDebug() << "Using " << json.object()["id"] << "as chromadb collection";
+ m_chromadb_url_collection = u"collections/%1/"_s.arg(json.object()["id"].toString())
+ .toLatin1();
+ m_connected = (httpCode == 200);
+ setError(httpCode != 200);
+ emit connectedChanged();
+ });
+ }
+}
+
+void ChromaDb::reset()
+{
+ QUrl url(m_chromadb_url_base + "collections/embeddings_db");
+ QNetworkRequest request(url);
+ m_restApi.deleteResource(request, this, [=](QRestReply &reply) {
+ if (reply.isHttpStatusSuccess()) {
+ if (m_connected) {
+ m_connected = false;
+ connect(true);
+ }
+ } else {
+ qDebug() << url << "deleted";
+ }
+
+ });
+}
+
+void ChromaDb::storeEmbeddings(QVariantList embeddings, QVariantList documents)
+{
+ if (connected()) {
+ int i = 0;
+ QStringList hashArray;
+ for (auto &&doc : documents) {
+ hashArray.append(QCryptographicHash::hash(doc.toString().toUtf8(), QCryptographicHash::Md5));
+ }
+ sendRequest(m_chromadb_url_base + m_chromadb_url_collection + "upsert",
+ QJsonObject({{"embeddings", QJsonArray::fromVariantList(embeddings)},
+ {"ids", QJsonArray::fromStringList(hashArray)},
+ {"documents", QJsonArray::fromVariantList(documents)}}),
+ [](auto json, int httpCode) {
+ qDebug() << "Embeddings stored:" << json << httpCode;
+ });
+ i++;
+ } else if(error()) {
+ qDebug() << "ChromaDb not supported";
+ } else {
+ qDebug() << "Not connected. Store embeddings later";
+ QTimer::singleShot(100, this, [=]() { storeEmbeddings(embeddings, documents); });
+ }
+
+}
+
+void ChromaDb::fetchEmbeddings(QVariantList hints)
+{
+ if (connected()) {
+ sendRequest(m_chromadb_url_base + m_chromadb_url_collection + "query",
+ QJsonObject({{"query_embeddings", QJsonArray::fromVariantList(hints)}, {"n_results", 3}}),
+ [this](auto json, int httpCode) {
+ qDebug() << json;
+ QVariantList embeddings;
+ if (httpCode == 200)
+ embeddings = json.object()["documents"].toArray().toVariantList();
+ emit embeddingsFound(embeddings);
+ });
+
+ } else if(error()) {
+ qDebug() << "ChromaDb not supported";
+ } else {
+ qDebug() << "Not connected. Fetch embeddings later";
+ QTimer::singleShot(100, this, [=]() { fetchEmbeddings(hints); });
+ }
+}
+
+bool ChromaDb::error() const
+{
+ return m_error;
+}
+
+void ChromaDb::setError(bool newError)
+{
+ if (m_error == newError)
+ return;
+ m_error = newError;
+ emit errorChanged();
+}
diff --git a/aimodel/chromadb.h b/aimodel/chromadb.h
new file mode 100644
index 0000000000000000000000000000000000000000..11b29068d527fdcbe226319cfba671df5aa14ce3
--- /dev/null
+++ b/aimodel/chromadb.h
@@ -0,0 +1,50 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#ifndef CHROMADB_H
+#define CHROMADB_H
+
+#include <QObject>
+#include <QRestAccessManager>
+
+class ChromaDb : public QObject
+{
+ Q_OBJECT
+ Q_PROPERTY(bool connected READ connected WRITE connect NOTIFY connectedChanged FINAL)
+ Q_PROPERTY(bool error READ error WRITE setError NOTIFY errorChanged FINAL)
+public:
+ explicit ChromaDb(QObject *parent = nullptr);
+
+ bool connected() const;
+ void connect(bool c);
+ Q_INVOKABLE void reset();
+
+ bool error() const;
+ void setError(bool newError);
+
+public Q_SLOTS:
+ void storeEmbeddings(QVariantList embeddings, QVariantList documents);
+ void fetchEmbeddings(QVariantList hints);
+
+signals:
+ void connectedChanged();
+ void embeddingsStored();
+ void embeddingsFound(QVariantList embeddings);
+
+ void errorChanged();
+
+private:
+ void sendRequest(
+ QString url,
+ QJsonObject object,
+ std::function<void(QJsonDocument, int)> lambda);
+private:
+ bool m_error {false};
+ bool m_connected {false};
+ QNetworkAccessManager m_manager;
+ QRestAccessManager m_restApi;
+ const QByteArray m_chromadb_url_base{"http://localhost:8000/api/v1/"};
+ QByteArray m_chromadb_url_collection{};
+};
+
+#endif // CHROMADB_H
diff --git a/aimodel/plugins/CMakeLists.txt b/aimodel/plugins/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0cf7b152013d91a73c116e49d995abb865cdc5fa
--- /dev/null
+++ b/aimodel/plugins/CMakeLists.txt
@@ -0,0 +1,6 @@
+
+add_subdirectory(asr)
+add_subdirectory(ollama)
+add_subdirectory(triton)
+add_subdirectory(yolo)
+add_subdirectory(piper-tts)
diff --git a/aimodel/plugins/asr/CMakeLists.txt b/aimodel/plugins/asr/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9c19a88be73c79ce86dcab3893ee6045e9bafed3
--- /dev/null
+++ b/aimodel/plugins/asr/CMakeLists.txt
@@ -0,0 +1,16 @@
+find_package(Qt6 REQUIRED COMPONENTS Core Network Quick)
+
+qt_add_plugin(QtAsrModel
+ CLASS_NAME QAiModelPluginFactory
+ qasraimodel_p.h qasraimodel_p.cpp
+ )
+set_target_properties(QtAsrModel PROPERTIES
+ LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/plugins"
+)
+target_link_libraries(QtAsrModel
+ PRIVATE
+ Qt6::Core
+ Qt6::Network
+ Qt6::Quick
+ QtAiModelPluginInterface)
+include_directories(../..)
diff --git a/aimodel/plugins/asr/plugin.json b/aimodel/plugins/asr/plugin.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ee770770751cb929683459fce055670aec2835f
--- /dev/null
+++ b/aimodel/plugins/asr/plugin.json
@@ -0,0 +1,3 @@
+{ "name": "asrplugin",
+ "supportedTypes": ["InputAudio", "OutputText"]
+}
diff --git a/aimodel/plugins/asr/qasraimodel_p.cpp b/aimodel/plugins/asr/qasraimodel_p.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a5f835cb91c861e1ebfd0f4d5fb6be470f2e4d08
--- /dev/null
+++ b/aimodel/plugins/asr/qasraimodel_p.cpp
@@ -0,0 +1,41 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#include "qaimodel.h"
+#include "qasraimodel_p.h"
+#include <QJsonDocument>
+#include <QJsonObject>
+#include <QNetworkReply>
+#include <QRestReply>
+
+QAsrAiModel::QAsrAiModel()
+ : AiModelPrivateInterface(),
+ m_manager(this)
+ , m_restApi(&m_manager)
+{
+}
+
+void QAsrAiModel::pushData(QVariantList data)
+{
+ qDebug() << "QAsrAiModel::pushData(): data:" << data;
+
+ if (data.isEmpty() || data.first().toUrl().isEmpty()) {
+ emit dataReceived(data.first().toUrl());
+ return;
+ }
+
+ QNetworkRequest request(QUrl("http://localhost:8003/send"));
+ request.setRawHeader("Content-Type", "application/json");
+ QJsonDocument doc;
+ QJsonObject obj = doc.object();
+ obj["model"] = m_owner->model();
+ obj["file"] = data.first().toUrl().path();
+ //obj["stream"] = false;
+ doc.setObject(obj);
+ m_restApi.post(request, doc.toJson(), this, [this](QRestReply &reply) {
+ if (auto json = reply.readJson()) {
+ qDebug() << "[\"response\"]=" << json->object()["response"].toString();
+ emit dataReceived(json->object()["response"].toString().toUtf8());
+ }
+ });
+}
diff --git a/aimodel/plugins/asr/qasraimodel_p.h b/aimodel/plugins/asr/qasraimodel_p.h
new file mode 100644
index 0000000000000000000000000000000000000000..44a57087569378bcf61ee3d73da9fc8209f70909
--- /dev/null
+++ b/aimodel/plugins/asr/qasraimodel_p.h
@@ -0,0 +1,33 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#ifndef QASRAIMODEL_P_H
+#define QASRAIMODEL_P_H
+
+#include <QObject>
+#include <QRestAccessManager>
+#include "qaimodelinterface_p.h"
+
+class QAsrAiModel : public AiModelPrivateInterface
+{
+ Q_OBJECT
+public:
+ QAsrAiModel();
+ void pushData(QVariantList data) override;
+
+private:
+ QNetworkAccessManager m_manager;
+ QRestAccessManager m_restApi;
+};
+
+class QAsrAiModelPlugin : public QAiModelPluginFactory
+{
+ Q_OBJECT
+ Q_PLUGIN_METADATA(IID "org.qt-project.Qt.QAiModelPluginFactory/1.0" FILE "plugin.json")
+ Q_INTERFACES(QAiModelPluginFactory)
+public:
+ QAsrAiModelPlugin() {}
+ AiModelPrivateInterface* createInterface() { return new QAsrAiModel(); }
+};
+
+#endif // QASRAIMODEL_P_H
diff --git a/aimodel/plugins/ollama/CMakeLists.txt b/aimodel/plugins/ollama/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..28a83e669c4166f68d267abf83fbf122550c7371
--- /dev/null
+++ b/aimodel/plugins/ollama/CMakeLists.txt
@@ -0,0 +1,17 @@
+find_package(Qt6 REQUIRED COMPONENTS Core Network Quick)
+
+qt_add_plugin(QtOllamaModel
+ CLASS_NAME QAiModelPluginFactory
+ qllmaimodel_p.h qllmaimodel_p.cpp
+ )
+
+set_target_properties(QtOllamaModel PROPERTIES
+ LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/plugins"
+)
+target_link_libraries(QtOllamaModel
+ PRIVATE
+ Qt6::Core
+ Qt6::Network
+ Qt6::Quick
+ QtAiModelPluginInterface)
+include_directories(../..)
diff --git a/aimodel/plugins/ollama/plugin.json b/aimodel/plugins/ollama/plugin.json
new file mode 100644
index 0000000000000000000000000000000000000000..3da918d83a893acef153d232bce435bf75f62414
--- /dev/null
+++ b/aimodel/plugins/ollama/plugin.json
@@ -0,0 +1,3 @@
+{ "name": "ollamaplugin",
+ "supportedTypes": ["InputImage", "InputText", "OutputText"]
+}
diff --git a/aimodel/plugins/ollama/qllmaimodel_p.cpp b/aimodel/plugins/ollama/qllmaimodel_p.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f1a57266fa4d3d75087af0a0f3735924be19528d
--- /dev/null
+++ b/aimodel/plugins/ollama/qllmaimodel_p.cpp
@@ -0,0 +1,127 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#include "qaimodel.h"
+#include "qllmaimodel_p.h"
+#include <QJsonArray>
+#include <QJsonDocument>
+#include <QJsonObject>
+#include <QNetworkReply>
+#include <QNetworkRequest>
+#include <QRestReply>
+#include <QString>
+#include <QFile>
+
+using namespace Qt::StringLiterals;
+
+QLlmAiModel::QLlmAiModel()
+ : AiModelPrivateInterface(),
+ m_manager(this)
+ , m_restApi(&m_manager)
+{
+ m_chromadb.reset();
+ m_chromadb.connect(true);
+ connect(this, &QLlmAiModel::embeddingsReceived,
+ &m_chromadb, &ChromaDb::storeEmbeddings);
+}
+
+static inline void sendRequest(
+ QRestAccessManager *restApi,
+ QString url,
+ QJsonObject object,
+ QLlmAiModel *owner,
+ std::function<void(QJsonDocument)> lambda)
+{
+ QNetworkRequest request(url);
+ request.setRawHeader("Content-Type", "application/json");
+ restApi->post(request, QJsonDocument(object).toJson(), owner, [=](QRestReply &reply) {
+ if (std::optional<QJsonDocument> json = reply.readJson()) {
+ lambda(json.value());
+ } else {
+ qDebug() << "Error. No data received from" << request.url() << reply;
+ }
+ });
+}
+
+void QLlmAiModel::pushData(QVariantList data)
+{
+ QString query = m_owner->prompt();
+ QJsonArray images;
+ for (auto &&i : data) {
+ query.append('\n');
+ query.append(QString::fromLatin1(i.toByteArray()));
+
+ if (i.canConvert<QUrl>()) {
+ QFile file(QUrl(i.toUrl()).path());
+ if (file.open(QIODevice::ReadOnly) != 0) {
+ QByteArray ba = file.readAll();
+ QByteArray ba2 = ba.toBase64();
+ QString s = QString::fromLatin1(ba2);
+ images.append(s);
+ }
+ }
+ }
+ qDebug() << this << "[\"prompt\"]: " << query << "[images]" << images.count();
+
+
+
+
+ auto promptResponseReceived = [=](auto json) {
+ emit dataReceived(json.object()["response"].toString().toUtf8());
+ };
+
+ if (m_chromadb.connected()) {
+ connect(&m_chromadb, &ChromaDb::embeddingsFound, this, [=](auto embeddings) {
+ QString documents;
+ for (auto &&i : embeddings) {
+ for (auto &&j : i.toList()) {
+ documents.append(j.toString() + '\n');
+ }
+ }
+ auto q = QString("Using this data: {%1}. Respond to this prompt: %2")
+ .arg(documents, query);
+ qDebug() << q;
+ sendRequest(&m_restApi,
+ m_ollama_url_base + "generate",
+ QJsonObject({{"model", m_owner->model()}, {"prompt", q}, {"stream", false}}),
+ this,
+ promptResponseReceived);
+ }, Qt::SingleShotConnection);
+
+ sendRequest(&m_restApi,
+ m_ollama_url_base + "embed",
+ QJsonObject({{"model", m_owner->model()}, {"input", query}}),
+ this,
+ [this](auto json) {
+ m_chromadb.fetchEmbeddings(json.object()["embeddings"].toArray().toVariantList());
+ });
+ } else {
+ sendRequest(&m_restApi,
+ m_ollama_url_base + "generate",
+ QJsonObject({{"model", m_owner->model()},
+ {"prompt", query},
+ {"stream", false},
+ {"images", images}}),
+ this,
+ promptResponseReceived);
+ }
+
+}
+void QLlmAiModel::setRag(QVariantList documentsToBeStored)
+{
+ if (documentsToBeStored.isEmpty())
+ return;
+
+ auto onEmbeddingsReceived = [=](auto json) {
+ auto arrayOfEmbeddings(json.object()["embeddings"].toArray().toVariantList());
+ emit embeddingsReceived(arrayOfEmbeddings, documentsToBeStored);
+ };
+
+ sendRequest(&m_restApi,
+ m_ollama_url_base + "embed",
+ QJsonObject({{"model", m_owner->model()},
+ {"input", QJsonArray::fromVariantList(documentsToBeStored)}}),
+ this,
+ onEmbeddingsReceived);
+
+}
diff --git a/aimodel/plugins/ollama/qllmaimodel_p.h b/aimodel/plugins/ollama/qllmaimodel_p.h
new file mode 100644
index 0000000000000000000000000000000000000000..e70180f17ae9abf8e9d138b5c0ca8504e8bf20e4
--- /dev/null
+++ b/aimodel/plugins/ollama/qllmaimodel_p.h
@@ -0,0 +1,38 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#ifndef QLLMAIMODEL_P_H
+#define QLLMAIMODEL_P_H
+
+#include <QObject>
+#include <QRestAccessManager>
+#include "qaimodelinterface_p.h"
+#include <chromadb.h>
+
+class QLlmAiModel : public AiModelPrivateInterface
+{
+ Q_OBJECT
+public:
+ QLlmAiModel();
+ void pushData(QVariantList data) override;
+ void setRag(QVariantList data) override;
+
+private:
+ ChromaDb m_chromadb;
+ QNetworkAccessManager m_manager;
+ QRestAccessManager m_restApi;
+ const QByteArray m_ollama_url_base{"http://localhost:11434/api/"};
+};
+
+
+class QLlmAiModelPlugin : public QAiModelPluginFactory
+{
+ Q_OBJECT
+ Q_PLUGIN_METADATA(IID "org.qt-project.Qt.QAiModelPluginFactory/1.0" FILE "plugin.json")
+ Q_INTERFACES(QAiModelPluginFactory)
+public:
+ QLlmAiModelPlugin() {}
+ AiModelPrivateInterface* createInterface() { return new QLlmAiModel(); }
+};
+
+#endif // QLLMAIMODEL_P_H
diff --git a/aimodel/plugins/piper-tts/CMakeLists.txt b/aimodel/plugins/piper-tts/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..651ef5be7c65d74728423dced40c6e23f78d5e0e
--- /dev/null
+++ b/aimodel/plugins/piper-tts/CMakeLists.txt
@@ -0,0 +1,17 @@
+find_package(Qt6 REQUIRED COMPONENTS Core Quick Multimedia)
+
+qt_add_plugin(QtPiperModel
+ CLASS_NAME QAiModelPluginFactory
+ qpiperaimodel_p.h qpiperaimodel_p.cpp
+ )
+
+set_target_properties(QtPiperModel PROPERTIES
+ LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/plugins"
+)
+target_link_libraries(QtPiperModel
+ PRIVATE
+ Qt6::Core
+ Qt6::Multimedia
+ Qt6::Quick
+ QtAiModelPluginInterface)
+include_directories(../..)
diff --git a/aimodel/plugins/piper-tts/plugin.json b/aimodel/plugins/piper-tts/plugin.json
new file mode 100644
index 0000000000000000000000000000000000000000..bb84b6f60d14db972274ccbcee444f96e56ffb26
--- /dev/null
+++ b/aimodel/plugins/piper-tts/plugin.json
@@ -0,0 +1,3 @@
+{ "name": "ttsplugin",
+ "supportedTypes": ["InputText", "OutputAudio"]
+}
diff --git a/aimodel/plugins/piper-tts/qpiperaimodel_p.cpp b/aimodel/plugins/piper-tts/qpiperaimodel_p.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..107f800324ebfb62da2ecfa4be62696d51493d37
--- /dev/null
+++ b/aimodel/plugins/piper-tts/qpiperaimodel_p.cpp
@@ -0,0 +1,50 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#include "qpiperaimodel_p.h"
+#include "qaimodel.h"
+#include <QJsonDocument>
+#include <QJsonObject>
+#include <QNetworkReply>
+#include <QRestReply>
+#include <QFile>
+
+QPiperAiModel::QPiperAiModel()
+ : AiModelPrivateInterface()
+ , m_manager(this)
+ , m_restApi(&m_manager)
+{
+ m_player.setAudioOutput(&m_audioOutput);
+}
+
+void QPiperAiModel::pushData(QVariantList data)
+{
+ qDebug() << "QPiperAiModel::pushData(): data:" << data;
+
+ if (data.isEmpty() || data.first().toByteArray().isEmpty()) {
+ emit dataReceived(data.first().toByteArray());
+ return;
+ }
+
+ QNetworkRequest request(QUrl("http://localhost:5000/send"));
+ request.setRawHeader("Content-Type", "application/json");
+ QJsonDocument doc;
+ QJsonObject obj = doc.object();
+ obj["text"] = QString::fromUtf8(data.first().toByteArray());
+ doc.setObject(obj);
+ m_restApi.post(request, doc.toJson(), this, [this](QRestReply &reply) {
+ if (auto json = reply.readJson()) {
+ //qDebug() << "[\"response\"]=" << json->object()["response"].toString();
+ //emit dataReceived(json->object()["response"].toString().toUtf8());
+ QFile file("test.wav");
+ file.open(QIODevice::WriteOnly);
+ file.write(QByteArray::fromBase64(json->object()["response"].toString().toUtf8()));
+ file.close();
+
+ m_player.stop();
+ m_player.setSource(QUrl::fromLocalFile("test.wav"));
+ m_player.play();
+ }
+ });
+
+}
diff --git a/aimodel/plugins/piper-tts/qpiperaimodel_p.h b/aimodel/plugins/piper-tts/qpiperaimodel_p.h
new file mode 100644
index 0000000000000000000000000000000000000000..5515abb6ceb181671b02cd918db197982ec206f0
--- /dev/null
+++ b/aimodel/plugins/piper-tts/qpiperaimodel_p.h
@@ -0,0 +1,37 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#ifndef QTEXT2SPEEHCAIMODEL_P_H
+#define QTEXT2SPEEHCAIMODEL_P_H
+
+#include <QObject>
+#include <QRestAccessManager>
+#include <QMediaPlayer>
+#include <QAudioOutput>
+#include "qaimodelinterface_p.h"
+
+class QPiperAiModel : public AiModelPrivateInterface
+{
+ Q_OBJECT
+public:
+ QPiperAiModel();
+ void pushData(QVariantList data) override;
+
+private:
+ QNetworkAccessManager m_manager;
+ QRestAccessManager m_restApi;
+ QMediaPlayer m_player;
+ QAudioOutput m_audioOutput;
+};
+
+
+class QLlmAiModelPlugin : public QAiModelPluginFactory
+{
+ Q_OBJECT
+ Q_PLUGIN_METADATA(IID "org.qt-project.Qt.QAiModelPluginFactory/1.0" FILE "plugin.json")
+ Q_INTERFACES(QAiModelPluginFactory)
+public:
+ AiModelPrivateInterface* createInterface() { return new QPiperAiModel(); }
+};
+
+#endif // QTEXT2SPEEHCAIMODEL_P_H
diff --git a/aimodel/plugins/triton/CMakeLists.txt b/aimodel/plugins/triton/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..fb1ca9435935a02d4dee91ecab7d78d7bf550b5d
--- /dev/null
+++ b/aimodel/plugins/triton/CMakeLists.txt
@@ -0,0 +1,20 @@
+find_package(Qt6 REQUIRED COMPONENTS Core Network Quick)
+find_package(OpenCV REQUIRED COMPONENTS core imgproc imgcodecs)
+
+qt_add_plugin(QtTritonModel
+ CLASS_NAME QAiModelPluginFactory
+ qtritonmodel_p.h qtritonmodel_p.cpp
+ )
+
+set_target_properties(QtTritonModel PROPERTIES
+ LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/plugins"
+)
+include_directories(../..)
+target_link_libraries(QtTritonModel
+ PRIVATE
+ Qt6::Core
+ Qt6::Network
+ Qt6::Quick
+ ${OpenCV_LIBS}
+ QtAiModelPluginInterface
+ )
diff --git a/aimodel/plugins/triton/plugin.json b/aimodel/plugins/triton/plugin.json
new file mode 100644
index 0000000000000000000000000000000000000000..456e30590fcea853217d2b90325dbb3000c78995
--- /dev/null
+++ b/aimodel/plugins/triton/plugin.json
@@ -0,0 +1,3 @@
+{ "name": "tritonplugin",
+ "supportedTypes": ["InputImage", "OutputJson"]
+}
diff --git a/aimodel/plugins/triton/qtritonmodel_p.cpp b/aimodel/plugins/triton/qtritonmodel_p.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..bd620bb512b4d2eca40afa7532f245fb77c16224
--- /dev/null
+++ b/aimodel/plugins/triton/qtritonmodel_p.cpp
@@ -0,0 +1,233 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#include "qtritonmodel_p.h"
+#include "qaimodel.h"
+#include <QJsonArray>
+#include <QJsonDocument>
+#include <QJsonObject>
+#include <QNetworkReply>
+#include <QNetworkRequest>
+#include <QRestReply>
+#include <QString>
+#include <QImage>
+#include <QBuffer>
+#include <QPixmap>
+#include <opencv2/core/version.hpp>
+#if CV_MAJOR_VERSION == 2
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+#elif CV_MAJOR_VERSION >= 3
+#include <opencv2/core.hpp>
+#include <opencv2/highgui.hpp>
+#include <opencv2/imgproc.hpp>
+#endif
+#include <iostream>
+#include <fstream>
+
+QTritonModel::QTritonModel()
+ : AiModelPrivateInterface(),
+ m_manager(this)
+ , m_restApi(&m_manager)
+{
+
+}
+
+
+#if CV_MAJOR_VERSION == 4
+#define GET_TRANSFORMATION_CODE(x) cv::COLOR_##x
+#else
+#define GET_TRANSFORMATION_CODE(x) CV_##x
+#endif
+enum ScaleType { NONE = 0, VGG = 1, INCEPTION = 2 };
+
+int
+Preprocess(
+ const cv::Mat& img, const std::string& format, int img_type1, int img_type3,
+ size_t img_channels, const cv::Size& img_size, const ScaleType scale,
+ std::vector<uint8_t>* input_data)
+{
+ // Image channels are in BGR order. Currently model configuration
+ // data doesn't provide any information as to the expected channel
+ // orderings (like RGB, BGR). We are going to assume that RGB is the
+ // most likely ordering and so change the channels to that ordering.
+
+ cv::Mat sample;
+ if ((img.channels() == 3) && (img_channels == 1)) {
+ cv::cvtColor(img, sample, GET_TRANSFORMATION_CODE(BGR2GRAY));
+ } else if ((img.channels() == 4) && (img_channels == 1)) {
+ cv::cvtColor(img, sample, GET_TRANSFORMATION_CODE(BGRA2GRAY));
+ } else if ((img.channels() == 3) && (img_channels == 3)) {
+ cv::cvtColor(img, sample, GET_TRANSFORMATION_CODE(BGR2RGB));
+ } else if ((img.channels() == 4) && (img_channels == 3)) {
+ cv::cvtColor(img, sample, GET_TRANSFORMATION_CODE(BGRA2RGB));
+ } else if ((img.channels() == 1) && (img_channels == 3)) {
+ cv::cvtColor(img, sample, GET_TRANSFORMATION_CODE(GRAY2RGB));
+ } else {
+ std::cerr << "unexpected number of channels " << img.channels()
+ << " in input image, model expects " << img_channels << "."
+ << std::endl;
+ return -1;
+ }
+
+ cv::Mat sample_resized;
+ if (sample.size() != img_size) {
+ cv::resize(sample, sample_resized, img_size);
+ } else {
+ sample_resized = sample;
+ }
+
+ cv::Mat sample_type;
+ sample_resized.convertTo(
+ sample_type, (img_channels == 3) ? img_type3 : img_type1);
+
+ cv::Mat sample_final;
+ if (scale == ScaleType::INCEPTION) {
+ if (img_channels == 1) {
+ sample_final = sample_type.mul(cv::Scalar(1 / 127.5));
+ sample_final = sample_final - cv::Scalar(1.0);
+ } else {
+ sample_final =
+ sample_type.mul(cv::Scalar(1 / 127.5, 1 / 127.5, 1 / 127.5));
+ sample_final = sample_final - cv::Scalar(1.0, 1.0, 1.0);
+ }
+ } else if (scale == ScaleType::VGG) {
+ if (img_channels == 1) {
+ sample_final = sample_type - cv::Scalar(128);
+ } else {
+ sample_final = sample_type - cv::Scalar(123, 117, 104);
+ }
+ } else {
+ sample_final = sample_type;
+ }
+
+ // Allocate a buffer to hold all image elements.
+ size_t img_byte_size = sample_final.total() * sample_final.elemSize();
+ size_t pos = 0;
+ input_data->resize(img_byte_size);
+
+ // For NHWC format Mat is already in the correct order but need to
+ // handle both cases of data being contiguous or not.
+ if (format.compare("FORMAT_NHWC") == 0) {
+ if (sample_final.isContinuous()) {
+ memcpy(&((*input_data)[0]), sample_final.datastart, img_byte_size);
+ pos = img_byte_size;
+ } else {
+ size_t row_byte_size = sample_final.cols * sample_final.elemSize();
+ for (int r = 0; r < sample_final.rows; ++r) {
+ memcpy(
+ &((*input_data)[pos]), sample_final.ptr<uint8_t>(r), row_byte_size);
+ pos += row_byte_size;
+ }
+ }
+ } else {
+ // (format.compare("FORMAT_NCHW") == 0)
+ //
+ // For CHW formats must split out each channel from the matrix and
+ // order them as BBBB...GGGG...RRRR. To do this split the channels
+ // of the image directly into 'input_data'. The BGR channels are
+ // backed by the 'input_data' vector so that ends up with CHW
+ // order of the data.
+ std::vector<cv::Mat> input_bgr_channels;
+ for (size_t i = 0; i < img_channels; ++i) {
+ input_bgr_channels.emplace_back(
+ img_size.height, img_size.width, img_type1, &((*input_data)[pos]));
+
+ pos += input_bgr_channels.back().total() *
+ input_bgr_channels.back().elemSize();
+ }
+
+ cv::split(sample_final, input_bgr_channels);
+ }
+
+ if (pos != img_byte_size) {
+ std::cerr << "unexpected total size of channels " << pos << ", expecting "
+ << img_byte_size << std::endl;
+ return -1;
+ }
+ return 0;
+}
+
+
+// KServe (Open Inference Protocol API)
+void QTritonModel::pushData(QVariantList data)
+{
+ // Load the specified image.
+ std::ifstream file(data.first().toByteArray().toStdString());
+ std::vector<char> data2;
+ file >> std::noskipws;
+ std::copy(
+ std::istream_iterator<char>(file), std::istream_iterator<char>(),
+ std::back_inserter(data2));
+ if (data2.empty()) {
+ qDebug() << "error: unable to read image file " << data;
+ return;
+ }
+
+ cv::Mat img = imdecode(cv::Mat(data2), 1);
+ const cv::Size cvSize(416, 416);
+ std::vector<uint8_t> convertedImage;
+ Preprocess(img, "FORMAT_NCHW", CV_32FC1, CV_32FC3, 3, cvSize, INCEPTION, &convertedImage);
+
+ QNetworkRequest request(QUrl("http://localhost:8000/v2/models/" + m_owner->model() + "/infer"));
+ //request.setRawHeader("Content-Type", "application/json");
+ request.setRawHeader("Content-Type", "application/octet-stream");
+ QJsonDocument doc;
+ QJsonObject input;
+ input["name"] = "input";
+ QJsonObject inputParameters;
+ //inputParameters["binary_data"] = false;
+ inputParameters["binary_data_size"] = static_cast<int>(convertedImage.size());
+ input["shape"] = QJsonArray({1, 3, 416, 416});
+ input["datatype"] = "FP32";
+ input["parameters"] = inputParameters;
+ QJsonObject output;
+ output["name"] = "confs";
+ QJsonObject outputParameters;
+ outputParameters["classification"] = 1;
+ outputParameters["binary_data"] = false;
+ output["parameters"] = outputParameters;
+ QJsonArray imageAsJson;
+
+#if 0
+ //QByteArrayView view(image.constBits(), image.sizeInBytes());
+ for (int i = 0; i < image.sizeInBytes() /3; i++) {
+ //imageAsJson.append(((double)view[(i * 3) + 0] * (1 / 127.5)) - 1.0);
+ imageAsJson.append(((float)view[(i * 3) + 0] / 255.0));
+ }
+ qDebug() << imageAsJson.first() << imageAsJson.last();
+ for (int i = 0; i < image.sizeInBytes() /3; i++) {
+ //imageAsJson.append(((double)view[(i * 3) + 1] * (1.0 / 127.5)) + 1.0);
+ imageAsJson.append(((float)view[(i * 3) + 1] / 255.0));
+ }
+ for (int i = 0; i < image.sizeInBytes() /3; i++) {
+ //imageAsJson.append(((double)view[(i * 3) + 2] * (1.0 / 127.5)) + 1.0);
+ imageAsJson.append(((float)view[(i * 3) + 2] / 255.0));
+ }
+ input["data"] = imageAsJson;
+#endif
+
+ QJsonObject obj = doc.object();
+ obj["id"] = "0";
+ obj["model"] = m_owner->model();
+ obj["model_version"] = "1";
+ obj["inputs"] = QJsonArray({input});
+ obj["outputs"] = QJsonArray({output});
+ doc.setObject(obj);
+ auto docAsJson = doc.toJson(QJsonDocument::Compact);
+ request.setRawHeader("Inference-Header-Content-Length", QByteArray::number(docAsJson.size()));
+ docAsJson.append(convertedImage);
+
+ m_restApi.post(request, docAsJson, this, [this](QRestReply &reply) {
+ QJsonParseError jsonError;
+ if (auto json = reply.readJson(&jsonError)) {
+ qDebug() << json;
+ emit dataReceived(json->toJson(QJsonDocument::Compact));
+ }
+ else
+ {
+ qDebug() << reply << jsonError.errorString();
+ }
+ });
+}
diff --git a/aimodel/plugins/triton/qtritonmodel_p.h b/aimodel/plugins/triton/qtritonmodel_p.h
new file mode 100644
index 0000000000000000000000000000000000000000..04f0127775cb0e87892ed989f9de2e60d5ae85c4
--- /dev/null
+++ b/aimodel/plugins/triton/qtritonmodel_p.h
@@ -0,0 +1,34 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#ifndef QTRITONMODEL_P_H
+#define QTRITONMODEL_P_H
+
+#include <QObject>
+#include <QRestAccessManager>
+#include "qaimodelinterface_p.h"
+
+class QTritonModel : public AiModelPrivateInterface
+{
+ Q_OBJECT
+public:
+ QTritonModel();
+ void pushData(QVariantList data) override;
+
+private:
+
+ QNetworkAccessManager m_manager;
+ QRestAccessManager m_restApi;
+};
+
+
+class QTritonAiModelPlugin : public QAiModelPluginFactory
+{
+ Q_OBJECT
+ Q_PLUGIN_METADATA(IID "org.qt-project.Qt.QAiModelPluginFactory/1.0" FILE "plugin.json")
+ Q_INTERFACES(QAiModelPluginFactory)
+public:
+ AiModelPrivateInterface* createInterface() { return new QTritonModel(); }
+};
+
+#endif // QTRITONMODEL_P_H
diff --git a/aimodel/plugins/tts/CMakeLists.txt b/aimodel/plugins/tts/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..92dcf3b25c51bf2e43c5adcb42e33d8ca6c505fd
--- /dev/null
+++ b/aimodel/plugins/tts/CMakeLists.txt
@@ -0,0 +1,17 @@
+find_package(Qt6 REQUIRED COMPONENTS Core Quick TextToSpeech)
+
+qt_add_plugin(QtTtsModel
+ CLASS_NAME QAiModelPluginFactory
+ qtext2speechaimodel_p.h qtext2speechaimodel_p.cpp
+ )
+
+set_target_properties(QtTtsModel PROPERTIES
+ LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/plugins"
+)
+target_link_libraries(QtTtsModel
+ PRIVATE
+ Qt6::Core
+ Qt6::TextToSpeech
+ Qt6::Quick
+ QtAiModelPluginInterface)
+include_directories(../..)
diff --git a/aimodel/plugins/tts/plugin.json b/aimodel/plugins/tts/plugin.json
new file mode 100644
index 0000000000000000000000000000000000000000..bb84b6f60d14db972274ccbcee444f96e56ffb26
--- /dev/null
+++ b/aimodel/plugins/tts/plugin.json
@@ -0,0 +1,3 @@
+{ "name": "ttsplugin",
+ "supportedTypes": ["InputText", "OutputAudio"]
+}
diff --git a/aimodel/plugins/tts/qtext2speechaimodel_p.cpp b/aimodel/plugins/tts/qtext2speechaimodel_p.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..da1890861f6dd96b0fbbde59f07d67f5e7030eab
--- /dev/null
+++ b/aimodel/plugins/tts/qtext2speechaimodel_p.cpp
@@ -0,0 +1,28 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#include <QTextToSpeech>
+#include <QTimer>
+#include "qtext2speechaimodel_p.h"
+
+QText2SpeechAiModel::QText2SpeechAiModel()
+ : AiModelPrivateInterface()
+{
+ m_speech.reset(new QTextToSpeech(this));
+ m_speech->setRate(0.0);
+ m_speech->setPitch(0.0);
+ m_speech->setLocale(QLocale(QLocale::English, QLocale::UnitedKingdom));
+ connect(m_speech.get(), &QTextToSpeech::stateChanged, [=](QTextToSpeech::State state) {
+ if (state == QTextToSpeech::Ready)
+ emit dataReceived("");
+ });
+}
+
+void QText2SpeechAiModel::pushData(QVariantList data)
+{
+ m_speech->stop();
+
+ if (!data.first().toByteArray().isEmpty()) {
+ m_speech->say(QString::fromUtf8(data.first().toByteArray()));
+ }
+}
diff --git a/aimodel/plugins/tts/qtext2speechaimodel_p.h b/aimodel/plugins/tts/qtext2speechaimodel_p.h
new file mode 100644
index 0000000000000000000000000000000000000000..2666fa89c28a46179a6928de8febc1c0641b0cfb
--- /dev/null
+++ b/aimodel/plugins/tts/qtext2speechaimodel_p.h
@@ -0,0 +1,34 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#ifndef QTEXT2SPEEHCAIMODEL_P_H
+#define QTEXT2SPEEHCAIMODEL_P_H
+
+#include <QObject>
+#include <QSharedPointer>
+#include "qaimodelinterface_p.h"
+
+class QTextToSpeech;
+
+class QText2SpeechAiModel : public AiModelPrivateInterface
+{
+ Q_OBJECT
+public:
+ QText2SpeechAiModel();
+ void pushData(QVariantList data) override;
+
+ QSharedPointer<QTextToSpeech> m_speech;
+};
+
+
+class QText2SpeechAiModelPlugin : public QAiModelPluginFactory
+{
+ Q_OBJECT
+ Q_PLUGIN_METADATA(IID "org.qt-project.Qt.QAiModelPluginFactory/1.0" FILE "plugin.json")
+ Q_INTERFACES(QAiModelPluginFactory)
+public:
+ AiModelPrivateInterface* createInterface() { return new QText2SpeechAiModel(); }
+};
+
+
+#endif // QTEXT2SPEEHCAIMODEL_P_H
diff --git a/aimodel/plugins/yolo/CMakeLists.txt b/aimodel/plugins/yolo/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..039fe5f59deaa457b8bde3c581a860f105c386ce
--- /dev/null
+++ b/aimodel/plugins/yolo/CMakeLists.txt
@@ -0,0 +1,18 @@
+find_package(Qt6 REQUIRED COMPONENTS Core Network Quick Multimedia)
+
+qt_add_plugin(QtYoloModel
+ CLASS_NAME QAiModelPluginFactory
+ qyoloaimodel.h qyoloaimodel.cpp
+ )
+
+set_target_properties(QtYoloModel PROPERTIES
+ LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/plugins"
+)
+target_link_libraries(QtYoloModel
+ PRIVATE
+ Qt6::Core
+ Qt6::Network
+ Qt6::Quick
+ Qt6::Multimedia
+ QtAiModelPluginInterface)
+include_directories(../..)
diff --git a/aimodel/plugins/yolo/plugin.json b/aimodel/plugins/yolo/plugin.json
new file mode 100644
index 0000000000000000000000000000000000000000..35bf07666dec0ea0e9bc4f077153f1522f10abed
--- /dev/null
+++ b/aimodel/plugins/yolo/plugin.json
@@ -0,0 +1,3 @@
+{ "name": "yoloplugin",
+ "supportedTypes": ["AiModelPrivateInterface::InputImage", "OutputJson"]
+}
diff --git a/aimodel/plugins/yolo/qyoloaimodel.cpp b/aimodel/plugins/yolo/qyoloaimodel.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..59b722405be24c2d18098655ce175040be37a9a4
--- /dev/null
+++ b/aimodel/plugins/yolo/qyoloaimodel.cpp
@@ -0,0 +1,64 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#include "qyoloaimodel.h"
+#include "qaimodel.h"
+#include <QJsonDocument>
+#include <QJsonObject>
+#include <QRestReply>
+#include <QImage>
+#include <QBuffer>
+#include <QPixmap>
+#include <QVideoFrame>
+
+QYoloAiModel::QYoloAiModel()
+ : AiModelPrivateInterface()
+ , m_manager(this)
+ , m_restApi(&m_manager)
+{
+
+}
+
+void QYoloAiModel::pushData(QVariantList data)
+{
+ QImage image;
+ if (data.first().canConvert<QImage>()) {
+ image = data.first().value<QImage>();
+ //image = imageTmp.convertToFormat(QImage::Format_RGB888).scaled(140, 80);
+ }else if (data.first().canConvert<QVideoFrame>()) {
+ image = data.first().value<QVideoFrame>().toImage();
+ //image = imageTmp.convertToFormat(QImage::Format_RGB888).scaled(140, 80);
+ } else {
+ qDebug() << "error" << data;
+ emit dataReceived("");
+ return;
+ }
+
+ if (!image.bits()) {
+ qDebug() << "error" << image;
+ emit dataReceived("");
+ return;
+ }
+ QBuffer bufferJpeg;
+ bufferJpeg.open(QIODevice::WriteOnly);
+ QPixmap pix = QPixmap::fromImage(image);
+ pix.save(&bufferJpeg, "JPG");
+ qDebug() << "QYoloAiModel::pushData(): data:" << image << "=> JPG:" << pix;
+
+ QNetworkRequest request(QUrl("http://localhost:8004/send"));
+ request.setRawHeader("Content-Type", "application/json");
+ QJsonDocument doc;
+ QJsonObject obj = doc.object();
+ obj["model"] = m_owner->model();
+ obj["frame"] = QString::fromLatin1(bufferJpeg.buffer().toBase64());
+ doc.setObject(obj);
+ m_restApi.post(request, doc.toJson(), this, [this](QRestReply &reply) {
+ if (auto json = reply.readJson()) {
+ emit dataReceived(json->toJson(QJsonDocument::Compact));
+ }
+ else
+ {
+ qDebug() << reply;
+ }
+ });
+}
diff --git a/aimodel/plugins/yolo/qyoloaimodel.h b/aimodel/plugins/yolo/qyoloaimodel.h
new file mode 100644
index 0000000000000000000000000000000000000000..e32c3f83c7a7896e912c370b8b76e0481a303545
--- /dev/null
+++ b/aimodel/plugins/yolo/qyoloaimodel.h
@@ -0,0 +1,34 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#ifndef QYOLOAIMODEL_H
+#define QYOLOAIMODEL_H
+
+#include <QObject>
+#include <QRestAccessManager>
+#include "qaimodelinterface_p.h"
+
+class QYoloAiModel : public AiModelPrivateInterface
+{
+ Q_OBJECT
+public:
+ QYoloAiModel();
+ void pushData(QVariantList data) override;
+
+private:
+ QNetworkAccessManager m_manager;
+ QRestAccessManager m_restApi;
+ const QByteArray m_yolo_url_base{"http://localhost:11434/api/"};
+};
+
+
+class QYoloAiModelPlugin : public QAiModelPluginFactory
+{
+ Q_OBJECT
+ Q_PLUGIN_METADATA(IID "org.qt-project.Qt.QAiModelPluginFactory/1.0" FILE "plugin.json")
+ Q_INTERFACES(QAiModelPluginFactory)
+public:
+ AiModelPrivateInterface* createInterface() { return new QYoloAiModel(); }
+};
+
+#endif // QYOLOAIMODEL_H
diff --git a/aimodel/qaimodel.cpp b/aimodel/qaimodel.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..411691d94f9bbc72d43279c8cb29e3491d1ec77f
--- /dev/null
+++ b/aimodel/qaimodel.cpp
@@ -0,0 +1,246 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#include "qaimodel.h"
+#include "qaimodelinterface_p.h"
+#include <QPluginLoader>
+#include <QIODevice>
+#include <QDir>
+#include <QJsonArray>
+#include <QList>
+
+QAiModel::QAiModel()
+{
+ qRegisterMetaType<AiModelPrivateInterface::AiModelTypes>();
+}
+
+AiModelPrivateInterface::AiModelTypes QAiModel::type() const
+{
+ return m_type;
+}
+
+static AiModelPrivateInterface::AiModelTypes constructAiModelTypeFlags(QVariantList list)
+{
+ auto&& metaEnum = QMetaEnum::fromType<AiModelPrivateInterface::AiModelType>();
+ AiModelPrivateInterface::AiModelTypes ret;
+ for (auto &&flagAsVariant : list) {
+ auto string = flagAsVariant.toString().toLatin1();
+ auto flag = static_cast<AiModelPrivateInterface::AiModelType>(metaEnum.keysToValue(string));
+ ret.setFlag(flag, true);
+ }
+ return ret;
+}
+
+void QAiModel::setType(const AiModelPrivateInterface::AiModelTypes &newType)
+{
+ qDebug() << "QAiModel::setType(newType: " << newType << ")";
+
+ if (type() == newType)
+ return;
+
+ const QObjectList &staticPlugins = QPluginLoader::staticInstances();
+ for (auto *plugin : staticPlugins)
+ qDebug() << "Static plugin: " << plugin;
+
+
+ m_interface.clear();
+ QDir pluginsDir(QDir::currentPath() + "/qt-ai-inference-api/aimodel/plugins");
+ //QDir pluginsDir(QDir::currentPath() + "/aimodel/plugins");
+ qDebug() << "Plugins dir: " << pluginsDir.absolutePath();
+ const auto entryList = pluginsDir.entryList(QDir::Files);
+ for (const QString &fileName : entryList) {
+ qDebug() << "Loading " << fileName << "...";
+ QPluginLoader loader(pluginsDir.absoluteFilePath(fileName));
+ QJsonObject object{ loader.metaData().value("MetaData").toObject() };
+ qDebug() << "Metadata for " << fileName << ": " << object;
+ if (!object.value("supportedTypes").isArray()) {
+ qDebug() << "Incorrect json format in" << loader.metaData()
+ << "for plugin:" << fileName;
+ continue;
+ }
+ auto flagArray = object.value("supportedTypes").toArray().toVariantList();
+ auto pluginFlags = constructAiModelTypeFlags(flagArray);
+
+ qDebug() << pluginFlags;
+ if (pluginFlags.testFlags(newType)) {
+ auto *instance = loader.instance();
+ QAiModelPluginFactory *plugin = qobject_cast<QAiModelPluginFactory*>(instance);
+ if (plugin) {
+ qDebug() << plugin << "created";
+ m_interface.reset(plugin->createInterface());
+ m_interface->init(this);
+ break;
+ } else {
+ qDebug() << "Could not convert" << instance << "to AiModelPrivateInterface*";
+ }
+ }
+
+ }
+
+ if (!m_interface.isNull()) {
+ connect(m_interface.get(),
+ &AiModelPrivateInterface::dataReceived,
+ this,
+ &QAiModel::dataReceived);
+ connect(m_interface.get(),
+ &AiModelPrivateInterface::processingChanged,
+ this,
+ &QAiModel::processingChanged);
+
+ m_type = newType;
+ emit typeChanged();
+ } else {
+ qWarning() << "No plugin found for flags:" << newType;
+ }
+}
+
+bool QAiModel::processing() const
+{
+ return m_processing;
+}
+
+void QAiModel::setPrompt(const QString &newPrompt)
+{
+ if (prompt() == newPrompt)
+ return;
+
+ m_prompt = newPrompt;
+ emit promptChanged();
+}
+
+void QAiModel::setModel(const QString &newModel)
+{
+ if (model() == newModel)
+ return;
+
+ m_model = newModel;
+ emit modelChanged();
+}
+
+QVector<QAiModel*> QAiModel::inputs() const
+{
+ return m_inputs;
+}
+
+void QAiModel::setInputs(
+ QVector<QAiModel*> newInputs)
+{
+ if (inputs() == newInputs)
+ return;
+ m_inputs = newInputs;
+ if (!m_interface.isNull()) {
+ for (auto input: newInputs) {
+ input->m_output = this;
+ }
+ emit inputsChanged();
+ }
+}
+
+void QAiModel::processCombinedData(QVariant data)
+{
+ int dataAvailableCounter = 0;
+ int optionalInputsWithData = 0;
+ QVariantList dataFromAllInputs;
+
+ for (auto &&input : m_inputs) {
+ if (!input->m_buffer.isNull()) {
+ dataAvailableCounter++;
+ dataFromAllInputs.append(input->m_buffer);
+ } else if (input->optional() && !input->processing()) {
+ qDebug() << "Skipping optional input" << input->interface();
+ }
+ if (input->processing()) {
+ qDebug() << input->interface() << "still processing -> wait for result";
+ return;
+ }
+
+ }
+
+ pushData(dataFromAllInputs);
+
+ for (auto &&input : m_inputs) {
+ m_buffer.clear();
+ }
+}
+
+void QAiModel::dataReceived(QVariant data)
+{
+ qDebug() << interface() << ":" << __func__ << "(): data:" << data;
+
+ m_buffer = data;
+ m_processing = false;
+ emit processingChanged();
+ emit gotResult(data);
+
+ if (m_output) {
+ m_output->processCombinedData(data);
+ }
+}
+
+void QAiModel::pushData(QVariant data)
+{
+ if (!m_interface.isNull()) {
+ if (m_processing) {
+ qDebug() << m_interface << "busy. Returning";
+ return;
+ }
+ m_buffer.clear();
+ m_processing = true;
+ emit processingChanged();
+ if (QByteArrayView(data.typeName()) == "QVariantList")
+ interface()->pushData(data.value<QVariantList>());
+ else
+ interface()->pushData({data});
+ }
+}
+
+
+QVariantList QAiModel::rag() const
+{
+ return m_rag;
+}
+
+void QAiModel::setRag(const QVariantList &newRag)
+{
+ if (m_rag == newRag)
+ return;
+ m_rag = newRag;
+ qDebug() << newRag << m_output;
+ if (!m_interface.isNull()) {
+ m_interface->setRag(newRag);
+ }
+
+ emit ragChanged();
+}
+
+
+bool QAiModel::buffered() const
+{
+ return m_buffered;
+}
+
+void QAiModel::setBuffered(bool newBuffered)
+{
+ if (m_buffered == newBuffered)
+ return;
+ m_buffered = newBuffered;
+ emit bufferedChanged();
+}
+
+bool QAiModel::optional() const
+{
+ return m_optional;
+}
+
+void QAiModel::setOptional(bool newOptional)
+{
+ if (m_optional == newOptional)
+ return;
+ m_optional = newOptional;
+ emit optionalChanged();
+}
+
+void QAiModel::clearBuffer()
+{
+ m_buffer.clear();
+}
diff --git a/aimodel/qaimodel.h b/aimodel/qaimodel.h
new file mode 100644
index 0000000000000000000000000000000000000000..620bbc7fd74b63cb77e04799a1e8bdcaecb32cff
--- /dev/null
+++ b/aimodel/qaimodel.h
@@ -0,0 +1,110 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#ifndef QAIMODEL_H
+#define QAIMODEL_H
+
+#include <QObject>
+#include <QQmlEngine>
+#include "qaimodelinterface_p.h"
+
+class AiModelPrivateInterface;
+
+class QAiModel : public QObject
+{
+ Q_OBJECT
+ QML_NAMED_ELEMENT(
+ MultiModal)
+
+ Q_PROPERTY(
+ AiModelPrivateInterface::AiModelTypes type READ type WRITE setType NOTIFY typeChanged FINAL REQUIRED)
+ Q_PROPERTY(
+ QString prompt READ prompt WRITE setPrompt NOTIFY promptChanged FINAL)
+ Q_PROPERTY(
+ QString model READ model WRITE setModel NOTIFY modelChanged FINAL)
+ Q_PROPERTY(
+ QVariantList rag READ rag WRITE setRag NOTIFY ragChanged FINAL)
+ Q_PROPERTY(
+ QVector<QAiModel*> inputs READ inputs WRITE setInputs NOTIFY inputsChanged FINAL)
+ Q_PROPERTY(
+ bool processing READ processing NOTIFY processingChanged FINAL)
+ Q_PROPERTY(
+ bool buffered READ buffered WRITE setBuffered NOTIFY bufferedChanged FINAL)
+ Q_PROPERTY(
+ bool optional READ optional WRITE setOptional NOTIFY optionalChanged FINAL)
+
+public:
+
+ QAiModel();
+
+ AiModelPrivateInterface::AiModelTypes type() const;
+ void setType(const AiModelPrivateInterface::AiModelTypes &newType);
+
+ bool processing() const;
+
+ QString prompt() const { return m_prompt; }
+ void setPrompt(const QString &newPrompt);
+
+ QString model() const { return m_model; }
+ void setModel(const QString &newModel);
+
+ QVector<QAiModel*> inputs() const;
+ void setInputs(QVector<QAiModel*>newInputs);
+
+ QVariantList rag() const;
+ void setRag(const QVariantList &newRag);
+
+ Q_INVOKABLE void pushData(QVariant data);
+ Q_INVOKABLE void clearBuffer();
+
+signals:
+ void typeChanged();
+
+ void promptChanged();
+
+ void modelChanged();
+
+ void inputsChanged();
+
+ void ragChanged();
+
+ void processingChanged();
+
+ void gotResult(QVariant result);
+
+ void bufferedChanged();
+
+ void optionalChanged();
+
+private Q_SLOTS:
+ void dataReceived(QVariant data);
+
+public:
+ AiModelPrivateInterface* interface() const { return m_interface.get(); }
+
+ bool buffered() const;
+ void setBuffered(bool newBuffered);
+
+ bool optional() const;
+ void setOptional(bool newOptional);
+
+private:
+ void processCombinedData(QVariant data);
+
+private:
+ QString m_model{};
+ QString m_prompt{};
+ QVector<QAiModel*> m_inputs;
+ QString m_inputModel{};
+ QAiModel* m_output{nullptr};
+ QSharedPointer<AiModelPrivateInterface> m_interface;
+ QVariantList m_rag;
+ AiModelPrivateInterface::AiModelTypes m_type;
+ QVariant m_buffer; // TODO: replace with QVariant
+ bool m_processing {false};
+ bool m_buffered {false};
+ bool m_optional {false};
+};
+
+
+#endif // QAIMODEL_H
diff --git a/aimodel/qaimodelinterface_p.h b/aimodel/qaimodelinterface_p.h
new file mode 100644
index 0000000000000000000000000000000000000000..177cc11ffb4e462ad97f2b33d0713bd989db9173
--- /dev/null
+++ b/aimodel/qaimodelinterface_p.h
@@ -0,0 +1,62 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#ifndef QAIMODELINTERFACE_P_H
+#define QAIMODELINTERFACE_P_H
+
+#include <QObject>
+#include <QVariant>
+
+class QAiModel;
+
+class AiModelPrivateInterface : public QObject
+{
+ Q_OBJECT
+public:
+ enum AiModelType {
+ InputText = 0x00001,
+ InputAudio = 0x00002,
+ InputVideo = 0x00004,
+ InputImage = 0x00008,
+ InputJson = 0x00010,
+
+ OutputText = 0x00100,
+ OutputAudio = 0x00200,
+ OutputVideo = 0x00400,
+ OutputImage = 0x00800,
+ OutputJson = 0x01000,
+ };
+ Q_FLAG(AiModelType);
+ Q_DECLARE_FLAGS(AiModelTypes, AiModelType)
+
+ AiModelPrivateInterface() {}
+ void init(QAiModel *owner) { m_owner = owner;}
+ virtual ~AiModelPrivateInterface() {}
+
+ virtual void pushData(QVariantList data) = 0;
+ virtual void setRag(QVariantList data) {}
+
+
+Q_SIGNALS:
+ void dataReceived(QVariant data);
+ void processingChanged();
+ void embeddingsReceived(QVariantList embeddings, QVariantList documents);
+
+public:
+ QAiModel *m_owner;
+};
+Q_DECLARE_OPERATORS_FOR_FLAGS(AiModelPrivateInterface::AiModelTypes)
+
+class QAiModelPluginFactory : public QObject
+{
+ Q_OBJECT
+public:
+ QAiModelPluginFactory() {}
+
+ virtual AiModelPrivateInterface* createInterface() = 0;
+
+};
+#define QAiModelPluginFactory_iid "org.qt-project.Qt.QAiModelFactoryInterface"
+Q_DECLARE_INTERFACE(QAiModelPluginFactory, QAiModelPluginFactory_iid)
+
+#endif // QAIMODELINTERFACE_P_H
diff --git a/aimodel/tts_server/piper_server.py b/aimodel/tts_server/piper_server.py
new file mode 100644
index 0000000000000000000000000000000000000000..97328a6495e11d119d53aabeba5895ee722477e6
--- /dev/null
+++ b/aimodel/tts_server/piper_server.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2025 The Qt Company Ltd.
+# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+import argparse
+import io
+import logging
+import wave
+from pathlib import Path
+from typing import Any, Dict
+
+from flask import Flask, request, jsonify
+import base64,json
+
+from piper import PiperVoice
+from piper.download import ensure_voice_exists, find_voice, get_voices
+
+_LOGGER = logging.getLogger()
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--host", default="localhost", help="HTTP server host")
+ parser.add_argument("--port", type=int, default=5000, help="HTTP server port")
+ #
+ parser.add_argument("-m", "--model", default="./en_US-lessac-high.onnx", help="Path to Onnx model file")
+ parser.add_argument("-c", "--config", default="./en_US-lessac-high.onnx.json", help="Path to model config file")
+ #
+ parser.add_argument("-s", "--speaker", type=int, help="Id of speaker (default: 0)")
+ parser.add_argument(
+ "--length-scale", "--length_scale", type=float, help="Phoneme length"
+ )
+ parser.add_argument(
+ "--noise-scale", "--noise_scale", type=float, help="Generator noise"
+ )
+ parser.add_argument(
+ "--noise-w", "--noise_w", type=float, help="Phoneme width noise"
+ )
+ #
+ parser.add_argument("--cuda", action="store_true", help="Use GPU")
+ #
+ parser.add_argument(
+ "--sentence-silence",
+ "--sentence_silence",
+ type=float,
+ default=0.0,
+ help="Seconds of silence after each sentence",
+ )
+ #
+ parser.add_argument(
+ "--data-dir",
+ "--data_dir",
+ action="append",
+ default=[str(Path.cwd())],
+ help="Data directory to check for downloaded models (default: current directory)",
+ )
+ parser.add_argument(
+ "--download-dir",
+ "--download_dir",
+ help="Directory to download voices into (default: first data dir)",
+ )
+ #
+ parser.add_argument(
+ "--update-voices",
+ action="store_true",
+ help="Download latest voices.json during startup",
+ )
+ #
+ parser.add_argument(
+ "--debug", action="store_true", help="Print DEBUG messages to console"
+ )
+ args = parser.parse_args()
+ logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO)
+ _LOGGER.debug(args)
+
+ if not args.download_dir:
+ # Download to first data directory by default
+ args.download_dir = args.data_dir[0]
+
+ # Download voice if file doesn't exist
+ model_path = Path(args.model)
+ if not model_path.exists():
+ # Load voice info
+ voices_info = get_voices(args.download_dir, update_voices=args.update_voices)
+
+ # Resolve aliases for backwards compatibility with old voice names
+ aliases_info: Dict[str, Any] = {}
+ for voice_info in voices_info.values():
+ for voice_alias in voice_info.get("aliases", []):
+ aliases_info[voice_alias] = {"_is_alias": True, **voice_info}
+
+ voices_info.update(aliases_info)
+ ensure_voice_exists(args.model, args.data_dir, args.download_dir, voices_info)
+ args.model, args.config = find_voice(args.model, args.data_dir)
+
+ # Load voice
+ voice = PiperVoice.load(args.model, config_path=args.config, use_cuda=args.cuda)
+ synthesize_args = {
+ "speaker_id": args.speaker,
+ "length_scale": args.length_scale,
+ "noise_scale": args.noise_scale,
+ "noise_w": args.noise_w,
+ "sentence_silence": args.sentence_silence,
+ }
+
+ # Create web server
+ app = Flask(__name__)
+
+ @app.route("/send", methods=["GET", "POST"])
+ def app_handler():
+ text = request.get_json()
+
+ _LOGGER.debug("get text: %s", text["text"])
+ json_response = {}
+ with io.BytesIO() as wav_io:
+ with wave.open(wav_io, "wb") as wav_file:
+ voice.synthesize(text["text"], wav_file, **synthesize_args)
+
+ json_response["response"] = base64.b64encode(wav_io.getvalue()).decode("utf-8")
+ #audio_str = "";
+ #for audio_bytes in voice.synthesize_stream_raw(text["text"], **synthesize_args):
+ # audio_str += (base64.b64encode(audio_bytes).decode("utf-8"))
+ #json_response["response"] = audio_str
+ return jsonify(json_response)
+
+ app.run(host=args.host, port=args.port)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/aimodel/yolo_server/yolo_server.py b/aimodel/yolo_server/yolo_server.py
new file mode 100755
index 0000000000000000000000000000000000000000..4c3d32f306615f7d9ea36e0a3dde1bce51b87270
--- /dev/null
+++ b/aimodel/yolo_server/yolo_server.py
@@ -0,0 +1,73 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2025 The Qt Company Ltd.
+# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+from http.server import BaseHTTPRequestHandler,HTTPServer
+from os import curdir, sep
+import re
+import cgi
+import torch
+import simplejson
+import base64
+from io import StringIO
+from io import BytesIO
+from PIL import Image
+import numpy as np
+
+from ultralytics import YOLO
+
+
+PORT_NUMBER = 8004
+
+#This class will handles any incoming request from
+#the browser
+class myHandler(BaseHTTPRequestHandler):
+
+ #Handler for the POST requests
+ def do_POST(self):
+ print("do_POST");
+ if self.path=="/send":
+ self.data_string = self.rfile.read(int(self.headers['Content-Length']))
+ print("data_string: " + self.data_string.decode())
+ json_data = simplejson.loads(self.data_string)
+ #print("json_data: " + simplejson.dumps(json_data))
+ print("MODEL: " + json_data["model"])
+
+ # Perform object detection
+ base64_image_data = json_data["frame"]
+ frameInB64 = base64.b64decode(base64_image_data)
+ print("Frame length:" + str(len(frameInB64)))
+ frame = Image.open(BytesIO(frameInB64))
+ results = model(frame)
+
+ # Render results on the frame
+ #frame = results[0].plot()
+ print(results[0].to_json())
+
+ print("Sending response")
+ self.send_response(200)
+ self.end_headers()
+
+ json_response_text = results[0].to_json()
+ self.wfile.write(json_response_text.encode("utf-8"))
+ return
+
+
+try:
+ #Create a web server and define the handler to manage the
+ #incoming request
+ server = HTTPServer(('', PORT_NUMBER), myHandler)
+ print('Started httpserver on port ' , PORT_NUMBER)
+
+ # Load YOLOv9 model
+ model = YOLO('yolov9c.pt')
+
+ #Wait forever for incoming htto requests
+ server.serve_forever()
+
+except KeyboardInterrupt:
+ print('^C received, shutting down the web server')
+ server.socket.close()
+
+
diff --git a/main.cpp b/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..52e40facce31fb86bb5f7a10582c4e96bc4da8bc
--- /dev/null
+++ b/main.cpp
@@ -0,0 +1,31 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only
+
+#include <QtCore>
+#include <QGuiApplication>
+#include <QQmlApplicationEngine>
+
+
+int main(int argc, char *argv[])
+{
+ QGuiApplication app(argc, argv);
+
+ QQmlApplicationEngine engine;
+ const QUrl url("qrc:///App.qml");
+ QObject::connect(
+ &engine, &QQmlApplicationEngine::objectCreated, &app,
+ [url](QObject *obj, const QUrl &objUrl) {
+ if (!obj && url == objUrl)
+ QCoreApplication::exit(-1);
+ }, Qt::QueuedConnection);
+
+ engine.addImportPath("qml");
+ engine.loadFromModule("qtaiinferenceapi", "App");
+
+ qDebug() << "Standard path for pictures: " << QStandardPaths::standardLocations(QStandardPaths::PicturesLocation);
+
+ if (engine.rootObjects().isEmpty())
+ return -1;
+
+ return app.exec();
+}
diff --git a/qtquickcontrols2.conf b/qtquickcontrols2.conf
new file mode 100644
index 0000000000000000000000000000000000000000..87a95d01144526e8cea0ad44d529c98e36deb177
--- /dev/null
+++ b/qtquickcontrols2.conf
@@ -0,0 +1,6 @@
+; This file can be edited to change the style of the application
+; Read "Qt Quick Controls 2 Configuration File" for details:
+; http://doc.qt.io/qt-5/qtquickcontrols2-configuration.html
+
+[Controls]
+Style=Basic