benchmark.cpp 8.44 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48
/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** $QT_BEGIN_LICENSE:BSD$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** BSD License Usage
** Alternatively, you may use this file under the terms of the BSD license
** as follows:
**
** "Redistribution and use in source and binary forms, with or without
** modification, are permitted provided that the following conditions are
** met:
**   * Redistributions of source code must retain the above copyright
**     notice, this list of conditions and the following disclaimer.
**   * Redistributions in binary form must reproduce the above copyright
**     notice, this list of conditions and the following disclaimer in
**     the documentation and/or other materials provided with the
**     distribution.
**   * Neither the name of The Qt Company Ltd nor the names of its
**     contributors may be used to endorse or promote products derived
**     from this software without specific prior written permission.
**
**
** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
** OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
** LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
**
** $QT_END_LICENSE$
**
****************************************************************************/

49 50 51 52 53 54
#include <benchmark.h>
#include <QtDebug>
#include <QElapsedTimer>
#include <QQmlApplicationEngine>
#include <QQuickWindow>
#include <QGuiApplication>
Daniel Smith's avatar
Daniel Smith committed
55 56
#include <fstream>
#include <unistd.h>
57 58
#include <cmath>
#include <iomanip>
59
#include <algorithm>
60 61 62 63 64 65

void Benchmark::startTimer(){

}

void Benchmark::countFrame(){
66 67 68
    //Save the frame time, reset the timer, and take a memory measurement.
    qint64 frametime = timer.nsecsElapsed();
    timer.start();
Daniel Smith's avatar
Daniel Smith committed
69 70
    if (frame % 10 == 0)
        emit measureMemory();
71
    float thisframe = static_cast<float>(frametime)/1000000;
72 73 74 75 76
    try{
        frametimes[frame] = thisframe;
    } catch (int e){
        //No need to handle this. We probably just rendered extra frames before finished() completed.
    }
77
    qDebug() << thisframe; //Turn this off, except the slot doesn't seem to execute at the correct time if we don't output anything.
78
//    qDebug() << "frame " << frame << ": " << thisframe << " ms"; //Turn this off for real runs.
79 80 81 82 83 84
    frame+=1;
    if (frame >= maxframes){
        this->finished();
    }
}

85 86 87 88 89 90
void Benchmark::resetFrameTimer(){
    //restart the timer at the end of the render loop, before the next set of render actions.
    timer.restart();
    qDebug() << "Restart Timer."; //Turn this off, except the slot doesn't seem to execute at the correct time if we don't output anything.
}

91
void Benchmark::measureMem(){
92
    //Uses OS-specific method to pull memory for this process
93 94 95 96 97 98 99 100 101 102 103 104 105 106
#ifdef _WIN32
   //define something for Windows (32-bit and 64-bit, this part is common)
#elif __APPLE__
    #include "TargetConditionals.h"
    #if TARGET_IPHONE_SIMULATOR
         // iOS Simulator
    #elif TARGET_OS_IPHONE
        // iOS device
    #elif TARGET_OS_MAC
        // Other kinds of Mac OS
    #else
    #   error "Unknown Apple platform"
    #endif
#elif __linux__
Daniel Smith's avatar
Daniel Smith committed
107 108 109 110 111
    int tSize = 0, resident = 0, share = 0;
    std::ifstream buffer("/proc/self/statm");
    buffer >> tSize >> resident >> share;
    buffer.close();

112 113 114
    int page_size_kb = static_cast<int>(sysconf(_SC_PAGE_SIZE) / 1024); // in case x86-64 is configured to use 2MB pages
    int rss = static_cast<int>(resident * page_size_kb);
    int shared_memKB = (share * page_size_kb);
115 116 117
#else
#   error "Unknown compiler"
#endif
118 119 120 121 122 123 124 125 126 127
    if (beforeFirstFrame) { //Write to the 0 element for our initial measurement.
        virtualMem[frame / 10] = shared_memKB;
        privateMem[frame / 10] = rss - shared_memKB;
        beforeFirstFrame = false;
    }else {
        virtualMem[(frame / 10) + 1] = shared_memKB;
        privateMem[(frame / 10) + 1] = rss - shared_memKB;
    }

    qDebug() << frame << (frame / 10) << "Virtual Memory (KB): " << shared_memKB << " RSS (KB): " << rss << " Private Bytes (KB): " << (rss - shared_memKB);
128 129
}

130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
float Benchmark::calculateSD(float data[]){
    //calculate the standard deviation for our results.
    float sum = 0.0, mean, standardDeviation = 0.0;

    int i;

    for(i = 0; i < maxframes; ++i)
    {
        sum += data[i];
    }

    mean = sum/maxframes;

    for(i = 0; i < maxframes; ++i)
        standardDeviation += pow(data[i] - mean, 2);
    return sqrt(standardDeviation / maxframes);
}


149
void Benchmark::finished(){
150 151
    qDebug() << "\n-------- FINISHED TEST RUN --------\n";

152 153 154 155 156 157 158 159 160 161 162
    //int averageVMem = std::accumulate(virtualMem, virtualMem+sizeof(virtualMem)/sizeof(virtualMem[0]), 0) / (maxframes / 10);
    int maxVMem = *std::max_element(virtualMem, virtualMem + (maxframes / 10) + 1);
    //int averagePMem = std::accumulate(privateMem, privateMem+sizeof(privateMem)/sizeof(privateMem[0]), 0) / (maxframes / 10);
    int maxPMem = *std::max_element(privateMem, privateMem + (maxframes / 10) + 1 );

    qDebug() << "Max private memory (KB)" << maxPMem;
    qDebug() << "Max Virtual memory (KB)" << maxVMem;
    qDebug() << "Max QML application private memory (KB)" << maxPMem - privateMem[0];
    qDebug() << "Max QML application Virtual memory (KB)" << maxVMem - virtualMem[0];

    qDebug() << "";
163 164 165 166 167 168 169 170
    QString frametimesCS;
    for (int i = 0; i < maxframes; i++){
        frametimesCS += QString::number(static_cast<double>(frametimes[i]), 'f', 2) + ((i < maxframes - 1) ? ",": "");
    }
    qDebug() << "Frame times (ms):" << frametimesCS;
    int startuptime = static_cast<int>(frametimes[0]);
    qDebug() << "Start-up time:" << startuptime << "ms";

171
    frametimes[0] = frametimes[1]; // hack to omit start-up time from our standard deviation calcuation.
172

173 174 175 176 177
    float runtime = std::accumulate(frametimes, frametimes+maxframes, 0.0);
    qDebug() << "Finished" << maxframes << "frames in" << runtime << "ms" << "at" << (maxframes*1000) / runtime << "fps";

    float meanFrametime = static_cast<float>(runtime) / static_cast<float>(maxframes);
    qDebug() << "Average Frame time:" << QString::number(static_cast<double>(meanFrametime), 'f', 2).toFloat() << "ms";
178 179
    qDebug() << "";

180 181 182 183
    float runtimeStdDev = calculateSD(frametimes);
    qDebug() << "Standard Deviation:" << runtimeStdDev; // << "over" << maxframes << "frames";
    qDebug() << "Coefficient of Variation:" << (runtimeStdDev / meanFrametime) * 100;

184 185 186 187
    QCoreApplication::quit();
}

int Benchmark::runBenchmark(QGuiApplication &app){
Daniel Smith's avatar
Daniel Smith committed
188 189 190 191
    measureMem();
    QObject::connect(this, &Benchmark::measureMemory,
       this, &Benchmark::measureMem);

192
        qDebug() << "Started timer.";
193
        this->timer.start();
194

195 196 197
        QQmlApplicationEngine engine(QUrl("qrc:/main.qml"));
        if (engine.rootObjects().size() != 1)
            return -1;
198

199
        QQuickWindow *window = qobject_cast<QQuickWindow*>(engine.rootObjects().first());
200 201 202 203
//        Take our measurement when rendering is complete, but before the framebuffer is swapped.
//        If rendering is faster than 16ms, we won't be able to measure by taking the measurement at
//        QQuickWindow::frameSwapped
        QObject::connect(window, &QQuickWindow::afterRendering,
204
           this, &Benchmark::countFrame);
205 206 207 208 209
//        Reset the timer when the frame is finally displayed so we can count sync events as well as render
//        events starting at the beginning of the next loop.
        QObject::connect(window, &QQuickWindow::frameSwapped,
           this, &Benchmark::resetFrameTimer);

210 211 212
        if (app.exec() != 0)
            return -1;
}