regressionFinder.py 54.4 KB
Newer Older
1
import argparse
2
import atexit
3
import json
4 5
import os
import platform
6
import re
7 8 9
import shutil
import stat
import subprocess
10
from time import sleep
11

12
import packaging
13 14 15 16 17
import requests
import thriftpy
import thriftpy.utils

from thrift import storagestructs
18 19 20 21 22 23

args = []
basedir = os.getcwd()
builddir = os.path.join(basedir, "QtBuild")
installdir = os.path.join(builddir, "Install")
isWindowsOS = (platform.system() == 'Windows')
24 25
compiler = ["BuildConsole", f"/COMMAND={os.path.join(basedir, 'JOM', 'jom.exe')}"] if isWindowsOS else ["make", f"-j{os.environ.get('BUILD_CORES') or os.cpu_count()}"]
installer = [compiler[0], compiler[1] + " install"] if isWindowsOS else compiler + ["install"]
26 27 28 29 30 31 32 33 34
exeExt = '.exe' if isWindowsOS else ''
goodBaselineScore = 0
badBaselineScore = 0
regressionTargetScore = 0
observedRegressionPercent = 0
refQtbaseRev = ""
testType = ""
noRebuild = False
finalResults = []
35 36
rawJSONResults = {}
extraLogData = ""
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61

# Bisect tracking items
revisionList = []
bisectResults = [[], ]
bisectPosition = 0
bisectGoodPos = 0
bisectBadPos = 0
bisectLastStatus = ""
bisectLastDirection = ""


def on_rm_error(func, path, exc_info):
    # Handler for files that fail to be removed.
    # path contains the path of the file that couldn't be removed
    # let's just assume that it's read-only and unlink it.
    try:
        os.chmod(path, stat.S_IWRITE)
        os.unlink(path)
    except Exception as e:
        print("There was an error removing a file from disk. Exception: {0}".format(e))


def parseArgs():
    parser = argparse.ArgumentParser()
    parser.add_argument("--setupEnv", dest="setupEnv", action="store_true", help="Run with --setupEnv to initialize the build environment by cloning needed repos or pulling updates as needed. Exits upon completion.")
62
    parser.add_argument("--branch", dest="branch", type=str, default="dev", help="Branch of Qt such as \'5.12\' or \'dev\'")
63 64 65 66 67 68 69
    parser.add_argument("--moduleToTest", dest="module", type=str, help="Module where regression is suspected such as \'qtbase\'.")
    parser.add_argument("--knownBadRev", dest="knownBadRev", type=str, help="Known bad revision in the module where a regression is expected.")
    parser.add_argument("--knownGoodRev", dest="knownGoodRev", type=str, help="Known good revision in the module where a regression is expected")
    parser.add_argument("--regressionTarget", dest="regressionTarget", type=int, help="Expected regression, in percent, to assist in finding the correct regressive commit. Be conservative if unsure.")
    parser.add_argument("--fuzzyRange", dest="fuzzyRange", type=int, default=2, help="Provide a value to use as a fuzzy range to check the regression against. Using \'--regressionTarget 20 --fuzzyRange 5\' will prefer regressions between 15-25%% as the bad commit")
    parser.add_argument("--buildCores", dest="buildCores", type=int, default=8, help="Number of build cores to use when building")
    parser.add_argument("--wipeWorkspace", dest="wipeWorkspace", action="store_true", help="Clear the entire workspace and force re-cloning of all modules")
70
    parser.add_argument("--VSDevEnv", dest="VSDevEnv", help="Full path to Visual studio VsDevCmd.bat file for windows build environments. Defaults to default installation for VS 2017 Build Tools", default="C:/Program Files (x86)/Microsoft Visual Studio/2019/BuildTools/Common7/Tools/VsDevCmd.bat")
71 72 73 74 75
    parser.add_argument("--benchmark", dest="benchmark", type=str, help="path to single benchmark to run. Only run one benchmark at a time!")
    parser.add_argument("--testSingleCommit", dest="testSingleCommit", type=str, help="Set this parameter to a commit ID to benchmark. Setting this parameter overrides bisecting behavior.")
    parser.add_argument("--testTwoCommit", dest="testTwoCommit", type=str, help="Set this parameter to a comma-separated list of two commit IDs to benchmark and compare. Setting this parameter overrides bisecting behavior.")
    parser.add_argument("--FirstBuildOnHead", dest="firstBuildOnHead", action="store_true", help="Enable this parameter to build the first commit against branch HEAD instead of searching for a COIN integration.")
    parser.add_argument("--SecondBuildOnHead", dest="secondBuildOnHead", action="store_true", help="Enable this parameter to build the second commit against branch HEAD instead of searching for a COIN integration.")
76
    parser.add_argument("--OpenGLBackend", dest="openGLBackend", type=str, default="desktop", help="Render backend options. Valid options are \'desktop\', \'angle\', \'software\'")
77
    parser.add_argument("--jobName", dest="jobName", type=str, help="unique job name used for writing results file to logs directory. Typically a hash of the job to be run.")
Daniel Smith's avatar
Daniel Smith committed
78
    parser.add_argument("--patches", dest="patches", type=str, help="List of patches to apply before building")
79 80 81 82 83 84
    parser.add_argument("--environment", dest="environment", type=str, help="Comma separated list of environment variables and values to use for the build and test environment.")

    return parser.parse_args()


def initRepository(repo):
85

86 87
    # Clone any needed repositories. If they already exist, sweep them and pull changes.
    module = repo[repo.index('/') + 1:]
88
    branch = args.branch if module != "qmlbench" else "dev"
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
    if not branch:
        branch = "dev"

    print(repo, branch)

    def cloneRepo():
        subprocess.run(["git", "clone", "-b", branch, f'https://code.qt.io/{repo}.git'], stderr=subprocess.PIPE, universal_newlines=True, cwd=builddir)
        subprocess.run(["git", "submodule", "update", "--init"], stderr=subprocess.PIPE, universal_newlines=True, cwd=os.path.join(builddir, module))

    # clone modules if necessary, otherwise just pull latest.
    if not os.path.exists(os.path.join(builddir, module)):
        cloneRepo()
    else:
        subprocess.run(["git", "clean", "-dqfx"], stderr=subprocess.PIPE, universal_newlines=True, cwd=os.path.join(builddir, module))
        subprocess.run(["git", "reset", "--hard", f"origin/{args.branch}", "--quiet"], stderr=subprocess.PIPE, universal_newlines=True, cwd=os.path.join(builddir, module))
        subprocess.run(["git", "pull", "origin", args.branch, "--quiet"], stderr=subprocess.PIPE, universal_newlines=True, cwd=os.path.join(builddir, module))
        subprocess.run(["git", "checkout", branch], stderr=subprocess.PIPE, universal_newlines=True, cwd=os.path.join(builddir, module))

    if repo == "qt/qt5":
        # Run the reset and pull again. Sometimes the previous pull leads to merge conflicts.
        # We don't care, just needed the first pull to get the lastest submodule list
        # and then reset and pull again to get the latest changes.
        subprocess.run(["git", "reset", "--hard", f"origin/{args.branch}", "--quiet"], stderr=subprocess.PIPE, universal_newlines=True, cwd=os.path.join(builddir, module))
        subprocess.run(["git", "pull", "origin", args.branch, "--quiet"], stderr=subprocess.PIPE, universal_newlines=True, cwd=os.path.join(builddir, module))
        subprocess.run(["git", "submodule", "update", "--init"], stderr=subprocess.PIPE, universal_newlines=True, cwd=builddir)

115

116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
def validateTest():
    # Verify the selected test exists
    global args
    if not os.path.exists(os.path.join(builddir, 'qmlbench', args.benchmark)):
        args.benchmark = os.path.join("benchmarks", args.benchmark)
        if not os.path.exists(os.path.join(builddir, 'qmlbench', args.benchmark)):
            print(f"Specified benchmark at {os.path.join(builddir, 'qmlbench', args.benchmark)} does not exist. Please check the path and try again.")
            return False
    return True


def validateCommits():
    # Verify that the provided commit(s) exist in the Qt repository.

    def revparse(rev):
        # Rev-parse our given SHA1 and see if Git returns anything. If it pushes anything to stdout, a match was found.
        proc = subprocess.run(["git", "rev-parse", rev], cwd=os.path.join(builddir, args.module),
                              stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True, shell=isWindowsOS)
        if proc.stderr:
            if lookupUnmergedCommit(rev, args.module):
                return revparse(rev)
            else:
                print(f"Unable to verify commit {rev}. Please check to make sure you have the correct repository and there are no typos.")
                return False
        else:
            print(proc.stdout.strip().split("\n"))
            if len(proc.stdout.strip().split("\n")) > 1:
                # Rev-parse will print one matching SHA per line.
                print(f"Provided commit {rev} matches more than one commit. Try using the full SHA1.")
                return False
            else:
                return proc.stdout.strip()

    global args
    global noRebuild

    # Do some logic to determine which type of test we're running.
    if args.knownGoodRev:
        args.knownGoodRev = args.knownGoodRev.strip()  # Cleanup inputs
        args.knownBadRev = args.knownBadRev.strip()
        # Try to see if we have at least one commit between the known good and bad commits. This will fail if the known bad is not an ancestor of the known good.
        proc = subprocess.run(["git", "rev-list", f"{args.knownGoodRev}..{args.knownBadRev}", f"^{args.knownGoodRev}"], cwd=os.path.join(builddir, args.module),
                              stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True, shell=isWindowsOS)

        if proc.returncode != 0:
            print(f"ERROR: Unable to verify selected commits. Please double check your input.\nGit error: {proc.stderr}")
162
            reportStatusUpdate("ERROR: Commits not seen in qt5. Please check commits and restage.")
163 164 165 166
            return False

        try:
            revisionList = proc.stdout.split('\n')
167
        except Exception:
168
            reportStatusUpdate("ERROR: No commits between good and bad commits. Nothing to test.")
169 170 171 172
            print(f"ERROR: No commits between good and bad commits. Nothing to test. Hint: Known bad commit {args.knownBadRev} might be the bad commit.")
            return False

        if revisionList:
173
            revisionList.pop()  # rev-list will return the list with the known bad commit as the head element. Get rid of it permanently.
174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252

        if len(revisionList) < 1:
            print(f"ERROR: No commits between good and bad commits. Nothing to test. Hint: Known bad commit {args.knownBadRev} might be the bad commit.")
            return False
        if args.module == "qtbase":
            print("Module to test is Qtbase. Verifying that COIN data is available for other modules before proceeding...")
            if not findRevToBuild(args.knownGoodRev, "qtdeclarative", args.branch) or not findRevToBuild(args.knownBadRev, "qtdeclarative", args.branch):
                return False
            else:
                print("COIN data verified. Continuing.")
        return True

    elif args.testSingleCommit:
        args.testSingleCommit = revparse(args.testSingleCommit.strip())
        if not args.testSingleCommit:
            return False
        else:
            if args.module == "qtbase":
                print("Module to test is Qtbase. Verifying that COIN data is available for other modules before proceeding...")
                if not findRevToBuild(args.testSingleCommit, "qtdeclarative", args.branch):
                    return False
                else:
                    print("COIN data verified. Continuing.")
            return True
    elif args.testTwoCommit:
        firstCommit = revparse(args.testTwoCommit.split(',')[0].strip())
        secondCommit = args.testTwoCommit.split(',')[1].strip()
        if secondCommit == "PARENT":
            noRebuild = True
            secondCommit = findRevToBuild(firstCommit, args.module, "", True)
            print(f"Second Commit found: {secondCommit}")
            args.testTwoCommit = f"{secondCommit},{firstCommit}"  # Reverse the order of testing when the user selected the easy regression test option. Expect regression from parent to child, not improvement.
        else:
            secondCommit = revparse(secondCommit)
            args.testTwoCommit = f"{firstCommit},{secondCommit}"
        if not firstCommit and secondCommit:
            return False
        else:
            if args.module == "qtbase":
                print("Module to test is Qtbase. Verifying that COIN data is available for other modules before proceeding...")
                if not findRevToBuild(firstCommit, "qtdeclarative", args.branch) or not findRevToBuild(secondCommit, "qtdeclarative", args.branch):
                    return False
                else:
                    print("COIN data verified. Continuing.")
            return True


def setWindowsEnv():
    # runs the vsDevCmd file from the visual studio installation
    vars = subprocess.check_output([args.VSDevEnv, '&&', 'set'])

    # splits the output of the batch file and saves PATH variables from the batch to the local os.environ
    for var in vars.splitlines():
        var = var.decode('cp1252')
        k, _, v = map(str.strip, var.strip().partition('='))
        if k.startswith('?'):
            continue
        os.environ[k] = v

    os.environ["PATH"] += (";" + builddir).replace("/", "\\")
    os.environ["PATH"] += (";" + basedir + "/flex_bison/").replace("/", "\\")
    os.environ["QTDIR"] = (builddir).replace("/", "\\")


def prepareEnv():
    # A place to do OS specific actions on startup.
    if args.environment:
        for variable in args.environment.split(','):
            varName, varValue = variable.split('=')
            os.environ[varName] = varValue

    if isWindowsOS:
        # set up windows build env
        print('Setting up windows build environment.')
        setWindowsEnv()


def buildModule(module):
    # Build the specified Qt module
253
    global extraLogData
254 255 256
    print(f"Preparing to build {module}")
    os.chdir(os.path.join(builddir, module))
    if (module == "qtbase"):
257
        reportStatusUpdate("Configuring QtBase")
258
        if isWindowsOS:
259
            configurecmd = ["configure.bat", "-prefix", installdir, "-no-pch", "-nomake", "tests", "-nomake", "examples", "-release", "-opensource", "-confirm-license", "-no-warnings-are-errors", "-opengl", "dynamic"]
260 261

        else:
262
            configurecmd = ["./configure", "-prefix", installdir, "-no-pch", "-developer-build", "-nomake", "tests", "-nomake", "examples", "-release", "-opensource", "-confirm-license", "-no-warnings-are-errors"]
263
        print("Running Configure for Qtbase")
264 265 266 267 268 269
        proc = subprocess.run(configurecmd, check=False, shell=isWindowsOS)
        if proc.returncode:
            reportStatusUpdate("ERROR: An error occurred during configure. Check the log.")
            try:
                with open("configure.summary") as configureSummary:
                    extraLogData = configureSummary.read()
270
            except IOError:
271 272 273 274 275 276
                try:
                    with open("config.log") as configLog:
                        extraLogData = '\n'.join(configLog.readlines()[-20:])
                except IOError:
                    extraLogData = "Unknown error when running Configure. Check full logs."
            return False
277 278 279 280

    else:
        # qmake
        print(f"Running QMake for {module}")
281
        if subprocess.run([os.path.join(installdir, "bin", f"qmake{exeExt}", ), f"{module}.pro"],
282
                          universal_newlines=True, shell=isWindowsOS).returncode:
283
            return False
284 285

    # build it!
286 287
    reportStatusUpdate(f"Running make for {module}")
    print(f"Building {module}")
288
    proc = subprocess.run(compiler, universal_newlines=True, shell=isWindowsOS)
289 290 291 292 293 294 295 296 297
    if proc.returncode and not module == "qmlbench":
        try:
            with open(os.path.normpath(os.path.join("../../logs", platform.node(), args.jobName + ".txt"))) as fullLog:
                fullLogData = fullLog.readlines()
                extraLogData = '\n'.join(fullLogData[-20:])
                for line in fullLogData:
                    if "recipe for target" in line:
                        reportStatusUpdate(f"ERROR: {line} in {module}")
                        break
298
        except IOError:
299 300
            extraLogData = f"ERROR: Unknown error occurred while running make for {module}"
            reportStatusUpdate(f"ERROR: Unknown error occurred while running make for {module}")
301
        if module in (args.module, "qtbase"):
302 303 304 305 306
            # If qtbase didn't fail, and this isn't the module to test, it's worth continuing
            # with errors.
            return False
        else:
            print(f"*** {module} failed to build! Continuing with errors! ***")
307 308 309 310
    if not module == "qmlbench":
        installModule(module)

    if isWindowsOS and module == "qmlbench":
311
        reportStatusUpdate("Building and installing WinDeployQt")
312
        print("Building WinDeployQt")
313
        # Also build and run winDeployQt on qmlbench
314 315 316
        if subprocess.run([os.path.join(installdir, "bin", f"qmake{exeExt}", )],
                          universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, "qttools")).returncode:
            return False
317
        if subprocess.run([os.path.join(installdir, "bin", f"qmake{exeExt}", ), "windeployqt.pro"],
318
                          universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, "qttools", "src", "windeployqt")).returncode:
319
            return False
320 321
        if subprocess.run(compiler,
                          universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, "qttools", "src", "windeployqt")).returncode:
322
            return False
323
        subprocess.run(installer,
324
                       universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, "qttools", "src", "windeployqt")).returncode
325
        print("Running WinDeployQt against QMLBench")
326 327 328 329 330
        proc = subprocess.run(["windeployqt.exe", "--qmldir", os.path.join(builddir, "qmlbench", "benchmarks"),
                               os.path.join(builddir, "qmlbench", "src", "release", "qmlbench.exe")],
                              universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(installdir, "bin"), check=True)

    os.chdir(builddir)
331
    return True
332 333 334


def installModule(module):
335
    reportStatusUpdate(f"Installing {module}")
336 337
    # Run make install on the module.
    print(f"Installing {module}")
338
    subprocess.run(installer, universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, module))
339 340 341


def runBenchmark():
342
    reportStatusUpdate("Running QMLBench test")
343 344 345 346 347 348 349 350
    # Actually execute qmlbench
    benchmarkEnv = os.environ.copy()
    if isWindowsOS:
        benchmarkEnv["QT_OPENGL"] = args.openGLBackend
        print(f"OpenGL Backend set to {benchmarkEnv['QT_OPENGL']}")

    print("Cooling off for 20 seconds before benchmarking...")
    sleep(20)
351
    print("Starting Benchmark")
352 353
    with open(os.path.join(builddir, "qmlbench", "results.json"), mode='wb') as results:  # Write output as JSON to results file for later collection.
        if isWindowsOS:
354
            proc = subprocess.run([os.path.join(builddir, "qmlbench", "src", "release", f"qmlbench{exeExt}"), "--json", "--shell", "frame-count", "--repeat", "10", os.path.join(builddir, "qmlbench", args.benchmark)], env=benchmarkEnv, stdout=results, stderr=subprocess.PIPE, shell=isWindowsOS)
355
        else:
356
            proc = subprocess.run([os.path.join(builddir, "qmlbench", "src", f"qmlbench{exeExt}"), "--json", "--shell", "frame-count", "--repeat", "10", os.path.join(builddir, "qmlbench", args.benchmark)], stdout=results, stderr=subprocess.PIPE, shell=isWindowsOS)
357

358 359
    if proc.stderr:
        reportStatusUpdate(proc.stderr.splitlines()[0])
360
    print("Completed Benchmark")
361 362


363
def parseResults(commit=""):
364
    # Open the results file and find the "average" result key.
365
    print("Parsing Benchmark results")
366 367 368 369 370
    resultsJSON = {}  # Typing.Dict()
    benchmark = args.benchmark.replace('\\', '/')
    with open(os.path.join(builddir, "qmlbench", "results.json")) as results:
        resultsJSON = json.loads(results.read())

371 372 373 374
    global rawJSONResults
    rawJSONResults[commit] = resultsJSON
    print(f"Raw results output for commit: {commit}\n{json.dumps(resultsJSON, sort_keys=True, indent=4)}\n")

375 376 377 378 379 380 381
    for key in resultsJSON:
        if benchmark in key:
            return int(resultsJSON[key]['average'])

    return False


382
def findRevToBuild(refRevision, module, branch="", returnParent=False, getQt5BaseSha=False):
383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
    # Do a whole bunch of stuff to associate the revision we should build of other qt modules.

    def getParentRev(commitJSON):
        for commit in commitJSON:
            parentRev = commit["revisions"][refRevision]["commit"]["parents"][0]["commit"]
            if parentRev:
                return parentRev

    # Search codereview for the change and return some specific details. Start the saved response from codereview at position 4 to ignore some garbage in the response.
    commitRaw = requests.get(f"https://codereview.qt-project.org/changes/?q=commit:{refRevision}&o=CURRENT_REVISION&o=CURRENT_COMMIT").text[4:]
    if not commitRaw:
        return False
    try:
        commitJSON = json.loads(commitRaw)  # Try to parse the JSON
    except IndexError:
        print("Unable to find revision on codereview.")
        saveResult(refRevision, "RevNotFound")
        return False

    if returnParent:  # Just return the direct parent of refRevision. This would usually be used by the TwoCommit mode for easy regression testing against a new commit's parent.
        return getParentRev(commitJSON)

    changeID = ""
    commitDetailsRaw = ""

    # At least we parsed the response into valid JSON. Now, Look through the responses, validate that things are correct, and retreive the comments list.
    for index, commit in enumerate(commitJSON):
410 411 412 413 414
        if not getQt5BaseSha:
            if branch not in commit["branch"]:
                print(f"ERROR: Selected revision of {module} was not committed on the {branch} branch. It was committed to {commit['branch']}. Please correct and resubmit the job.")
                if index >= (len(commitJSON) - 1):
                    reportStatusUpdate(f"ERROR: Selected revision of {module} was not committed on the {branch} branch. It was committed to {commit['branch']}.")
415
                    return False
416 417 418 419 420 421
                else:
                    continue
            elif args.module not in commit["project"]:
                print(f"ERROR: Selected revision is not in {args.module}. It is part of {commit['project'].split('/')[1]}. Please correct and resubmit the job.")
                if index >= (len(commitJSON) - 1):
                    reportStatusUpdate(f"ERROR: Selected revision is not in {args.module}. It is part of {commit['project'].split('/')[1]}")
422
                    return False
423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438
                else:
                    continue
            elif branch in commit["branch"] and module == args.module:
                # This logic branch will return immediately because it's the module we're testing a specific commit.
                # We'll check it out directly and just wanted to verify it exists.
                # But if the module to test is qtquick3d, verify that the commit date is after the first
                # known integration and the branch to test is new enough.
                if module == "qtquick3d":
                    if subprocess.run(["git", "show", "-s", "--format=%ct", refRevision], cwd=os.path.join(builddir, "qt5", args.module), stdout=subprocess.PIPE).stdout < 1566565560:
                        reportStatusUpdate("Commit too old to be built with qtquick3d")
                        return False
                    if (packaging.version.parse(args.branch) if not args.branch == "dev" else False) < packaging.version.parse(5.14):
                        reportStatusUpdate("Cannot build. Commit must be 5.14 or later")
                        return False
                else:
                    return True
439 440 441 442

        changeID = commit["change_id"]
        print(f"Found Change ID: {changeID} based on commit {refRevision}")
        # Pull the comments history on the change so we can look for a COIN integration ID.
443
        request = requests.get(f"https://codereview.qt-project.org/changes/qt%2F{args.module}~{args.branch}~{changeID}/detail")
444 445 446 447 448 449 450 451 452 453 454 455 456 457 458
        if request.status_code != 200:
            continue  # Something's fishy about the commit and codereview doesn't recognize it. Try the next commit in the list of responses from our original query.
        else:
            commitDetailsRaw = request.text[4:]  # Gerrit responded favorably. Save the response, but still look at the others in the list from the original query.
            if commitDetailsRaw:
                break

    # Try to load the commit's JSON
    try:
        commitDetails = json.loads(commitDetailsRaw)
    except json.decoder.JSONDecodeError:
        saveResult(refRevision, "RevFoundButBadAPIDetailResponse")

        # Try again with the reference commit's parent. It should have been integrated.
        parentRev = getParentRev(commitJSON)
459 460 461 462
        sha = ""
        if parentRev:
            print(f"INFO: Found Parent {parentRev}, trying it instead")
            sha = findRevToBuild(parentRev, module, branch)
463 464 465 466 467
        if sha:
            print("WARN: Using the reference commit's parent to build against. This should only happen if you're testing a change that hasn't been merged yet.")
            return sha
        else:
            print("ERROR: Failed to load commit details from gerrit, and failed to use commit parent as backup. Please report this error.")
468
            reportStatusUpdate("ERROR: Failed to load commit details from gerrit, and failed to use commit parent as backup.")
469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
            return False

    integrationId = ""

    # Scan the comments section for COIN's telltale message.
    for message in commitDetails["messages"]:
        if "Continuous Integration: Passed" in message["message"]:
            integrationId = re.search(r"[0-9]{10}", message["message"]).group(0)
            print(f"Found COIN integration {integrationId}")
            break  # Found it. No need to look at other comments.

    # Some changes get pushed directly for some reason. They're untestible because we can't ask COIN what to build against unless it has a parent commit that integrated normally.
    if not(integrationId):
        print("WARN: The change ID selected did not have a COIN integration.")
        for message in commitDetails["messages"]:
            if "Change has been successfully cherry-picked" in message["message"]:
485
                print("WARN: Change was Cherry-picked. Manual Review suggested.")
486 487 488 489
                saveResult(refRevision, "NoIntegrationCherryPicked")
                break
        # Try again with the reference commit's parent. It should have been integrated.
        parentRev = getParentRev(commitJSON)
490 491 492 493
        sha = ""
        if parentRev:
            print(f"INFO: Found Parent {parentRev}, trying it instead")
            sha = findRevToBuild(parentRev, module, branch)
494 495 496 497 498
        if sha:
            print("WARN: Using the reference commit's parent to build against. This should only happen if you're testing a change that hasn't been merged yet.")
            return sha
        else:
            print("ERROR: Failed to load commit details from gerrit, and failed to use commit parent as backup. Please report this error.")
499
            reportStatusUpdate("ERROR: Failed to load commit details from gerrit, and failed to use commit parent as backup.")
500 501 502 503 504 505 506 507 508
            return False

    # Try to pull the saved integration data from coin for the integration we found. It contains the shas of all modules it built with at the time.
    data = requests.get(f"https://testresults.qt.io/logs/qt/{args.module}/tasks/{integrationId}.thrift_bin").content
    if b"404 Not Found" in data:
        print(f"ERROR: Failed to find COIN data for integration ID {integrationId}. Maybe the commit we're trying isn't in the selected module of {args.module}. Trying the parent commit's integration data just in case.")

        # Try again with the reference commit's parent. It should have been integrated.
        parentRev = getParentRev(commitJSON)
509 510 511 512
        sha = ""
        if parentRev:
            print(f"INFO: Found Parent {parentRev}, trying it instead")
            sha = findRevToBuild(parentRev, module, branch)
513 514 515 516 517
        if sha:
            print("WARN: Using the reference commit's parent to build against because the target commit had no COIN data.")
            return sha
        else:
            print(f"ERROR: Failed to load parent COIN integration data. This is probably permanently fatal. Commit {refRevision} can't be reliably tested.")
518
            reportStatusUpdate("ERROR: Failed to load parent COIN integration data.")
519 520
            return False
        saveResult(refRevision, "BadCOINData")
521
        reportStatusUpdate(f"Error: Coin has no record of integration ID {integrationId} for {args.module}")
522 523 524 525 526
        return False
    else:
        try:
            # Convert the integration's binary thrift_bin file back into a workitem object.
            workitem = thriftpy.utils.deserialize(storagestructs.Task(), data)
527
        except Exception:
528 529 530
            print(f"Failed to extract COIN data for integration ID {integrationId}.")
            # Try again with the reference commit's parent. It should have been integrated.
            parentRev = getParentRev(commitJSON)
531 532 533 534
            sha = ""
            if parentRev:
                print(f"INFO: Found Parent {parentRev}, trying it instead")
                sha = findRevToBuild(parentRev, module, branch)
535 536 537 538 539
            if sha:
                print("WARN: Using the reference commit's parent to build against because the target commit had bad COIN data.")
                return sha
            else:
                print(f"ERROR: Failed to load parent COIN integration data. This is probably permanently fatal. Commit {refRevision} can't be reliably tested.")
540
                reportStatusUpdate("ERROR: Failed to load parent COIN integration data.")
541 542 543 544 545
                return False

        # Ask git to identify the SHA1 of our target module to build based on the sha of the change we're testing. This uses the Qt5 supermodule.

        try:
546
            print(f"Running git ls-tree to find the sha1 to build for {module}")
547 548 549 550 551 552 553
            proc = subprocess.run(["git", "ls-tree", workitem.product.sha1, module, "-z"], cwd=os.path.join(builddir, "qt5"), stdout=subprocess.PIPE)
            sha = str(proc.stdout).split(" ")[2].split("\\t")[0]
            print(f'Found {module} sha1: {sha}')
            return sha
        except Exception:
            # Couldn't find the qt5 product sha from the module to build.
            print(f"ERROR: Couldn't find COIN product sha {workitem.product.sha1} for module {module} in qt5.")
554
            reportStatusUpdate(f"ERROR: Couldn't find COIN product sha {workitem.product.sha1} for module {module} in qt5.")
555 556 557
            saveResult(refRevision, "BadCOINData")
            return False
        #####################################################################################################################
558
        # Try to use the internal COIN as a backup. This code will not be functional outside of COIN's local network!!! ###
559 560 561 562 563 564 565 566 567 568 569 570 571 572 573
        #####################################################################################################################

        # data = requests.get(f"https://testresults.qt.io/coin/api/item/qt/{args.module}/tasks/{integrationId}.thrift_bin").content
        # print(data)
        # try:
        #     workitem = thriftpy.utils.deserialize(storagestructs.Task(), data)
        #     print(workitem.product)
        #     proc = subprocess.run(["git", "ls-tree", workitem.product.sha1, module, "-z"], cwd=os.path.join(builddir, "qt5"), stdout=subprocess.PIPE)
        #     sha = str(proc.stdout).split(" ")[2].split("\\t")[0]
        #     print(f'Found {module} sha1: {sha} from internal coin.')
        #     return sha
        # except:
        #     print(f"Also Failed to import COIN data from internal COIN for integration ID {integrationId}. Maybe the commit we're trying isn't in the selected module of {args.module}")

        ############################
574
        # End of internal code ###
575 576 577 578 579
        ############################


def determineBuildEligibility(revision, module, branch):
    # Verify that the SHA provided for our module to test is valid and can be built.
580 581 582
    revision = findRevToBuild(revision, module, branch)
    if not revision:
        reportStatusUpdate(f"Revision {revision} ")
583 584


585 586 587 588
def fetchChange(url, ref, module):
    subprocess.run(["git", "fetch", url, ref], universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, module))


Daniel Smith's avatar
Daniel Smith committed
589
def cherryPickChange(module, sha=""):
590 591 592 593
    subprocess.run(["git", "cherry-pick", sha if sha else "FETCH_HEAD", "--no-commit"], cwd=os.path.join(builddir, module), stderr=subprocess.PIPE,
                   universal_newlines=True, shell=isWindowsOS)


Daniel Smith's avatar
Daniel Smith committed
594
def lookupUnmergedCommit(revision, module=""):
595
    commitRaw = requests.get(f"https://codereview.qt-project.org/changes/?q=commit:{revision}&o=ALL_REVISIONS&o=CURRENT_COMMIT").text[4:]
596
    ref = ""
597
    if not commitRaw:
598
        return (False, ref, module)
599 600 601 602
    try:
        commitJSON = json.loads(commitRaw)[0]
    except IndexError:
        print(f"ERROR: Unable to find revision on codereview: {revision}")
603
        return (False, ref, module)
604 605 606 607 608 609 610
    try:
        # Get the fetch addresses so we can pull and check out the revision.
        for item in commitJSON["revisions"]:
            if revision in item:
                revision = item
        url = commitJSON["revisions"][revision]["fetch"]["anonymous http"]["url"]
        ref = commitJSON["revisions"][revision]["fetch"]["anonymous http"]["ref"]
611 612
        if not module:
            module = commitJSON["project"][3:]
613 614
    except Exception:
        print(f"ERROR: Unable to get ref fetch data for commit to test {revision}")
615
        reportStatusUpdate(f"ERROR: Unable to get ref fetch data for commit to test {revision}")
616
        return (False, ref, module)
617
    print(f"Fetching {module} patch ref {ref}")
618 619 620
    return (url, ref, module)


Daniel Smith's avatar
Daniel Smith committed
621
def lookupCommitByRef(ref, module=""):
622 623 624 625
    matches = re.findall(r"(\d{6})(?:/(\d{1,}))?", ref)
    changeNumber = ""
    patchset = ""
    if matches:
Daniel Smith's avatar
Daniel Smith committed
626 627 628 629 630 631
        changeNumber = matches[0][0]
    else:
        print(f"ERROR: Unable to parse ref: {ref}")
        return (False, module)
    if matches[0][1]:
        patchset = matches[0][1]
632 633 634 635 636 637 638 639 640 641 642 643 644 645 646
    commitRaw = requests.get(f"https://codereview.qt-project.org/changes/?q={changeNumber}&o=ALL_REVISIONS&o=CURRENT_COMMIT").text[4:]
    if not commitRaw:
        return (False, module)
    try:
        commitJSON = json.loads(commitRaw)[0]
    except IndexError:
        print(f"ERROR: Unable to find ref on codereview: {ref}")
        return (False, module)

    if not module:
        module = commitJSON["project"][3:]
    if patchset:
        try:
            # Get the fetch addresses so we can pull and check out the revision.
            for item in commitJSON["revisions"]:
647
                if patchset == commitJSON["revisions"][item]["_number"]:
648 649 650 651 652 653 654 655 656 657 658
                    return (commitJSON["revisions"][item]["fetch"]["anonymous http"]["url"], module)
        except Exception:
            print(f"Unable to get fetch URL for desired patchset {patchset}")
    try:
        return (commitJSON["revisions"][commitJSON["current_revision"]]["fetch"]["anonymous http"]["url"], module)
    except Exception:
        print(f"WARN: Failed to fall back to the current revision of {ref}")
        return (False, module)


def applyPatches(currentModule):
Daniel Smith's avatar
Daniel Smith committed
659
    print("Applying patches...")
660 661 662
    if not args.patches:
        return
    else:
Daniel Smith's avatar
Daniel Smith committed
663
        for patch in args.patches.split(","):
664 665 666 667 668 669 670 671 672 673 674 675
            url = ""
            ref = patch
            module = ""
            if "refs/changes/" in patch:
                url, module = lookupCommitByRef(patch)
                if not url:
                    continue
            else:
                url, ref, module = lookupUnmergedCommit(patch)
                if not url:
                    continue
            if module != currentModule:
Daniel Smith's avatar
Daniel Smith committed
676
                print(f"Skipping patch {ref} since it applies to {module}.")
677 678 679 680
                continue
            fetchChange(url, ref, module)
            print(f"Cherry-picking qt/{module} ref/sha {ref} as a patch.")
            cherryPickChange(module)
681 682 683 684 685 686 687 688 689


def checkout(revision, module, bisecting=False, branch="", buildOnHead=False):
    # Run git checkouts
    print(f"Running Checkout for {module}, using revision {revision} as the reference")
    print(f"This module should {'' if buildOnHead  else 'not '}be built on head ")
    if module == args.module:
        print(f"This is the module to test. Cleaning {module}.")
        subprocess.run(["git", "clean", "-dqfx"], universal_newlines=True, cwd=os.path.join(builddir, module))
690 691 692
        qt5baseRev = findRevToBuild(revision, module, branch, False, True)
        subprocess.run(["git", "checkout", qt5baseRev], cwd=os.path.join(builddir, module), stderr=subprocess.PIPE,
                       universal_newlines=True, shell=isWindowsOS)
693
        print(f"Trying checkout of module to test with revision {revision}")
694
        proc = subprocess.run(["git", "cherry-pick", revision, "--no-commit"], cwd=os.path.join(builddir, module), stderr=subprocess.PIPE,
695 696 697
                              universal_newlines=True, shell=isWindowsOS)
        if "reference is not a tree" in proc.stderr:
            # We must be trying to build a commit that's not yet merged. Let's look it up.
698 699
            url, ref, module = lookupUnmergedCommit(revision, module)
            fetchChange(url, ref, module)
700
            print(f"Checking out unmerged commit {revision}")
701 702
            cherryPickChange(module)
            applyPatches(module)
703 704
            return True

705
        applyPatches(module)
706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723
        return True

    else:
        # This is one of the dependent modules
        if buildOnHead:
            revToBuild = "HEAD"
        else:
            revToBuild = findRevToBuild(revision, module, branch)

        if not revToBuild:
            return False
        else:
            proc = subprocess.run(["git", "checkout", revToBuild], cwd=os.path.join(builddir, module),
                                  universal_newlines=True, shell=isWindowsOS)

            actualRevision = subprocess.run(["git", "show", "-s", "--date=short", "--format=%cd"], cwd=os.path.join(builddir, module),
                                            stdout=subprocess.PIPE, universal_newlines=True, shell=isWindowsOS).stdout.strip()
            print(f"Checking out {module} from date {actualRevision} with SHA/tag {revToBuild}")
724
            applyPatches(module)
725 726 727 728 729
            return True


def cleanup(prepForBuild=False, noRebuild=False):
    # Clean up our files and start fresh each time, or just uninstall and clean the module to test if we already specified to build against parent.
730
    reportStatusUpdate("Cleaning up from the last build")
731

732 733 734 735 736 737 738 739 740 741 742 743
    def clean(module):
        subprocess.run(["git", "clean", "-dqfx"], universal_newlines=True, cwd=os.path.join(builddir, module))
        if os.path.exists(os.path.join(builddir, module, '.git', 'index.lock')):
            # Force git to accept a new process. If this file exists, it probably means an earlier job was cancelled or crashed while doing git stuff.
            os.remove(os.path.join(builddir, module, '.git', 'index.lock'))
            proc = subprocess.run(["git", "reset", "--hard", "origin/head"], universal_newlines=True, cwd=os.path.join(builddir, module))
            if proc.returncode != 0:
                print(f"ERROR: The git directory for {module} is corrupted. We'll try to remove it and reclone.")
                shutil.rmtree(os.path.join(builddir, module), on_rm_error)
                initRepository(module)

    if noRebuild:
744
        subprocess.run(compiler.append("uninstall"), universal_newlines=True, cwd=os.path.join(builddir, args.module))
745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765
        clean(args.module)
    else:
        shutil.rmtree(os.path.join(builddir, "Install"), on_rm_error)
        if prepForBuild:
            for module in next(os.walk(builddir, followlinks=False))[1]:
                print(f"cleaning {module}")
                clean(module)


def gatherBaselines():
    # In a bisect operation, test known good and known bad and set our regression threshold.
    global goodBaselineScore
    global badBaselineScore
    global regressionTargetScore
    global observedRegressionPercent

    cleanup(True)
    if not buildChanges(args.knownGoodRev):
        print("Error!!! Good baseline failed to build! Maybe you had the wrong module to test selected.")
        exit(1)
    runBenchmark()
766
    goodBaselineScore = parseResults(commit=args.knownGoodRev)
767 768 769 770 771 772 773
    print(f'Good baseline scored {goodBaselineScore}')

    cleanup(True)
    if not buildChanges(args.knownBadRev):
        print("Error!!! Good baseline failed to build! Maybe you had the wrong module to test selected.")
        exit(1)
    runBenchmark()
774
    badBaselineScore = parseResults(args.knownBadRev)
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909
    print(f'Bad baseline scored {badBaselineScore}\n')

    print(f'Expected regression is {args.regressionTarget}%.')
    regressionTargetScore = int(goodBaselineScore - (goodBaselineScore * (args.regressionTarget / 100)))
    observedRegressionPercent = int((((goodBaselineScore - badBaselineScore) / goodBaselineScore) * 100))

    if badBaselineScore > regressionTargetScore:
        print("The \'bad\' commit provided did not meet the expected target regression percentage. Are you sure this is correct?")
        print(f"The provided \'bad\' commit was only {observedRegressionPercent}% lower than the \'good\' commit.")
        print("Aborting the test. Please adjust the expected regression target or provide a new \'bad\' commit ID.\n")
        exit(2)  # Reserve exit status of "2" for this use.

    else:
        severity = abs(args.regressionTarget - observedRegressionPercent)
        print(f"Observed regression of {observedRegressionPercent} is {severity}% greater than the expected target regression.\nYou may need to run this tool again with a different \'good\' later.\n")
        print("Continuing... We'll try to find the single commit that caused a regression close to the target.")


def gatherRevisionsToTest():
    # In a bisect, find the full list of commits between known good and known bad.
    global revisionList

    revisionList = subprocess.run(["git", "rev-list", f"{args.knownGoodRev}..{args.knownBadRev}", f"^{args.knownGoodRev}"], cwd=os.path.join(builddir, args.module),
                                  stdout=subprocess.PIPE, universal_newlines=True, shell=isWindowsOS).stdout.split('\n')

    if not revisionList:
        return False

    revisionList.pop()  # Git will return our known bad revision at the top of the list. Remove it since we already test it as part of the baseline process.
    revisionList.reverse()  # Reverse the list so that the known bad is at the end and known good at the start.


def bisectOperator(action):
    # Bisect logic that acts like Git's bisect command.
    # Return commit info if we're done bisecting, otherwise, false.
    global bisectPosition
    global bisectBadPos
    global bisectGoodPos
    global bisectLastStatus
    global bisectLastDirection

    if action == "start":
        gatherRevisionsToTest()
        bisectPosition = int(len(revisionList) / 2)
        bisectGoodPos = 0
        bisectBadPos = len(revisionList) - 1
        return False

    elif action == "good":
        bisectGoodPos = bisectPosition
        bisectLastStatus = "good"
        bisectLastDirection = "plus"
        if (bisectBadPos - bisectGoodPos) < 2:
            return revisionList[bisectPosition + 1]
        bisectPosition = bisectPosition + int((bisectBadPos - bisectGoodPos) / 2)
        return False

    elif action == "bad":
        bisectBadPos = bisectPosition
        bisectLastStatus = "bad"
        bisectLastDirection = "minus"
        if (bisectBadPos - bisectGoodPos) < 2:
            return revisionList[bisectPosition]
        bisectPosition = bisectPosition - int((bisectBadPos - bisectGoodPos) / 2)
        return False

    elif action == "skip":
        if bisectLastStatus == "good":
            bisectPosition += 1
            bisectLastDirection = "plus"
        elif bisectLastStatus == "bad":
            bisectPosition -= 1
            bisectLastDirection = "minus"
        elif bisectLastStatus == "skip":
            print("received skip...")
            if bisectLastDirection == "minus":
                print("is Minus")
                bisectPosition -= 1
            elif bisectLastDirection == "plus":
                print("is Plus")
                bisectPosition += 1
            else:
                print(f"is Other? ...{bisectLastDirection}...")
                bisectPosition += 1
        else:
            print(f"Bisect Last Direction is something else ...{bisectLastDirection}...")
            bisectPosition += 1

        bisectLastStatus = "skip"
        if (bisectPosition <= bisectGoodPos and bisectLastDirection == "minus") or (bisectPosition >= bisectBadPos and bisectLastDirection == "plus"):
            revisions = []
            for revision in revisionList[bisectGoodPos:bisectBadPos]:
                revisions.append(revision)
            return revisions

        return False

    print(f"ERROR: Action was invalid. This should not happen. Ever. Action: {action}")


def saveResult(revision, status, score=None):
    # Add the bisect result to our list of results.
    global bisectResults
    bisectResults[0].append({"revision": revision, "status": status, "score": score})


def manualBisect():
    # Do the bisect action. Good and Bad baselines have already been taken.
    global finalResults
    print("Beginning bisect process")

    def iterate(branch):
        # The logic
        global goodBaselineScore
        global badBaselineScore
        global regressionTargetScore
        global observedRegressionPercent

        print(f"++++++ Testing new index {bisectPosition} with revision {revisionList[bisectPosition]}\nCurrent Known Good index: {bisectGoodPos} with revision {revisionList[bisectGoodPos]}\n\
Current Known Bad index: {bisectBadPos} with revision {revisionList[bisectBadPos]}\nLast revision was {bisectLastStatus} and we adjusted the index to the {bisectLastDirection}")

        currentRevision = revisionList[bisectPosition]
        cleanup(True)
        if not buildChanges(currentRevision, True, branch):
            # The build either failed or the module revision doesn't point to a COIN integration.
            print("Skipping bisected checkout.")
            foundRegression = bisectOperator("skip")
            if foundRegression:
                print(f"Skipping this iteration completed the bisect, but returned more than one commit. Please investigate the following commits\n{foundRegression}")
                bisectResults.append(foundRegression)
                return foundRegression
            else:
                iterate(branch)
        else:
            runBenchmark()
910
            resultScore = parseResults(commit=currentRevision)
911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949
            if not resultScore:
                saveResult(currentRevision, "fail")
                print("Unable to parse a result. Maybe the test crashed with this revision. Skipping revision.")
                foundRegression = bisectOperator("skip")
                if foundRegression:
                    print(f"Skipping this iteration completed the bisect, but returned more than one commit. Please investigate the following commits\n{foundRegression}")
                    bisectResults.append(foundRegression)
                    return foundRegression
                else:
                    iterate(branch)
            else:
                print(f"Iteration scored {resultScore}, {int((((goodBaselineScore - resultScore)/goodBaselineScore)*100))}% off from good baseline score of {goodBaselineScore}")
                if resultScore > regressionTargetScore:
                    saveResult(currentRevision, "good", resultScore)
                    foundRegression = bisectOperator("good")
                    if foundRegression:
                        print(f"We found a single commit that caused the regression. Please investigate {foundRegression}")
                        bisectResults.append([foundRegression])
                        return [foundRegression]
                    else:
                        iterate(branch)
                else:
                    saveResult(currentRevision, "bad", resultScore)
                    foundRegression = bisectOperator("bad")
                    if foundRegression:
                        print(f"We found a single commit that caused the regression. Please investigate {foundRegression}")
                        bisectResults.append([foundRegression])
                        return [foundRegression]
                    else:
                        iterate(branch)

    bisectOperator("start")
    commit = iterate(args.branch)
    return commit


def testSingleCommit(commit, buildOnHead=False, noRebuild=False):
    # We're testing one commit by itself
    cleanup(True, noRebuild)
950
    if not buildChanges(commit, buildOnHead=buildOnHead, noRebuild=noRebuild):
951
        exit(1)
952 953 954
    else:
        runBenchmark()
    resultScore = parseResults(commit=commit)
955 956
    if not resultScore:
        print("Unable to parse a result. Maybe the test crashed with this revision.")
957
        # reportStatusUpdate("Tried to run QMLbench but was unable to parse a result. Maybe the test crashed with this revision.")
958 959 960 961 962 963 964
        return False
    print(f"Commit {commit} scored {resultScore}")
    return resultScore


def initRepositories():
    # Prepare the system for building. Clone required repos
965
    reportStatusUpdate("Cleaning and updating Git repositories")
966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
    if args.wipeWorkspace:
        shutil.rmtree(os.path.join(builddir, "."), on_rm_error)
        for dir in os.walk(builddir):
            print(dir)

    if not os.path.exists(builddir):
        os.makedirs(builddir)

    initRepository("qt/qt5")
    initRepository("qt/qtbase")
    initRepository("qt/qtdeclarative")
    initRepository("qt/qtquickcontrols")
    initRepository("qt/qtquickcontrols2")
    initRepository("qt/qtgraphicaleffects")
    initRepository("qt/qtsvg")
    initRepository("qt/qtquick3d")
    initRepository("qt-labs/qmlbench")

    if (isWindowsOS):
        initRepository("qt/qttools")

    if args.setupEnv:
        # RegressionFinder.py was called with --setupEnv and will exit after cloning repos
        exit(0)


def resultWriter():
    # Dump our test results to a log file
    if not os.path.exists(os.path.join(basedir, "logs", platform.node(), "results")):
        os.mkdir(os.path.join(basedir, "logs", platform.node(), "results"))

    data = {}
    data['jobName'] = args.jobName
    data['testType'] = testType
    data['testHost'] = platform.node()
    data['commits'] = args.testSingleCommit if testType == "singleCommit" else args.testTwoCommit if testType == "twoCommit" else f"{args.knownGoodRev},{args.knownBadRev}"
    data['branch'] = args.branch
    data['module'] = args.module
    if testType == 'bisect':
        data["bisectResults"] = bisectResults
        data["baselineResults"] = [goodBaselineScore, badBaselineScore, observedRegressionPercent]
    data['finalResults'] = finalResults
1008 1009
    data['QMLBenchRawJSON'] = rawJSONResults
    data['extraLogData'] = extraLogData
1010 1011 1012 1013 1014 1015 1016 1017

    with open(os.path.join(basedir, "logs", platform.node(), "results", f"{args.jobName}.json"), mode="w") as outfile:
        json.dump(data, outfile)


def on_exit():
    # At exit time, whether successful or not, print results to screen and try to write the results file.
    if not args.setupEnv:
1018 1019 1020 1021 1022 1023
        resultWriter()
        if finalResults or bisectResults[0]:
            print("Complete! Results:")
        if bisectResults[0]:
            for tested in bisectResults[0]:
                print(tested)
1024 1025 1026 1027 1028 1029 1030 1031
        if finalResults:
            print(finalResults)


def buildChanges(revision, bisecting=False, branch="", buildOnHead=False, noRebuild=False):
    if noRebuild:
        # Just build the module under test. This is used when regression testing against parent, since all other modules will be the same SHA anyway.
        if not checkout(revision, args.module, branch, buildOnHead=buildOnHead):
1032
            reportStatusUpdate(f"Failed to check out {args.module} with revision {revision}")
1033 1034 1035 1036 1037 1038 1039
            return False
        buildModule(args.module)
    else:
        # Build all the modules.
        if bisecting:
            if not determineBuildEligibility(revision, args.module, branch):  # Make sure that the revision we're going to test is valid and can be built, so we don't waste time.
                return False
1040
        modulesToBuild = ["qtbase", "qtdeclarative", "qtquickcontrols2", "qtgraphicaleffects", "qtsvg"]
1041 1042 1043 1044 1045
        if args.module == "qtquick3d":
            modulesToBuild.append("qtquick3d")
        for module in modulesToBuild:  # Qtbase must be built first.
            if not checkout(revision, module, branch, buildOnHead=buildOnHead):
                return False
1046 1047
            if not buildModule(module):
                return False
1048 1049 1050 1051 1052

    buildModule("qmlbench")
    return True


1053
def reportStatusUpdate(status_msg=""):
1054 1055
    try:
        requests.post(f"http://localhost:{os.getenv('HTTP_PORT', 8080)}/updateLocalRunningJobStatus",
1056 1057
                      json={'status_msg': status_msg})
    except Exception:
1058 1059
        pass

1060

1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
if __name__ == "__main__":
    atexit.register(on_exit)
    args = parseArgs()
    prepareEnv()
    initRepositories()
    if not validateCommits() or not validateTest():  # Validate inputs!
        exit(1)
    if args.testSingleCommit:
        print("Entering single commit benchmark mode...")
        testType = "singleCommit"
        print(f"Building on head? {args.firstBuildOnHead}")
        result = testSingleCommit(args.testSingleCommit, args.firstBuildOnHead)
        if not result:
            exit(1)
        finalResults = [result]
    elif args.testTwoCommit:
        print("Entering two commit compare mode...")
        testType = "twoCommit"
        firstCommit = args.testTwoCommit.split(',')[0].strip()
        secondCommit = args.testTwoCommit.split(',')[1].strip()
        result1 = testSingleCommit(firstCommit, args.firstBuildOnHead)
        if not result1:
            exit(1)
        result2 = testSingleCommit(secondCommit, args.secondBuildOnHead, noRebuild)
        if not result2:
            exit(1)
        observedRegressionPercent = 0
        if secondCommit == "PARENT":
            observedRegressionPercent = int((((result2 - result1) / result2) * 100))
        else:
            observedRegressionPercent = int((((result1 - result2) / result1) * 100))
        print(f"First commit scored {result1}. Second commit scored {result2}, a {observedRegressionPercent}% difference.")
        finalResults = [result1, result2, observedRegressionPercent]
    else:
        testType = "bisect"
        gatherBaselines()
        manualBisect()

# sample string
# python .\regressionFinder.py --branch dev --buildCores 8 --benchmark benchmarks/auto/js/sum10k.qml --regressionTarget 20 --knownGoodRev 85fc49612816dcfc81c9dc265b146b0b90b0f184 --knownBadRev eace041161a03a849d3896af65493b7885cecc04 --moduleToTest qtdeclarative