Commit 1fefbbee authored by Daniel Smith's avatar Daniel Smith

Add support for patch dependencies

parent 14eaa701
This is the changelog for jom 1.1.2, the parallel make tool.
This is the changelog for jom 1.1.3, the parallel make tool.
Changes since jom 1.1.2
- Removed the /KEEPTEMPFILES option. This option only worked for top-level make
files anyway and was less useful than intended. Use the /U option to display
the content of inline files instead.
- Fixed an issue where jom.exe would try to load qt.conf from drive E.
- Fixed handling of double backslash at the end of line (QTCREATORBUG-20550).
- Fixed handling of line continuations in preprocessor directives
(QTCREATORBUG-8621, QTCREATORBUG-18001).
- Fixed the CMake project file.
Changes since jom 1.1.1
- Fixed exit code propagation in xgejom.bat and ibjom.bat (QTCREATORBUG-16619).
......
No preview for this file type
import os
import subprocess
import argparse
import platform
import stat
import shutil
import atexit
import json
import requests
from thrift import storagestructs
import thriftpy.utils
import thriftpy
import os
import platform
import re
import shutil
import stat
import subprocess
from time import sleep
import atexit
import packaging
import requests
import thriftpy
import thriftpy.utils
from thrift import storagestructs
args = []
basedir = os.getcwd()
builddir = os.path.join(basedir, "QtBuild")
installdir = os.path.join(builddir, "Install")
isWindowsOS = (platform.system() == 'Windows')
compiler = basedir + "\\JOM\\jom.exe" if isWindowsOS else "make"
compiler = ["BuildConsole", f"/COMMAND={os.path.join(basedir, 'JOM', 'jom.exe')}"] if isWindowsOS else ["make", f"-j{os.environ.get('BUILD_CORES') or os.cpu_count()}"]
print(compiler)
installer = [compiler[0], compiler[1] + " install"] if isWindowsOS else compiler + ["install"]
print(installer)
exeExt = '.exe' if isWindowsOS else ''
goodBaselineScore = 0
badBaselineScore = 0
......@@ -64,24 +69,25 @@ def parseArgs():
parser.add_argument("--fuzzyRange", dest="fuzzyRange", type=int, default=2, help="Provide a value to use as a fuzzy range to check the regression against. Using \'--regressionTarget 20 --fuzzyRange 5\' will prefer regressions between 15-25%% as the bad commit")
parser.add_argument("--buildCores", dest="buildCores", type=int, default=8, help="Number of build cores to use when building")
parser.add_argument("--wipeWorkspace", dest="wipeWorkspace", action="store_true", help="Clear the entire workspace and force re-cloning of all modules")
parser.add_argument("--VSDevEnv", dest="VSDevEnv", help="Full path to Visual studio VsDevCmd.bat file for windows build environments. Defaults to default installation for VS 2017 Build Tools", default="C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/Common7/Tools/VsDevCmd.bat")
parser.add_argument("--VSDevEnv", dest="VSDevEnv", help="Full path to Visual studio VsDevCmd.bat file for windows build environments. Defaults to default installation for VS 2017 Build Tools", default="C:/Program Files (x86)/Microsoft Visual Studio/2019/BuildTools/Common7/Tools/VsDevCmd.bat")
parser.add_argument("--benchmark", dest="benchmark", type=str, help="path to single benchmark to run. Only run one benchmark at a time!")
parser.add_argument("--testSingleCommit", dest="testSingleCommit", type=str, help="Set this parameter to a commit ID to benchmark. Setting this parameter overrides bisecting behavior.")
parser.add_argument("--testTwoCommit", dest="testTwoCommit", type=str, help="Set this parameter to a comma-separated list of two commit IDs to benchmark and compare. Setting this parameter overrides bisecting behavior.")
parser.add_argument("--FirstBuildOnHead", dest="firstBuildOnHead", action="store_true", help="Enable this parameter to build the first commit against branch HEAD instead of searching for a COIN integration.")
parser.add_argument("--SecondBuildOnHead", dest="secondBuildOnHead", action="store_true", help="Enable this parameter to build the second commit against branch HEAD instead of searching for a COIN integration.")
parser.add_argument("--OpenGLBackend", dest="openGLBackend", type=str, default="desktop", help="Render backend options. Valid options are \'dekstop\', \'angle\', \'software\'")
parser.add_argument("--OpenGLBackend", dest="openGLBackend", type=str, default="desktop", help="Render backend options. Valid options are \'desktop\', \'angle\', \'software\'")
parser.add_argument("--jobName", dest="jobName", type=str, help="unique job name used for writing results file to logs directory. Typically a hash of the job to be run.")
parser.add_argument("--patches", dest="patches", type=str, help="List of patches to apply before building")
parser.add_argument("--environment", dest="environment", type=str, help="Comma separated list of environment variables and values to use for the build and test environment.")
return parser.parse_args()
def initRepository(repo):
# Clone any needed repositories. If they already exist, sweep them and pull changes.
module = repo[repo.index('/') + 1:]
branch = args.branch if module != "qmlbench" else "master"
branch = args.branch if module != "qmlbench" else "dev"
if not branch:
branch = "dev"
......@@ -90,7 +96,6 @@ def initRepository(repo):
def cloneRepo():
subprocess.run(["git", "clone", "-b", branch, f'https://code.qt.io/{repo}.git'], stderr=subprocess.PIPE, universal_newlines=True, cwd=builddir)
subprocess.run(["git", "submodule", "update", "--init"], stderr=subprocess.PIPE, universal_newlines=True, cwd=os.path.join(builddir, module))
# clone modules if necessary, otherwise just pull latest.
if not os.path.exists(os.path.join(builddir, module)):
......@@ -109,6 +114,7 @@ def initRepository(repo):
subprocess.run(["git", "pull", "origin", args.branch, "--quiet"], stderr=subprocess.PIPE, universal_newlines=True, cwd=os.path.join(builddir, module))
subprocess.run(["git", "submodule", "update", "--init"], stderr=subprocess.PIPE, universal_newlines=True, cwd=builddir)
def validateTest():
# Verify the selected test exists
global args
......@@ -160,13 +166,13 @@ def validateCommits():
try:
revisionList = proc.stdout.split('\n')
except Exception as e:
except Exception:
reportStatusUpdate("ERROR: No commits between good and bad commits. Nothing to test.")
print(f"ERROR: No commits between good and bad commits. Nothing to test. Hint: Known bad commit {args.knownBadRev} might be the bad commit.")
return False
if revisionList:
revisionList.pop() # rev-list will return the list with the known bad commit as the head element. Get rid of it permenantly.
revisionList.pop() # rev-list will return the list with the known bad commit as the head element. Get rid of it permanently.
if len(revisionList) < 1:
print(f"ERROR: No commits between good and bad commits. Nothing to test. Hint: Known bad commit {args.knownBadRev} might be the bad commit.")
......@@ -256,14 +262,14 @@ def buildModule(module):
else:
configurecmd = ["./configure", "-prefix", installdir, "-no-pch", "-developer-build", "-nomake", "tests", "-nomake", "examples", "-release", "-opensource", "-confirm-license", "-no-warnings-are-errors"]
print(f"Running Configure for Qtbase")
print("Running Configure for Qtbase")
proc = subprocess.run(configurecmd, check=False, shell=isWindowsOS)
if proc.returncode:
reportStatusUpdate("ERROR: An error occurred during configure. Check the log.")
try:
with open("configure.summary") as configureSummary:
extraLogData = configureSummary.read()
except IOError as e:
except IOError:
try:
with open("config.log") as configLog:
extraLogData = '\n'.join(configLog.readlines()[-20:])
......@@ -275,13 +281,13 @@ def buildModule(module):
# qmake
print(f"Running QMake for {module}")
if subprocess.run([os.path.join(installdir, "bin", f"qmake{exeExt}", ), f"{module}.pro"],
universal_newlines=True, shell=isWindowsOS).returncode:
universal_newlines=True, shell=isWindowsOS).returncode:
return False
# build it!
reportStatusUpdate(f"Running make for {module}")
print(f"Building {module}")
proc = subprocess.run([compiler, "-j", f"{args.buildCores}"], universal_newlines=True, shell=isWindowsOS)
proc = subprocess.run(compiler, universal_newlines=True, shell=isWindowsOS)
if proc.returncode and not module == "qmlbench":
try:
with open(os.path.normpath(os.path.join("../../logs", platform.node(), args.jobName + ".txt"))) as fullLog:
......@@ -291,26 +297,34 @@ def buildModule(module):
if "recipe for target" in line:
reportStatusUpdate(f"ERROR: {line} in {module}")
break
except IOError as e:
except IOError:
extraLogData = f"ERROR: Unknown error occurred while running make for {module}"
reportStatusUpdate(f"ERROR: Unknown error occurred while running make for {module}")
return False
if module == args.module and not module == "qtbase":
# If qtbase didn't fail, and this isn't the module to test, it's worth continuing
# with errors.
return False
else:
print(f"*** {module} failed to build! Continuing with errors! ***")
if not module == "qmlbench":
installModule(module)
if isWindowsOS and module == "qmlbench":
reportStatusUpdate("Building and installing WinDeployQt")
print(f"Building WinDeployQt")
print("Building WinDeployQt")
# Also build and run winDeployQt on qmlbench
if subprocess.run([os.path.join(installdir, "bin", f"qmake{exeExt}", )],
universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, "qttools")).returncode:
return False
if subprocess.run([os.path.join(installdir, "bin", f"qmake{exeExt}", ), "windeployqt.pro"],
universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, "qttools", "src", "windeployqt")).returncode:
universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, "qttools", "src", "windeployqt")).returncode:
return False
if subprocess.run([compiler, "-j", f"{args.buildCores}"],
universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, "qttools", "src", "windeployqt")).returncode:
if subprocess.run(compiler,
universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, "qttools", "src", "windeployqt")).returncode:
return False
subprocess.run([compiler, "install", "-j", f"{args.buildCores}"],
subprocess.run(installer,
universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, "qttools", "src", "windeployqt")).returncode
print(f"Running WinDeployQt against QMLBench")
print("Running WinDeployQt against QMLBench")
proc = subprocess.run(["windeployqt.exe", "--qmldir", os.path.join(builddir, "qmlbench", "benchmarks"),
os.path.join(builddir, "qmlbench", "src", "release", "qmlbench.exe")],
universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(installdir, "bin"), check=True)
......@@ -323,7 +337,7 @@ def installModule(module):
reportStatusUpdate(f"Installing {module}")
# Run make install on the module.
print(f"Installing {module}")
subprocess.run([compiler, "install", "-j", f"{int(args.buildCores) * 3 }"], universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, module))
subprocess.run(installer, universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, module))
def runBenchmark():
......@@ -336,21 +350,21 @@ def runBenchmark():
print("Cooling off for 20 seconds before benchmarking...")
sleep(20)
print(f"Starting Benchmark")
print("Starting Benchmark")
with open(os.path.join(builddir, "qmlbench", "results.json"), mode='wb') as results: # Write output as JSON to results file for later collection.
if isWindowsOS:
proc = subprocess.run([os.path.join(builddir, "qmlbench", "src", "release", f"qmlbench{exeExt}"), "--json", "--shell", "frame-count", "--repeat", "10", os.path.join(builddir, "qmlbench", args.benchmark)], env=benchmarkEnv, stdout=results, stderr=subprocess.PIPE, shell=isWindowsOS)
else:
proc = subprocess.run([os.path.join(builddir, "qmlbench", "src", f"qmlbench{exeExt}"), "--json", "--shell", "frame-count", "--repeat", "10", os.path.join(builddir, "qmlbench", args.benchmark)], stdout=results, stderr=subprocess.PIPE, shell=isWindowsOS)
if proc.stderr:
reportStatusUpdate(proc.stderr.splitlines()[0])
print(f"Completed Benchmark")
print("Completed Benchmark")
def parseResults(commit = ""):
def parseResults(commit=""):
# Open the results file and find the "average" result key.
print(f"Parsing Benchmark results")
print("Parsing Benchmark results")
resultsJSON = {} # Typing.Dict()
benchmark = args.benchmark.replace('\\', '/')
with open(os.path.join(builddir, "qmlbench", "results.json")) as results:
......@@ -367,7 +381,7 @@ def parseResults(commit = ""):
return False
def findRevToBuild(refRevision, module, branch="", returnParent=False):
def findRevToBuild(refRevision, module, branch="", returnParent=False, getQt5BaseSha=False):
# Do a whole bunch of stuff to associate the revision we should build of other qt modules.
def getParentRev(commitJSON):
......@@ -395,40 +409,40 @@ def findRevToBuild(refRevision, module, branch="", returnParent=False):
# At least we parsed the response into valid JSON. Now, Look through the responses, validate that things are correct, and retreive the comments list.
for index, commit in enumerate(commitJSON):
if branch not in commit["branch"]:
print(f"ERROR: Selected revision of {module} was not commited on the {branch} branch. It was commited to {commit['branch']}. Please correct and resubmit the job.")
if index >= (len(commitJSON) - 1):
reportStatusUpdate(f"ERROR: Selected revision of {module} was not commited on the {branch} branch. It was commited to {commit['branch']}.")
return False
else:
continue
elif args.module not in commit["project"]:
print(f"ERROR: Selected revision is not in {args.module}. It is part of {commit['project'].split('/')[1]}. Please correct and resubmit the job.")
if index >= (len(commitJSON) - 1):
reportStatusUpdate(f"ERROR: Selected revision is not in {args.module}. It is part of {commit['project'].split('/')[1]}")
return False
else:
continue
elif branch in commit["branch"] and module == args.module:
# This logic branch will return immediately because it's the module we're testing a specific commit.
# We'll check it out directly and just wanted to verify it exists.
# But if the module to test is qtquick3d, verify that the commit date is after the first
# known integration and the branch to test is new enough.
if module == "qtquick3d":
if subprocess.run(["git", "show", "-s", "--format=%ct", refRevision], cwd=os.path.join(builddir, "qt5", args.module), stdout=subprocess.PIPE).stdout < 1566565560:
reportStatusUpdate("Commit too old to be built with qtquick3d")
if not getQt5BaseSha:
if branch not in commit["branch"]:
print(f"ERROR: Selected revision of {module} was not committed on the {branch} branch. It was committed to {commit['branch']}. Please correct and resubmit the job.")
if index >= (len(commitJSON) - 1):
reportStatusUpdate(f"ERROR: Selected revision of {module} was not committed on the {branch} branch. It was committed to {commit['branch']}.")
return False
if (packaging.version.parse(args.branch) if not args.branch == "dev" else False) < packaging.version.parse(5.14):
reportStatusUpdate("Cannot build. Commit must be 5.14 or later")
else:
continue
elif args.module not in commit["project"]:
print(f"ERROR: Selected revision is not in {args.module}. It is part of {commit['project'].split('/')[1]}. Please correct and resubmit the job.")
if index >= (len(commitJSON) - 1):
reportStatusUpdate(f"ERROR: Selected revision is not in {args.module}. It is part of {commit['project'].split('/')[1]}")
return False
else:
return True
else:
continue
elif branch in commit["branch"] and module == args.module:
# This logic branch will return immediately because it's the module we're testing a specific commit.
# We'll check it out directly and just wanted to verify it exists.
# But if the module to test is qtquick3d, verify that the commit date is after the first
# known integration and the branch to test is new enough.
if module == "qtquick3d":
if subprocess.run(["git", "show", "-s", "--format=%ct", refRevision], cwd=os.path.join(builddir, "qt5", args.module), stdout=subprocess.PIPE).stdout < 1566565560:
reportStatusUpdate("Commit too old to be built with qtquick3d")
return False
if (packaging.version.parse(args.branch) if not args.branch == "dev" else False) < packaging.version.parse(5.14):
reportStatusUpdate("Cannot build. Commit must be 5.14 or later")
return False
else:
return True
changeID = commit["change_id"]
print(f"Found Change ID: {changeID} based on commit {refRevision}")
# Pull the comments history on the change so we can look for a COIN integration ID.
request = requests.get(f"https://codereview.qt-project.org/changes/{changeID}/detail")
request = requests.get(f"https://codereview.qt-project.org/changes/qt%2F{args.module}~{args.branch}~{changeID}/detail")
if request.status_code != 200:
continue # Something's fishy about the commit and codereview doesn't recognize it. Try the next commit in the list of responses from our original query.
else:
......@@ -470,7 +484,7 @@ def findRevToBuild(refRevision, module, branch="", returnParent=False):
print("WARN: The change ID selected did not have a COIN integration.")
for message in commitDetails["messages"]:
if "Change has been successfully cherry-picked" in message["message"]:
print(f"WARN: Change was Cherry-picked. Manual Review suggested.")
print("WARN: Change was Cherry-picked. Manual Review suggested.")
saveResult(refRevision, "NoIntegrationCherryPicked")
break
# Try again with the reference commit's parent. It should have been integrated.
......@@ -512,7 +526,7 @@ def findRevToBuild(refRevision, module, branch="", returnParent=False):
try:
# Convert the integration's binary thrift_bin file back into a workitem object.
workitem = thriftpy.utils.deserialize(storagestructs.Task(), data)
except Exception as e:
except Exception:
print(f"Failed to extract COIN data for integration ID {integrationId}.")
# Try again with the reference commit's parent. It should have been integrated.
parentRev = getParentRev(commitJSON)
......@@ -531,6 +545,7 @@ def findRevToBuild(refRevision, module, branch="", returnParent=False):
# Ask git to identify the SHA1 of our target module to build based on the sha of the change we're testing. This uses the Qt5 supermodule.
try:
print(f"Running git ls-tree to find the sha1 to build for {module}")
proc = subprocess.run(["git", "ls-tree", workitem.product.sha1, module, "-z"], cwd=os.path.join(builddir, "qt5"), stdout=subprocess.PIPE)
sha = str(proc.stdout).split(" ")[2].split("\\t")[0]
print(f'Found {module} sha1: {sha}')
......@@ -542,7 +557,7 @@ def findRevToBuild(refRevision, module, branch="", returnParent=False):
saveResult(refRevision, "BadCOINData")
return False
#####################################################################################################################
### Try to use the internal COIN as a backup. This code will not be functional outside of COIN's local network!!! ###
# Try to use the internal COIN as a backup. This code will not be functional outside of COIN's local network!!! ###
#####################################################################################################################
# data = requests.get(f"https://testresults.qt.io/coin/api/item/qt/{args.module}/tasks/{integrationId}.thrift_bin").content
......@@ -558,7 +573,7 @@ def findRevToBuild(refRevision, module, branch="", returnParent=False):
# print(f"Also Failed to import COIN data from internal COIN for integration ID {integrationId}. Maybe the commit we're trying isn't in the selected module of {args.module}")
############################
### End of internal code ###
# End of internal code ###
############################
......@@ -569,15 +584,25 @@ def determineBuildEligibility(revision, module, branch):
reportStatusUpdate(f"Revision {revision} ")
def fetchChange(url, ref, module):
subprocess.run(["git", "fetch", url, ref], universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, module))
def cherryPickChange(module, sha):
subprocess.run(["git", "cherry-pick", sha if sha else "FETCH_HEAD", "--no-commit"], cwd=os.path.join(builddir, module), stderr=subprocess.PIPE,
universal_newlines=True, shell=isWindowsOS)
def lookupUnmergedCommit(revision, module):
commitRaw = requests.get(f"https://codereview.qt-project.org/changes/?q=commit:{revision}&o=ALL_REVISIONS&o=CURRENT_COMMIT").text[4:]
ref = ""
if not commitRaw:
return False
return (False, ref, module)
try:
commitJSON = json.loads(commitRaw)[0]
except IndexError:
print(f"ERROR: Unable to find revision on codereview: {revision}")
return False
return (False, ref, module)
try:
# Get the fetch addresses so we can pull and check out the revision.
for item in commitJSON["revisions"]:
......@@ -585,13 +610,72 @@ def lookupUnmergedCommit(revision, module):
revision = item
url = commitJSON["revisions"][revision]["fetch"]["anonymous http"]["url"]
ref = commitJSON["revisions"][revision]["fetch"]["anonymous http"]["ref"]
if not module:
module = commitJSON["project"][3:]
except Exception:
print(f"ERROR: Unable to get ref fetch data for commit to test {revision}")
reportStatusUpdate(f"ERROR: Unable to get ref fetch data for commit to test {revision}")
return False
return (False, ref, module)
print(f"Fetching {module} patch ref {ref}")
subprocess.run(["git", "fetch", url, ref], universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, module))
return True
return (url, ref, module)
def lookupCommitByRef(ref, module):
matches = re.findall(r"(\d{6})(?:/(\d{1,}))?", ref)
changeNumber = ""
patchset = ""
if matches:
changeNumber = matches[0]
if matches[1]:
patchset = matches[1]
commitRaw = requests.get(f"https://codereview.qt-project.org/changes/?q={changeNumber}&o=ALL_REVISIONS&o=CURRENT_COMMIT").text[4:]
if not commitRaw:
return (False, module)
try:
commitJSON = json.loads(commitRaw)[0]
except IndexError:
print(f"ERROR: Unable to find ref on codereview: {ref}")
return (False, module)
if not module:
module = commitJSON["project"][3:]
if patchset:
try:
# Get the fetch addresses so we can pull and check out the revision.
for item in commitJSON["revisions"]:
if patchset == item["_number"]:
return (commitJSON["revisions"][item]["fetch"]["anonymous http"]["url"], module)
except Exception:
print(f"Unable to get fetch URL for desired patchset {patchset}")
try:
return (commitJSON["revisions"][commitJSON["current_revision"]]["fetch"]["anonymous http"]["url"], module)
except Exception:
print(f"WARN: Failed to fall back to the current revision of {ref}")
return (False, module)
def applyPatches(currentModule):
if not args.patches:
return
else:
for patch in args.patches:
url = ""
ref = patch
module = ""
if "refs/changes/" in patch:
url, module = lookupCommitByRef(patch)
if not url:
continue
else:
url, ref, module = lookupUnmergedCommit(patch)
if not url:
continue
if module != currentModule:
print(f"Skipping patch {ref}. We're in {currentModule}, but it applies to {module}.")
continue
fetchChange(url, ref, module)
print(f"Cherry-picking qt/{module} ref/sha {ref} as a patch.")
cherryPickChange(module)
def checkout(revision, module, bisecting=False, branch="", buildOnHead=False):
......@@ -601,32 +685,22 @@ def checkout(revision, module, bisecting=False, branch="", buildOnHead=False):
if module == args.module:
print(f"This is the module to test. Cleaning {module}.")
subprocess.run(["git", "clean", "-dqfx"], universal_newlines=True, cwd=os.path.join(builddir, module))
qt5baseRev = findRevToBuild(revision, module, branch, False, True)
subprocess.run(["git", "checkout", qt5baseRev], cwd=os.path.join(builddir, module), stderr=subprocess.PIPE,
universal_newlines=True, shell=isWindowsOS)
print(f"Trying checkout of module to test with revision {revision}")
proc = subprocess.run(["git", "checkout", revision], cwd=os.path.join(builddir, module), stderr=subprocess.PIPE,
proc = subprocess.run(["git", "cherry-pick", revision, "--no-commit"], cwd=os.path.join(builddir, module), stderr=subprocess.PIPE,
universal_newlines=True, shell=isWindowsOS)
if "reference is not a tree" in proc.stderr:
# We must be trying to build a commit that's not yet merged. Let's look it up.
lookupUnmergedCommit(revision, module)
url, ref, module = lookupUnmergedCommit(revision, module)
fetchChange(url, ref, module)
print(f"Checking out unmerged commit {revision}")
proc = subprocess.run(["git", "checkout", "FETCH_HEAD"], cwd=os.path.join(builddir, module), stderr=subprocess.PIPE,
universal_newlines=True, shell=isWindowsOS)
cherryPickChange(module)
applyPatches(module)
return True
########### NOT YET PROPERLY IMPLEMENTED ###########
# needPatch = False # TESTING
# if needPatch:
# print("Applying patches...")
# print("Applying patch refs/changes/77/249377/2: QtQuickControls2: QQuickPopupPositioner: fix crash on application exit")
# subprocess.run(["git", "fetch", "https://codereview.qt-project.org/qt/qtquickcontrols2", "refs/changes/77/249377/2"], universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, module))
# subprocess.run(["git", "cherry-pick", "FETCH_HEAD", "-X", "theirs"], universal_newlines=True, shell=isWindowsOS, cwd=os.path.join(builddir, module))
# Example of implementing hard-coded patch:
# try: # merge-base gives no output if not ancestor.
# subprocess.run(["git", "merge-base", "--is-ancestor", "fa4b1aa6024f5f3b2d0e0502561b1eaedddd0c78", "HEAD"], stdout = subprocess.PIPE, universal_newlines=True, shell=isWindowsOS).stdout.splitlines()[0]
# except IndexError:
# # QtQuickControls2: QQuickPopupPositioner: fix crash on application exit
# subprocess.run(["git", "cherry-pick", "fa4b1aa6024f5f3b2d0e0502561b1eaedddd0c78"], universal_newlines=True, shell=isWindowsOS)
applyPatches(module)
return True
else:
......@@ -645,12 +719,14 @@ def checkout(revision, module, bisecting=False, branch="", buildOnHead=False):
actualRevision = subprocess.run(["git", "show", "-s", "--date=short", "--format=%cd"], cwd=os.path.join(builddir, module),
stdout=subprocess.PIPE, universal_newlines=True, shell=isWindowsOS).stdout.strip()
print(f"Checking out {module} from date {actualRevision} with SHA/tag {revToBuild}")
applyPatches(module)
return True
def cleanup(prepForBuild=False, noRebuild=False):
# Clean up our files and start fresh each time, or just uninstall and clean the module to test if we already specified to build against parent.
reportStatusUpdate("Cleaning up from the last build")
def clean(module):
subprocess.run(["git", "clean", "-dqfx"], universal_newlines=True, cwd=os.path.join(builddir, module))
if os.path.exists(os.path.join(builddir, module, '.git', 'index.lock')):
......@@ -663,7 +739,7 @@ def cleanup(prepForBuild=False, noRebuild=False):
initRepository(module)
if noRebuild:
subprocess.run([compiler, "uninstall", "-j", f"{int(args.buildCores) * 3 }"], universal_newlines=True, cwd=os.path.join(builddir, args.module))
subprocess.run(compiler.append("uninstall"), universal_newlines=True, cwd=os.path.join(builddir, args.module))
clean(args.module)
else:
shutil.rmtree(os.path.join(builddir, "Install"), on_rm_error)
......@@ -959,7 +1035,7 @@ def buildChanges(revision, bisecting=False, branch="", buildOnHead=False, noRebu
if bisecting:
if not determineBuildEligibility(revision, args.module, branch): # Make sure that the revision we're going to test is valid and can be built, so we don't waste time.
return False
modulesToBuild = ["qtbase", "qtdeclarative", "qtquickcontrols", "qtquickcontrols2", "qtgraphicaleffects", "qtsvg"]
modulesToBuild = ["qtbase", "qtdeclarative", "qtquickcontrols2", "qtgraphicaleffects", "qtsvg"]
if args.module == "qtquick3d":
modulesToBuild.append("qtquick3d")
for module in modulesToBuild: # Qtbase must be built first.
......@@ -972,13 +1048,14 @@ def buildChanges(revision, bisecting=False, branch="", buildOnHead=False, noRebu
return True
def reportStatusUpdate(status_msg = ""):
def reportStatusUpdate(status_msg=""):
try:
requests.post(f"http://localhost:{os.getenv('HTTP_PORT', 8080)}/updateLocalRunningJobStatus",
json={'status_msg': status_msg})
except:
json={'status_msg': status_msg})
except Exception:
pass
if __name__ == "__main__":
atexit.register(on_exit)
args = parseArgs()
......
......@@ -456,6 +456,8 @@ function runTest(test) {
test.bad_commit,
"--knownGoodRev",
test.good_commit,
"--patches",
test.patches,
"--benchmark",
test.test_name ? test.test_name : test.custom_benchmark_file.tempFilePath.replace(re, "/"),
"--regressionTarget",
......@@ -472,6 +474,8 @@ function runTest(test) {
test.module,
"--testSingleCommit",
test.commit,
"--patches",
test.patches,
"--benchmark",
test.test_name ? test.test_name : test.custom_benchmark_file.tempFilePath.replace(re, "/"),
"--buildCores",
......@@ -488,6 +492,8 @@ function runTest(test) {
test.module,
"--testTwoCommit",
test.firstCommit + "," + test.secondCommit,
"--patches",
test.patches,
"--benchmark",
test.test_name ? test.test_name : test.custom_benchmark_file.tempFilePath.replace(re, "/"),
"--buildCores",
......@@ -736,6 +742,7 @@ function scheduleJobRequest(req, res) {
branch: req.body.branch,
module: req.body.module,
commit: req.body.commit,
patches: req.body.patches,
firstCommitBuildOnHead: req.body.firstCommitBuildOnHead,
secondCommitBuildOnHead: req.body.secondCommitBuildOnHead,
firstCommit: req.body.firstCommit,
......@@ -906,6 +913,7 @@ function queryJobs(req, res) {
branch: runningJob.branch,
module: runningJob.module,
commit: runningJob.commit,
patches: runningJob.patches,
firstCommitBuildOnHead: runningJob.firstCommitBuildOnHead,
secondCommitBuildOnHead: runningJob.secondCommitBuildOnHead,
firstCommit: runningJob.firstCommit,
......@@ -1207,6 +1215,18 @@ function homePage(req, res) {
});
}
function addJobRecursive(viewElement, joblist, index, emitter, doneSignal, callback) {
toolbox.jobFormatter(joblist[index], hosts, (job) => {
viewElement += job;
index += 1;
if (index < joblist.length)
addJobRecursive(viewElement, joblist, index, emitter, doneSignal, callback);
else
callback(viewElement);
});
}
if (isPrimaryHost && !req.query.status) {
// Serve the web page if we're the scheduler and status wasn't explicitly requested.
const readyEmitter = new events.EventEmitter();
......@@ -1328,21 +1348,14 @@ function homePage(req, res) {
if (completedJobs.length > 15)
completedJobs = completedJobs.slice(-15);
readyEmitter.on("completedJobFinished", () => {
if ((completedJobsFinishedCount += 1) == completedJobs.length) {
view.completedJobs += "</ul></br>";
readyChecklist.completedJobs = true;
readyEmitter.emit('readyCheck');
}
});
view.completedJobs = `<div style="padding-left:25px"><button onclick="window.location.href='/clearFinishedJobs'">Clean up old jobs</button></div>`;
view.completedJobs += "<ul>";
for (let i = completedJobs.length - 1; i >= 0; i--) {
toolbox.jobFormatter(completedJobs[i], hosts, (job) => {
view