Skip to content

Commit

Permalink
Specify path to python3.5 for perf jobs (dotnet/coreclr#20443)
Browse files Browse the repository at this point in the history 
* Specify path to python3.5 for perf jobs

* Change path to python in throughput testing


Commit migrated from dotnet/coreclr@383f22c
  • Loading branch information
@michellemcdaniel
michellemcdaniel authored Nov 8, 2018
1 parent 0d35937 commit 9ea73ee
Show file tree
Hide file tree
Showing 3 changed files with 35 additions and 42 deletions.
59 changes: 33 additions & 26 deletions src/coreclr/perf.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,13 @@ def static getOSGroup(def os) {
def architecture = arch
def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
def testEnv = ""
def python = "C:\\Python35\\python.exe"

def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
// Set the label.
if (isSmoketest) {
label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
python = "C:\\python3.7.0\\python.exe"
}
else {
label('windows_server_2016_clr_perf')
Expand Down Expand Up @@ -87,23 +89,23 @@ def static getOSGroup(def os) {
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
"set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"[email protected]\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"[email protected]\"\n" +
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")

batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")

def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""

// Run with just stopwatch: Profile=Off
batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")

// Run with the full set of counters enabled: Profile=On
if (opt_level != 'min_opt') {
batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
}
}
}
Expand Down Expand Up @@ -184,6 +186,8 @@ def static getOSGroup(def os) {
['full_opt', 'min_opt'].each { opt_level ->
def architecture = arch

def python = "C:\\Python35\\python.exe"

pgo_build = ""
pgo_test = ""
pgo_string = "pgo"
Expand Down Expand Up @@ -223,11 +227,11 @@ def static getOSGroup(def os) {
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
"set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"[email protected]\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"[email protected]\"\n" +
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
batchFile("${python} -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
}
}

Expand Down Expand Up @@ -634,6 +638,7 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {

def testEnv = ""
def python = "C:\\Python35\\python.exe"

// Set the label.
label('windows_server_2016_clr_perf')
Expand Down Expand Up @@ -670,21 +675,21 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
"set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"[email protected]\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"[email protected]\"\n" +
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")

batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")

def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"

// Profile=Off
batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")

// Profile=On
if (opt_level != 'min_opt') {
batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
}
}
}
Expand Down Expand Up @@ -768,6 +773,7 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
def coreRoot = "${testBin}\\Tests\\Core_Root"
def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
def python = "C:\\Python35\\python.exe"

steps {
// Install nuget and get BenchView tools
Expand All @@ -781,11 +787,11 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
"set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
"py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"[email protected]\"\n" +
"py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
"${python} \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"[email protected]\"\n" +
"${python} \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")

// Generate machine data from BenchView
batchFile("py \"${benchViewTools}\\machinedata.py\"")
batchFile("${python} \"${benchViewTools}\\machinedata.py\"")

// Build CoreCLR and gnerate test layout
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
Expand All @@ -795,11 +801,11 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")

// From sodbench.csv, create measurment.json, then submission.json
batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
batchFile("${python} \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
batchFile("${python} \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")

// If this is a PR, upload submission.json
batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
batchFile("${python} \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
}
}

Expand Down Expand Up @@ -841,6 +847,7 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')

def testEnv = ""
def python = "C:\\Python35\\python.exe"
wrappers {
credentialsBinding {
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
Expand Down Expand Up @@ -874,9 +881,9 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
"set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"[email protected]\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"[email protected]\"\n" +
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")

batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
Expand All @@ -885,7 +892,7 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {

// Scenario: ILLink
batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
"py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
"${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
}
}

Expand Down
14 changes: 1 addition & 13 deletions src/coreclr/tests/scripts/run-throughput-perf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,17 +98,6 @@
'Ubuntu16.04': 'Linux'
}

python_exe_list = {
'Windows_NT': {
'x64': 'py',
'x86': 'py'
},
'Linux': {
'x64': 'python3.5',
'arm': 'python3.6'
}
}

##########################################################################
# Argument Parser
##########################################################################
Expand Down Expand Up @@ -337,7 +326,6 @@ def main(args):
global dll_exclude_list
global jit_list
global os_group_list
global python_exe_list

architecture, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations, opt_level, jit_name, no_pgo = validate_args(args)
arch = architecture
Expand Down Expand Up @@ -371,7 +359,7 @@ def main(args):
# Replace assembly_root's System.Private.CoreLib with built System.Private.CoreLib.
shutil.copyfile(os.path.join(bin_path, 'System.Private.CoreLib.dll'), os.path.join(assembly_root, 'System.Private.CoreLib.dll'))

python_exe = python_exe_list[os_group][arch]
python_exe = sys.executable

# Run throughput testing
for dll_file_name in os.listdir(assembly_root):
Expand Down
4 changes: 1 addition & 3 deletions src/coreclr/tests/scripts/run-xunit-perf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -418,13 +418,11 @@ def main(args):
coreclrPerf, assemblyName, arch, operatingSystem, configuration, jitName, optLevel, runType, outputDir, stabilityPrefix, isScenarioTest, benchviewPath, isPgoOptimized, benchviewGroup, hasWarmupRun, collectionFlags, isLibrary, uploadToBenchview, better, sliceNumber, sliceConfigFile = validate_args(args)

platform = sys.platform
python = 'py'
python = sys.executable
if platform == 'linux' or platform == 'linux2':
platform = 'Linux'
python = 'python3'
elif platform == 'darwin':
platform = 'OSX'
python = 'python3'
elif platform == 'win32':
platform = "Windows_NT"
else:
Expand Down

0 comments on commit 9ea73ee

Please sign in to comment.