Skip to content

Commit

Permalink
Migrate common benchmark builders to jsonnet
Browse files Browse the repository at this point in the history
  • Loading branch information
@farquet
farquet committed Apr 22, 2021
1 parent 7b97b5c commit d219e79
Show file tree
Hide file tree
Showing 18 changed files with 651 additions and 306 deletions.
10 changes: 10 additions & 0 deletions ci-resources.libsonnet
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
// location of resources that can be easily overwritten
{
infra: {
ol8_bench_image: "<ol8_bench_image>",
benchmarking_config_repo: "<benchmarking_config_repo>",

notify_nexus_deploy: "<notify_nexus_deploy>",
notify_releaser_service: "<notify_releaser_service>"
}
}
9 changes: 6 additions & 3 deletions ci.jsonnet
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# Compiler
local compiler = import 'compiler/ci.jsonnet';

# GraalWasm
local wasm = import 'wasm/ci.jsonnet';

Expand All @@ -7,9 +10,9 @@ local espresso = import 'espresso/ci.jsonnet';
# Sulong
local sulong = import 'sulong/ci.jsonnet';
{
# ensure that public entries in common.jsonnet can be resolved
# ensure that entries in common.jsonnet can be resolved
_checkCommon: (import 'common.jsonnet'),
ci_resources:: (import 'ci-resources.libsonnet'),
specVersion: "2",
builds: wasm.builds + espresso.builds + sulong.builds
builds: compiler.builds + wasm.builds + espresso.builds + sulong.builds
}

130 changes: 113 additions & 17 deletions common.jsonnet
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

local mx_version = (import "graal-common.json").mx_version,
local common_json = composable(import "common.json"),
local repo_config = import 'repo-configuration.libsonnet',
local jdks = common_json.jdks,
local deps = common_json.deps,
local downloads = common_json.downloads,
Expand All @@ -14,24 +15,78 @@
}
},

oraclejdk8:: { downloads+: { JAVA_HOME : jdks.oraclejdk8, EXTRA_JAVA_HOMES : { pathlist :[ jdks["labsjdk-ee-11"] ]} }},
oraclejdk8Only:: { downloads+: { JAVA_HOME : jdks.oraclejdk8 }},
oraclejdk8Debug:: { downloads+: { JAVA_HOME : jdks.oraclejdk8Debug, EXTRA_JAVA_HOMES : { pathlist :[ jdks["labsjdk-ee-11"] ]} }},
oraclejdk8OnlyDebug:: { downloads+: { JAVA_HOME : jdks.oraclejdk8Debug }},
eclipse:: downloads.eclipse,
jdt:: downloads.jdt,

build_base:: {
// holds location of CI resources that can easily be overwritten
ci_resources:: (import "ci-resources.libsonnet"),
},

// Heap settings
// *************
local small_heap = "1G",
local default_heap = "8G",
local large_heap = "31G", // strictly smaller than 32G to keep compressed oops enabled
local large_young_gen_heap = "27G", // tuned to reduce latency of large apps like SpecJBB2015
heap:: {
small:: {
environment+: {
XMS: small_heap,
XMX: small_heap
}
},
default:: {
environment+: {
XMS: default_heap,
XMX: default_heap
}
},
large:: {
environment+: {
XMS: large_heap,
XMX: large_heap
}
},
large_with_large_young_gen:: {
environment+: {
XMS: large_heap,
XMX: large_heap,
XMN: large_young_gen_heap
}
}
},

// JDK definitions
// ***************
# jdk_version is an hidden field that can be used to generate job names
local jdk8 = { jdk_version:: 8},
local jdk11 = { jdk_version:: 11},
local jdk16 = { jdk_version:: 16},

oraclejdk8:: jdk8 + { downloads+: { JAVA_HOME : jdks.oraclejdk8, EXTRA_JAVA_HOMES : { pathlist :[ jdks["labsjdk-ee-11"] ]} }},
oraclejdk8Only:: jdk8 + { downloads+: { JAVA_HOME : jdks.oraclejdk8 }},
oraclejdk8Debug:: jdk8 + { downloads+: { JAVA_HOME : jdks.oraclejdk8Debug, EXTRA_JAVA_HOMES : { pathlist :[ jdks["labsjdk-ee-11"] ]} }},
oraclejdk8OnlyDebug:: jdk8 + { downloads+: { JAVA_HOME : jdks.oraclejdk8Debug }},

openjdk8:: jdk8 + { downloads+: { JAVA_HOME : jdks.openjdk8 }},

openjdk8:: { downloads+: { JAVA_HOME : jdks.openjdk8 }},
oraclejdk11:: jdk11 + { downloads+: { JAVA_HOME : jdks.oraclejdk11 }},
oraclejdk16:: jdk16 + { downloads+: { JAVA_HOME : jdks.oraclejdk16 }},
openjdk11:: jdk11 + { downloads+: { JAVA_HOME : jdks.openjdk11 }},

oraclejdk11:: { downloads+: { JAVA_HOME : jdks.oraclejdk11 }},
oraclejdk16:: { downloads+: { JAVA_HOME : jdks.oraclejdk16 }},
openjdk11:: { downloads+: { JAVA_HOME : jdks.openjdk11 }},
"labsjdk-ce-11":: jdk11 + { downloads+: { JAVA_HOME : jdks["labsjdk-ce-11"] }},
"labsjdk-ee-11":: jdk11 + { downloads+: { JAVA_HOME : jdks["labsjdk-ee-11"] }},
labsjdk11:: self["labsjdk-" + repo_config.graalvm_edition + "-11"],
"labsjdk-ce-16":: jdk16 + { downloads+: { JAVA_HOME : jdks["labsjdk-ce-16"] }},
"labsjdk-ee-16":: jdk16 + { downloads+: { JAVA_HOME : jdks["labsjdk-ee-16"] }},
labsjdk16:: self["labsjdk-" + repo_config.graalvm_edition + "-16"],
"labsjdk-ce-16Debug":: jdk16 + { downloads+: { JAVA_HOME : jdks["labsjdk-ce-16Debug"] }},
"labsjdk-ee-16Debug":: jdk16 + { downloads+: { JAVA_HOME : jdks["labsjdk-ee-16Debug"] }},

"labsjdk-ce-11":: { downloads+: { JAVA_HOME : jdks["labsjdk-ce-11"] }},
"labsjdk-ee-11":: { downloads+: { JAVA_HOME : jdks["labsjdk-ee-11"] }},
"labsjdk-ce-16":: { downloads+: { JAVA_HOME : jdks["labsjdk-ce-16"] }},
"labsjdk-ee-16":: { downloads+: { JAVA_HOME : jdks["labsjdk-ee-16"] }},
"labsjdk-ce-16Debug":: { downloads+: { JAVA_HOME : jdks["labsjdk-ce-16Debug"] }},
"labsjdk-ee-16Debug":: { downloads+: { JAVA_HOME : jdks["labsjdk-ee-16Debug"] }},

// Hardware definitions
// ********************
common:: deps.common + self.mx + {
# enforce self.os (useful for generating job names)
os:: error "self.os not set",
Expand Down Expand Up @@ -60,7 +115,7 @@

amd64:: {
arch::"amd64",
capabilities+: [self.arch],
capabilities+: [self.arch]
},

aarch64:: {
Expand All @@ -73,6 +128,47 @@
"windows-amd64":: self.windows + self.amd64,
"linux-aarch64":: self.linux + self.aarch64,

eclipse:: downloads.eclipse,
jdt:: downloads.jdt,
// Benchmarking building blocks
// ****************************
bench_hw:: {
_bench_machine:: {
targets+: ["bench"],
machine_name:: error "machine_name must be set!",
local _machine_name = self.machine_name,
capabilities+: [_machine_name],
environment+: { "MACHINE_NAME": _machine_name },
numa_nodes:: [],
is_numa:: std.length(self.numa_nodes) > 0,
},

x52:: $.linux + $.amd64 + self._bench_machine + {
machine_name:: "x52",
capabilities+: ["no_frequency_scaling", "tmpfs25g"],
numa_nodes:: [0, 1]
},
xgene3:: $.linux + $.aarch64 + self._bench_machine + {
machine_name:: "xgene3",
capabilities+: [],
},
a12c:: $.linux + $.aarch64 + self._bench_machine + {
machine_name:: "a12c",
capabilities+: ["no_frequency_scaling", "tmpfs25g"],
numa_nodes:: [0, 1]
}
},

hwlocIfNuma(numa, cmd, node=0)::
if numa then
["hwloc-bind", "--cpubind", "node:"+node, "--membind", "node:"+node, "--"] + cmd
else
cmd,

parallelHwloc(cmd_node0, cmd_node1)::
// Returns a list of commands that will run cmd_nod0 on NUMA node 0
// concurrently with cmd_node1 on NUMA node 1 and then wait for both to complete.
[
$.hwlocIfNuma(true, cmd_node0, node=0) + ["&"],
$.hwlocIfNuma(true, cmd_node1, node=1) + ["&"],
["wait"]
]
}
7 changes: 7 additions & 0 deletions compiler/ci.jsonnet
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
local graal_benchmarks = (import 'ci_common/benchmark-builders.jsonnet'),
local baseline_benchmarks = (import 'ci_includes/baseline-benchmarks.jsonnet'),
builds:
graal_benchmarks.builds +
baseline_benchmarks.builds
}
15 changes: 0 additions & 15 deletions compiler/ci_common/bench-aarch64.hocon
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,4 @@ aarch64.default-libgraal : ${aarch64.default} {
}

builds += [
# jargraal
${aarch64.default} ${bench-dacapo} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-dacapo-11-linux-aarch64" }
${aarch64.default} ${bench-scala-dacapo} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-scala-dacapo-11-linux-aarch64", timelimit: "45:00" }
${aarch64.default} ${bench-renaissance-no-db-shootout} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-renaissance-11-linux-aarch64" }
${aarch64.default} ${bench-specjvm2008-Single} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-specjvm2008-Single-11-linux-aarch64" }
${aarch64.default} ${bench-specjbb2005} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-specjbb2005-11-linux-aarch64" }
${aarch64.default} ${bench-specjbb2015} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-specjbb2015-11-linux-aarch64" }

# libgraal
${aarch64.default-libgraal} ${bench-dacapo} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-dacapo-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-scala-dacapo} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-scala-dacapo-libgraal-11-linux-aarch64", timelimit: "45:00" }
${aarch64.default-libgraal} ${bench-renaissance-no-db-shootout} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-renaissance-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-specjvm2008-Single} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-specjvm2008-Single-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-specjbb2005} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-specjbb2005-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-specjbb2015} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-specjbb2015-libgraal-11-linux-aarch64" }
]
Loading

0 comments on commit d219e79

Please sign in to comment.