Skip to content

Commit 569cf39

Browse files
upthewaterspoutupthewaterspout
authored
GEODE-9176: Automatically pass properties to benchmark JVMs (#149)
Converting all of the system properties we use in the benchmarks to start with the benchmark prefix. Changing the gradle build to copy all benchmark.* properties as system properties in the tst. Getting rid of problematic org.json dependency, and also making sure we capture *all* system properties that might effect the behavior of the test. Adding a way to automatically set system properties in test JVMs. Just add a property with the prefix benchmark.system.ROLE, where ROLE is the role of jvms to target. Eg benchmark.system.server.gemfire.disablePartitionedRegionBucketAck=true would set gemfire.disablePartitionedRegionBucketAck=true in the server JVMs. Logging all benchmark properties during test run
1 parent ec0a225 commit 569cf39

22 files changed

+211
-105
lines changed

‎geode-benchmarks/build.gradle

+39-15
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ test{
7373
logger.quiet "Executing test ${desc.className}.${desc.name} with result: ${result.resultType}"
7474
}
7575
useJUnitPlatform()
76+
systemProperty 'org.slf4j.simpleLogger.showDateTime', 'true'
77+
systemProperty 'org.slf4j.simpleLogger.dateTimeFormat', 'yyyy-MM-dd HH:mm:ss.SSS'
78+
systemProperty 'org.slf4j.simpleLogger.showThreadNam', 'false'
79+
systemProperty 'org.slf4j.simpleLogger.showShortLogName', 'true'
7680
}
7781

7882
task benchmark(type: Test) {
@@ -83,49 +87,69 @@ task benchmark(type: Test) {
8387
testClassesDirs = project.sourceSets.main.output.classesDirs
8488
classpath = project.sourceSets.main.runtimeClasspath
8589
useJUnitPlatform()
86-
testLogging { exceptionFormat = 'full' }
90+
testLogging {
91+
exceptionFormat = 'full'
92+
showStandardStreams = true
93+
}
8794

8895
exclude "**/NoopBenchmark.class"
8996
exclude "**/P2pPartitionedPutLongBenchmark.class"
9097

9198
forkEvery 1
9299
failFast = true
93100

101+
systemProperty 'org.slf4j.simpleLogger.showDateTime', 'true'
102+
systemProperty 'org.slf4j.simpleLogger.dateTimeFormat', 'yyyy-MM-dd HH:mm:ss.SSS'
103+
systemProperty 'org.slf4j.simpleLogger.showThreadName', 'false'
104+
systemProperty 'org.slf4j.simpleLogger.showShortLogName', 'true'
94105
systemProperty 'TEST_HOSTS', project.findProperty('hosts')
95106
systemProperty 'TEST_METADATA', project.findProperty('metadata')
96107
systemProperty 'OUTPUT_DIR', outputDir
108+
109+
//Set all project properties starting with "benchmark." as system properties in the test
110+
//JVM
111+
project.properties.findAll {
112+
it.key.startsWith("benchmark.")
113+
}.each {
114+
systemProperty(it.getKey(), it.getValue())
115+
}
116+
117+
//------------------------------------------------------------
118+
//Legacy properties - these properties were added before the benchmark.
119+
//prefix convention. They will be passed on to the JVM for now to not break
120+
//CI or peoples scripts. Remove these soon!
121+
//------------------------------------------------------------
97122
if (project.hasProperty('withGc')) {
98-
systemProperty 'withGc', project.findProperty('withGc')
123+
systemProperty 'benchmark.withGc', project.findProperty('withGc')
99124
}
100125
if (project.hasProperty('withHeap')) {
101-
systemProperty 'withHeap', project.findProperty('withHeap')
126+
systemProperty 'benchmark.withHeap', project.findProperty('withHeap')
102127
}
103128
if (project.hasProperty('withThreads')) {
104-
systemProperty 'withThreads', project.findProperty('withThreads')
129+
systemProperty 'benchmark.withThreads', project.findProperty('withThreads')
105130
}
106131
if (project.hasProperty('withWarmup')) {
107-
systemProperty 'withWarmup', project.findProperty('withWarmup')
132+
systemProperty 'benchmark.withWarmup', project.findProperty('withWarmup')
108133
}
109134
if (project.hasProperty('withDuration')) {
110-
systemProperty 'withDuration', project.findProperty('withDuration')
135+
systemProperty 'benchmark.withDuration', project.findProperty('withDuration')
111136
}
112137

113-
systemProperty 'withSsl', project.hasProperty('withSsl')
114-
systemProperty 'withSslProtocols', project.findProperty('withSslProtocols')
115-
systemProperty 'withSslCiphers', project.findProperty('withSslCiphers')
138+
systemProperty 'benchmark.withSsl', project.hasProperty('withSsl')
139+
systemProperty 'benchmark.withSslProtocols', project.findProperty('withSslProtocols')
140+
systemProperty 'benchmark.withSslCiphers', project.findProperty('withSslCiphers')
116141

117142
if (project.hasProperty('withSniProxy')) {
118-
systemProperty 'withSniProxy', project.findProperty('withSniProxy')
143+
systemProperty 'benchmark.withSniProxy', project.findProperty('withSniProxy')
119144
}
120-
systemProperty 'withSniProxyImage', project.findProperty('withSniProxyImage')
145+
systemProperty 'benchmark.withSniProxyImage', project.findProperty('withSniProxyImage')
121146

122147
if (project.hasProperty('withRouter')) {
123-
systemProperty 'withRouter', project.findProperty('withRouter')
148+
systemProperty 'benchmark.withRouter', project.findProperty('withRouter')
124149
}
125-
systemProperty 'withRouterImage', project.findProperty('withRouterImage')
150+
systemProperty 'benchmark.withRouterImage', project.findProperty('withRouterImage')
126151

127-
systemProperty 'withSecurityManager', project.hasProperty('withSecurityManager')
128-
systemProperty 'benchmark.profiler.argument', project.findProperty('benchmark.profiler.argument')
152+
systemProperty 'benchmark.withSecurityManager', project.hasProperty('withSecurityManager')
129153

130154

131155
doFirst {

‎geode-benchmarks/src/main/java/org/apache/geode/benchmark/parameters/GcParameters.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public class GcParameters {
2828

2929
public static void configure(final TestConfig testConfig) {
3030
final GcImplementation gcImplementation =
31-
GcImplementation.valueOf(System.getProperty("withGc", "CMS"));
31+
GcImplementation.valueOf(System.getProperty("benchmark.withGc", "CMS"));
3232
logger.info("Configuring {} GC.", gcImplementation);
3333
switch (gcImplementation) {
3434
case CMS:

‎geode-benchmarks/src/main/java/org/apache/geode/benchmark/parameters/HeapParameters.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public class HeapParameters {
2626
private static final Logger logger = LoggerFactory.getLogger(HeapParameters.class);
2727

2828
public static void configure(final TestConfig testConfig) {
29-
final String heap = System.getProperty("withHeap", "8g");
29+
final String heap = System.getProperty("benchmark.withHeap", "8g");
3030
logger.info("Configuring heap parameters {}.", heap);
3131
configureGeodeProductJvms(testConfig, "-Xmx" + heap, "-Xms" + heap);
3232
}

‎geode-benchmarks/src/main/java/org/apache/geode/benchmark/topology/ClientServerTopologyWithRouterAndSniProxy.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@
3535
import org.apache.geode.perftest.TestConfig;
3636

3737
public class ClientServerTopologyWithRouterAndSniProxy extends ClientServerTopologyWithSniProxy {
38-
public static final String WITH_ROUTER_PROPERTY = "withRouter";
39-
public static final String WITH_ROUTER_IMAGE_PROPERTY = "withRouterImage";
38+
public static final String WITH_ROUTER_PROPERTY = "benchmark.withRouter";
39+
public static final String WITH_ROUTER_IMAGE_PROPERTY = "benchmark.withRouterImage";
4040

4141
private static final int NUM_LOCATORS = 1;
4242
private static final int NUM_SERVERS = 2;

‎geode-benchmarks/src/main/java/org/apache/geode/benchmark/topology/ClientServerTopologyWithSniProxy.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,8 @@
4040
import org.apache.geode.perftest.TestConfig;
4141

4242
public class ClientServerTopologyWithSniProxy extends Topology {
43-
public static final String WITH_SNI_PROXY_PROPERTY = "withSniProxy";
44-
public static final String WITH_SNI_PROXY_IMAGE_PROPERTY = "withSniProxyImage";
43+
public static final String WITH_SNI_PROXY_PROPERTY = "benchmark.withSniProxy";
44+
public static final String WITH_SNI_PROXY_IMAGE_PROPERTY = "benchmark.withSniProxyImage";
4545

4646
private static final int NUM_LOCATORS = 1;
4747
private static final int NUM_SERVERS = 2;

‎geode-benchmarks/src/main/java/org/apache/geode/benchmark/topology/Topology.java

+6-6
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,14 @@
2525
import org.apache.geode.perftest.TestConfig;
2626

2727
public abstract class Topology {
28-
public static final String WITH_SSL_PROPERTY = "withSsl";
29-
static final String WITH_SSL_ARGUMENT = "-DwithSsl=true";
28+
public static final String WITH_SSL_PROPERTY = "benchmark.withSsl";
29+
static final String WITH_SSL_ARGUMENT = "-Dbenchmark.withSsl=true";
3030

31-
public static final String WITH_SSL_PROTOCOLS_PROPERTY = "withSslProtocols";
32-
public static final String WITH_SSL_CIPHERS_PROPERTY = "withSslCiphers";
31+
public static final String WITH_SSL_PROTOCOLS_PROPERTY = "benchmark.withSslProtocols";
32+
public static final String WITH_SSL_CIPHERS_PROPERTY = "benchmark.withSslCiphers";
3333

34-
public static final String WITH_SECURITY_MANAGER_PROPERTY = "withSecurityManager";
35-
static final String WITH_SECURITY_MANAGER_ARGUMENT = "-DwithSecurityManager=true";
34+
public static final String WITH_SECURITY_MANAGER_PROPERTY = "benchmark.withSecurityManager";
35+
static final String WITH_SECURITY_MANAGER_ARGUMENT = "-Dbenchmark.withSecurityManager=true";
3636

3737
static void configureCommon(TestConfig config) {
3838
JvmParameters.configure(config);

‎geode-benchmarks/src/test/java/org/apache/geode/benchmark/parameters/GcParametersTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
import org.apache.geode.perftest.TestConfig;
3131

3232
class GcParametersTest {
33-
private static final String WITH_GC = "withGc";
33+
private static final String WITH_GC = "benchmark.withGc";
3434
private static final String JAVA_RUNTIME_VERSION = "java.runtime.version";
3535
private static final String XX_USE_ZGC = "-XX:+UseZGC";
3636
private static final String XX_USE_G_1_GC = "-XX:+UseG1GC";

‎geode-benchmarks/src/test/java/org/apache/geode/benchmark/parameters/HeapParametersTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030

3131
class HeapParametersTest {
3232

33-
private static final String WITH_HEAP = "withHeap";
33+
private static final String WITH_HEAP = "benchmark.withHeap";
3434

3535
private Properties systemProperties;
3636

‎geode-benchmarks/src/test/java/org/apache/geode/benchmark/tests/ClientServerBenchmarkTest.java

+6-6
Original file line numberDiff line numberDiff line change
@@ -56,40 +56,40 @@ public void beforeEach() {
5656

5757
@AfterAll
5858
public static void afterAll() {
59-
System.clearProperty("withSniProxy");
59+
System.clearProperty("benchmark.withSniProxy");
6060
}
6161

6262
@Test
6363
public void withoutSniProxy() {
64-
System.clearProperty("withSniProxy");
64+
System.clearProperty("benchmark.withSniProxy");
6565
config = ClientServerBenchmark.createConfig();
6666
assertThat(config.getBefore()).doesNotContain(startHAProxyStep, startEnvoyStep);
6767
}
6868

6969
@Test
7070
public void withSniProxyInvalid() {
71-
System.setProperty("withSniProxy", "invalid");
71+
System.setProperty("benchmark.withSniProxy", "invalid");
7272
assertThatThrownBy(() -> ClientServerBenchmark.createConfig())
7373
.isInstanceOf(IllegalArgumentException.class);
7474
}
7575

7676
@Test
7777
public void withSniProxyDefault() {
78-
System.setProperty("withSniProxy", "");
78+
System.setProperty("benchmark.withSniProxy", "");
7979
config = ClientServerBenchmark.createConfig();
8080
assertThat(config.getBefore()).contains(startHAProxyStep).doesNotContain(startEnvoyStep);
8181
}
8282

8383
@Test
8484
public void withSniProxyHAProxy() {
85-
System.setProperty("withSniProxy", "HAProxy");
85+
System.setProperty("benchmark.withSniProxy", "HAProxy");
8686
config = ClientServerBenchmark.createConfig();
8787
assertThat(config.getBefore()).contains(startHAProxyStep);
8888
}
8989

9090
@Test
9191
public void withSniProxyEnvoy() {
92-
System.setProperty("withSniProxy", "Envoy");
92+
System.setProperty("benchmark.withSniProxy", "Envoy");
9393
config = ClientServerBenchmark.createConfig();
9494
assertThat(config.getBefore()).contains(startEnvoyStep);
9595
}

‎geode-benchmarks/src/test/java/org/apache/geode/benchmark/topology/ClientServerTopologyTest.java

+13-10
Original file line numberDiff line numberDiff line change
@@ -43,45 +43,48 @@ public void afterEach() {
4343

4444
@Test
4545
public void configWithSsl() {
46-
System.setProperty("withSsl", "true");
46+
System.setProperty("benchmark.withSsl", "true");
4747
TestConfig testConfig = new TestConfig();
4848
ClientServerTopology.configure(testConfig);
49-
assertThat(testConfig.getJvmArgs().get(CLIENT.name())).contains("-DwithSsl=true");
49+
assertThat(testConfig.getJvmArgs().get(CLIENT.name())).contains("-Dbenchmark.withSsl=true");
5050
}
5151

5252
@Test
5353
public void configWithNoSsl() {
5454
TestConfig testConfig = new TestConfig();
5555
ClientServerTopology.configure(testConfig);
56-
assertThat(testConfig.getJvmArgs().get(CLIENT.name())).doesNotContain("-DwithSsl=true");
56+
assertThat(testConfig.getJvmArgs().get(CLIENT.name()))
57+
.doesNotContain("-Dbenchmark.withSsl=true");
5758
}
5859

5960
@Test
6061
public void configWithoutSecurityManager() {
6162
TestConfig testConfig = new TestConfig();
6263
ClientServerTopology.configure(testConfig);
6364
assertThat(testConfig.getJvmArgs().get(CLIENT.name()))
64-
.doesNotContain("-DwithSecurityManager=true");
65+
.doesNotContain("-Dbenchmark.withSecurityManager=true");
6566
}
6667

6768
@Test
6869
public void configWithSecurityManager() {
69-
System.setProperty("withSecurityManager", "true");
70+
System.setProperty("benchmark.withSecurityManager", "true");
7071
TestConfig testConfig = new TestConfig();
7172
ClientServerTopology.configure(testConfig);
72-
assertThat(testConfig.getJvmArgs().get(CLIENT.name())).contains("-DwithSecurityManager=true");
73+
assertThat(testConfig.getJvmArgs().get(CLIENT.name()))
74+
.contains("-Dbenchmark.withSecurityManager=true");
7375
}
7476

7577
@Test
7678
public void configWithSecurityManagerAndSslAndJava11() {
77-
System.setProperty("withSecurityManager", "true");
79+
System.setProperty("benchmark.withSecurityManager", "true");
7880
System.setProperty("java.runtime.version", "11.0.4+11");
79-
System.setProperty("withSsl", "true");
81+
System.setProperty("benchmark.withSsl", "true");
8082
TestConfig testConfig = new TestConfig();
8183

8284
ClientServerTopology.configure(testConfig);
8385

84-
assertThat(testConfig.getJvmArgs().get(CLIENT.name())).contains("-DwithSecurityManager=true");
85-
assertThat(testConfig.getJvmArgs().get(CLIENT.name())).contains("-DwithSsl=true");
86+
assertThat(testConfig.getJvmArgs().get(CLIENT.name()))
87+
.contains("-Dbenchmark.withSecurityManager=true");
88+
assertThat(testConfig.getJvmArgs().get(CLIENT.name())).contains("-Dbenchmark.withSsl=true");
8689
}
8790
}

‎geode-benchmarks/src/test/java/org/apache/geode/benchmark/topology/ClientServerTopologyWithSniProxyTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public void configWithNoSsl(final SniProxyImplementation sniProxyImplementation)
5151
System.setProperty(WITH_SNI_PROXY_PROPERTY, sniProxyImplementation.name());
5252
final TestConfig testConfig = new TestConfig();
5353
ClientServerTopologyWithSniProxy.configure(testConfig);
54-
assertThat(testConfig.getJvmArgs().get(CLIENT.name())).contains("-DwithSsl=true");
54+
assertThat(testConfig.getJvmArgs().get(CLIENT.name())).contains("-Dbenchmark.withSsl=true");
5555
}
5656

5757
}

‎harness/build.gradle

-1
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@ dependencies {
5555
compile(group: 'commons-io', name: 'commons-io', version: project.'commons-io.version')
5656
compile(group: 'org.yardstickframework', name: 'yardstick', version: project.'yardstick.version')
5757
compile(group: 'org.hdrhistogram', name: 'HdrHistogram', version: project.'HdrHistogram.version')
58-
compile(group: 'org.json', name: 'json', version: project.'JSON.version')
5958
compile(group: 'org.apache.geode', name: 'geode-core', version: geodeVersion)
6059
testCompile(group: 'org.mockito', name: 'mockito-all', version: project.'mockito-all.version')
6160
testCompile(group: 'org.awaitility', name: 'awaitility', version: project.'awaitility.version')

‎harness/src/main/java/org/apache/geode/perftest/BenchmarkProperties.java

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.geode.perftest;
19+
20+
import java.util.ArrayList;
21+
import java.util.HashMap;
22+
import java.util.List;
23+
import java.util.Map;
24+
25+
public class BenchmarkProperties {
26+
27+
/**
28+
* Read the current system properties and find any system properties that start
29+
* with "benchmark.system.ROLE" Returns a map of ROLE-> system properties for that
30+
* role.
31+
*/
32+
public static Map<String, List<String>> getDefaultJVMArgs() {
33+
Map<String, List<String>> results = new HashMap<>();
34+
System.getProperties().stringPropertyNames().stream()
35+
.filter(name -> name.startsWith("benchmark.system."))
36+
.forEach(name -> {
37+
String shortName = name.replace("benchmark.system.", "");
38+
String[] roleAndProperty = shortName.split("\\.", 2);
39+
String role = roleAndProperty[0];
40+
String property = roleAndProperty[1];
41+
String value = System.getProperty(name);
42+
List<String> roleProperties = results.computeIfAbsent(role, key -> new ArrayList<>());
43+
roleProperties.add("-D" + property + "=" + value);
44+
});
45+
return results;
46+
}
47+
}

‎harness/src/main/java/org/apache/geode/perftest/TestConfig.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import java.util.ArrayList;
2222
import java.util.Arrays;
2323
import java.util.Collections;
24-
import java.util.HashMap;
2524
import java.util.LinkedHashMap;
2625
import java.util.List;
2726
import java.util.Map;
@@ -39,7 +38,7 @@ public class TestConfig implements Serializable {
3938

4039
private final WorkloadConfig workloadConfig = new WorkloadConfig();
4140
private Map<String, Integer> roles = new LinkedHashMap<>();
42-
private Map<String, List<String>> jvmArgs = new HashMap<>();
41+
private Map<String, List<String>> jvmArgs = BenchmarkProperties.getDefaultJVMArgs();
4342
private List<TestStep> before = new ArrayList<>();
4443
private List<TestStep> workload = new ArrayList<>();
4544
private List<TestStep> after = new ArrayList<>();

‎harness/src/main/java/org/apache/geode/perftest/TestRunners.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@
3838
*/
3939
public class TestRunners {
4040

41-
public static final String TEST_HOSTS = "TEST_HOSTS";
42-
public static final String OUTPUT_DIR = "OUTPUT_DIR";
41+
public static final String TEST_HOSTS = "benchmark.TEST_HOSTS";
42+
public static final String OUTPUT_DIR = "benchmark.OUTPUT_DIR";
4343

4444
public static final String[] JVM_ARGS_SMALL_SIZE = new String[] {
4545
"-XX:CMSInitiatingOccupancyFraction=60",
@@ -91,7 +91,7 @@ public static TestRunner defaultRunner() {
9191
static TestRunner defaultRunner(String testHosts, File outputDir) {
9292
if (testHosts == null) {
9393
throw new IllegalStateException(
94-
"You must set the TEST_HOSTS system property to a comma separated list of hosts to run the benchmarks on.");
94+
"You must set the benchmark.TEST_HOSTS system property to a comma separated list of hosts to run the benchmarks on.");
9595
}
9696

9797
String userName = System.getProperty("user.name");

0 commit comments

Comments
 (0)