Skip to content

Commit

Permalink
GEODE-10052: Fix flakiness in OutOfMemoryDUnitTest (apache#7369)
Browse files Browse the repository at this point in the history
- Add larger values to get over critical memory threshold faster and
 make it more likely to stay there
 - Force GC before each attempt to add a large value to make it less
 likely that we'll drop back below critical threshold due to small
 objects being GCd while over critical threshold
  • Loading branch information
DonalEvans authored Feb 17, 2022
1 parent 5d72863 commit 920268b
Showing 1 changed file with 14 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ public class OutOfMemoryDUnitTest {

private static final long KEY_TTL_SECONDS = 10;
private static final int MAX_ITERATION_COUNT = 4000;
private static final int LARGE_VALUE_SIZE = 128 * 1024;
private static final int LARGE_VALUE_SIZE = 1024 * 1024;
private static final String LARGE_VALUE = StringUtils.leftPad("a", LARGE_VALUE_SIZE);
private static final String KEY = "key";

private static String locatorPort;
Expand Down Expand Up @@ -265,11 +266,10 @@ public void shouldAllowWriteOperations_afterDroppingBelowCriticalThreshold() {

memoryPressure.cancel(true);

String value = StringUtils.leftPad("a", LARGE_VALUE_SIZE);
await().untilAsserted(() -> assertThatNoException().isThrownBy(
() -> {
removeAllKeysAndForceGC();
jedis.set(server1Tag + "newKey", value);
jedis.set(server1Tag + "newKey", LARGE_VALUE);
}));
}

Expand All @@ -285,11 +285,10 @@ public void shouldAllowWriteOperations_onOtherServer_afterDroppingBelowCriticalT

memoryPressure.cancel(true);

String value = StringUtils.leftPad("a", LARGE_VALUE_SIZE);
await().untilAsserted(() -> assertThatNoException().isThrownBy(
() -> {
removeAllKeysAndForceGC();
jedis.set(server2Tag + "newKey", value);
jedis.set(server2Tag + "newKey", LARGE_VALUE);
}));
}

Expand Down Expand Up @@ -338,34 +337,26 @@ public void shouldAllowPublish_afterDroppingBelowCriticalThreshold() {
}

private void fillServer1Memory(JedisCluster jedis, boolean withExpiration) {
String valueString = StringUtils.leftPad("a", LARGE_VALUE_SIZE);

while (valueString.length() > 1) {
addMultipleKeysToServer1UntilOOMExceptionIsThrown(jedis, valueString, withExpiration);
valueString = valueString.substring(valueString.length() / 2);
}
}

private void maintainMemoryPressure(JedisCluster jedis, boolean withExpiration) {
while (!Thread.interrupted()) {
fillServer1Memory(jedis, withExpiration);
}
}

private void addMultipleKeysToServer1UntilOOMExceptionIsThrown(JedisCluster jedis,
String valueString, boolean withExpiration) {
assertThatThrownBy(() -> {
// First force GC to reduce the chances of dropping back below critical accidentally
gfsh.execute("gc");
for (int count = 0; count < MAX_ITERATION_COUNT; ++count) {
if (withExpiration) {
jedis.setex(server1Tag + KEY + numberOfKeys.get(), KEY_TTL_SECONDS, valueString);
jedis.setex(server1Tag + KEY + numberOfKeys.get(), KEY_TTL_SECONDS, LARGE_VALUE);
} else {
jedis.set(server1Tag + KEY + numberOfKeys.get(), valueString);
jedis.set(server1Tag + KEY + numberOfKeys.get(), LARGE_VALUE);
}
numberOfKeys.incrementAndGet();
}
}).hasMessageContaining("OOM command not allowed");
}

private void maintainMemoryPressure(JedisCluster jedis, boolean withExpiration) {
while (!Thread.interrupted()) {
fillServer1Memory(jedis, withExpiration);
}
}

void removeAllKeysAndForceGC() throws Exception {
// Remove all the keys to allow memory to drop below critical
Set<String> keys = jedis.keys(server1Tag + "*");
Expand Down

0 comments on commit 920268b

Please sign in to comment.