Skip to content

Commit

Permalink
GEODE-6304: Refactor memory monitor to properly reset tolerance count…
Browse files Browse the repository at this point in the history 
…er (apache#3102)

The tolerance counter was not properly being reset between
non-consecutive EVICTION and CRITICAL events.  To fix this and make it
more understandable, the threshold logic was combined with the logic to
compute the next state in the MemoryThreshold state machine.  The
counter is now reset between non-consecutive EVICTION and CRITICAL
events.
  • Loading branch information
@mcmellawatt
mcmellawatt authored Jan 30, 2019
1 parent 054687a commit 546abdd
Show file tree
Hide file tree
Showing 10 changed files with 414 additions and 235 deletions.
136 changes: 74 additions & 62 deletions ...src/distributedTest/java/org/apache/geode/cache/management/MemoryThresholdsDUnitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,7 @@ public void testEventDelivery() throws Exception {
public Object call() throws Exception {
InternalCache gfCache = getCache();
getCache().getLogger().fine(addExpectedExString);
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(950);
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(950, "test");
getCache().getLogger().fine(removeExpectedExString);
return null;
}
Expand All @@ -491,7 +491,7 @@ public Object call() throws Exception {
public Object call() throws Exception {
InternalCache gfCache = getCache();
getCache().getLogger().fine(addExpectedBelow);
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(850);
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(850, "test");
getCache().getLogger().fine(removeExpectedBelow);
return null;
}
Expand All @@ -508,7 +508,7 @@ public Object call() throws Exception {
@Override
public Object call() throws Exception {
InternalCache gfCache = getCache();
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(840);
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(840, "test");
return null;
}
});
Expand All @@ -524,7 +524,7 @@ public Object call() throws Exception {
@Override
public Object call() throws Exception {
InternalCache gfCache = getCache();
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(750);
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(750, "test");
return null;
}
});
Expand All @@ -541,7 +541,7 @@ public Object call() throws Exception {
@Override
public Object call() throws Exception {
InternalCache gfCache = getCache();
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(950);
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(950, "test");
return null;
}
});
Expand All @@ -557,7 +557,7 @@ public Object call() throws Exception {
@Override
public Object call() throws Exception {
InternalCache gfCache = getCache();
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(750);
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(750, "test");
return null;
}
});
Expand All @@ -574,7 +574,7 @@ public Object call() throws Exception {
@Override
public Object call() throws Exception {
InternalCache gfCache = getCache();
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(850);
gfCache.getInternalResourceManager().getHeapMonitor().updateStateAndSendEvent(850, "test");
return null;
}
});
Expand Down Expand Up @@ -1499,7 +1499,7 @@ private void setUsageAboveCriticalThreshold(VM vm) {
public Object call() throws Exception {
getCache().getLogger().fine(addExpectedExString);
((GemFireCacheImpl) getCache()).getInternalResourceManager().getHeapMonitor()
.updateStateAndSendEvent(950);
.updateStateAndSendEvent(950, "test");
HeapMemoryMonitor.setTestBytesUsedForThresholdSet(950);
getCache().getLogger().fine(removeExpectedExString);
return null;
Expand All @@ -1514,7 +1514,7 @@ public Object call() throws Exception {
getCache().getLogger().fine(addExpectedBelow);
HeapMemoryMonitor.setTestBytesUsedForThresholdSet(850);
((GemFireCacheImpl) getCache()).getInternalResourceManager().getHeapMonitor()
.updateStateAndSendEvent(850);
.updateStateAndSendEvent(850, "test");
getCache().getLogger().fine(removeExpectedBelow);
return null;
}
Expand All @@ -1527,7 +1527,7 @@ private void setUsageBelowEviction(VM vm) {
public Object call() throws Exception {
getCache().getLogger().fine(addExpectedBelow);
((GemFireCacheImpl) getCache()).getInternalResourceManager().getHeapMonitor()
.updateStateAndSendEvent(750);
.updateStateAndSendEvent(750, "test");
getCache().getLogger().fine(removeExpectedBelow);
return null;
}
Expand Down Expand Up @@ -1746,7 +1746,7 @@ public Object call() throws Exception {
public Object call() throws Exception {
InternalResourceManager irm = ((GemFireCacheImpl) getCache()).getInternalResourceManager();
// Reset CRITICAL_UP by informing all that heap usage is now 1 byte (0 would disable).
irm.getHeapMonitor().updateStateAndSendEvent(1);
irm.getHeapMonitor().updateStateAndSendEvent(1, "test");
Set<ResourceListener> listeners = irm.getResourceListeners(ResourceType.HEAP_MEMORY);
Iterator<ResourceListener> it = listeners.iterator();
while (it.hasNext()) {
Expand Down Expand Up @@ -1943,7 +1943,7 @@ public void close() {}
// 10%
assertTrue(fakeHeapUsage > 0);
assertTrue(fakeHeapUsage <= fakeHeapMaxSize);
hmm.updateStateAndSendEvent(fakeHeapUsage);
hmm.updateStateAndSendEvent(fakeHeapUsage, "test");
getCache().getLogger().fine(removeExpectedExString);

assertTrue(hmm.getState().isCritical());
Expand All @@ -1961,7 +1961,7 @@ public void close() {}
// by 30%
assertTrue(fakeHeapMaxSize > 0);
getCache().getLogger().fine(addExpectedBelow);
hmm.updateStateAndSendEvent(fakeHeapUsage);
hmm.updateStateAndSendEvent(fakeHeapUsage, "test");
getCache().getLogger().fine(removeExpectedBelow);
assertFalse(hmm.getState().isCritical());
{
Expand Down Expand Up @@ -2110,7 +2110,7 @@ public void close() {}
// 10%
assertTrue(newfakeHeapUsage > 0);
assertTrue(newfakeHeapUsage <= fakeHeapMaxSize);
hmm.updateStateAndSendEvent(newfakeHeapUsage);
hmm.updateStateAndSendEvent(newfakeHeapUsage, "test");
assertTrue(hmm.getState().isCritical());
{
Integer k = new Integer(2);
Expand All @@ -2122,7 +2122,7 @@ public void close() {}
// by 30%
assertTrue(fakeHeapMaxSize > 0);
getCache().getLogger().fine(addExpectedBelow);
hmm.updateStateAndSendEvent(newfakeHeapUsage);
hmm.updateStateAndSendEvent(newfakeHeapUsage, "test");
getCache().getLogger().fine(removeExpectedBelow);
assertFalse(hmm.getState().isCritical());
{
Expand Down Expand Up @@ -2175,7 +2175,7 @@ public Object call() throws Exception {
// by 10%
assertTrue(newfakeHeapUsage > 0);
assertTrue(newfakeHeapUsage <= fakeHeapMaxSize);
hmm.updateStateAndSendEvent(newfakeHeapUsage);
hmm.updateStateAndSendEvent(newfakeHeapUsage, "test");
assertTrue(hmm.getState().isCritical());
{
Integer k = new Integer(5);
Expand All @@ -2189,7 +2189,7 @@ public Object call() throws Exception {
// 30%
assertTrue(fakeHeapMaxSize > 0);
getCache().getLogger().fine(addExpectedBelow);
hmm.updateStateAndSendEvent(newfakeHeapUsage);
hmm.updateStateAndSendEvent(newfakeHeapUsage, "test");
getCache().getLogger().fine(removeExpectedBelow);
assertFalse(hmm.getState().isCritical());
{
Expand Down Expand Up @@ -2314,7 +2314,7 @@ public Object call() throws Exception {
assertTrue(newfakeHeapUsage <= fakeHeapMaxSize);
HeapMemoryMonitor hmm =
((InternalResourceManager) getCache().getResourceManager()).getHeapMonitor();
hmm.updateStateAndSendEvent(newfakeHeapUsage);
hmm.updateStateAndSendEvent(newfakeHeapUsage, "test");
assertTrue(hmm.getState().isCritical());
final Integer k = new Integer(2); // reload with same key again and again
final Integer expectedInvocations3 = new Integer(expectedInvocations.getAndIncrement());
Expand Down Expand Up @@ -2361,7 +2361,7 @@ public Object call() throws Exception {
// 30%
assertTrue(fakeHeapMaxSize > 0);
getCache().getLogger().fine(addExpectedBelow);
hmm.updateStateAndSendEvent(newfakeHeapUsage);
hmm.updateStateAndSendEvent(newfakeHeapUsage, "test");
getCache().getLogger().fine(removeExpectedBelow);
assertFalse(hmm.getState().isCritical());
Integer k = new Integer(3); // same key as previously used, this time is should stick
Expand Down Expand Up @@ -2396,7 +2396,7 @@ public Object call() throws Exception {
// by 10%
assertTrue(newfakeHeapUsage > 0);
assertTrue(newfakeHeapUsage <= fakeHeapMaxSize);
hmm.updateStateAndSendEvent(newfakeHeapUsage);
hmm.updateStateAndSendEvent(newfakeHeapUsage, "test");
getCache().getLogger().fine(removeExpectedExString);
assertTrue(hmm.getState().isCritical());
k = new Integer(5);
Expand All @@ -2411,7 +2411,7 @@ public Object call() throws Exception {
// 30%
assertTrue(fakeHeapMaxSize > 0);
getCache().getLogger().fine(addExpectedBelow);
hmm.updateStateAndSendEvent(newfakeHeapUsage);
hmm.updateStateAndSendEvent(newfakeHeapUsage, "test");
getCache().getLogger().fine(removeExpectedBelow);
assertFalse(hmm.getState().isCritical());
return expectedInvocations10;
Expand Down Expand Up @@ -2504,63 +2504,75 @@ public void run() {

@Test
public void testCriticalMemoryEventTolerance() {
final Host host = Host.getHost(0);
final VM vm = host.getVM(0);
vm.invoke(new SerializableCallable() {
@Override
public Object call() throws Exception {
int defaultTolerance = 1;
HeapMemoryMonitor.setTestDisableMemoryUpdates(false);
GemFireCacheImpl cache = (GemFireCacheImpl) getCache();
InternalResourceManager irm = cache.getInternalResourceManager();
HeapMemoryMonitor hmm = irm.getHeapMonitor();
hmm.setTestMaxMemoryBytes(100);
HeapMemoryMonitor.setTestBytesUsedForThresholdSet(1);
irm.setCriticalHeapPercentage(95);
for (int i = 0; i < defaultTolerance; i++) {
hmm.updateStateAndSendEvent(96);
assertFalse(hmm.getState().isCritical());
}
getCache().getLogger().fine(addExpectedExString);
hmm.updateStateAndSendEvent(96);
assertTrue(hmm.getState().isCritical());
getCache().getLogger().fine(removeExpectedExString);
getCache().getLogger().fine(addExpectedBelow);
hmm.updateStateAndSendEvent(92);
getCache().getLogger().fine(removeExpectedBelow);
assertFalse(hmm.getState().isCritical());
HeapMemoryMonitor.setTestDisableMemoryUpdates(true);
return null;
}
});
testMemoryEventTolerance(true);
}

@Test
public void testEvictionMemoryEventTolerance() {
testMemoryEventTolerance(false);
}

private void testMemoryEventTolerance(boolean isCritical) {
final Host host = Host.getHost(0);
final VM vm = host.getVM(0);
vm.invoke(new SerializableCallable() {
@Override
public Object call() throws Exception {
HeapMemoryMonitor.setTestDisableMemoryUpdates(false);
String vendor = System.getProperty("java.vendor");
boolean isSun = (vendor.contains("Sun") || vendor.contains("Oracle"));
int defaultTolerance = isSun ? 1 : 5;
GemFireCacheImpl cache = (GemFireCacheImpl) getCache();
InternalResourceManager irm = cache.getInternalResourceManager();
HeapMemoryMonitor hmm = irm.getHeapMonitor();
hmm.setTestMaxMemoryBytes(100);
HeapMemoryMonitor.setTestBytesUsedForThresholdSet(1);
irm.setEvictionHeapPercentage(50);
for (int i = 0; i < defaultTolerance; i++) {
hmm.updateStateAndSendEvent(55);
assertFalse(hmm.getState().isEviction());

if (isCritical) {
irm.setCriticalHeapPercentage(95);
} else {
irm.setEvictionHeapPercentage(50);
}

int previousMemoryStateChangeTolerance = hmm.getMemoryStateChangeTolerance();
final int criticalBytesUsed = 96;
final int evictionBytesUsed = 55;
final int memoryStateChangeTolerance = 3;

try {
hmm.setMemoryStateChangeTolerance(memoryStateChangeTolerance);

for (int i = 0; i < memoryStateChangeTolerance; i++) {
if (isCritical) {
hmm.updateStateAndSendEvent(criticalBytesUsed, "test");
assertFalse(hmm.getState().isCritical());
} else {
hmm.updateStateAndSendEvent(evictionBytesUsed, "test");
assertFalse(hmm.getState().isEviction());
}
}
if (isCritical) {
// Adding expected strings so we do not fail the
// test prematurely
getCache().getLogger().fine(addExpectedExString);
hmm.updateStateAndSendEvent(criticalBytesUsed, "test");
assertTrue(hmm.getState().isCritical());
getCache().getLogger().fine(removeExpectedExString);
getCache().getLogger().fine(addExpectedBelow);
final int belowCriticalBytes = 92;
hmm.updateStateAndSendEvent(belowCriticalBytes, "test");
getCache().getLogger().fine(removeExpectedBelow);
assertFalse(hmm.getState().isCritical());
} else {
hmm.updateStateAndSendEvent(evictionBytesUsed, "test");
assertTrue(hmm.getState().isEviction());
final int belowEvictionBytes = 45;
hmm.updateStateAndSendEvent(belowEvictionBytes, "test");
assertFalse(hmm.getState().isEviction());

}

HeapMemoryMonitor.setTestDisableMemoryUpdates(true);
} finally {
hmm.setMemoryStateChangeTolerance(previousMemoryStateChangeTolerance);
}
hmm.updateStateAndSendEvent(55);
assertTrue(hmm.getState().isEviction());
hmm.updateStateAndSendEvent(45);
assertFalse(hmm.getState().isEviction());
HeapMemoryMonitor.setTestDisableMemoryUpdates(true);
return null;
}
});
Expand Down
10 changes: 5 additions & 5 deletions ...est/java/org/apache/geode/cache/query/dunit/ResourceManagerWithQueryMonitorDUnitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ public Object call() throws Exception {
public Object call() throws Exception {
InternalResourceManager irm = ((GemFireCacheImpl) getCache()).getInternalResourceManager();
// Reset CRITICAL_UP by informing all that heap usage is now 1 byte (0 would disable).
irm.getHeapMonitor().updateStateAndSendEvent(NORMAL_HEAP_USED);
irm.getHeapMonitor().updateStateAndSendEvent(NORMAL_HEAP_USED, "test");
Set<ResourceListener> listeners = irm.getResourceListeners(ResourceType.HEAP_MEMORY);
Iterator<ResourceListener> it = listeners.iterator();
while (it.hasNext()) {
Expand Down Expand Up @@ -1043,7 +1043,7 @@ private void vmHitsCriticalHeap(VM vm) {
public void run2() {
InternalResourceManager resourceManager =
(InternalResourceManager) getCache().getResourceManager();
resourceManager.getHeapMonitor().updateStateAndSendEvent(CRITICAL_HEAP_USED);
resourceManager.getHeapMonitor().updateStateAndSendEvent(CRITICAL_HEAP_USED, "test");
}
});
}
Expand All @@ -1054,7 +1054,7 @@ private void vmRecoversFromCriticalHeap(VM vm) {
public void run2() {
InternalResourceManager resourceManager =
(InternalResourceManager) getCache().getResourceManager();
resourceManager.getHeapMonitor().updateStateAndSendEvent(NORMAL_HEAP_USED);
resourceManager.getHeapMonitor().updateStateAndSendEvent(NORMAL_HEAP_USED, "test");
}
});
}
Expand Down Expand Up @@ -1336,7 +1336,7 @@ public void doTestHook(final SPOTS spot, final DefaultQuery _ignored) {
if (count++ == numObjectsBeforeCancel) {
InternalResourceManager resourceManager =
(InternalResourceManager) getCache().getResourceManager();
resourceManager.getHeapMonitor().updateStateAndSendEvent(CRITICAL_HEAP_USED);
resourceManager.getHeapMonitor().updateStateAndSendEvent(CRITICAL_HEAP_USED, "test");
triggeredOOME = true;
}
break;
Expand All @@ -1356,7 +1356,7 @@ public void doTestHook(final SPOTS spot, final DefaultQuery _ignored) {
if (triggeredOOME == false) {
InternalResourceManager resourceManager =
(InternalResourceManager) getCache().getResourceManager();
resourceManager.getHeapMonitor().updateStateAndSendEvent(CRITICAL_HEAP_USED);
resourceManager.getHeapMonitor().updateStateAndSendEvent(CRITICAL_HEAP_USED, "test");
triggeredOOME = true;
try {
Thread.sleep(1000);
Expand Down
5 changes: 1 addition & 4 deletions ...tributedTest/java/org/apache/geode/internal/cache/PartitionedRegionEvictionDUnitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@
import org.apache.geode.cache.util.CacheListenerAdapter;
import org.apache.geode.cache.util.ObjectSizer;
import org.apache.geode.cache30.CacheSerializableRunnable;
import org.apache.geode.distributed.internal.DistributionConfig;
import org.apache.geode.internal.OSProcess;
import org.apache.geode.internal.cache.control.HeapMemoryMonitor;
import org.apache.geode.internal.cache.control.InternalResourceManager.ResourceType;
Expand Down Expand Up @@ -203,20 +202,18 @@ public String description() {
protected void raiseFakeNotification() {
((GemFireCacheImpl) getCache()).getHeapEvictor().setTestAbortAfterLoopCount(1);
HeapMemoryMonitor.setTestDisableMemoryUpdates(true);
System.setProperty(DistributionConfig.GEMFIRE_PREFIX + "memoryEventTolerance", "0");

setEvictionPercentage(85);
HeapMemoryMonitor hmm =
((GemFireCacheImpl) getCache()).getInternalResourceManager().getHeapMonitor();
hmm.setTestMaxMemoryBytes(100);

hmm.updateStateAndSendEvent(90);
hmm.updateStateAndSendEvent(90, "test");
}

protected void cleanUpAfterFakeNotification() {
((GemFireCacheImpl) getCache()).getHeapEvictor().setTestAbortAfterLoopCount(Integer.MAX_VALUE);
HeapMemoryMonitor.setTestDisableMemoryUpdates(false);
System.clearProperty(DistributionConfig.GEMFIRE_PREFIX + "memoryEventTolerance");
}

@Test
Expand Down
2 changes: 1 addition & 1 deletion .../src/distributedTest/java/org/apache/geode/internal/cache/eviction/EvictionDUnitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ private static int sendEventAndWaitForExpectedEviction(String regionName) {
} else {
HeapMemoryMonitor hmm = cache.getInternalResourceManager().getHeapMonitor();
hmm.setTestMaxMemoryBytes(100);
hmm.updateStateAndSendEvent(90);
hmm.updateStateAndSendEvent(90, "test");
}

int entrySize = ENTRY_SIZE + 100;
Expand Down
Loading

0 comments on commit 546abdd

Please sign in to comment.