Skip to content

Commit

Permalink
PHOENIX-4239 Fix flapping test in PartialIndexRebuilderIT
Browse files Browse the repository at this point in the history
  • Loading branch information
@JamesRTaylor
JamesRTaylor committed Sep 27, 2017
1 parent 4969794 commit 176f541
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 13 deletions.
8 changes: 5 additions & 3 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ public static void doSetup() throws Exception {
Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(10);
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_ATTRIB, Boolean.TRUE.toString());
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_INTERVAL_ATTRIB, Long.toString(REBUILD_INTERVAL));
serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, "300000"); // give up rebuilding after 5 minutes
serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, "50000000");
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_PERIOD, Long.toString(REBUILD_PERIOD)); // batch at 50 seconds
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB, Long.toString(WAIT_AFTER_DISABLED));
setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), ReadOnlyProps.EMPTY_PROPS);
Expand Down Expand Up @@ -897,7 +897,7 @@ private void testIndexWriteFailureDuringRebuild(PIndexState indexStateOnFailure)
TestUtil.removeCoprocessor(conn, fullIndexName, WriteFailingRegionObserver.class);

runIndexRebuilder();
assertTrue(TestUtil.checkIndexState(conn, fullIndexName, indexStateOnFailure == PIndexState.DISABLE ? PIndexState.INACTIVE : PIndexState.ACTIVE, null));
assertEquals(indexStateOnFailure == PIndexState.DISABLE ? PIndexState.INACTIVE : PIndexState.ACTIVE, TestUtil.getIndexState(conn, fullIndexName));
clock.time += WAIT_AFTER_DISABLED;

// First batch should have been processed again because we started over
Expand All @@ -907,7 +907,9 @@ private void testIndexWriteFailureDuringRebuild(PIndexState indexStateOnFailure)
clock.time += 2 * REBUILD_PERIOD;
// Second batch should have been processed now
runIndexRebuilder();
assertTrue(TestUtil.checkIndexState(conn, fullIndexName, PIndexState.ACTIVE, 0L));
clock.time += 2 * REBUILD_PERIOD;
runIndexRebuilder();
TestUtil.assertIndexState(conn, fullIndexName, PIndexState.ACTIVE, 0L);

// Verify that other batches were processed
IndexScrutiny.scrutinizeIndex(conn, fullTableName, fullIndexName);
Expand Down
51 changes: 41 additions & 10 deletions phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -888,14 +888,25 @@ public static void waitForIndexRebuild(Connection conn, String fullIndexName, PI
waitForIndexState(conn, fullIndexName, indexState, 0L);
}

private enum IndexStateCheck {SUCCESS, FAIL, KEEP_TRYING};
private static class IndexStateCheck {
public final PIndexState indexState;
public final Long indexDisableTimestamp;
public final Boolean success;

public IndexStateCheck(PIndexState indexState, Long indexDisableTimestamp, Boolean success) {
this.indexState = indexState;
this.indexDisableTimestamp = indexDisableTimestamp;
this.success = success;
}
}

public static void waitForIndexState(Connection conn, String fullIndexName, PIndexState expectedIndexState, Long expectedIndexDisableTimestamp) throws InterruptedException, SQLException {
int maxTries = 60, nTries = 0;
do {
Thread.sleep(1000); // sleep 1 sec
IndexStateCheck state = checkIndexStateInternal(conn, fullIndexName, expectedIndexState, expectedIndexDisableTimestamp);
if (state != IndexStateCheck.KEEP_TRYING) {
if (state == IndexStateCheck.SUCCESS) {
if (state.success != null) {
if (Boolean.TRUE.equals(state.success)) {
return;
}
fail("Index state will not become " + expectedIndexState);
Expand All @@ -905,8 +916,26 @@ public static void waitForIndexState(Connection conn, String fullIndexName, PInd
}

public static boolean checkIndexState(Connection conn, String fullIndexName, PIndexState expectedIndexState, Long expectedIndexDisableTimestamp) throws SQLException {
return checkIndexStateInternal(conn,fullIndexName, expectedIndexState, expectedIndexDisableTimestamp) == IndexStateCheck.SUCCESS;
return Boolean.TRUE.equals(checkIndexStateInternal(conn,fullIndexName, expectedIndexState, expectedIndexDisableTimestamp).success);
}

public static void assertIndexState(Connection conn, String fullIndexName, PIndexState expectedIndexState, Long expectedIndexDisableTimestamp) throws SQLException {
IndexStateCheck state = checkIndexStateInternal(conn,fullIndexName, expectedIndexState, expectedIndexDisableTimestamp);
if (!Boolean.TRUE.equals(state.success)) {
if (expectedIndexState != null) {
assertEquals(expectedIndexState, state.indexState);
}
if (expectedIndexDisableTimestamp != null) {
assertEquals(expectedIndexDisableTimestamp, state.indexDisableTimestamp);
}
}
}

public static PIndexState getIndexState(Connection conn, String fullIndexName) throws SQLException {
IndexStateCheck state = checkIndexStateInternal(conn, fullIndexName, null, null);
return state.indexState;
}

private static IndexStateCheck checkIndexStateInternal(Connection conn, String fullIndexName, PIndexState expectedIndexState, Long expectedIndexDisableTimestamp) throws SQLException {
String schema = SchemaUtil.getSchemaNameFromFullName(fullIndexName);
String index = SchemaUtil.getTableNameFromFullName(fullIndexName);
Expand All @@ -915,19 +944,21 @@ private static IndexStateCheck checkIndexStateInternal(Connection conn, String f
+ ") = (" + "'" + schema + "','" + index + "') "
+ "AND " + PhoenixDatabaseMetaData.COLUMN_FAMILY + " IS NULL AND " + PhoenixDatabaseMetaData.COLUMN_NAME + " IS NULL";
ResultSet rs = conn.createStatement().executeQuery(query);
Long actualIndexDisableTimestamp = null;
PIndexState actualIndexState = null;
if (rs.next()) {
Long actualIndexDisableTimestamp = rs.getLong(1);
PIndexState actualIndexState = PIndexState.fromSerializedValue(rs.getString(2));
actualIndexDisableTimestamp = rs.getLong(1);
actualIndexState = PIndexState.fromSerializedValue(rs.getString(2));
boolean matchesExpected = (expectedIndexDisableTimestamp == null || Objects.equal(actualIndexDisableTimestamp, expectedIndexDisableTimestamp))
&& actualIndexState == expectedIndexState;
&& (expectedIndexState == null || actualIndexState == expectedIndexState);
if (matchesExpected) {
return IndexStateCheck.SUCCESS;
return new IndexStateCheck(actualIndexState, actualIndexDisableTimestamp, Boolean.TRUE);
}
if (ZERO.equals(actualIndexDisableTimestamp)) {
return IndexStateCheck.FAIL;
return new IndexStateCheck(actualIndexState, actualIndexDisableTimestamp, Boolean.FALSE);
}
}
return IndexStateCheck.KEEP_TRYING;
return new IndexStateCheck(actualIndexState, actualIndexDisableTimestamp, null);
}

public static long getRowCount(Connection conn, String tableName) throws SQLException {
Expand Down

0 comments on commit 176f541

Please sign in to comment.