PARQUET-2059: Handle resource-intensive tests in CI (apache#915)

parquet-column/src/test/java/org/apache/parquet/ResourceIntensiveTestRule.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet;
+
+import static org.junit.Assume.assumeTrue;
+
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+/**
+ * A test rule for resource intensive tests that are not executed on every environment (e.g. CI). It is managed by the
+ * system property {@code enableResourceIntensiveTests}. If the value is {@code true} the related tests are executed,
+ * skipped otherwise.
+ */
+public class ResourceIntensiveTestRule implements TestRule {
+
+  private static final TestRule INSTANCE = new ResourceIntensiveTestRule(
+      Boolean.getBoolean("enableResourceIntensiveTests"));
+
+  public static TestRule get() {
+    return INSTANCE;
+  }
+
+  private final boolean enabled;
+
+  private ResourceIntensiveTestRule(boolean enabled) {
+    this.enabled = enabled;
+  }
+
+  @Override
+  public Statement apply(Statement base, Description description) {
+    return enabled ? base : new Statement() {
+      @Override
+      public void evaluate() {
+        assumeTrue("Resource intensive test is not executed due to the limitations of the environment", enabled);
+      }
+    };
+  }
+
+}

parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestLargeColumnChunk.java

@@ -35,6 +35,7 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.parquet.ResourceIntensiveTestRule;
 import org.apache.parquet.example.data.Group;
 import org.apache.parquet.example.data.GroupFactory;
 import org.apache.parquet.example.data.simple.SimpleGroupFactory;
@@ -47,13 +48,12 @@
 import org.apache.parquet.schema.MessageType;
 import org.junit.*;
 import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
 
 /**
  * This test is to test parquet-mr working with potential int overflows (when the sizes are greater than
- * Integer.MAX_VALUE). The test requires ~3GB memory so it is likely to fail in the CI environment, so these
- * tests are flagged to be ignored.
+ * Integer.MAX_VALUE).
  */
-@Ignore
 public class TestLargeColumnChunk {
   private static final MessageType SCHEMA = buildMessage().addFields(
       required(INT64).named("id"),
@@ -71,6 +71,9 @@ public class TestLargeColumnChunk {
   private static Binary VALUE_NOT_IN_DATA;
   private static Path file;
 
+  @ClassRule
+  public static TestRule maySkip = ResourceIntensiveTestRule.get();
+
   @ClassRule
   public static TemporaryFolder folder = new TemporaryFolder();
 

pom.xml

@@ -112,6 +112,9 @@
 
     <!-- properties for the profiles -->
     <surefire.logLevel>INFO</surefire.logLevel>
+
+    <!-- Resource intesive tests are enabled by default but disabled in the CI envrionment -->
+    <enableResourceIntensiveTests>true</enableResourceIntensiveTests>
   </properties>
 
   <modules>
@@ -418,6 +421,7 @@
 
             <!-- Configure log level for Hadoop -->
             <hadoop.logLevel>${surefire.logLevel}</hadoop.logLevel>
+            <enableResourceIntensiveTests>${enableResourceIntensiveTests}</enableResourceIntensiveTests>
           </systemPropertyVariables>
           <excludes>
             <exclude>**/benchmark/*.java</exclude>
@@ -617,6 +621,7 @@
       <properties>
         <surefire.logLevel>WARN</surefire.logLevel>
         <surefire.argLine>-XX:MaxJavaStackTraceDepth=10</surefire.argLine>
+        <enableResourceIntensiveTests>false</enableResourceIntensiveTests>
       </properties>
     </profile>