HIVE-7519 : Refactor QTestUtil to remove its duplication with QFileCl…

…ient for qtest setup and teardown (Ashish via Szehon)

git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1616913 13f79535-47bb-0310-9956-ffa450edef68

ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java

@@ -141,6 +141,10 @@ public boolean accept(File filePath) {
 
   private String hadoopVersion;
 
+  private String initScript;
+
+  private String cleanupScript;
+
   public void setHadoopVersion(String ver) {
     this.hadoopVersion = ver;
   }
@@ -197,6 +201,22 @@ public String getTemplate() {
     return template;
   }
 
+  public String getInitScript() {
+    return initScript;
+  }
+
+  public void setInitScript(String initScript) {
+    this.initScript = initScript;
+  }
+
+  public String getCleanupScript() {
+    return cleanupScript;
+  }
+
+  public void setCleanupScript(String cleanupScript) {
+    this.cleanupScript = cleanupScript;
+  }
+
   public void setHiveRootDirectory(File hiveRootDirectory) {
     try {
       this.hiveRootDirectory = hiveRootDirectory.getCanonicalPath();
@@ -444,6 +464,8 @@ public void execute() throws BuildException {
       ctx.put("clusterMode", clusterMode);
       ctx.put("hiveConfDir", escapePath(hiveConfDir));
       ctx.put("hadoopVersion", hadoopVersion);
+      ctx.put("initScript", initScript);
+      ctx.put("cleanupScript", cleanupScript);
 
       File outFile = new File(outDir, className + ".java");
       FileWriter writer = new FileWriter(outFile);

data/conf/hive-site.xml

@@ -111,6 +111,12 @@
   <description></description>
 </property>
 
+<property>
+  <name>test.data.scripts</name>
+  <value>${hive.root}/data/scripts</value>
+  <description></description>
+</property>
+
 <property>
   <name>hive.jar.path</name>
   <value>${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>

data/conf/tez/hive-site.xml

@@ -120,6 +120,12 @@
   <description></description>
 </property>
 
+<property>
+  <name>test.data.scripts</name>
+  <value>${hive.root}/data/scripts</value>
+  <description></description>
+</property>
+
 <property>
   <name>hive.jar.path</name>
   <value>${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>

data/scripts/q_test_cleanup.sql

@@ -7,4 +7,12 @@ DROP TABLE IF EXISTS srcbucket;
 DROP TABLE IF EXISTS srcbucket2;
 DROP TABLE IF EXISTS srcpart;
 DROP TABLE IF EXISTS primitives;
-
+DROP TABLE IF EXISTS dest1;
+DROP TABLE IF EXISTS dest2;
+DROP TABLE IF EXISTS dest3;
+DROP TABLE IF EXISTS dest4;
+DROP TABLE IF EXISTS dest4_sequencefile;
+DROP TABLE IF EXISTS dest_j1;
+DROP TABLE IF EXISTS dest_g1;
+DROP TABLE IF EXISTS dest_g2;
+DROP TABLE IF EXISTS fetchtask_ioexception;

data/scripts/q_test_init.sql

@@ -3,7 +3,7 @@
 --
 DROP TABLE IF EXISTS src;
 
-CREATE TABLE src (key STRING, value STRING) STORED AS TEXTFILE;
+CREATE TABLE src (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE;
 
 LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" INTO TABLE src;
 
@@ -12,7 +12,7 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" INTO TABLE src;
 --
 DROP TABLE IF EXISTS src1;
 
-CREATE TABLE src1 (key STRING, value STRING) STORED AS TEXTFILE;
+CREATE TABLE src1 (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE;
 
 LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv3.txt" INTO TABLE src1;
 
@@ -21,7 +21,7 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv3.txt" INTO TABLE src1;
 --
 DROP TABLE IF EXISTS src_json;
 
-CREATE TABLE src_json (json STRING) STORED AS TEXTFILE;
+CREATE TABLE src_json (json STRING COMMENT 'default') STORED AS TEXTFILE;
 
 LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/json.txt" INTO TABLE src_json;
 
@@ -31,7 +31,7 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/json.txt" INTO TABLE src_json;
 --
 DROP TABLE IF EXISTS src_sequencefile;
 
-CREATE TABLE src_sequencefile (key STRING, value STRING) STORED AS SEQUENCEFILE;
+CREATE TABLE src_sequencefile (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS SEQUENCEFILE;
 
 LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.seq" INTO TABLE src_sequencefile;
 
@@ -45,7 +45,7 @@ CREATE TABLE src_thrift
 ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer'
 WITH SERDEPROPERTIES (
   'serialization.class' = 'org.apache.hadoop.hive.serde2.thrift.test.Complex',
-  'serialization.format' = 'com.facebook.thrift.protocol.TBinaryProtocol')
+  'serialization.format' = 'org.apache.thrift.protocol.TBinaryProtocol')
 STORED AS SEQUENCEFILE;
 
 LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/complex.seq" INTO TABLE src_thrift;
@@ -75,14 +75,16 @@ STORED AS TEXTFILE;
 
 LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket20.txt" INTO TABLE srcbucket2;
 LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket21.txt" INTO TABLE srcbucket2;
+LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket22.txt" INTO TABLE srcbucket2;
+LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket23.txt" INTO TABLE srcbucket2;
 
 
 --
 -- Table srcpart
 --
 DROP TABLE IF EXISTS srcpart;
 
-CREATE TABLE srcpart (key STRING, value STRING)
+CREATE TABLE srcpart (key STRING COMMENT 'default', value STRING COMMENT 'default')
 PARTITIONED BY (ds STRING, hr STRING)
 STORED AS TEXTFILE;
 
@@ -99,20 +101,46 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt"
 OVERWRITE INTO TABLE srcpart PARTITION (ds="2008-04-09", hr="12");
 
 
+--
+-- Table alltypesorc
+--
+DROP TABLE IF EXISTS alltypesorc;
+CREATE TABLE alltypesorc(
+    ctinyint TINYINT,
+    csmallint SMALLINT,
+    cint INT,
+    cbigint BIGINT,
+    cfloat FLOAT,
+    cdouble DOUBLE,
+    cstring1 STRING,
+    cstring2 STRING,
+    ctimestamp1 TIMESTAMP,
+    ctimestamp2 TIMESTAMP,
+    cboolean1 BOOLEAN,
+    cboolean2 BOOLEAN)
+    STORED AS ORC;
+
+LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/alltypesorc"
+OVERWRITE INTO  TABLE alltypesorc;
+
+
+--
+-- Table primitives
+--
 DROP TABLE IF EXISTS primitives;
 CREATE TABLE primitives (
-  id INT,
-  bool_col BOOLEAN,
-  tinyint_col TINYINT,
-  smallint_col SMALLINT,
-  int_col INT,
-  bigint_col BIGINT,
-  float_col FLOAT,
-  double_col DOUBLE,
-  date_string_col STRING,
-  string_col STRING,
-  timestamp_col TIMESTAMP)
-PARTITIONED BY (year INT, month INT)
+  id INT COMMENT 'default',
+  bool_col BOOLEAN COMMENT 'default',
+  tinyint_col TINYINT COMMENT 'default',
+  smallint_col SMALLINT COMMENT 'default',
+  int_col INT COMMENT 'default',
+  bigint_col BIGINT COMMENT 'default',
+  float_col FLOAT COMMENT 'default',
+  double_col DOUBLE COMMENT 'default',
+  date_string_col STRING COMMENT 'default',
+  string_col STRING COMMENT 'default',
+  timestamp_col TIMESTAMP COMMENT 'default')
+PARTITIONED BY (year INT COMMENT 'default', month INT COMMENT 'default')
 ROW FORMAT DELIMITED
   FIELDS TERMINATED BY ','
   ESCAPED BY '\\'
@@ -130,3 +158,60 @@ OVERWRITE INTO TABLE primitives PARTITION(year=2009, month=3);
 LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/types/primitives/090401.txt"
 OVERWRITE INTO TABLE primitives PARTITION(year=2009, month=4);
 
+--
+-- Function qtest_get_java_boolean
+--
+DROP FUNCTION IF EXISTS qtest_get_java_boolean;
+CREATE FUNCTION qtest_get_java_boolean AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaBoolean';
+
+--
+-- Table dest1
+--
+DROP TABLE IF EXISTS dest1;
+
+CREATE TABLE dest1 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+--
+-- Table dest2
+--
+DROP TABLE IF EXISTS dest2;
+
+CREATE TABLE dest2 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+--
+-- Table dest3
+--
+DROP TABLE IF EXISTS dest3;
+
+CREATE TABLE dest3 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+PARTITIONED BY (ds STRING, hr STRING)
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+ALTER TABLE dest3 ADD PARTITION (ds='2008-04-08',hr='12');
+
+--
+-- Table dest4
+--
+DROP TABLE IF EXISTS dest4;
+
+CREATE TABLE dest4 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+--
+-- Table dest4_sequencefile
+--
+DROP TABLE IF EXISTS dest4_sequencefile;
+
+CREATE TABLE dest4_sequencefile (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.SequenceFileInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.mapred.SequenceFileOutputFormat';

hbase-handler/src/test/templates/TestHBaseCliDriver.vm

@@ -44,9 +44,12 @@ public class $className extends TestCase {
   protected void setUp() {
 
     MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+    String initScript = "$initScript";
+    String cleanupScript = "$cleanupScript";
 
     try {
-      qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, setup);
+      qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
+      setup, initScript, cleanupScript);
     } catch (Exception e) {
       System.err.println("Exception: " + e.getMessage());
       e.printStackTrace();

hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm

@@ -45,9 +45,12 @@ public class $className extends TestCase {
   protected void setUp() {
 
     MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+    String initScript = "$initScript";
+    String cleanupScript = "$cleanupScript";
 
     try {
-      qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, setup);
+      qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
+      setup, initScript, cleanupScript);
     } catch (Exception e) {
       System.err.println("Exception: " + e.getMessage());
       e.printStackTrace();

itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java

@@ -88,7 +88,7 @@ public CheckResults(String outDir, String logDir, MiniClusterType miniMr,
         String hadoopVer, String locationSubdir)
       throws Exception
     {
-      super(outDir, logDir, miniMr, hadoopVer);
+      super(outDir, logDir, miniMr, hadoopVer, "", "");
       this.locationSubdir = locationSubdir;
     }
   }