Provide data for ci tests

airflow/utils.py

@@ -182,6 +182,11 @@ def initdb():
             conn_id='airflow_db', conn_type='mysql',
             host='localhost', login='root',
             schema='airflow'))
+    merge_conn(
+        models.Connection(
+            conn_id='airflow_ci', conn_type='mysql',
+            host='localhost', login='root',
+            schema='airflow_ci'))
     merge_conn(
         models.Connection(
             conn_id='beeline_default', conn_type='beeline', port="10000",

scripts/ci/data/baby_names.csv

@@ -0,0 +1,54 @@
+"year","name","percent","sex"
+1880,"John",0.081541,"boy"
+1880,"William",0.080511,"boy"
+1880,"James",0.050057,"boy"
+1880,"Charles",0.045167,"boy"
+1880,"George",0.043292,"boy"
+1880,"Frank",0.02738,"boy"
+1880,"Joseph",0.022229,"boy"
+1880,"Thomas",0.021401,"boy"
+1880,"Henry",0.020641,"boy"
+1880,"Robert",0.020404,"boy"
+1880,"Edward",0.019965,"boy"
+1880,"Harry",0.018175,"boy"
+1880,"Walter",0.014822,"boy"
+1880,"Arthur",0.013504,"boy"
+1880,"Fred",0.013251,"boy"
+1880,"Albert",0.012609,"boy"
+1880,"Samuel",0.008648,"boy"
+1880,"David",0.007339,"boy"
+1880,"Louis",0.006993,"boy"
+1880,"Joe",0.006174,"boy"
+1880,"Charlie",0.006165,"boy"
+1880,"Clarence",0.006165,"boy"
+1880,"Richard",0.006148,"boy"
+1880,"Andrew",0.005439,"boy"
+1880,"Daniel",0.00543,"boy"
+1880,"Ernest",0.005194,"boy"
+1880,"Will",0.004966,"boy"
+1880,"Jesse",0.004805,"boy"
+1880,"Oscar",0.004594,"boy"
+1880,"Lewis",0.004366,"boy"
+1880,"Peter",0.004189,"boy"
+1880,"Benjamin",0.004138,"boy"
+1880,"Frederick",0.004079,"boy"
+1880,"Willie",0.00402,"boy"
+1880,"Alfred",0.003961,"boy"
+1880,"Sam",0.00386,"boy"
+1880,"Roy",0.003716,"boy"
+1880,"Herbert",0.003581,"boy"
+1880,"Jacob",0.003412,"boy"
+1880,"Tom",0.00337,"boy"
+1880,"Elmer",0.00315,"boy"
+1880,"Carl",0.003142,"boy"
+1880,"Lee",0.003049,"boy"
+1880,"Howard",0.003015,"boy"
+1880,"Martin",0.003015,"boy"
+1880,"Michael",0.00299,"boy"
+1880,"Bert",0.002939,"boy"
+1880,"Herman",0.002931,"boy"
+1880,"Jim",0.002914,"boy"
+1880,"Francis",0.002905,"boy"
+1880,"Harvey",0.002905,"boy"
+1880,"Earl",0.002829,"boy"
+1880,"Eugene",0.00277,"boy"

scripts/ci/data/mysql_schema.sql

@@ -0,0 +1,8 @@
+USE airflow_ci;
+
+CREATE TABLE IF NOT EXISTS baby_names (
+  org_year integer(4),
+  baby_name VARCHAR(25),
+  rate FLOAT(7,6),
+  sex VARCHAR(4)
+)

scripts/ci/load_data.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#!/usr/bin/env bash
+set -o verbose
+
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+DATA_DIR="${DIR}/data"
+DATA_FILE="${DATA_DIR}/baby_names.csv"
+DATABASE=airflow_ci
+
+mysqladmin -u root create ${DATABASE}
+mysql -u root < ${DATA_DIR}/mysql_schema.sql
+mysqlimport -u root --fields-optionally-enclosed-by="\"" --fields-terminated-by=, --ignore-lines=1 ${DATABASE} ${DATA_FILE}
+

tests/core.py

@@ -1423,10 +1423,10 @@ def test_clear(self):
         def test_mysql_to_hive(self):
             # import airflow.operators
             from airflow.operators.mysql_to_hive import MySqlToHiveTransfer
-            sql = "SELECT * FROM task_instance LIMIT 1000;"
+            sql = "SELECT * FROM baby_names LIMIT 1000;"
             t = MySqlToHiveTransfer(
                 task_id='test_m2h',
-                mysql_conn_id='airflow_db',
+                mysql_conn_id='airflow_ci',
                 hive_cli_conn_id='beeline_default',
                 sql=sql,
                 hive_table='test_mysql_to_hive',
@@ -1444,7 +1444,7 @@ def test_mysql_to_hive_partition(self):
                 hive_cli_conn_id='beeline_default',
                 sql=sql,
                 hive_table='test_mysql_to_hive_part',
-                partition={'ds': DEFAULT_DATE_DS},
+                partition={'org_year': DEFAULT_DATE_DS},
                 recreate=False,
                 create=True,
                 delimiter=",",

tox.ini

@@ -51,6 +51,7 @@ commands =
   {toxinidir}/scripts/ci/setup_env.sh
   {toxinidir}/scripts/ci/ldap.sh
   {toxinidir}/scripts/ci/load_fixtures.sh
+  {toxinidir}/scripts/ci/load_data.sh
   {toxinidir}/scripts/ci/run_tests.sh []
   {toxinidir}/scripts/ci/check-license.sh
   coveralls