Merge pull request mozilla-services#2700 from rhelmer/bug1149236-post…

…gres-correlations

fix bug 1149236 - pull processed JSON from postgres, hbase is gone from mozilla infra

scripts/crons/cron_libraries.sh

@@ -53,8 +53,8 @@ do
   for J in $VERSIONS
   do
     techo "Phase 1: Version: $J start"
-    techo "Running socorro copy_processed"
-    $PYTHON $SOCORRO_DIR/socorro-virtualenv/bin/socorro copy_processed --admin.conf=/etc/socorro/copy_processed.ini --new_crash_source.crash_id_query="select uuid from reports_${WEEK} where completed_datetime < $SQL_DATE and completed_datetime > ($SQL_DATE - interval '24 hours') and product = '${I}' and version = '${J}'" --destination.tarball_name=$TMPDIR/${I}_${J}.tar > $TMPDIR/${I}_${J}.log 2>&1
+    techo "Pulling processed JSON from PostgreSQL"
+    psql -t -U $databaseUserName -h $databaseHost $databaseName -c "copy (select processed_crashes.uuid, processed_crash from processed_crashes join reports_${WEEK} on (reports_${WEEK}.uuid::uuid = processed_crashes.uuid) where completed_datetime < $SQL_DATE and completed_datetime > ($SQL_DATE - interval '24 hours') and product = '${I}' and version = '${J}') to stdout with csv quote e'\x01' delimiter e'\x02'" | /home/socorro/tar_crashes.py $TMPDIR/${I}_${J}.tar
     techo "per-crash-core-count.py > $TMPDIR/${DATE}_${I}_${J}-core-counts.txt"
     $PYTHON /data/crash-data-tools/per-crash-core-count.py -p ${I} -r ${J} -f $TMPDIR/${I}_${J}.tar > $TMPDIR/${DATE}_${I}_${J}-core-counts.txt
     techo "per-crash-interesting-modules.py > $TMPDIR/${DATE}_${I}_${J}-interesting-modules.txt"
@@ -78,8 +78,8 @@ do
   for J in $MANUAL_VERSION_OVERRIDE
   do
     techo "Phase 1: Version: $J start"
-    techo "Running socorro copy_processed"
-    $PYTHON $SOCORRO_DIR/socorro-virtualenv/bin/socorro copy_processed --admin.conf=/etc/socorro/copy_processed.ini --new_crash_source.crash_id_query="select uuid from reports_${WEEK} where completed_datetime < $SQL_DATE and completed_datetime > ($SQL_DATE - interval '24 hours') and product = '${I}' and version = '${J}'" --destination.tarball_name=$TMPDIR/${I}_${J}.tar > $TMPDIR/${I}_${J}.log 2>&1
+    techo "Pulling processed JSON from PostgreSQL"
+    psql -t -U $databaseUserName -h $databaseHost $databaseName -c "copy (select processed_crashes.uuid, processed_crash from processed_crashes join reports_${WEEK} on (reports_${WEEK}.uuid::uuid = processed_crashes.uuid) where completed_datetime < $SQL_DATE and completed_datetime > ($SQL_DATE - interval '24 hours') and product = '${I}' and version = '${J}') to stdout with csv quote e'\x01' delimiter e'\x02'" | /home/socorro/tar_crashes.py $TMPDIR/${I}_${J}.tar
     techo "per-crash-core-count.py > $TMPDIR/${DATE}_${I}_${J}-core-counts.txt"
     $PYTHON /data/crash-data-tools/per-crash-core-count.py -p ${I} -r ${J} -f $TMPDIR/${I}_${J}.tar > $TMPDIR/${DATE}_${I}_${J}-core-counts.txt
     techo "per-crash-interesting-modules.py > $TMPDIR/${DATE}_${I}_${J}-interesting-modules.txt"