Fixes for new correlations and adding RabbitMQ host to config

config/rabbitmq-middleware.ini-dist

@@ -134,17 +134,41 @@ application='MiddlewareApp'
     # converter: configman.converters.class_converter
     transaction_executor_class='socorro.database.transaction_executor.TransactionExecutor'
 
+[http]
+
+    [[correlations]]
+
+        # name: base_url
+        # doc: Base URL where correlations text files are
+        # converter: str
+        base_url='https://crash-analysis.mozilla.com/crash_analysis/'
+
+        # name: save_download
+        # doc: Whether files downloaded for correlations should be temporary stored on disk
+        # converter: configman.converters.boolean_converter
+        save_download='True'
+
+        # name: save_root
+        # doc: Directory where the temporary downloads are stored (if left empty will become the systems tmp directory)
+        # converter: str
+        save_root=''
+
+        # name: save_seconds
+        # doc: Number of seconds that the downloaded .txt file is stored in a temporary place
+        # converter: int
+        save_seconds='600'
+
 [implementations]
 
     # name: implementation_list
     # doc: list of packages for service implementations
     # converter: items_list_converter
-    implementation_list='psql:socorro.external.postgresql, fs:socorro.external.fs'
+    implementation_list='psql: socorro.external.postgresql, fs: socorro.external.fs, http:socorro.external.http'
 
     # name: service_overrides
     # doc: comma separated list of class overrides, e.g `Crashes: hbase`
     # converter: items_list_converter
-    service_overrides='CrashData:fs'
+    service_overrides='CrashData: fs, Correlations: http, CorrelationsSignatures: http'
 
 [logging]
 

scripts/build.sh

@@ -37,8 +37,10 @@ then
   exit 1
 fi
 
-# Override database hostname
+# Override hostnames for jenkins
 export DB_HOST="jenkins-pg92"
+export RABBITMQ_HOST="rabbitmq-zlb.webapp.phx"
+
 # RHEL postgres 9 RPM installs pg_config here, psycopg2 needs it
 export PATH=/usr/pgsql-9.2/bin:$PATH
 echo "My path is $PATH"
@@ -56,7 +58,8 @@ mv breakpad stackwalk
 
 # run socorro integration test
 echo "Running integration test..."
-./scripts/monitor-integration-test.sh --destroy
+#./scripts/monitor-integration-test.sh --destroy
+./scripts/rabbitmq-integration-test.sh --destroy
 
 # package socorro.tar.gz for distribution
 mkdir builds/

scripts/rabbitmq-integration-test.sh

@@ -30,10 +30,15 @@ then
   DB_PASSWORD="aPassword"
 fi
 
+if [ -z "$RABBITMQ_HOST" ]
+then
+  RABBITMQ_HOST="localhost"
+fi
+
 function cleanup() {
   echo "INFO: cleaning up crash storage directories"
   rm -rf ./primaryCrashStore/ ./processedCrashStore/
-#  rm -rf ./crashes/
+  rm -rf ./crashes/
 
   echo "INFO: Terminating background jobs"
 
@@ -103,8 +108,8 @@ done
 echo " Done."
 
 echo -n "INFO: starting up collector, processor and middleware..."
-python socorro/collector/collector_app.py --admin.conf=./config/rabbitmq-collector.ini > collector.log 2>&1 &
-python socorro/processor/processor_app.py --admin.conf=./config/rabbitmq-processor.ini > processor.log 2>&1 &
+python socorro/collector/collector_app.py --admin.conf=./config/rabbitmq-collector.ini --storage.storage1.host=$RABBITMQ_HOST > collector.log 2>&1 &
+python socorro/processor/processor_app.py --admin.conf=./config/rabbitmq-processor.ini --new_crash_source.host=$RABBITMQ_HOST > processor.log 2>&1 &
 sleep 1
 python socorro/middleware/middleware_app.py --admin.conf=./config/rabbitmq-middleware.ini --database.database_host=$DB_HOST > middleware.log 2>&1 &
 echo " Done."
@@ -178,7 +183,6 @@ fi
 count=0
 while true
 do
-  curl -s "http://localhost:8883/crash/uuid/${CRASHID}"
   curl -s "http://localhost:8883/crash/uuid/${CRASHID}"  | grep '"total": 1' > /dev/null
   if [ $? != 0 ]
   then