chore(ci) track jobs failure statistics

From Kong#3857

.ci/run_tests.sh

@@ -14,18 +14,19 @@ elif [ "$KONG_TEST_DATABASE" == "cassandra" ]; then
     export TEST_CMD="bin/busted $BUSTED_ARGS,postgres"
 fi
 
+mkdir output
 if [ "$TEST_SUITE" == "integration" ]; then
-    eval "$TEST_CMD" spec/02-integration/
+    eval "$TEST_CMD" spec/02-integration/ | tee output/integration.txt
 fi
 if [ "$TEST_SUITE" == "plugins" ]; then
-    eval "$TEST_CMD" spec/03-plugins/
+    eval "$TEST_CMD" spec/03-plugins/ | tee output/plugins.txt
 fi
 if [ "$TEST_SUITE" == "old-integration" ]; then
-    eval "$TEST_CMD" spec-old-api/02-integration/
+    eval "$TEST_CMD" spec-old-api/02-integration/ | tee output/old_integration.txt
 fi
 if [ "$TEST_SUITE" == "old-plugins" ]; then
-    eval "$TEST_CMD" spec-old-api/03-plugins/
+    eval "$TEST_CMD" spec-old-api/03-plugins/ | tee output/old_plugins.txt
 fi
 if [ "$TEST_SUITE" == "pdk" ]; then
-    TEST_NGINX_RANDOMIZE=1 prove -I. -j$JOBS -r t/01-pdk
+    TEST_NGINX_RANDOMIZE=1 prove -I. -j$JOBS -r t/01-pdk | tee output/pdk.txt
 fi

.ci/track_failures.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if ! [[ "$TRAVIS_BRANCH" == "master" || "$TRAVIS_BRANCH" == "next" ]] ; then
+  exit 0
+fi
+
+pip install datadog
+cat >~/.dogrc <<EOL
+[Connection]
+apikey = $DD_API_KEY
+appkey = $DD_APP_KEY
+EOL
+
+cat ~/output/* | grep 'FAILED  ' | grep -v listed | grep -v ms > collated_output.txt
+cat collated_output.txt | sed 's/^.*spec\///; s/\.lua.*$//' | sed 's/^//' > parsed_output.txt
+cat parsed_output.txt | sed 's/-/_/g' | sed 's/[^a-zA-Z0-9_]/\./g' > dd_compatible.txt
+<dd_compatible.txt xargs -I % dog metric post travis_ci.kong.failure --type count --tags "test:%" 1

.travis.yml

@@ -1,10 +1,7 @@
 dist: trusty
 sudo: false
 
-language: java
-
-jdk:
-  - oraclejdk8
+language: python
 
 notifications:
   email: false
@@ -81,3 +78,6 @@ jobs:
 
 script:
   - .ci/run_tests.sh
+
+after_failure:
+  - .ci/track_failures.sh