Generate test reports to monitor performance (2600hz#6380)

* Generate test reports to monitor performance CircleCI has a nifty 'insights' tool that can ingest test reports generated in XML (see eunit's eunit_surefire for format). This change includes the generation of those XML files when running eunit tests and instructing CircleCI to ingest them for reporting purposes. Eventually I would like kazoo_proper to also generate these types of files so we can track performance of those tests as well. * find TEST-*.xml * update per-app CI configs * document the sed command * run eunit in parallel * timings werid
pqkazoo · Mar 13, 2020 · 8762cc3 · 8762cc3
1 parent 2fca2ce
commit 8762cc3
Show file tree

Hide file tree

Showing 42 changed files with 460 additions and 13 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -106,6 +106,16 @@ jobs:
       - run:
           name: Running Eunit/Proper tests
           command: make eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ \;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/.gitignore b/.gitignore
@@ -74,7 +74,9 @@ compile_commands.json
 /make/.deps.mk.*
 /make/circleci
 /core/*/cover/*.html
+/core/*/cover/*.xml
 /applications/*/cover/*.html
+/applications/*/cover/*.xml
 
 /applications/crossbar/priv/couchdb/schemas/port_requests.to_scheduled.json
 

diff --git a/applications/acdc/.circleci/config.yml b/applications/acdc/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/ananke/.circleci/config.yml b/applications/ananke/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/blackhole/.circleci/config.yml b/applications/blackhole/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/braintree/.circleci/config.yml b/applications/braintree/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/call_inspector/.circleci/config.yml b/applications/call_inspector/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/callflow/.circleci/config.yml b/applications/callflow/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/camper/.circleci/config.yml b/applications/camper/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/cccp/.circleci/config.yml b/applications/cccp/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/cdr/.circleci/config.yml b/applications/cdr/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/conference/.circleci/config.yml b/applications/conference/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/crossbar/.circleci/config.yml b/applications/crossbar/.circleci/config.yml
@@ -249,6 +249,16 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/doodle/.circleci/config.yml b/applications/doodle/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/ecallmgr/.circleci/config.yml b/applications/ecallmgr/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/edr/.circleci/config.yml b/applications/edr/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/fax/.circleci/config.yml b/applications/fax/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/frontier/.circleci/config.yml b/applications/frontier/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/hangups/.circleci/config.yml b/applications/hangups/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/hotornot/.circleci/config.yml b/applications/hotornot/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/jonny5/.circleci/config.yml b/applications/jonny5/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report

diff --git a/applications/konami/.circleci/config.yml b/applications/konami/.circleci/config.yml
@@ -249,6 +249,17 @@ jobs:
       - run:
           name: Running Eunit tests
           command: make -C "$KAZOO_ROOT/$APP_PATH" eunit
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -name "TEST-*.xml" -exec cp {} ~/test-results/junit/ ;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit/
+
       ## disabling this for now until we figure out why it is crashing, and how to set token
       # - run:
       #     name: Uploading Coverage Report