Continue running network tests after failure (ansible#26407)

Previously tests would stop running on the first failure. With this PR we move onto running the tests for the next module. This allows DCI runs to have a fuller picture of what's broken. This only effects manual and DCI runs, it doesn't effect Shippable runs as this the modules are tested directly.
snhrdt · Jul 4, 2017 · 50f2313 · 50f2313
1 parent 1a9b0ae
commit 50f2313
Show file tree

Hide file tree

Showing 10 changed files with 596 additions and 77 deletions.
diff --git a/test/integration/asa.yaml b/test/integration/asa.yaml
@@ -7,7 +7,36 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: asa_command, when: "limit_to in ['*', 'asa_command']" }
-    - { role: asa_config, when: "limit_to in ['*', 'asa_config']" }
-    - { role: asa_acl, when: "limit_to in ['*', 'asa_acl']" }
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: asa_command
+        when: "limit_to in ['*', 'asa_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: asa_config
+        when: "limit_to in ['*', 'asa_config']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: asa_acl
+        when: "limit_to in ['*', 'asa_acl']"
+      rescue:
+        - set_fact: test_failed=true
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
diff --git a/test/integration/eos.yaml b/test/integration/eos.yaml
@@ -7,14 +7,86 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: eos_banner, when: "limit_to in ['*', 'eos_banner']" }
-    - { role: eos_command, when: "limit_to in ['*', 'eos_command']" }
-    - { role: eos_config, when: "limit_to in ['*', 'eos_config']" }
-    - { role: eos_template, when: "limit_to in ['*', 'eos_template']" }
-    - { role: eos_facts, when: "limit_to in ['*', 'eos_facts']" }
-    - { role: eos_eapi, debug: yes, when: "limit_to in ['*', 'eos_eapi']" }
-    - { role: eos_system, debug: yes, when: "limit_to in ['*', 'eos_system']" }
-    - { role: eos_user, when: "limit_to in ['*', 'eos_user']" }
-    - { role: eos_vlan, debug: yes, when: "limit_to in ['*', 'eos_vlan']" }
-    - { role: eos_vrf, debug: yes, when: "limit_to in ['*', 'eos_vrf']" }
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: eos_banner
+        when: "limit_to in ['*', 'eos_banner']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_command
+        when: "limit_to in ['*', 'eos_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_config
+        when: "limit_to in ['*', 'eos_config']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_template
+        when: "limit_to in ['*', 'eos_template']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_facts
+        when: "limit_to in ['*', 'eos_facts']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_eapi
+        when: "limit_to in ['*', 'eos_eapi']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_system
+        when: "limit_to in ['*', 'eos_system']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_user
+        when: "limit_to in ['*', 'eos_user']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_vlan
+        when: "limit_to in ['*', 'eos_vlan']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_vrf
+        when: "limit_to in ['*', 'eos_vrf']"
+      rescue:
+        - set_fact: test_failed=true
+
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
diff --git a/test/integration/ios.yaml b/test/integration/ios.yaml
@@ -6,13 +6,72 @@
   vars:
     limit_to: "*"
     debug: false
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
 
-  roles:
-    - { role: ios_banner, when: "limit_to in ['*', 'ios_banner']" }
-    - { role: ios_command, when: "limit_to in ['*', 'ios_command']" }
-    - { role: ios_config, when: "limit_to in ['*', 'ios_config']" }
-    - { role: ios_facts, when: "limit_to in ['*', 'ios_facts']" }
-    - { role: ios_template, when: "limit_to in ['*', 'ios_template']" }
-    - { role: ios_system, when: "limit_to in ['*', 'ios_system']" }
-    - { role: ios_user, when: "limit_to in ['*', 'ios_user']" }
-    - { role: ios_static_route, when: "limit_to in ['*', 'ios_static_route']" }
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: ios_banner
+        when: "limit_to in ['*', 'ios_banner']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_command
+        when: "limit_to in ['*', 'ios_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_config
+        when: "limit_to in ['*', 'ios_config']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_facts
+        when: "limit_to in ['*', 'ios_facts']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_template
+        when: "limit_to in ['*', 'ios_template']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_system
+        when: "limit_to in ['*', 'ios_system']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_user
+        when: "limit_to in ['*', 'ios_user']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: ios_static_route
+        when: "limit_to in ['*', 'ios_static_route']"
+      rescue:
+        - set_fact: test_failed=true
+
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed
diff --git a/test/integration/iosxr.yaml b/test/integration/iosxr.yaml
@@ -7,11 +7,64 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: iosxr_command, when: "limit_to in ['*', 'iosxr_command']" }
-    - { role: iosxr_config, when: "limit_to in ['*', 'iosxr_config']" }
-    - { role: iosxr_facts, when: "limit_to in ['*', 'iosxr_facts']" }
-    - { role: iosxr_template, when: "limit_to in ['*', 'iosxr_template']" }
-    - { role: iosxr_system, when: "limit_to in ['*', 'iosxr_system']" }
-    - { role: iosxr_user, when: "limit_to in ['*', 'iosxr_user']" }
-    - { role: iosxr_banner, when: "limit_to in ['*', 'iosxr_banner']" }
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: iosxr_command
+        when: "limit_to in ['*', 'iosxr_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: iosxr_config
+        when: "limit_to in ['*', 'iosxr_config']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: iosxr_facts
+        when: "limit_to in ['*', 'iosxr_facts']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: iosxr_template
+        when: "limit_to in ['*', 'iosxr_template']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: iosxr_system
+        when: "limit_to in ['*', 'iosxr_system']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: iosxr_user
+        when: "limit_to in ['*', 'iosxr_user']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: iosxr_banner
+        when: "limit_to in ['*', 'iosxr_banner']"
+      rescue:
+        - set_fact: test_failed=true
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed