Improve CI (ansible-collections#528)

* Update CI scripts to be more close to the ones in ansible-core. * Extend CI matrix. * Make sure that docker daemon is running (when not in a container). * Make sure that connection plugin tests do not uninstall Docker daemon. * Check some conditions. * Fix error ignores. * Skip SSH test on Alpine VMs. * Take care of more errors. * Adjust for more errors. * Improve conditions. * Remove new entries from CI matrix; make CI matrix nicer.
Henkhogan · Dec 11, 2022 · 02915cd · 02915cd
1 parent 7e21320
commit 02915cd
Show file tree

Hide file tree

Showing 17 changed files with 162 additions and 45 deletions.
diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml
@@ -187,7 +187,7 @@ stages:
               test: debian-bullseye/3.9
             - name: ArchLinux
               test: archlinux/3.10
-            - name: CentOS Stream 8
+            - name: CentOS Stream 8 w/ Python 3.9
               test: centos-stream8/3.9
           groups:
             - 4
@@ -200,11 +200,12 @@ stages:
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: RHEL {0}
-          testFormat: devel/rhel/{0}
+          testFormat: devel/{0}
           targets:
-            - test: '7.9'
-            - test: '9.0-pypi-latest'
+            - name: RHEL 7.9
+              test: rhel/7.9
+            - name: RHEL 9.0 w/ latest Docker SDK from PyPi
+              test: rhel/9.0-pypi-latest
           groups:
             - 1
             - 2
@@ -217,10 +218,10 @@ stages:
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: RHEL {0}
-          testFormat: 2.14/rhel/{0}
+          testFormat: 2.14/{0}
           targets:
-            - test: '9.0'
+            - name: RHEL 9.0
+              test: rhel/9.0
           groups:
             - 1
             - 2
@@ -233,10 +234,10 @@ stages:
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: RHEL {0}
-          testFormat: 2.13/rhel/{0}
+          testFormat: 2.13/{0}
           targets:
-            - test: '8.5'
+            - name: RHEL 8.5
+              test: rhel/8.5
           groups:
             - 1
             - 2
@@ -249,10 +250,10 @@ stages:
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: RHEL {0}
-          testFormat: 2.12/rhel/{0}
+          testFormat: 2.12/{0}
           targets:
-            - test: '8.4'
+            - name: RHEL 8.4
+              test: rhel/8.4
           groups:
             - 1
             - 2

diff --git a/tests/integration/targets/connection_docker/setup.yml b/tests/integration/targets/connection_docker/setup.yml
@@ -6,7 +6,7 @@
 - hosts: localhost
   connection: local
   vars:
-    docker_skip_cleanup: yes
+    docker_skip_cleanup: true
 
   tasks:
     - name: Setup docker

diff --git a/tests/integration/targets/connection_docker/shutdown.yml b/tests/integration/targets/connection_docker/shutdown.yml
@@ -6,7 +6,7 @@
 - hosts: localhost
   connection: local
   vars:
-    docker_skip_cleanup: yes
+    docker_skip_cleanup: true
 
   tasks:
     - name: Remove docker packages
@@ -17,3 +17,4 @@
           - docker-ce
           - docker-ce-cli
         state: absent
+      when: not docker_skip_cleanup
diff --git a/tests/integration/targets/connection_docker_api/setup.yml b/tests/integration/targets/connection_docker_api/setup.yml
@@ -6,7 +6,7 @@
 - hosts: localhost
   connection: local
   vars:
-    docker_skip_cleanup: yes
+    docker_skip_cleanup: true
 
   tasks:
     - name: Setup docker

diff --git a/tests/integration/targets/connection_docker_api/shutdown.yml b/tests/integration/targets/connection_docker_api/shutdown.yml
@@ -6,7 +6,7 @@
 - hosts: localhost
   connection: local
   vars:
-    docker_skip_cleanup: yes
+    docker_skip_cleanup: true
 
   tasks:
     - name: Remove docker packages
@@ -17,3 +17,4 @@
           - docker-ce
           - docker-ce-cli
         state: absent
+      when: not docker_skip_cleanup
diff --git a/tests/integration/targets/connection_nsenter/setup.yml b/tests/integration/targets/connection_nsenter/setup.yml
@@ -6,7 +6,7 @@
 - hosts: localhost
   connection: local
   vars:
-    docker_skip_cleanup: yes
+    docker_skip_cleanup: true
 
   tasks:
     - name: Setup docker

diff --git a/tests/integration/targets/connection_nsenter/shutdown.yml b/tests/integration/targets/connection_nsenter/shutdown.yml
@@ -6,7 +6,7 @@
 - hosts: localhost
   connection: local
   vars:
-    docker_skip_cleanup: yes
+    docker_skip_cleanup: true
 
   tasks:
     - name: Remove docker packages
@@ -17,3 +17,4 @@
           - docker-ce
           - docker-ce-cli
         state: absent
+      when: not docker_skip_cleanup
diff --git a/tests/integration/targets/docker_container/tasks/tests/options.yml b/tests/integration/targets/docker_container/tasks/tests/options.yml
@@ -53,6 +53,7 @@
     state: started
     blkio_weight: 123
   register: blkio_weight_1
+  ignore_errors: true
 
 - name: blkio_weight (idempotency)
   docker_container:
@@ -62,6 +63,7 @@
     state: started
     blkio_weight: 123
   register: blkio_weight_2
+  ignore_errors: true
 
 - name: blkio_weight (change)
   docker_container:
@@ -72,6 +74,7 @@
     blkio_weight: 234
     force_kill: yes
   register: blkio_weight_3
+  ignore_errors: true
 
 - name: cleanup
   docker_container:
@@ -80,10 +83,18 @@
     force_kill: yes
   diff: no
 
-- assert:
+- when: blkio_weight_1 is failed
+  assert:
+    that:
+    - "'setting cgroup config for procHooks process caused: failed to write' in blkio_weight_1.msg"
+
+- when: blkio_weight_1 is not failed
+  assert:
     that:
     - blkio_weight_1 is changed
+    - blkio_weight_2 is not failed
     - "blkio_weight_2 is not changed or 'Docker warning: Your kernel does not support Block I/O weight or the cgroup is not mounted. Weight discarded.' in (blkio_weight_2.warnings | default([]))"
+    - blkio_weight_3 is not failed
     - blkio_weight_3 is changed
 
 ####################################################################
@@ -197,8 +208,16 @@
     that:
     - cgroupns_mode_1 is changed
     - cgroupns_mode_2 is not changed and cgroupns_mode_2 is not failed
-    - "cgroupns_mode_3 is changed or 'Docker warning: Your kernel does not support cgroup namespaces.  Cgroup namespace setting discarded.' in (cgroupns_mode_3.warnings | default([]))"
-  when: docker_api_version is version('1.41', '>=')
+    - >-
+      cgroupns_mode_3 is changed or
+      'Docker warning: Your kernel does not support cgroup namespaces.  Cgroup namespace setting discarded.' in (cgroupns_mode_3.warnings | default([])) or
+      (cgroupns_mode_3 is failed and 'error mounting "cgroup" to rootfs at "/sys/fs/cgroup"' in cgroupns_mode_3.msg)
+  when: docker_api_version is version('1.41', '>=') and cgroupns_mode_1 is not failed
+- assert:
+    that:
+    - >-
+      'error mounting "cgroup" to rootfs at "/sys/fs/cgroup"' in cgroupns_mode_1.msg
+  when: docker_api_version is version('1.41', '>=') and cgroupns_mode_1 is failed
 - assert:
     that:
     - cgroupns_mode_1 is failed
@@ -861,6 +880,7 @@
       - path: /dev/urandom
         rate: 10K
   register: device_read_bps_1
+  ignore_errors: true
 
 - name: device_read_bps (idempotency)
   docker_container:
@@ -874,6 +894,7 @@
       - path: /dev/random
         rate: 20M
   register: device_read_bps_2
+  ignore_errors: true
 
 - name: device_read_bps (lesser entries)
   docker_container:
@@ -885,6 +906,7 @@
       - path: /dev/random
         rate: 20M
   register: device_read_bps_3
+  ignore_errors: true
 
 - name: device_read_bps (changed)
   docker_container:
@@ -899,6 +921,7 @@
         rate: 5K
     force_kill: yes
   register: device_read_bps_4
+  ignore_errors: true
 
 - name: cleanup
   docker_container:
@@ -907,13 +930,23 @@
     force_kill: yes
   diff: no
 
-- assert:
+- when: device_read_bps_1 is not failed
+  assert:
     that:
+    - device_read_bps_1 is not failed
     - device_read_bps_1 is changed
+    - device_read_bps_2 is not failed
     - device_read_bps_2 is not changed
+    - device_read_bps_3 is not failed
     - device_read_bps_3 is not changed
+    - device_read_bps_4 is not failed
     - device_read_bps_4 is changed
 
+- when: device_read_bps_1 is failed
+  assert:
+    that:
+    - "'error setting cgroup config for procHooks process' in device_read_bps_1.msg and 'blkio.throttle.read_bps_device: no such device' in device_read_bps_1.msg"
+
 ####################################################################
 ## device_read_iops ################################################
 ####################################################################
@@ -930,6 +963,7 @@
       - path: /dev/urandom
         rate: 20
   register: device_read_iops_1
+  ignore_errors: true
 
 - name: device_read_iops (idempotency)
   docker_container:
@@ -943,6 +977,7 @@
       - path: /dev/random
         rate: 10
   register: device_read_iops_2
+  ignore_errors: true
 
 - name: device_read_iops (less)
   docker_container:
@@ -954,6 +989,7 @@
       - path: /dev/random
         rate: 10
   register: device_read_iops_3
+  ignore_errors: true
 
 - name: device_read_iops (changed)
   docker_container:
@@ -968,6 +1004,7 @@
         rate: 50
     force_kill: yes
   register: device_read_iops_4
+  ignore_errors: true
 
 - name: cleanup
   docker_container:
@@ -976,13 +1013,23 @@
     force_kill: yes
   diff: no
 
-- assert:
+- when: device_read_iops_1 is not failed
+  assert:
     that:
+    - device_read_iops_1 is not failed
     - device_read_iops_1 is changed
+    - device_read_iops_2 is not failed
     - device_read_iops_2 is not changed
+    - device_read_iops_3 is not failed
     - device_read_iops_3 is not changed
+    - device_read_iops_4 is not failed
     - device_read_iops_4 is changed
 
+- when: device_read_iops_1 is failed
+  assert:
+    that:
+    - "'error setting cgroup config for procHooks process' in device_read_iops_1.msg and 'blkio.throttle.read_iops_device: no such device' in device_read_iops_1.msg"
+
 ####################################################################
 ## device_write_bps and device_write_iops ##########################
 ####################################################################
@@ -1000,6 +1047,7 @@
       - path: /dev/urandom
         rate: 30
   register: device_write_limit_1
+  ignore_errors: true
 
 - name: device_write_bps and device_write_iops (idempotency)
   docker_container:
@@ -1014,6 +1062,7 @@
       - path: /dev/urandom
         rate: 30
   register: device_write_limit_2
+  ignore_errors: true
 
 - name: device_write_bps device_write_iops (changed)
   docker_container:
@@ -1029,6 +1078,7 @@
         rate: 100
     force_kill: yes
   register: device_write_limit_3
+  ignore_errors: true
 
 - name: cleanup
   docker_container:
@@ -1037,12 +1087,19 @@
     force_kill: yes
   diff: no
 
-- assert:
+- when: device_write_limit_1 is not failed
+  assert:
     that:
+    - device_write_limit_1 is not failed and device_write_limit_2 is not failed and device_write_limit_3 is not failed
     - device_write_limit_1 is changed
     - device_write_limit_2 is not changed
     - device_write_limit_3 is changed
 
+- when: device_write_limit_1 is failed
+  assert:
+    that:
+    - "'error setting cgroup config for procHooks process' in device_write_limit_1.msg and 'blkio.throttle.write_bps_device: no such device' in device_write_limit_1.msg"
+
 ####################################################################
 ## device_requests #################################################
 ####################################################################