CI: add wait for autohttps acquired certs to be saved during upgrade …

…tests

.github/workflows/test.yml

@@ -119,6 +119,7 @@ jobs:
           helm install jupyterhub jupyterhub/jupyterhub --values dev-config.yaml
           await_jupyterhub
           await_autohttps_tls_cert_acquisition
+          await_autohttps_tls_cert_save
 
       - name: Install JupyterHub dev chart
         run: |

ci/common

@@ -42,7 +42,7 @@ await_jupyterhub() {
 
 await_autohttps_tls_cert_acquisition() {
     i=0; while [ $i -ne 60 ]; do
-        kubectl logs deploy/pebble -c pebble | grep "Issued certificate" \
+        kubectl logs deploy/autohttps -c traefik | grep "Adding certificate" \
         && acquired_cert=true && break \
         || acquired_cert=false && sleep 0.5 && i=$((i + 1))
     done
@@ -62,6 +62,25 @@ await_autohttps_tls_cert_acquisition() {
     sleep 1
 }
 
+await_autohttps_tls_cert_save() {
+    i=0; while [ $i -ne 60 ]; do
+        kubectl logs deploy/autohttps -c secret-sync | grep "Created secret" \
+        && saved_cert=true && break \
+        || saved_cert=false && sleep 0.5 && i=$((i + 1))
+    done
+    if [ "$saved_cert" != "true" ]; then
+        echo "Certificate acquisition failed!"
+        kubectl get service,networkpolicy,configmap,pod
+        kubectl describe pod -l app.kubernetes.io/name=pebble
+        kubectl logs deploy/pebble --all-containers # --prefix
+        kubectl describe pod -l app.kubernetes.io/name=pebble-coredns
+        kubectl logs deploy/pebble-coredns --all-containers # --prefix
+        kubectl describe pod -l component=autohttps
+        kubectl logs deploy/autohttps --all-containers # --prefix
+        exit 1
+    fi
+}
+
 setup_kubeval () {
     echo "setup kubeval ${KUBEVAL_VERSION}"
     curl -sfL https://github.com/instrumenta/kubeval/releases/download/${KUBEVAL_VERSION}/kubeval-linux-amd64.tar.gz | tar xz kubeval
@@ -106,10 +125,6 @@ full_namespace_report () {
     echo ""
     echo "Just while debugging intermittent issue, lets output the logs of the proxy pod."
     kubectl logs --all-containers deploy/proxy
-
-    echo ""
-    echo "Just while debugging intermittent issue, lets output the logs of the autohttps pod."
-    kubectl logs --all-containers deploy/autohttps
 }
 
 install_and_run_chartpress_and_pebble () {