Create presubmit job for build cluster validation (openshift#22161)

* create presubmit job for build cluster validation * permissions update for bash script * create make target and prompt to use it Co-authored-by: smg247 <[email protected]>
ashishmax31 · Sep 24, 2021 · f444f03 · f444f03
1 parent 8b17a51
commit f444f03
Show file tree

Hide file tree

Showing 12 changed files with 158 additions and 241 deletions.
diff --git a/Makefile b/Makefile
@@ -236,6 +236,11 @@ build_farm_credentials_folder:
 	oc --context app.ci -n ci extract secret/config-updater --to=$(build_farm_credentials_folder) --confirm
 .PHONY: build_farm_credentials_folder
 
+update-ci-build-clusters:
+	$(CONTAINER_ENGINE) pull registry.ci.openshift.org/ci/cluster-init:latest
+	$(CONTAINER_ENGINE) run --rm -v "$(CURDIR):/release:z" registry.ci.openshift.org/ci/cluster-init:latest -release-repo=/release -create-pr=false -update=true
+.PHONY: update-ci-build-clusters
+
 verify-app-ci:
 	true
 
@@ -257,10 +262,6 @@ secrets:
 serviceaccount-secret-rotation:
 	make job JOB=periodic-rotate-serviceaccount-secrets
 
-ci-secret-bootstrap-config:
-	hack/generate-pull-secret-entries.py core-services/ci-secret-bootstrap/_config.yaml
-.PHONY: ci-secret-bootstrap-config
-
 # generate the manifets for cluster pools admins
 # example: make TEAM=hypershift OWNERS=dmace,petr new-pool-admins
 new-pool-admins:

diff --git a/ci-operator/jobs/infra-periodics.yaml b/ci-operator/jobs/infra-periodics.yaml
@@ -1603,8 +1603,8 @@ periodics:
       - --bw-allow-unused=dptp/[email protected] mailing list
       - --bw-allow-unused=dptp/AWS ci-longlivedcluster-bot
       - --bw-allow-unused=dptp/bugzilla.redhat.com
-      - --bw-allow-unused=dptp/build_farm_01_cluster
-      - --bw-allow-unused=dptp/build_farm_02_cluster
+      - --bw-allow-unused=dptp/build_farm_build01
+      - --bw-allow-unused=dptp/build_farm_build02
       - --bw-allow-unused=dptp/kata-jenkins-ci.westus2.cloudapp.azure.com
       - --bw-allow-unused=dptp/quay.io
       - --bw-allow-unused=dptp/quay.io/multi-arch

diff --git a/ci-operator/jobs/openshift/release/openshift-release-master-presubmits.yaml b/ci-operator/jobs/openshift/release/openshift-release-master-presubmits.yaml
@@ -176,6 +176,30 @@ presubmits:
           requests:
             cpu: 10m
     trigger: (?m)^/test( | .* )boskos-config-generation,?($|\s.*)
+  - agent: kubernetes
+    always_run: true
+    branches:
+    - master
+    cluster: build02
+    context: ci/prow/build-clusters
+    decorate: true
+    labels:
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-release-master-build-clusters
+    rerun_command: /test build-clusters
+    spec:
+      containers:
+      - args:
+        - ./
+        command:
+        - hack/validate-ci-build-clusters.sh
+        image: registry.ci.openshift.org/ci/cluster-init:latest
+        imagePullPolicy: Always
+        name: ""
+        resources:
+          requests:
+            cpu: 10m
+    trigger: ((?m)^/test build-clusters,?(\s+|$))
   - agent: kubernetes
     always_run: true
     branches:

diff --git a/clusters/build-clusters/01_cluster/Makefile b/clusters/build-clusters/01_cluster/Makefile
@@ -21,7 +21,7 @@ install-dptp-managed-cluster:
 	@echo "installing dptp-managed cluster with version $(ocp_version) ..."
 	./_install/install_cluster.sh $(ocp_version)
 
-#Saved in BW build_farm_01_cluster
+#Saved in BW build_farm_build01
 client_id := "change_me"
 client_secret := "change_me"
 

diff --git a/clusters/build-clusters/01_cluster/README.md b/clusters/build-clusters/01_cluster/README.md
@@ -28,7 +28,7 @@ $ make install-dptp-managed-cluster
 Post-install action:
 
 Once we install the cluster, store the installation directory somewhere in case we need to destroy the cluster later on.
-Update password of `kubeadmin` in bitwarden (searching for item called `build_farm_01_cluster `).
+Update password of `kubeadmin` in bitwarden (searching for item called `build_farm_build01 `).
 The cert-based kubeconfig file is also uploaded to the same BW item (attachement `b01.admin.cert.kubeconfig`).
 
 ## OAuth provider: github

diff --git a/clusters/build-clusters/02_cluster/README.md b/clusters/build-clusters/02_cluster/README.md
@@ -2,7 +2,7 @@
 
 [02-Cluster](https://console-openshift-console.apps.build02.gcp.ci.openshift.org) is an OpenShift-cluster managed by DPTP-team. It is one of the clusters for running Prow job pods.
 
-The secrets have been uploaded to BitWarden item `build_farm_02_cluster`:
+The secrets have been uploaded to BitWarden item `build_farm_build02`:
 
 * the key file for the service account `ocp-cluster-installer`
 * the SSH key pair (`id_rsa` and `id_rsa.pub`)

diff --git a/core-services/ci-secret-bootstrap/_config.yaml b/core-services/ci-secret-bootstrap/_config.yaml
@@ -1720,10 +1720,10 @@ secret_configs:
 - from:
     build01_github_client_id:
       field: github_client_id
-      item: build_farm_01_cluster
+      item: build_farm_build01
     build02_github_client_id:
       field: github_client_id
-      item: build_farm_02_cluster
+      item: build_farm_build02
     vsphere_github_client_id:
       field: github_client_id
       item: build_farm_vsphere_cluster
@@ -2655,6 +2655,9 @@ secret_configs:
       - auth_field: token_image-puller_build01_reg_auth_value.txt
         item: build_farm
         registry_url: image-registry.openshift-image-registry.svc:5000
+      - auth_field: token_image-puller_build01_reg_auth_value.txt
+        item: build_farm
+        registry_url: registry.build01.ci.openshift.org
       - auth_field: auth
         email_field: email
         item: cloud.openshift.com-pull-secret
@@ -2677,9 +2680,6 @@ secret_configs:
       - auth_field: token_image-puller_arm01_reg_auth_value.txt
         item: build_farm
         registry_url: registry.arm-build01.arm-build.devcluster.openshift.com
-      - auth_field: token_image-puller_build01_reg_auth_value.txt
-        item: build_farm
-        registry_url: registry.build01.ci.openshift.org
       - auth_field: token_image-puller_build02_reg_auth_value.txt
         item: build_farm
         registry_url: registry.build02.ci.openshift.org
@@ -2704,6 +2704,9 @@ secret_configs:
       - auth_field: token_image-puller_build02_reg_auth_value.txt
         item: build_farm
         registry_url: image-registry.openshift-image-registry.svc:5000
+      - auth_field: token_image-puller_build02_reg_auth_value.txt
+        item: build_farm
+        registry_url: registry.build02.ci.openshift.org
       - auth_field: auth
         email_field: email
         item: cloud.openshift.com-pull-secret
@@ -2729,9 +2732,6 @@ secret_configs:
       - auth_field: token_image-puller_build01_reg_auth_value.txt
         item: build_farm
         registry_url: registry.build01.ci.openshift.org
-      - auth_field: token_image-puller_build02_reg_auth_value.txt
-        item: build_farm
-        registry_url: registry.build02.ci.openshift.org
       - auth_field: token_image-puller_vsphere_reg_auth_value.txt
         item: build_farm
         registry_url: registry.apps.build01-us-west-2.vmc.ci.openshift.org
@@ -2943,15 +2943,15 @@ secret_configs:
 - from:
     clientSecret:
       field: github_client_secret
-      item: build_farm_01_cluster
+      item: build_farm_build01
   to:
   - cluster: build01
     name: github-client-secret
     namespace: openshift-config
 - from:
     clientSecret:
       field: github_client_secret
-      item: build_farm_02_cluster
+      item: build_farm_build02
   to:
   - cluster: build02
     name: github-client-secret

diff --git a/core-services/ci-secret-generator/_config.yaml b/core-services/ci-secret-generator/_config.yaml
@@ -1,6 +1,6 @@
 - fields:
-  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config sa create-kubeconfig --namespace ci $(service_account)
-      | sed "s/$(service_account)/$(cluster)/g"
+  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config
+      sa create-kubeconfig --namespace ci $(service_account) | sed "s/$(service_account)/$(cluster)/g"
     name: sa.$(service_account).$(cluster).config
   item_name: build_farm
   params:
@@ -19,8 +19,8 @@
     - ci-operator
     - promoted-image-governor
 - fields:
-  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config sa create-kubeconfig --namespace ci $(service_account)
-      | sed "s/$(service_account)/$(cluster)/g"
+  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config
+      sa create-kubeconfig --namespace ci $(service_account) | sed "s/$(service_account)/$(cluster)/g"
     name: sa.$(service_account).$(cluster).config
   item_name: build_farm
   params:
@@ -36,8 +36,8 @@
     - pj-rehearse
     - ci-operator
 - fields:
-  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config sa create-kubeconfig --namespace ci $(service_account)
-      | sed "s/$(service_account)/$(cluster)/g"
+  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config
+      sa create-kubeconfig --namespace ci $(service_account) | sed "s/$(service_account)/$(cluster)/g"
     name: sa.$(service_account).$(cluster).config
   item_name: release-controller
   params:
@@ -54,9 +54,9 @@
     - release-controller-ocp-arm64
     - release-controller-ocp-arm64-priv
 - fields:
-  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config get secrets --sort-by=.metadata.creationTimestamp
-      --namespace ci -o json | jq '.items[] | select(.type=="kubernetes.io/dockercfg")
-      | select(.metadata.annotations["kubernetes.io/service-account.name"]=="image-puller")'|
+  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config
+      get secrets --sort-by=.metadata.creationTimestamp --namespace ci -o json | jq
+      '.items[] | select(.type=="kubernetes.io/dockercfg") | select(.metadata.annotations["kubernetes.io/service-account.name"]=="image-puller")'|
       jq --slurp '.[-1] | .data[".dockercfg"]' --raw-output | base64 --decode | jq
       '.["image-registry.openshift-image-registry.svc:5000"].auth' --raw-output |
       tr -d '\n'
@@ -70,9 +70,9 @@
     - build02
     - vsphere
 - fields:
-  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config get secrets --sort-by=.metadata.creationTimestamp
-      --namespace ci -o json | jq '.items[] | select(.type=="kubernetes.io/dockercfg")
-      | select(.metadata.annotations["kubernetes.io/service-account.name"]=="image-pusher")'
+  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config
+      get secrets --sort-by=.metadata.creationTimestamp --namespace ci -o json | jq
+      '.items[] | select(.type=="kubernetes.io/dockercfg") | select(.metadata.annotations["kubernetes.io/service-account.name"]=="image-pusher")'
       | jq --slurp '.[-1] | .data[".dockercfg"]' --raw-output | base64 --decode |
       jq '.["image-registry.openshift-image-registry.svc.cluster.local:5000"].auth'
       --raw-output | tr -d '\n'
@@ -82,8 +82,8 @@
     cluster:
     - app.ci
 - fields:
-  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config sa create-kubeconfig --namespace ci $(service_account)
-      | sed "s/$(service_account)/$(cluster)/g"
+  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config
+      sa create-kubeconfig --namespace ci $(service_account) | sed "s/$(service_account)/$(cluster)/g"
     name: sa.$(service_account).$(cluster).config
   item_name: ci-chat-bot
   params:
@@ -95,20 +95,20 @@
     service_account:
     - ci-chat-bot
 - fields:
-  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config sa create-kubeconfig --namespace ci $(service_account)
-      | sed "s/$(service_account)/$(cluster)/g"
+  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config
+      sa create-kubeconfig --namespace ci $(service_account) | sed "s/$(service_account)/$(cluster)/g"
     name: sa.$(service_account).$(cluster).config
   item_name: pod-scaler
   params:
     cluster:
+    - app.ci
     - build01
     - build02
-    - app.ci
     service_account:
     - pod-scaler
 - fields:
-  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config sa create-kubeconfig --namespace bparees $(service_account)
-      | sed "s/$(service_account)/$(cluster)/g"
+  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config
+      sa create-kubeconfig --namespace bparees $(service_account) | sed "s/$(service_account)/$(cluster)/g"
     name: sa.$(service_account).$(cluster).config
   item_name: endurance_cluster
   params:
@@ -117,8 +117,8 @@
     service_account:
     - endurance
 - fields:
-  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config sa create-kubeconfig --namespace ci $(service_account)
-      | sed "s/$(service_account)/$(cluster)/g"
+  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config
+      sa create-kubeconfig --namespace ci $(service_account) | sed "s/$(service_account)/$(cluster)/g"
     name: sa.$(service_account).$(cluster).config
   item_name: build_farm
   params:
@@ -129,8 +129,8 @@
     - config-updater
     - hive
 - fields:
-  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config sa create-kubeconfig --namespace ci $(service_account)
-      | sed "s/$(service_account)/$(cluster)/g"
+  - cmd: oc --kubeconfig /tmp/build-farm-credentials/sa.config-updater.$(cluster).config
+      sa create-kubeconfig --namespace ci $(service_account) | sed "s/$(service_account)/$(cluster)/g"
     name: sa.$(service_account).$(cluster).config
   item_name: build_farm
   params:

diff --git a/core-services/sanitize-prow-jobs/_config.yaml b/core-services/sanitize-prow-jobs/_config.yaml
@@ -4702,70 +4702,70 @@ default: build01
 groups:
   app.ci:
     jobs:
-    - periodic-openshift-release-fast-forward
-    - periodic-openshift-release-private-org-sync
-    - periodic-openshift-release-merge-blockers
+    - branch-ci-openshift-config-master-group-update
+    - branch-ci-openshift-config-master-org-sync
+    - branch-ci-openshift-release-master-app-ci-apply
+    - branch-ci-openshift-release-master-arm01-apply
+    - branch-ci-openshift-release-master-build01-apply
+    - branch-ci-openshift-release-master-build02-apply
+    - branch-ci-openshift-release-master-config-change-trigger
+    - branch-ci-openshift-release-master-hive-apply
+    - branch-ci-openshift-release-master-label-sync
+    - branch-ci-openshift-release-master-release-controller-annotate
+    - branch-ci-openshift-release-master-vsphere-apply
+    - openshift-release-master-config-bootstrapper
     - periodic-auto-private-org-peribolos-sync
+    - periodic-auto-prow-job-dispatcher
     - periodic-auto-publicize-config
-    - periodic-ocp-build-data-enforcer
+    - periodic-branch-protector
+    - periodic-branch-protector-openshift-org
     - periodic-bugzilla-refresh
+    - periodic-ci-operator-yaml-creator
+    - periodic-ci-secret-bootstrap
+    - periodic-ci-secret-generator
     - periodic-daily-bugzilla-refresh
-    - periodic-retester
-    - periodic-issue-close
-    - periodic-issue-rotten
-    - periodic-issue-stale
     - periodic-enhancements-close
     - periodic-enhancements-rotten
     - periodic-enhancements-stale
-    - periodic-prow-image-autobump
-    - periodic-prow-auto-config-brancher
-    - periodic-ci-operator-yaml-creator
-    - periodic-prow-auto-owners
-    - periodic-ci-secret-bootstrap
-    - periodic-ci-secret-generator
-    - periodic-rotate-serviceaccount-secrets
-    - branch-ci-openshift-release-master-app-ci-apply
-    - pull-ci-openshift-release-master-app-ci-config-dry
-    - pull-ci-openshift-ci-tools-master-secret-bootstrapper-validation
-    - pull-ci-openshift-release-master-arm01-dry
-    - branch-ci-openshift-release-master-arm01-apply
-    - pull-ci-openshift-release-master-build01-dry
-    - branch-ci-openshift-release-master-build01-apply
-    - pull-ci-openshift-release-master-build02-dry
-    - branch-ci-openshift-release-master-build02-apply
-    - pull-ci-openshift-release-master-hive-dry
-    - branch-ci-openshift-release-master-hive-apply
-    - pull-ci-openshift-release-master-vsphere-dry
-    - branch-ci-openshift-release-master-vsphere-apply
-    - branch-ci-openshift-config-master-group-update
+    - periodic-imagestream-importer
+    - periodic-issue-close
+    - periodic-issue-rotten
+    - periodic-issue-stale
+    - periodic-label-sync
+    - periodic-manage-clonerefs
+    - periodic-ocp-build-data-enforcer
+    - periodic-openshift-library-import
     - periodic-openshift-priv-group-update
-    - branch-ci-openshift-release-master-config-change-trigger
-    - periodic-prow-auto-testgrid-generator
+    - periodic-openshift-release-fast-forward
+    - periodic-openshift-release-master-accept-invitations-cherrypick-robot
+    - periodic-openshift-release-master-accept-invitations-ci-robot
+    - periodic-openshift-release-master-accept-invitations-merge-robot
     - periodic-openshift-release-master-app-ci-apply
     - periodic-openshift-release-master-arm01-apply
-    - periodic-openshift-release-master-hive-apply
     - periodic-openshift-release-master-build01-apply
     - periodic-openshift-release-master-build02-apply
+    - periodic-openshift-release-master-hive-apply
     - periodic-openshift-release-master-vsphere-apply
-    - branch-ci-openshift-release-master-label-sync
-    - periodic-label-sync
-    - periodic-branch-protector
-    - periodic-branch-protector-openshift-org
-    - periodic-manage-clonerefs
-    - periodic-sprint-automation
-    - periodic-imagestream-importer
-    - branch-ci-openshift-release-master-release-controller-annotate
-    - openshift-release-master-config-bootstrapper
-    - branch-ci-openshift-config-master-org-sync
+    - periodic-openshift-release-merge-blockers
+    - periodic-openshift-release-private-org-sync
     - periodic-org-sync
-    - periodic-auto-prow-job-dispatcher
-    - periodic-openshift-library-import
+    - periodic-promoted-image-governor
+    - periodic-prow-auto-config-brancher
+    - periodic-prow-auto-owners
+    - periodic-prow-auto-testgrid-generator
+    - periodic-prow-image-autobump
+    - periodic-retester
+    - periodic-rotate-serviceaccount-secrets
+    - periodic-sprint-automation
+    - pull-ci-openshift-ci-tools-master-secret-bootstrapper-validation
     - pull-ci-openshift-release-ci-secret-bootstrap-config-validation
+    - pull-ci-openshift-release-master-app-ci-config-dry
+    - pull-ci-openshift-release-master-arm01-dry
+    - pull-ci-openshift-release-master-build01-dry
+    - pull-ci-openshift-release-master-build02-dry
     - pull-ci-openshift-release-master-config
-    - periodic-openshift-release-master-accept-invitations-cherrypick-robot
-    - periodic-openshift-release-master-accept-invitations-merge-robot
-    - periodic-openshift-release-master-accept-invitations-ci-robot
-    - periodic-promoted-image-governor
+    - pull-ci-openshift-release-master-hive-dry
+    - pull-ci-openshift-release-master-vsphere-dry
     paths:
     - infra-image-mirroring.yaml
   build01: