[ci] ci uses batch2 to build steps (hail-is#7466)

* [ci] ci uses batch2 to build steps

* increase timeout to test_ci

* fix delete batch2 instances

* move delete_batch2_instances to end of build.yaml

* increase number of instances in tests

batch2/batch/batch_configuration.py

@@ -10,6 +10,6 @@
 WORKER_TYPE = os.environ.get('WORKER_TYPE', 'standard')
 WORKER_CORES = int(os.environ.get('WORKER_CORES', 8))
 WORKER_DISK_SIZE_GB = int(os.environ.get('WORKER_DISK_SIZE_GB', 100))
-POOL_SIZE = int(os.environ.get('POOL_SIZE', 100))
-MAX_INSTANCES = int(os.environ.get('MAX_INSTANCES', 5))
+POOL_SIZE = int(os.environ.get('POOL_SIZE', 10))
+MAX_INSTANCES = int(os.environ.get('MAX_INSTANCES', 12))
 KUBERNETES_SERVER_URL = os.environ['KUBERNETES_SERVER_URL']

batch2/deployment.yaml

@@ -53,9 +53,9 @@ spec:
          - name: WORKER_DISK_SIZE_GB
            value: "10"
          - name: POOL_SIZE
-           value: "1"
-         - name: MAX_INSTANCES
            value: "2"
+         - name: MAX_INSTANCES
+           value: "3"
 {% endif %}
         ports:
          - containerPort: 5000

build.yaml

@@ -832,32 +832,6 @@ steps:
     - test_batch2_image
     - ci_utils_image
     - deploy_ci_agent
- - kind: runImage
-   name: delete_batch2_instances
-   image:
-     valueFrom: base_image.image
-   alwaysRun: true     
-   script: |
-     set -ex
-     ZONE=$(curl http://metadata.google.internal/computeMetadata/v1/instance/zone -H 'Metadata-Flavor: Google')
-     gcloud -q auth activate-service-account --key-file=/test-gsa-key/privateKeyData
-     gcloud -q compute instances list \
-         --filter 'tags.items=batch2-agent AND labels.namespace={{ default_ns.name }}' \
-         --format="value(name)" \
-       | xargs -r gcloud -q compute instances delete --zone $ZONE
-   secrets:
-    - name: test-gsa-key
-      namespace:
-        valueFrom: batch_pods_ns.name
-      mountPath: /test-gsa-key
-   scopes:
-    - dev
-    - test
-   dependsOn:
-    - default_ns
-    - batch_pods_ns
-    - base_image    
-    - test_batch2
  - kind: runImage
    name: create_ci_test_repo
    image:
@@ -926,7 +900,7 @@ steps:
     - ci_database
     - create_ci_tables
     - deploy_auth
-    - deploy_batch
+    - deploy_batch2
     - create_ci_test_repo
     - deploy_ci_agent
     - deploy_ci_agent_default
@@ -975,7 +949,7 @@ steps:
    dependsOn:
     - default_ns
     - batch_pods_ns
-    - deploy_batch
+    - deploy_batch2
     - test_pipeline_image
     - setup_pipeline
  - kind: runImage
@@ -1193,4 +1167,31 @@ steps:
    dependsOn:
     - default_ns
     - deploy_router
-
+ - kind: runImage
+   name: delete_batch2_instances
+   image:
+     valueFrom: base_image.image
+   alwaysRun: true     
+   script: |
+     set -ex
+     ZONE=$(curl http://metadata.google.internal/computeMetadata/v1/instance/zone -H 'Metadata-Flavor: Google')
+     gcloud -q auth activate-service-account --key-file=/test-gsa-key/privateKeyData
+     gcloud -q compute instances list \
+         --filter 'tags.items=batch2-agent AND labels.namespace={{ default_ns.name }}' \
+         --format="value(name)" \
+       | xargs -r gcloud -q compute instances delete --zone $ZONE
+   secrets:
+    - name: test-gsa-key
+      namespace:
+        valueFrom: batch_pods_ns.name
+      mountPath: /test-gsa-key
+   scopes:
+    - dev
+    - test
+   dependsOn:
+    - default_ns
+    - batch_pods_ns
+    - base_image 
+    - test_batch2
+    - test_ci
+    - test_pipeline

ci/ci/build.py

@@ -461,7 +461,7 @@ def __init__(self, params, namespace_name, admin_service_account, public, secret
         self.secrets = secrets
         self.job = None
 
-        if DEFAULT_NAMESPACE != 'default':
+        if is_test_deployment:
             self._name = DEFAULT_NAMESPACE
             return
 
@@ -789,6 +789,9 @@ def __init__(self, params, database_name, namespace):
         self.namespace = get_namespace(namespace, self.input_config(params.code, params.scope))
         self.job = None
 
+        if is_test_deployment:
+            self.namespace = DEFAULT_NAMESPACE
+
         # MySQL user name can be up to 16 characters long before MySQL 5.7.8 (32 after)
         if params.scope == 'deploy':
             self._name = database_name

ci/ci/ci.py

@@ -323,7 +323,7 @@ async def on_startup(app):
         timeout=aiohttp.ClientTimeout(total=60))
     app['client_session'] = session
     app['github_client'] = gh_aiohttp.GitHubAPI(session, 'ci', oauth_token=oauth_token)
-    app['batch_client'] = await BatchClient(session=session)
+    app['batch_client'] = await BatchClient(session=session, _service='batch2')
 
     with open('/ci-user-secret/sql-config.json', 'r') as f:
         config = json.loads(f.read().strip())

ci/test/resources/build.yaml

@@ -31,6 +31,13 @@ steps:
      python3 -m flake8 /hello/hello.py
    dependsOn:
      - hello_image
+ - kind: createDatabase
+   name: hello_database
+   databaseName: hello
+   namespace:
+     valueFrom: default_ns.name
+   dependsOn:
+    - default_ns
  - kind: deploy
    name: deploy_hello
    namespace:
@@ -44,3 +51,4 @@ steps:
     - default_ns
     - hello_image
     - check_hello
+    - hello_database

ci/test/test_ci.py

@@ -24,7 +24,7 @@
 
 
 def wait_for_hello():
-    wait_cmd = f'python3 wait-for.py 300 {namespace} Service -p 80 hello'
+    wait_cmd = f'python3 wait-for.py 900 {namespace} Service -p 80 hello'
     result = sp.run(wait_cmd, stdout=sp.PIPE, stderr=sp.PIPE, shell=True)
     if result.returncode != 0:
         raise Exception(f'hello service was not deployed: {result!r}')

pipeline/test/test_pipeline.py

@@ -291,7 +291,7 @@ def test_resource_group_mentioned(self):
 
 class BatchTests(unittest.TestCase):
     def setUp(self):
-        self.backend = BatchBackend()
+        self.backend = BatchBackend(_service='batch2')
 
     def tearDown(self):
         self.backend.close()