[CI] Rebalance/split cigroups and speed up overall CI time (elastic#1…

…13676)

.buildkite/pipelines/es_snapshots/verify.yml

@@ -29,7 +29,7 @@ steps:
     label: 'Default CI Group'
     parallelism: 13
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 150
     key: default-cigroup
@@ -41,7 +41,7 @@ steps:
   - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Docker CI Group'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     key: default-cigroup-docker
@@ -77,7 +77,7 @@ steps:
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'
     agents:
-      queue: jest
+      queue: n2-2
     timeout_in_minutes: 120
     key: api-integration
 

.buildkite/pipelines/flaky_tests/runner.js

@@ -78,7 +78,7 @@ for (const testSuite of testSuites) {
         steps.push({
           command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/xpack_cigroup.sh`,
           label: `Default CI Group ${CI_GROUP}`,
-          agents: { queue: 'ci-group-6' },
+          agents: { queue: 'n2-4' },
           depends_on: 'build',
           parallelism: RUN_COUNT,
           concurrency: concurrency,
@@ -103,7 +103,7 @@ for (const testSuite of testSuites) {
       steps.push({
         command: `.buildkite/scripts/steps/functional/${IS_XPACK ? 'xpack' : 'oss'}_firefox.sh`,
         label: `${IS_XPACK ? 'Default' : 'OSS'} Firefox`,
-        agents: { queue: IS_XPACK ? 'ci-group-6' : 'ci-group-4d' },
+        agents: { queue: IS_XPACK ? 'n2-4' : 'ci-group-4d' },
         depends_on: 'build',
         parallelism: RUN_COUNT,
         concurrency: concurrency,
@@ -118,7 +118,7 @@ for (const testSuite of testSuites) {
           IS_XPACK ? 'xpack' : 'oss'
         }_accessibility.sh`,
         label: `${IS_XPACK ? 'Default' : 'OSS'} Accessibility`,
-        agents: { queue: IS_XPACK ? 'ci-group-6' : 'ci-group-4d' },
+        agents: { queue: IS_XPACK ? 'n2-4' : 'ci-group-4d' },
         depends_on: 'build',
         parallelism: RUN_COUNT,
         concurrency: concurrency,

.buildkite/pipelines/hourly.yml

@@ -19,7 +19,7 @@ steps:
     label: 'Default CI Group'
     parallelism: 13
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 250
     key: default-cigroup
@@ -31,7 +31,7 @@ steps:
   - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Docker CI Group'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     key: default-cigroup-docker
@@ -67,7 +67,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
     label: 'Default Accessibility Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -89,7 +89,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_firefox.sh
     label: 'Default Firefox Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -100,7 +100,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -111,7 +111,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
     label: 'Saved Object Field Metrics'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:

.buildkite/pipelines/pull_request/base.yml

@@ -15,9 +15,9 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 13
+    parallelism: 27
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 150
     key: default-cigroup
@@ -29,7 +29,7 @@ steps:
   - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Docker CI Group'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     key: default-cigroup-docker
@@ -65,7 +65,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
     label: 'Default Accessibility Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -87,7 +87,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_firefox.sh
     label: 'Default Firefox Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -98,7 +98,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -109,7 +109,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
     label: 'Saved Object Field Metrics'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -156,7 +156,7 @@ steps:
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
     agents:
-      queue: c2-4
+      queue: c2-8
     key: checks
     timeout_in_minutes: 120
 

.buildkite/scripts/steps/checks/type_check_plugin_public_api_docs.sh

@@ -11,9 +11,27 @@ checks-reporter-with-killswitch "Build TS Refs" \
     --no-cache \
     --force
 
-echo --- Check Types
 checks-reporter-with-killswitch "Check Types" \
-  node scripts/type_check
+  node scripts/type_check &> target/check_types.log &
+check_types_pid=$!
+
+node --max-old-space-size=12000 scripts/build_api_docs &> target/build_api_docs.log &
+api_docs_pid=$!
+
+wait $check_types_pid
+check_types_exit=$?
+
+wait $api_docs_pid
+api_docs_exit=$?
+
+echo --- Check Types
+cat target/check_types.log
+if [[ "$check_types_exit" != "0" ]]; then echo "^^^ +++"; fi
 
 echo --- Building api docs
-node --max-old-space-size=12000 scripts/build_api_docs
+cat target/build_api_docs.log
+if [[ "$api_docs_exit" != "0" ]]; then echo "^^^ +++"; fi
+
+if [[ "${api_docs_exit}${check_types_exit}" != "00" ]]; then
+  exit 1
+fi

.ci/ci_groups.yml

@@ -25,4 +25,18 @@ xpack:
   - ciGroup11
   - ciGroup12
   - ciGroup13
+  - ciGroup14
+  - ciGroup15
+  - ciGroup16
+  - ciGroup17
+  - ciGroup18
+  - ciGroup19
+  - ciGroup20
+  - ciGroup21
+  - ciGroup22
+  - ciGroup23
+  - ciGroup24
+  - ciGroup25
+  - ciGroup26
+  - ciGroup27
   - ciGroupDocker

test/functional/apps/management/index.ts

@@ -22,7 +22,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
     });
 
     describe('', function () {
-      this.tags('ciGroup7');
+      this.tags('ciGroup9');
 
       loadTestFile(require.resolve('./_create_index_pattern_wizard'));
       loadTestFile(require.resolve('./_index_pattern_create_delete'));

test/functional/apps/visualize/index.ts

@@ -74,8 +74,8 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
       loadTestFile(require.resolve('./_metric_chart'));
     });
 
-    describe('visualize ciGroup4', function () {
-      this.tags('ciGroup4');
+    describe('visualize ciGroup1', function () {
+      this.tags('ciGroup1');
 
       loadTestFile(require.resolve('./_pie_chart'));
       loadTestFile(require.resolve('./_shared_item'));

x-pack/test/alerting_api_integration/security_and_spaces/tests/index.ts

@@ -58,7 +58,7 @@ export async function tearDown(getService: FtrProviderContext['getService']) {
 // eslint-disable-next-line import/no-default-export
 export default function alertingApiIntegrationTests({ loadTestFile }: FtrProviderContext) {
   describe('alerting api integration security and spaces enabled', function () {
-    this.tags('ciGroup5');
+    this.tags('ciGroup17');
 
     loadTestFile(require.resolve('./actions'));
     loadTestFile(require.resolve('./alerting'));

x-pack/test/api_integration/apis/index.ts

@@ -9,7 +9,7 @@ import { FtrProviderContext } from '../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('apis', function () {
-    this.tags('ciGroup6');
+    this.tags('ciGroup18');
 
     loadTestFile(require.resolve('./search'));
     loadTestFile(require.resolve('./es'));
@@ -27,12 +27,12 @@ export default function ({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./maps'));
     loadTestFile(require.resolve('./security_solution'));
     loadTestFile(require.resolve('./lens'));
-    loadTestFile(require.resolve('./ml'));
     loadTestFile(require.resolve('./transform'));
     loadTestFile(require.resolve('./lists'));
     loadTestFile(require.resolve('./upgrade_assistant'));
     loadTestFile(require.resolve('./searchprofiler'));
     loadTestFile(require.resolve('./painless_lab'));
     loadTestFile(require.resolve('./file_upload'));
+    loadTestFile(require.resolve('./ml'));
   });
 }

x-pack/test/api_integration/apis/security/index.ts

@@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('security', function () {
-    this.tags('ciGroup6');
+    this.tags('ciGroup18');
 
     // Updates here should be mirrored in `./security_basic.ts` if tests
     // should also run under a basic license.

x-pack/test/api_integration/apis/spaces/index.ts

@@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('spaces', function () {
-    this.tags('ciGroup6');
+    this.tags('ciGroup18');
 
     loadTestFile(require.resolve('./get_active_space'));
     loadTestFile(require.resolve('./saved_objects'));

x-pack/test/case_api_integration/security_and_spaces/tests/basic/index.ts

@@ -12,7 +12,7 @@ import { createSpacesAndUsers, deleteSpacesAndUsers } from '../../../common/lib/
 export default ({ loadTestFile, getService }: FtrProviderContext): void => {
   describe('cases security and spaces enabled: basic', function () {
     // Fastest ciGroup for the moment.
-    this.tags('ciGroup13');
+    this.tags('ciGroup27');
 
     before(async () => {
       await createSpacesAndUsers(getService);

x-pack/test/case_api_integration/security_and_spaces/tests/trial/index.ts

@@ -11,8 +11,7 @@ import { createSpacesAndUsers, deleteSpacesAndUsers } from '../../../common/lib/
 // eslint-disable-next-line import/no-default-export
 export default ({ loadTestFile, getService }: FtrProviderContext): void => {
   describe('cases security and spaces enabled: trial', function () {
-    // Fastest ciGroup for the moment.
-    this.tags('ciGroup13');
+    this.tags('ciGroup25');
 
     before(async () => {
       await createSpacesAndUsers(getService);

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/index.ts

@@ -11,7 +11,7 @@ import { FtrProviderContext } from '../../../common/ftr_provider_context';
 export default ({ loadTestFile }: FtrProviderContext): void => {
   describe('Detection exceptions data types and operators', function () {
     describe('', function () {
-      this.tags('ciGroup11');
+      this.tags('ciGroup23');
 
       loadTestFile(require.resolve('./date'));
       loadTestFile(require.resolve('./double'));
@@ -20,7 +20,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => {
     });
 
     describe('', function () {
-      this.tags('ciGroup12');
+      this.tags('ciGroup24');
 
       loadTestFile(require.resolve('./ip'));
       loadTestFile(require.resolve('./ip_array'));

x-pack/test/functional/apps/dashboard/index.ts

@@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('dashboard', function () {
-    this.tags('ciGroup7');
+    this.tags('ciGroup19');
 
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./preserve_url'));

x-pack/test/functional/apps/discover/index.ts

@@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('discover', function () {
-    this.tags('ciGroup1');
+    this.tags('ciGroup25');
 
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./preserve_url'));

x-pack/test/functional/apps/lens/index.ts

@@ -50,10 +50,6 @@ export default function ({ getService, loadTestFile, getPageObjects }: FtrProvid
     describe('', function () {
       this.tags(['ciGroup4', 'skipFirefox']);
 
-      loadTestFile(require.resolve('./add_to_dashboard'));
-      loadTestFile(require.resolve('./table'));
-      loadTestFile(require.resolve('./runtime_fields'));
-      loadTestFile(require.resolve('./dashboard'));
       loadTestFile(require.resolve('./colors'));
       loadTestFile(require.resolve('./chart_data'));
       loadTestFile(require.resolve('./time_shift'));
@@ -69,5 +65,14 @@ export default function ({ getService, loadTestFile, getPageObjects }: FtrProvid
       // has to be last one in the suite because it overrides saved objects
       loadTestFile(require.resolve('./rollup'));
     });
+
+    describe('', function () {
+      this.tags(['ciGroup16', 'skipFirefox']);
+
+      loadTestFile(require.resolve('./add_to_dashboard'));
+      loadTestFile(require.resolve('./table'));
+      loadTestFile(require.resolve('./runtime_fields'));
+      loadTestFile(require.resolve('./dashboard'));
+    });
   });
 }

x-pack/test/functional/apps/maps/index.js

@@ -73,8 +73,13 @@ export default function ({ loadTestFile, getService }) {
     });
 
     describe('', function () {
-      this.tags('ciGroup10');
+      this.tags('ciGroup22');
       loadTestFile(require.resolve('./es_geo_grid_source'));
+      loadTestFile(require.resolve('./embeddable'));
+    });
+
+    describe('', function () {
+      this.tags('ciGroup10');
       loadTestFile(require.resolve('./es_pew_pew_source'));
       loadTestFile(require.resolve('./joins'));
       loadTestFile(require.resolve('./mapbox_styles'));
@@ -83,7 +88,6 @@ export default function ({ loadTestFile, getService }) {
       loadTestFile(require.resolve('./add_layer_panel'));
       loadTestFile(require.resolve('./import_geojson'));
       loadTestFile(require.resolve('./layer_errors'));
-      loadTestFile(require.resolve('./embeddable'));
       loadTestFile(require.resolve('./visualize_create_menu'));
       loadTestFile(require.resolve('./discover'));
     });