test: remove fleet dependency from system tests (elastic#11696)

* test: remove fleet dependency from system tests

* test: remove unnecessary goroutine

* test: remove fleet server plugin from local dev setup

docker-compose.yml

@@ -52,43 +52,13 @@ services:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-kibana_system_user}"
       ELASTICSEARCH_PASSWORD: "${KIBANA_ES_PASS:-changeme}"
-      XPACK_FLEET_AGENTS_FLEET_SERVER_HOSTS: '["https://fleet-server:8220"]'
       XPACK_FLEET_AGENTS_ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
     depends_on:
       elasticsearch: { condition: service_healthy }
     volumes:
       - "./testing/docker/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml"
     logging: *default-logging
 
-  fleet-server:
-    image: docker.elastic.co/beats/elastic-agent:8.11.0-6aa5b988-SNAPSHOT
-    ports:
-      - 8220:8220
-    healthcheck:
-      test: ["CMD-SHELL", "curl -s -k https://localhost:8220/api/status | grep -q 'HEALTHY'"]
-      retries: 300
-      interval: 1s
-    environment:
-      FLEET_SERVER_ENABLE: "1"
-      FLEET_SERVER_POLICY_ID: "fleet-server-apm"
-      FLEET_SERVER_ELASTICSEARCH_HOST: http://elasticsearch:9200
-      FLEET_SERVER_ELASTICSEARCH_USERNAME: "${ES_SUPERUSER_USER:-admin}"
-      FLEET_SERVER_ELASTICSEARCH_PASSWORD: "${ES_SUPERUSER_PASS:-changeme}"
-      FLEET_SERVER_CERT: /etc/pki/tls/certs/fleet-server.pem
-      FLEET_SERVER_CERT_KEY: /etc/pki/tls/private/fleet-server-key.pem
-      FLEET_URL: https://fleet-server:8220
-      KIBANA_FLEET_SETUP: "true"
-      KIBANA_FLEET_HOST: "http://kibana:5601"
-      KIBANA_FLEET_USERNAME: "${ES_SUPERUSER_USER:-admin}"
-      KIBANA_FLEET_PASSWORD: "${ES_SUPERUSER_PASS:-changeme}"
-    depends_on:
-      elasticsearch: { condition: service_healthy }
-      kibana: { condition: service_healthy }
-    volumes:
-      - "./testing/docker/fleet-server/certificate.pem:/etc/pki/tls/certs/fleet-server.pem"
-      - "./testing/docker/fleet-server/key.pem:/etc/pki/tls/private/fleet-server-key.pem"
-    logging: *default-logging
-
   metricbeat:
     image: docker.elastic.co/beats/metricbeat:8.11.0-6aa5b988-SNAPSHOT
     environment:
@@ -97,7 +67,6 @@ services:
       ELASTICSEARCH_PASSWORD: "${KIBANA_ES_PASS:-changeme}"
     depends_on:
       elasticsearch: { condition: service_healthy }
-      fleet-server: { condition: service_healthy }
     volumes:
       - "./testing/docker/metricbeat/elasticsearch-xpack.yml://usr/share/metricbeat/modules.d/elasticsearch-xpack.yml"
       - "./testing/docker/metricbeat/apm-server.yml://usr/share/metricbeat/modules.d/apm-server.yml"

systemtest/containers.go

@@ -41,7 +41,6 @@ import (
 	"github.com/docker/go-connections/nat"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/wait"
-	"golang.org/x/sync/errgroup"
 )
 
 const (
@@ -69,22 +68,19 @@ func init() {
 func StartStackContainers() error {
 	cmd := exec.Command(
 		"docker-compose", "-f", "../docker-compose.yml",
-		"up", "-d", "elasticsearch", "kibana", "fleet-server",
+		"up", "-d", "elasticsearch", "kibana",
 	)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {
 		return err
 	}
 
-	// Wait for up to 5 minutes for Kibana and Fleet Server to become healthy,
+	// Wait for up to 5 minutes for Kibana to become healthy,
 	// which implies Elasticsearch is healthy too.
 	ctx, cancel := context.WithTimeout(context.Background(), startContainersTimeout)
 	defer cancel()
-	g, ctx := errgroup.WithContext(ctx)
-	g.Go(func() error { return waitContainerHealthy(ctx, "kibana") })
-	g.Go(func() error { return waitContainerHealthy(ctx, "fleet-server") })
-	return g.Wait()
+	return waitContainerHealthy(ctx, "kibana")
 }
 
 func waitContainerHealthy(ctx context.Context, serviceName string) error {
@@ -212,18 +208,11 @@ func NewUnstartedElasticAgentContainer(opts ContainerConfig) (*ElasticAgentConta
 	}
 	vcsRef := agentImageDetails.Config.Labels["org.label-schema.vcs-ref"]
 
-	containerCACertPath := "/etc/pki/tls/certs/fleet-ca.pem"
-	hostCACertPath := filepath.Join(systemtestDir, "../testing/docker/fleet-server/certificate.pem") // self-signed
 	req := testcontainers.ContainerRequest{
 		Name:       opts.Name,
 		Image:      agentImage,
 		AutoRemove: true,
 		Networks:   networks,
-		Mounts:     testcontainers.Mounts(testcontainers.BindMount(hostCACertPath, testcontainers.ContainerMountTarget(containerCACertPath))),
-		Env: map[string]string{
-			"FLEET_URL": "https://fleet-server:8220",
-			"FLEET_CA":  containerCACertPath,
-		},
 		SkipReaper: true, // we use our own reaping logic
 	}
 	return &ElasticAgentContainer{

testing/docker/kibana/kibana.yml

@@ -8,19 +8,3 @@ xpack.encryptedSavedObjects.encryptionKey: fhjskloppd678ehkdfdlliverpoolfcr
 # Disable the Fleet agent ID verification pipeline,
 # which sets Fleet-specific fields that we don't want.
 xpack.fleet.agentIdVerificationEnabled: false
-
-xpack.fleet.packages:
-  - name: fleet_server
-    version: latest
-xpack.fleet.agentPolicies:
-  - name: Fleet Server (APM)
-    id: fleet-server-apm
-    is_default_fleet_server: true
-    is_managed: false
-    namespace: default
-    package_policies:
-      - name: fleet_server-apm
-        id: default-fleet-server
-        package:
-          name: fleet_server
-

testing/infra/k8s/base/stack/kibana.yaml

@@ -44,40 +44,5 @@ spec:
       version: latest
     - name: elastic_agent
       version: latest
-    - name: fleet_server
-      version: latest
     - name: kubernetes
       version: latest
-    xpack.fleet.agentPolicies:
-    - name: Fleet Server on ECK policy
-      id: eck-fleet-server
-      is_default_fleet_server: true
-      namespace: default
-      monitoring_enabled:
-      - logs
-      - metrics
-      unenroll_timeout: 900
-      package_policies:
-      - name: fleet_server-1
-        id: fleet_server-1
-        package:
-          name: fleet_server
-    - name: APM Server on ECK policy
-      id: eck-apm-server
-      namespace: default
-      monitoring_enabled:
-      - logs
-      - metrics
-      unenroll_timeout: 900
-      is_default: true
-      package_policies:
-      - name: apm
-        id: apm-1
-        package:
-          name: apm
-        inputs:
-        - type: apm
-          enabled: true
-          vars:
-          - name: host
-            value: :8200