Add Splunk Platform functional test for logs (signalfx#269)

.github/workflows/functional_test.yaml

@@ -0,0 +1,104 @@
+name: Splunk Platform Functional Test
+
+on: pull_request
+
+jobs:
+  e2e-test:
+    name: ${{ matrix.container_runtime }}
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        container_runtime:
+          - "docker"
+          - "containerd"
+          - "cri-o"
+    env:
+      CI_SPLUNK_PORT: 8089
+      CI_SPLUNK_USERNAME: admin
+      CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959
+      CI_SPLUNK_PASSWORD: helloworld
+      CI_INDEX_EVENTS: ci_events
+      CONTAINER_RUNTIME: ${{ matrix.container_runtime }}
+      KUBERNETES_VERSION: v1.21.2
+      MINIKUBE_VERSION: v1.22.0
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Minikube
+        run: |
+          # Install Kubectl
+          curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl
+          chmod +x kubectl
+          sudo mv kubectl /usr/local/bin/
+          mkdir -p ${HOME}/.kube
+          touch ${HOME}/.kube/config
+          # Install Minikube
+          curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
+          chmod +x minikube
+          sudo mv minikube /usr/local/bin/
+          sudo sysctl fs.protected_regular=0
+          # Start Minikube and Wait
+          minikube start --driver=docker --container-runtime=${CONTAINER_RUNTIME} --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --no-vtx-check
+          kubectl apply -f https://docs.projectcalico.org/v3.14/manifests/calico.yaml
+          export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
+          until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
+            sleep 1;
+          done
+
+      - name: Install Splunk
+        run: |
+          # Wait until default service account is created
+          until kubectl -n default get serviceaccount default -o name; do
+            sleep 1;
+          done
+          # Install Splunk on minikube
+          kubectl apply -f ci_scripts/k8s-splunk.yml
+          # Wait until splunk is ready
+          until kubectl logs splunk --tail=2 | grep -q 'Ansible playbook complete'; do
+            sleep 1;
+          done
+          export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
+          # Setup Indexes
+          curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_EVENTS -d datatype=event
+          curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=ns-anno -d datatype=event
+          curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=pod-anno -d datatype=event
+          curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_METRICS -d datatype=metric
+          # Enable HEC services
+          curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable
+          # Create new HEC token
+          curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token&token=a6b5e77f-d5f6-415a-bd43-930cecb12959&disabled=0&index=main&indexes=main,ci_events,ns-anno,pod-anno" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http
+          # Restart Splunk
+          curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/server/control/restart -X POST
+
+      - name: Deploy sck otel connector
+        run: |
+          export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
+          ci_scripts/deploy_connector.sh
+
+      - name: Deploy log generator
+        run: |
+          kubectl apply -f test/test_setup.yaml
+          sleep 60
+          kubectl get pods --all-namespaces
+          kubectl logs -l component=agent-collector
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Run functional tests
+        run: |
+          kubectl get nodes
+          export PYTHONWARNINGS="ignore:Unverified HTTPS request"
+          export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
+          cd test
+          pip install --upgrade pip
+          pip install -r requirements.txt
+          echo "Running functional tests....."
+          python -m pytest \
+          	--splunkd-url https://$CI_SPLUNK_HOST:8089 \
+          	--splunk-user admin \
+          	--splunk-password $CI_SPLUNK_PASSWORD \
+          	-p no:warnings -s

.pre-commit-config.yaml

@@ -12,10 +12,10 @@ repos:
     rev: v3.2.0
     hooks:
     - id: trailing-whitespace
-      exclude: "^rendered"
+      exclude: "^rendered|^test"
     - id: end-of-file-fixer
-      exclude: "^rendered"
+      exclude: "^rendered|^test"
     - id: check-yaml
       # Can't check source yaml since it has go templates in it.
-      exclude: "^helm-charts"
+      exclude: "^helm-charts|^test"
     - id: check-added-large-files

ci_scripts/deploy_connector.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -e
+curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+
+#Make sure to check and clean previously failed deployment
+echo "Checking if previous deployment exist..."
+if [ "`helm ls --short`" == "" ]; then
+   echo "Nothing to clean, ready for deployment"
+else
+   helm delete $(helm ls --short)
+fi
+echo "Deploying Splunk OTel Collector for Kubernetes"
+helm install ci-sck --set splunkPlatform.index=$CI_INDEX_EVENTS \
+--set splunkPlatform.token=$CI_SPLUNK_HEC_TOKEN \
+--set splunkPlatform.endpoint=https://$CI_SPLUNK_HOST:8088/services/collector \
+-f ci_scripts/sck_otel_values.yaml helm-charts/splunk-otel-collector/
+#--set containerLogs.containerRuntime=$CONTAINER_RUNTIME \
+#wait for deployment to finish
+until kubectl get pod | grep Running | [[ $(wc -l) == 1 ]]; do
+   sleep 1;
+done

ci_scripts/k8s-splunk.yml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: splunk
+spec:
+  hostNetwork: true
+  securityContext:
+    runAsUser: 0
+    runAsGroup: 0
+  containers:
+  - name: splunk
+    image: splunk/splunk:8.2.0
+    ports:
+    - containerPort: 8000
+      hostPort: 8000
+      protocol: TCP
+    - containerPort: 8088
+      hostPort: 8088
+      protocol: TCP
+    - containerPort: 8089
+      hostPort: 8089
+      protocol: TCP
+    env:
+    - name: SPLUNK_START_ARGS
+      value: --accept-license
+    - name: SPLUNK_USER
+      value: root
+    - name: SPLUNK_PASSWORD
+      value: helloworld
+    - name: SPLUNK_LAUNCH_CONF
+      value: OPTIMISTIC_ABOUT_FILE_LOCKING=1

ci_scripts/run_functional_tests.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+pyenv global 3.9.1
+pip install --upgrade pip
+pip install -r test/requirements.txt
+cd test
+#Run pytests
+echo "Running functional tests....."
+python -m pytest \
+	--splunkd-url https://$CI_SPLUNK_HOST:8089 \
+	--splunk-user admin \
+	--splunk-password $CI_SPLUNK_PASSWORD \
+	-p no:warnings -s

ci_scripts/sck_otel_values.yaml

@@ -0,0 +1,70 @@
+# Default values for sck-otel.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# Splunk Data-to-Everything Platform configuration.
+# Fields `endpoint` and `token` are required to collect and send logs to Splunk Cloud or Splunk Enterprise.
+splunkPlatform:
+  # URL to a Splunk instance to send data to. ie) "http://X.X.X.X:8088/services/collector"
+  endpoint:
+  # Splunk HTTP Event Collector token.
+  token:
+  # Optional. Name of the Splunk index targeted.
+  index: "ci_events"
+  metrics_index: "ci_metrics"
+  # Optional. Default value for `source` field.
+  source: "kubernetes"
+  # Optional. Default value for `sourcetype` field. For container logs, it will be container name.
+  sourcetype:
+  # Maximum HTTP connections to use simultaneously when sending data. Defaults to 200.
+  max_connections: 200
+  # Whether to disable gzip compression over HTTP. Defaults to true.
+  disable_compression: true
+  # HTTP timeout when sending data. Defaults to 10s.
+  timeout: 10s
+  # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to true.
+  insecure_skip_verify: true
+  # Options to disable particular telemetry data types that will be sent to
+  # Splunk Platform.
+  logsEnabled: true
+  metricsEnabled: true
+
+logsEngine: otel
+
+clusterName: "functional-test"
+
+# Metadata to be set on the telemetry data from Kubernetes objects.
+# https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sprocessor.
+#k8sMetadata:
+#  # Boolean for enriching all telemetry with k8s metadata (labels, annotations, etc)
+#  enabled: true
+#  # This option is to fetch metadata from extra annotations.
+#  annotations:
+#    - key: splunk.com/customField
+#      tag_name: customField
+#  labels:
+#    - key: app
+#      from: pod
+#    - key: app
+#      from: namespace
+
+# List of key/value pairs to set additional metadata set uniformly across the collected telemetry.
+# Can be used to define things such as cloud_account_id, cloud_account_region, etc.
+# Should be set using https://github.com/open-telemetry/opentelemetry-collector/tree/main/processor/resourceprocessor.
+extraAttributes:
+  fromAnnotations:
+    - key: splunk.com/customField
+      tag_name: customField
+  fromLabels:
+    - key: app
+      from: pod
+    - key: app
+      from: namespace
+
+  custom:
+   - name: "cluster_name"
+     value: "ci-k8s-cluster"
+   - name: "customfield1"
+     value: "customvalue1"
+   - name: "customfield2"
+     value: "customvalue2"

test/README.md

@@ -0,0 +1,98 @@
+# Splunk Platform Functional Test Environment Setup
+
+## Prerequsite
+* Python version must be > 3.x
+* Kubectl = v1.15.2
+* Minikube = v1.20.0
+* Helm = 3.3.x
+* libseccomp2 and cri-o (optional)
+
+## Setup local environment
+
+#### Start Minikube
+    # Set environment variable for container runtime. Options are: docker, cri-o and containerd
+      export CONTAINER_RUNTIME=docker
+
+    # Start minikube
+      minikube start --driver=docker --container-runtime=$CONTAINER_RUNTIME --cpus 3 --memory 8192 --kubernetes-version=v1.15.2 --no-vtx-check
+
+#### Install Splunk on minikube
+    # Use ci_scripts/k8s-splunk.yml file to deploy splunk on minikube
+    kubectl apply -f ci_scripts/k8s-splunk.yml
+
+    # Run following command to check if Splunk is ready. User should see "Ansible playbook complete, will begin streaming splunkd_stderr.log"
+    kubectl logs splunk -f
+
+    # To be abel to interact with Splunk pod from local workstation, you need to forward local ports to the ports on the Splunk Pod
+    # Start a new terminal concole, run following command and keep it running in the background
+    kubectl port-forward pods/splunk 8089
+
+    # Setup Indexes
+    curl -k -u admin:helloworld https://localhost:8089/services/data/indexes -d name=ci_events -d datatype=event
+    curl -k -u admin:helloworld https://localhost:8089/services/data/indexes -d name=ns-anno -d datatype=event
+    curl -k -u admin:helloworld https://localhost:8089/services/data/indexes -d name=pod-anno -d datatype=event
+
+    # Enable HEC services
+    curl -X POST -u admin:helloworld -k https://localhost:8089/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable
+
+    # Create new HEC token
+    curl -X POST -u admin:helloworld -k -d "name=splunk_hec_token&token=a6b5e77f-d5f6-415a-bd43-930cecb12959&disabled=0&index=main&indexes=main,ci_events,ns-anno,pod-anno" https://localhost:8089/servicesNS/nobody/splunk_httpinput/data/inputs/http
+
+    # Restart Splunk
+    curl -k -u admin:helloworld https://localhost:8089/services/server/control/restart -X POST
+
+    # Start a new terminal concole, forward local port 8000 to the port on Splunk pod (for debugging)
+    kubectl port-forward pods/splunk 8000
+    You can then vistit Splunk web page: https://localhost:8000
+
+#### Deploy sck otel connector
+    # Get Splunk Host IP
+    export SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
+
+    # Use ci_scripts/sck_otel_values.yaml file to deploy sck otel connector
+    # Default image repository: quay.io/signalfx/splunk-otel-collector
+    helm install ci-sck --set splunkPlatform.index=$CI_INDEX_EVENTS \
+    --set splunkPlatform.token=$CI_SPLUNK_HEC_TOKEN \
+    --set splunkPlatform.endpoint=https://$CI_SPLUNK_HOST:8088/services/collector \
+    -f ci_scripts/sck_otel_values.yaml helm-charts/splunk-otel-collector/
+
+#### Deploy log generator
+    # Use test/test_setup.yaml file to deploy log generator
+    kubectl apply -f test/test_setup.yaml
+
+#### Check data on Splunk
+    To see the test events generaged in Splunk, you can vistit Splunk web page: https://localhost:8000
+    Search for events by index.
+    For example: `index=ci_events`
+
+## Testing Instructions
+1. (Optional) Use a virtual environment for the test
+    ```
+    virtualenv --python=python3.6 venv
+    source venv/bin/activate
+    ```
+2. Install the dependencies
+    ```
+    pip install -r requirements.txt
+    export PYTHONWARNINGS="ignore:Unverified HTTPS request"
+    ```
+3. Start the test with the required options configured
+    ```
+    python -m pytest \
+    --splunkd-url https://localhost:8089 \
+    --splunk-user admin --splunk-password helloworld \
+    -p no:warnings -s
+    ```
+    **Options are:**
+    --splunkd-url
+    * Description: splunkd url used to send test data to.
+    * Default: https://localhost:8089
+
+    --splunk-user
+    * Description: splunk username
+    * Default: admin
+
+    --splunk-password
+    * Description: splunk user password
+    * Default: changeme
+