Adding Journald functional tests (signalfx#404)

.github/workflows/functional_test.yaml

@@ -45,7 +45,7 @@ jobs:
           sudo mv minikube /usr/local/bin/
           sudo sysctl fs.protected_regular=0
           # Start Minikube and Wait
-          minikube start --driver=docker --container-runtime=${CONTAINER_RUNTIME} --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --no-vtx-check
+          minikube start --container-runtime=${CONTAINER_RUNTIME} --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --no-vtx-check
           kubectl apply -f https://docs.projectcalico.org/v3.14/manifests/calico.yaml
           export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
           until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do

ci_scripts/sck_otel_values.yaml

@@ -71,3 +71,20 @@ extraAttributes:
      value: "customvalue1"
    - name: "customfield2"
      value: "customvalue2"
+
+logsCollection:
+  # Configuration for collecting journald logs using otel collector
+  journald:
+    enabled: true
+    # Please update directory path for journald if it's different from below default value "/var/log/journal"
+    directory: /run/log/journal
+    # List of service units to collect journald logs for and configuration for each.
+    units:
+      - name: kubelet
+        priority: info
+      - name: docker
+        priority: info
+      - name: containerd
+        priority: info
+    # Route journald logs to its own Splunk Index by specifying the index value below, else leave it blank. Please make sure the index exist in Splunk and is configured to receive HEC traffic. Not applicable to Splunk Observability.
+    index: ""

test/k8s_logging_tests/test_config_logging.py

@@ -272,3 +272,52 @@ def test_custom_metadata_fields_annotations(setup, label, index, value, expected
     logger.info("Splunk received %s events in the last minute",
                 len(events))
     assert len(events) >= expected
+
+@pytest.mark.parametrize("test_input,expected", [
+    ("test_journald_data", 1)
+])
+def test_journald_logs(setup, test_input, expected):
+    '''
+    Test that user specified index can successfully index the
+    journald log stream from k8s. If no index is specified, default
+    index "ci_events" will be used.
+    '''
+    logger.info("testing test_journald_logs input={0} expected={1} event(s)".format(
+        test_input, expected))
+    index_logging = os.environ["CI_INDEX_EVENTS"] if os.environ["CI_INDEX_EVENTS"] else "ci_events"
+    search_query = "index=" + index_logging + " sourcetype=kube:journald*"
+
+    events = check_events_from_splunk(start_time="-1h@h",
+                                      url=setup["splunkd_url"],
+                                      user=setup["splunk_user"],
+                                      query=["search {0}".format(
+                                          search_query)],
+                                      password=setup["splunk_password"])
+    logger.info("Splunk received %s events in the last hour",
+                len(events))
+    assert len(events) >= expected
+
+@pytest.mark.parametrize("test_input,expected", [
+    ("containerd.service", 1),
+    ("docker.service", 1),
+    ("kubelet.service", 1),
+    ("empty_unit", 0)
+])
+def test_journald_unit(setup, test_input, expected):
+    '''
+    Test that all configured journald units are present in target index.
+    '''
+    logger.info("testing for presence of journald_unit={0} expected={1} event(s)".format(
+        test_input, expected))
+    index_logging = os.environ["CI_INDEX_EVENTS"] if os.environ["CI_INDEX_EVENTS"] else "ci_events"
+    search_query = "index=" + index_logging + " sourcetype=kube:journald:" + test_input
+    events = check_events_from_splunk(start_time="-1h@h",
+                                      url=setup["splunkd_url"],
+                                      user=setup["splunk_user"],
+                                      query=["search {0}".format(
+                                          search_query)],
+                                      password=setup["splunk_password"])
+    logger.info("Splunk received %s events in the last hour",
+                    len(events))
+    assert len(events) >= expected if test_input != "empty_unit" else len(
+        events) == expected