Fix generation of service.name log attribute in istio environment (si…

…gnalfx#176) Istio has an involved logic for generating service name for traces. So we cannot just rely on "app" pod label. This commit adds the same logic to generate service.name logs attribute as istio uses for traces. It's needed to make correlation between logs and traces work OOTB in Splunk o11y suite.
dmitryax · Jul 10, 2021 · 401d6ad · 401d6ad
1 parent 46c1d72
commit 401d6ad
Show file tree

Hide file tree

Showing 44 changed files with 71 additions and 54 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+## [0.29.1] - 2021-07-08
+
+### Fixed
+
+- Fix generation of service.name log attribute in istio environment (#176)
+
 ## [0.29.0] - 2021-07-08
 
 ### Changed

diff --git a/helm-charts/splunk-otel-collector/Chart.yaml b/helm-charts/splunk-otel-collector/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: splunk-otel-collector
-version: 0.29.0
+version: 0.29.1
 description: Splunk OpenTelemetry Connector for Kubernetes
 icon: https://github.com/signalfx/splunk-otel-collector-chart/tree/main/splunk.png
 type: application

diff --git a/helm-charts/splunk-otel-collector/templates/configmap-fluentd.yaml b/helm-charts/splunk-otel-collector/templates/configmap-fluentd.yaml
@@ -210,14 +210,23 @@ data:
           k8s.pod.labels.{{ . }} ${record.dig("kubernetes","labels","{{ . }}")}
           {{- end }}
 
-          {{- if .Values.autodetect.istio }}
-          # Automaically compose service.name attribute the same way as done in istio for traces
-          service.name ${record.dig("kubernetes","labels","app")}.${record.dig("kubernetes","namespace_name")}
-          {{- end }}
-
           denylist ${record.dig("kubernetes", "annotations", "splunk.com/exclude") ? record.dig("kubernetes", "annotations", "splunk.com/exclude") : record.dig("kubernetes", "namespace_annotations", "splunk.com/exclude") ? (record["kubernetes"]["namespace_annotations"]["splunk.com/exclude"]) : ("false")}
         </record>
       </filter>
+
+      {{- if .Values.autodetect.istio }}
+      # This filter adds "service.name" attribute, if it's not present, the same way as istio
+      # prepares service name for generated traces:
+      # https://github.com/istio/istio/blob/77e00b47a61f0e995a29b33438c9dae7e8aace82/galley/pkg/config/analysis/analyzers/testdata/common/sidecar-injector-configmap.yaml#L110-L115
+      # Name of the k8s deployment (or another object owning the pod, if any) is taken from "k8s.pod.name" attribute
+      # using regex to filter out name suffix according to the k8s name generation rules:
+      # https://github.com/kubernetes/apimachinery/blob/ff522ab81c745a9ac5f7eeb7852fac134194a3b6/pkg/util/rand/rand.go#L92-L127
+      <filter tail.containers.**>
+        @type jq_transformer
+        jq '.record["service.name"] = (.record["service.name"] // (.record.kubernetes.labels.app // (.record["k8s.pod.name"] // "istio-proxy" | (capture("^(?<owner_obj>.+?)-(?:(?:[0-9bcdf]+-)?[bcdfghjklmnpqrstvwxz2456789]{5}|[0-9]+)$") | .owner_obj) // .)) + "." + (.record["k8s.namespace.name"] // "default")) | .record'
+      </filter>
+      {{- end }}
+
       <filter tail.containers.**>
         # Exclude all logs that are denylisted
         @type grep

diff --git a/rendered/manifests/agent-only/clusterRole.yaml b/rendered/manifests/agent-only/clusterRole.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 rules:

diff --git a/rendered/manifests/agent-only/clusterRoleBinding.yaml b/rendered/manifests/agent-only/clusterRoleBinding.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 roleRef:

diff --git a/rendered/manifests/agent-only/configmap-fluentd-cri.yaml b/rendered/manifests/agent-only/configmap-fluentd-cri.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-fluentd-cri
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 data:

diff --git a/rendered/manifests/agent-only/configmap-fluentd-json.yaml b/rendered/manifests/agent-only/configmap-fluentd-json.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-fluentd-json
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 data:

diff --git a/rendered/manifests/agent-only/configmap-fluentd.yaml b/rendered/manifests/agent-only/configmap-fluentd.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-fluentd
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 data:
@@ -268,6 +268,7 @@ data:
           denylist ${record.dig("kubernetes", "annotations", "splunk.com/exclude") ? record.dig("kubernetes", "annotations", "splunk.com/exclude") : record.dig("kubernetes", "namespace_annotations", "splunk.com/exclude") ? (record["kubernetes"]["namespace_annotations"]["splunk.com/exclude"]) : ("false")}
         </record>
       </filter>
+
       <filter tail.containers.**>
         # Exclude all logs that are denylisted
         @type grep

diff --git a/rendered/manifests/agent-only/configmap-otel-agent.yaml b/rendered/manifests/agent-only/configmap-otel-agent.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-otel-agent
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 data:

diff --git a/rendered/manifests/agent-only/configmap-otel-k8s-cluster-receiver.yaml b/rendered/manifests/agent-only/configmap-otel-k8s-cluster-receiver.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-otel-k8s-cluster-receiver
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 data:

diff --git a/rendered/manifests/agent-only/daemonset.yaml b/rendered/manifests/agent-only/daemonset.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-agent
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
     engine: fluentd
@@ -23,7 +23,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: a1b886bef2816a9293cc136642338a78e7da0b2111f8ce66f47cd7937c298d16
+        checksum/config: 826663a823f84bf1225877b27bffc55e8471bf244e93cc8ee5a441c3012a9f37
     spec:
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet

diff --git a/rendered/manifests/agent-only/deployment-k8s-cluster-receiver.yaml b/rendered/manifests/agent-only/deployment-k8s-cluster-receiver.yaml
@@ -7,7 +7,7 @@ metadata:
   labels:
     app: splunk-otel-collector
     component: otel-k8s-cluster-receiver
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 spec:
@@ -24,7 +24,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d9c45723a74160ada19f7b91af3bcc935ebe526fd883fb977717297ca1c6d33f
+        checksum/config: 9be884aefa99593fd91546948340bf6a039d0b9bd4e5389da31226179262af90
     spec:
       serviceAccountName: default-splunk-otel-collector
       containers:

diff --git a/rendered/manifests/agent-only/secret.yaml b/rendered/manifests/agent-only/secret.yaml
@@ -6,7 +6,7 @@ metadata:
   name: splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 type: Opaque

diff --git a/rendered/manifests/agent-only/serviceAccount.yaml b/rendered/manifests/agent-only/serviceAccount.yaml
@@ -6,6 +6,6 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
diff --git a/rendered/manifests/gateway-only/clusterRole.yaml b/rendered/manifests/gateway-only/clusterRole.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 rules:

diff --git a/rendered/manifests/gateway-only/clusterRoleBinding.yaml b/rendered/manifests/gateway-only/clusterRoleBinding.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 roleRef:

diff --git a/rendered/manifests/gateway-only/configmap-otel-collector.yaml b/rendered/manifests/gateway-only/configmap-otel-collector.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 data:

diff --git a/rendered/manifests/gateway-only/deployment-collector.yaml b/rendered/manifests/gateway-only/deployment-collector.yaml
@@ -7,7 +7,7 @@ metadata:
   labels:
     app: splunk-otel-collector
     component: otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 spec:
@@ -24,7 +24,7 @@ spec:
         component: otel-collector
         release: default
       annotations:
-        checksum/config: 74f33a1a30a2162f8a82b6b81efe14821cd6a270d8f41e13d80dd0df5c6d546f
+        checksum/config: c62bfb843e168c5ae848390a1f6de78823d8ea4259dc5817cd16c50b305267cf
     spec:
       serviceAccountName: default-splunk-otel-collector
       containers:

diff --git a/rendered/manifests/gateway-only/secret.yaml b/rendered/manifests/gateway-only/secret.yaml
@@ -6,7 +6,7 @@ metadata:
   name: splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 type: Opaque

diff --git a/rendered/manifests/gateway-only/service.yaml b/rendered/manifests/gateway-only/service.yaml
@@ -7,7 +7,7 @@ metadata:
   labels:
     app: splunk-otel-collector
     component: otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 spec:

diff --git a/rendered/manifests/gateway-only/serviceAccount.yaml b/rendered/manifests/gateway-only/serviceAccount.yaml
@@ -6,6 +6,6 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
diff --git a/rendered/manifests/logs-only/clusterRole.yaml b/rendered/manifests/logs-only/clusterRole.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 rules:

diff --git a/rendered/manifests/logs-only/clusterRoleBinding.yaml b/rendered/manifests/logs-only/clusterRoleBinding.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 roleRef:

diff --git a/rendered/manifests/logs-only/configmap-fluentd-cri.yaml b/rendered/manifests/logs-only/configmap-fluentd-cri.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-fluentd-cri
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 data:

diff --git a/rendered/manifests/logs-only/configmap-fluentd-json.yaml b/rendered/manifests/logs-only/configmap-fluentd-json.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-fluentd-json
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 data:

diff --git a/rendered/manifests/logs-only/configmap-fluentd.yaml b/rendered/manifests/logs-only/configmap-fluentd.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-fluentd
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 data:
@@ -268,6 +268,7 @@ data:
           denylist ${record.dig("kubernetes", "annotations", "splunk.com/exclude") ? record.dig("kubernetes", "annotations", "splunk.com/exclude") : record.dig("kubernetes", "namespace_annotations", "splunk.com/exclude") ? (record["kubernetes"]["namespace_annotations"]["splunk.com/exclude"]) : ("false")}
         </record>
       </filter>
+
       <filter tail.containers.**>
         # Exclude all logs that are denylisted
         @type grep

diff --git a/rendered/manifests/logs-only/configmap-otel-agent.yaml b/rendered/manifests/logs-only/configmap-otel-agent.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-otel-agent
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 data:

diff --git a/rendered/manifests/logs-only/daemonset.yaml b/rendered/manifests/logs-only/daemonset.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-agent
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
     engine: fluentd
@@ -23,7 +23,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: ab8acc9c6f81b9179a4b9219de716d5f715bd13fcf693be3a8a08bb92621d426
+        checksum/config: 45d855c74c87edf260c527d28e7acc8bb75d16a5d7c17ccf3b3c5290fae94003
     spec:
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet

diff --git a/rendered/manifests/logs-only/secret.yaml b/rendered/manifests/logs-only/secret.yaml
@@ -6,7 +6,7 @@ metadata:
   name: splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 type: Opaque

diff --git a/rendered/manifests/logs-only/serviceAccount.yaml b/rendered/manifests/logs-only/serviceAccount.yaml
@@ -6,6 +6,6 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
diff --git a/rendered/manifests/metrics-only/clusterRole.yaml b/rendered/manifests/metrics-only/clusterRole.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 rules:

diff --git a/rendered/manifests/metrics-only/clusterRoleBinding.yaml b/rendered/manifests/metrics-only/clusterRoleBinding.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 roleRef:

diff --git a/rendered/manifests/metrics-only/configmap-otel-agent.yaml b/rendered/manifests/metrics-only/configmap-otel-agent.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-otel-agent
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 data:

diff --git a/rendered/manifests/metrics-only/configmap-otel-k8s-cluster-receiver.yaml b/rendered/manifests/metrics-only/configmap-otel-k8s-cluster-receiver.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-otel-k8s-cluster-receiver
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 data:

diff --git a/rendered/manifests/metrics-only/daemonset.yaml b/rendered/manifests/metrics-only/daemonset.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-agent
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.29.0
+    chart: splunk-otel-collector-0.29.1
     release: default
     heritage: Helm
 spec:
@@ -22,7 +22,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: bf9ebe156eb421f3430803c320e9138f6fd47c775b362a7ba872b074ebdeb8b5
+        checksum/config: 81f5de159129a1ac8223e1b2bc17f0544bc85a7d08604f83346be894eb743dfc
     spec:
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet