Automatically detect container runtime and apply relevant parsing con…

…fig (signalfx#154) This change makes transition from deprecated docker to containerd runtime not disruptive. Otherwise, users have to change criTimeFormat property and upgrade the chart release after they upgraded k8s cluster to a new version.
caronsteve · May 29, 2021 · efa9a34 · efa9a34
1 parent e103feb
commit efa9a34
Show file tree

Hide file tree

Showing 49 changed files with 390 additions and 119 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,15 @@
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.26.2] - 2021-05-28
+
+### Changed
+
+- Automatically detect container runtime using initContainers and apply
+  relevant parsing config instead of asking user to specify criTimeFormat.
+  This is an important change to enable smooth transition from deprecated docker
+  to containerd runtime (#154)
+
 ## [0.26.1] - 2021-05-25
 
 ### Added

diff --git a/README.md b/README.md
@@ -95,21 +95,6 @@ While this helm chart should work for other Kubernetes distributions, it may
 require additional configurations applied to
 [values.yaml](helm-charts/splunk-otel-collector/values.yaml).
 
----
-**IMPORTANT**
-
-By default, the chart is configured to collect logs from k8s clusters with
-a **docker** runtime, which will be deprecated starting with Kubernetes
-version 1.20.
-
-If your cluster is running with a **containerd** or **cri-o** runtime, make sure you
-add this option to the installation script:
-
-```
---set fluentd.config.containers.logFormatType=cri
-```
----
-
 ## Getting Started
 
 ### Prerequisites

diff --git a/helm-charts/splunk-otel-collector/Chart.yaml b/helm-charts/splunk-otel-collector/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: splunk-otel-collector
-version: 0.26.1
+version: 0.26.2
 description: Splunk OpenTelemetry Connector for Kubernetes
 icon: https://github.com/signalfx/splunk-otel-collector-chart/tree/main/splunk.png
 type: application

diff --git a/helm-charts/splunk-otel-collector/templates/configmap-fluentd-cri.yaml b/helm-charts/splunk-otel-collector/templates/configmap-fluentd-cri.yaml
@@ -0,0 +1,49 @@
+{{ if and .Values.logsEnabled .Values.otelAgent.enabled }}
+{{/*
+Fluentd config parts applied only to clusters with containerd/cri-o runtime.
+*/}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "splunk-otel-collector.fullname" . }}-fluentd-cri
+  labels:
+    app: {{ template "splunk-otel-collector.name" . }}
+    chart: {{ template "splunk-otel-collector.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+
+  source.containers.parse.conf: |-
+    @type regexp
+    expression /^(?<time>.+) (?<stream>stdout|stderr)( (?<partial_flag>[FP]))? (?<log>.*)$/
+    time_format {{ .Values.fluentd.config.containers.criTimeFormat }}
+
+  output.concat.conf: |-
+    # = handle cri/containerd multiline format =
+    <filter tail.containers.var.log.containers.**>
+      @type concat
+      key log
+      partial_key partial_flag
+      partial_value P
+      separator ''
+      timeout_label @SPLUNK
+    </filter>
+
+  output.transform.conf: |-
+    # extract pod_uid and container_name for CRIO runtime
+    # currently CRI does not produce log paths with all the necessary
+    # metadata to parse out pod, namespace, container_name, container_id.
+    # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031
+    <filter tail.containers.var.log.pods.**>
+      @type jq_transformer
+      jq '.record | . + (.source | capture("/var/log/pods/(?<pod_uid>[^/]+)/(?<container_name>[^/]+)/(?<container_retry>[0-9]+).log")) | .sourcetype = ("kube:container:" + .container_name)'
+    </filter>
+    # rename pod_uid and container_name to otel semantics.
+    <filter tail.containers.var.log.pods.**>
+      @type record_transformer
+      <record>
+        k8s.pod.uid ${record["pod_uid"]}
+        k8s.container.name ${record["container_name"]}
+      </record>
+    </filter>
+{{- end }}
diff --git a/helm-charts/splunk-otel-collector/templates/configmap-fluentd-json.yaml b/helm-charts/splunk-otel-collector/templates/configmap-fluentd-json.yaml
@@ -0,0 +1,22 @@
+{{ if and .Values.logsEnabled .Values.otelAgent.enabled }}
+{{/*
+Fluentd config parts applied only to clusters with docker runtime.
+*/}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "splunk-otel-collector.fullname" . }}-fluentd-json
+  labels:
+    app: {{ template "splunk-otel-collector.name" . }}
+    chart: {{ template "splunk-otel-collector.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  source.containers.parse.conf: |-
+    @type json
+    time_format %Y-%m-%dT%H:%M:%S.%NZ
+
+  output.filter.conf: ""
+
+  output.transform.conf: ""
+{{- end }}
diff --git a/helm-charts/splunk-otel-collector/templates/configmap-fluentd.yaml b/helm-charts/splunk-otel-collector/templates/configmap-fluentd.yaml
@@ -71,14 +71,7 @@ data:
       path_key source
       read_from_head true
       <parse>
-      {{- if eq .Values.fluentd.config.containers.logFormatType "cri" }}
-        @type regexp
-        expression /^(?<time>.+) (?<stream>stdout|stderr)( (?<partial_flag>[FP]))? (?<log>.*)$/
-        time_format {{ .Values.fluentd.config.containers.criTimeFormat }}
-      {{- else if eq .Values.fluentd.config.containers.logFormatType "json" }}
-        @type json
-        time_format %Y-%m-%dT%H:%M:%S.%NZ
-      {{- end }}
+        @include source.containers.parse.conf
         time_key time
         time_type string
         localtime false
@@ -157,17 +150,7 @@ data:
   output.conf: |-
     #Events are emitted to the CONCAT label from the container, file and journald sources for multiline processing.
     <label @CONCAT>
-      # = handle cri/containerd multiline format =
-      {{- if eq .Values.fluentd.config.containers.logFormatType "cri" }}
-      <filter tail.containers.var.log.containers.**>
-        @type concat
-        key log
-        partial_key partial_flag
-        partial_value P
-        separator ''
-        timeout_label @SPLUNK
-      </filter>
-      {{- end }}
+      @include output.filter.conf
       # = handle custom multiline logs =
       {{- range $name, $logDef := .Values.fluentd.config.logs }}
       {{- if and $logDef.from.pod $logDef.multiline }}
@@ -238,24 +221,8 @@ data:
           pattern /^true$/
         </exclude>
       </filter>
-      # extract pod_uid and container_name for CRIO runtime
-      # currently CRI does not produce log paths with all the necessary
-      # metadata to parse out pod, namespace, container_name, container_id.
-      # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031
-      {{- if eq .Values.fluentd.config.containers.logFormatType "cri" }}
-      <filter tail.containers.var.log.pods.**>
-        @type jq_transformer
-        jq '.record | . + (.source | capture("/var/log/pods/(?<pod_uid>[^/]+)/(?<container_name>[^/]+)/(?<container_retry>[0-9]+).log")) | .sourcetype = ("kube:container:" + .container_name)'
-      </filter>
-      # rename pod_uid and container_name to otel semantics.
-      <filter tail.containers.var.log.pods.**>
-        @type record_transformer
-        <record>
-          k8s.pod.uid ${record["pod_uid"]}
-          k8s.container.name ${record["container_name"]}
-        </record>
-      </filter>
-      {{- end }}
+
+      @include output.transform.conf
 
       # create source and sourcetype
       {{- if $checks.hasJournald }}

diff --git a/helm-charts/splunk-otel-collector/templates/daemonset.yaml b/helm-charts/splunk-otel-collector/templates/daemonset.yaml
@@ -41,6 +41,37 @@ spec:
       tolerations:
         {{ toYaml . | nindent 8 }}
       {{- end }}
+      {{- if .Values.logsEnabled }}
+      initContainers:
+        - name: prepare-fluentd-config
+          image: busybox:1.33
+          command: [ "sh", "-c"]
+          args:
+            - if [ -z "${LOG_FORMAT_TYPE}" ]; then
+                if [ "$(ls /hostfs/var/lib/docker/containers/*/*json.log 2>/dev/null | wc -l)" != "0" ]; then
+                  export LOG_FORMAT_TYPE=json;
+                else
+                  export LOG_FORMAT_TYPE=cri;
+                fi;
+              fi;
+              cp /fluentd/etc/common/* /fluentd/etc/${LOG_FORMAT_TYPE}/* /fluentd/etc/
+          env:
+            - name: LOG_FORMAT_TYPE
+              value: "{{ .Values.fluentd.config.containers.logFormatType }}"
+          volumeMounts:
+            - mountPath: /hostfs
+              name: hostfs
+              readOnly: true
+              mountPropagation: HostToContainer
+            - name: fluentd-config
+              mountPath: /fluentd/etc
+            - name: fluentd-config-common
+              mountPath: /fluentd/etc/common
+            - name: fluentd-config-json
+              mountPath: /fluentd/etc/json
+            - name: fluentd-config-cri
+              mountPath: /fluentd/etc/cri
+      {{- end }}
       containers:
       {{- if .Values.logsEnabled }}
       - name: fluentd
@@ -79,7 +110,7 @@ spec:
         - name: journallogpath
           mountPath: {{ .Values.fluentd.config.journalLogPath | quote }}
           readOnly: true
-        - name: fluentd-configmap
+        - name: fluentd-config
           mountPath: /fluentd/etc
         - name: secrets
           mountPath: /fluentd/etc/splunk
@@ -194,9 +225,17 @@ spec:
       - name: secrets
         secret:
           secretName: {{ template "splunk-otel-collector.secret" . }}
-      - name: fluentd-configmap
+      - name: fluentd-config
+        emptyDir: {}
+      - name: fluentd-config-common
         configMap:
           name: {{ template "splunk-otel-collector.fullname" . }}-fluentd
+      - name: fluentd-config-cri
+        configMap:
+          name: {{ template "splunk-otel-collector.fullname" . }}-fluentd-cri
+      - name: fluentd-config-json
+        configMap:
+          name: {{ template "splunk-otel-collector.fullname" . }}-fluentd-json
       {{- end}}
       {{- if .Values.metricsEnabled }}
       - name: hostfs

diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml
@@ -221,8 +221,10 @@ fluentd:
       path: /var/log
       # Final volume destination of container log symlinks
       pathDest: /var/lib/docker/containers
-      # Log format type, "json" or "cri"
-      logFormatType: json
+      # Log format type, "json" or "cri".
+      # If omitted (default), the value is detected automatically based on container runtime.
+      # "json" is set if docker runtime detected, otherwise it defaults to "cri".
+      logFormatType: ""
       # Specify the log format for "cri" logFormatType
       # It can be "%Y-%m-%dT%H:%M:%S.%N%:z" for openshift and "%Y-%m-%dT%H:%M:%S.%NZ" for IBM IKS
       criTimeFormat: "%Y-%m-%dT%H:%M:%S.%N%:z"

diff --git a/rendered/manifests/agent-only/clusterRole.yaml b/rendered/manifests/agent-only/clusterRole.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.26.1
+    chart: splunk-otel-collector-0.26.2
     release: default
     heritage: Helm
 rules:

diff --git a/rendered/manifests/agent-only/clusterRoleBinding.yaml b/rendered/manifests/agent-only/clusterRoleBinding.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.26.1
+    chart: splunk-otel-collector-0.26.2
     release: default
     heritage: Helm
 roleRef:

diff --git a/rendered/manifests/agent-only/configmap-fluentd-cri.yaml b/rendered/manifests/agent-only/configmap-fluentd-cri.yaml
@@ -0,0 +1,46 @@
+---
+# Source: splunk-otel-collector/templates/configmap-fluentd-cri.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: default-splunk-otel-collector-fluentd-cri
+  labels:
+    app: splunk-otel-collector
+    chart: splunk-otel-collector-0.26.2
+    release: default
+    heritage: Helm
+data:
+
+  source.containers.parse.conf: |-
+    @type regexp
+    expression /^(?<time>.+) (?<stream>stdout|stderr)( (?<partial_flag>[FP]))? (?<log>.*)$/
+    time_format %Y-%m-%dT%H:%M:%S.%N%:z
+
+  output.concat.conf: |-
+    # = handle cri/containerd multiline format =
+    <filter tail.containers.var.log.containers.**>
+      @type concat
+      key log
+      partial_key partial_flag
+      partial_value P
+      separator ''
+      timeout_label @SPLUNK
+    </filter>
+
+  output.transform.conf: |-
+    # extract pod_uid and container_name for CRIO runtime
+    # currently CRI does not produce log paths with all the necessary
+    # metadata to parse out pod, namespace, container_name, container_id.
+    # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031
+    <filter tail.containers.var.log.pods.**>
+      @type jq_transformer
+      jq '.record | . + (.source | capture("/var/log/pods/(?<pod_uid>[^/]+)/(?<container_name>[^/]+)/(?<container_retry>[0-9]+).log")) | .sourcetype = ("kube:container:" + .container_name)'
+    </filter>
+    # rename pod_uid and container_name to otel semantics.
+    <filter tail.containers.var.log.pods.**>
+      @type record_transformer
+      <record>
+        k8s.pod.uid ${record["pod_uid"]}
+        k8s.container.name ${record["container_name"]}
+      </record>
+    </filter>
diff --git a/rendered/manifests/agent-only/configmap-fluentd-json.yaml b/rendered/manifests/agent-only/configmap-fluentd-json.yaml
@@ -0,0 +1,19 @@
+---
+# Source: splunk-otel-collector/templates/configmap-fluentd-json.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: default-splunk-otel-collector-fluentd-json
+  labels:
+    app: splunk-otel-collector
+    chart: splunk-otel-collector-0.26.2
+    release: default
+    heritage: Helm
+data:
+  source.containers.parse.conf: |-
+    @type json
+    time_format %Y-%m-%dT%H:%M:%S.%NZ
+
+  output.filter.conf: ""
+
+  output.transform.conf: ""
diff --git a/rendered/manifests/agent-only/configmap-fluentd.yaml b/rendered/manifests/agent-only/configmap-fluentd.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-fluentd
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.26.1
+    chart: splunk-otel-collector-0.26.2
     release: default
     heritage: Helm
 data:
@@ -69,8 +69,7 @@ data:
       path_key source
       read_from_head true
       <parse>
-        @type json
-        time_format %Y-%m-%dT%H:%M:%S.%NZ
+        @include source.containers.parse.conf
         time_key time
         time_type string
         localtime false
@@ -139,7 +138,7 @@ data:
   output.conf: |-
     #Events are emitted to the CONCAT label from the container, file and journald sources for multiline processing.
     <label @CONCAT>
-      # = handle cri/containerd multiline format =
+      @include output.filter.conf
       # = handle custom multiline logs =
       <filter tail.containers.var.log.containers.dns-controller*.log>
         @type concat
@@ -277,10 +276,8 @@ data:
           pattern /^true$/
         </exclude>
       </filter>
-      # extract pod_uid and container_name for CRIO runtime
-      # currently CRI does not produce log paths with all the necessary
-      # metadata to parse out pod, namespace, container_name, container_id.
-      # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031
+
+      @include output.transform.conf
 
       # create source and sourcetype
       <filter journald.**>

diff --git a/rendered/manifests/agent-only/configmap-otel-agent.yaml b/rendered/manifests/agent-only/configmap-otel-agent.yaml
@@ -6,7 +6,7 @@ metadata:
   name: default-splunk-otel-collector-otel-agent
   labels:
     app: splunk-otel-collector
-    chart: splunk-otel-collector-0.26.1
+    chart: splunk-otel-collector-0.26.2
     release: default
     heritage: Helm
 data: