filebeat-kubernetes.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-dynamic-config
  namespace: kube-system
  labels:
    k8s-app: filebeat-dynamic
    kubernetes.io/cluster-service: "true"
data:
  filebeat.yml: |-
    setup.dashboards.enabled: true
    setup.template.enabled: true

    setup.template.settings:
      index.number_of_shards: 1

    filebeat.modules:
      - module: system
        syslog:
          enabled: true
          #var.paths: ["/var/log/syslog"]
        auth:
          enabled: true
          #var.paths: ["/var/log/authlog"]

    filebeat.autodiscover:
      providers:
        - type: kubernetes
          templates:
            - condition.equals:
                kubernetes.labels.app: redis
              config:
                - module: redis
                  log:
                    input:
                      type: docker
                      containers.ids:
                        - ${data.kubernetes.container.id}
                  slowlog:
                    enabled: true
                    var.hosts: ["${data.host}:${data.port}"]
            - condition.contains:
                kubernetes.labels.tier: frontend
              config:
                - module: apache2
                  access:
                    input:
                      type: docker
                      containers.ids:
                        - ${data.kubernetes.container.id}
                  error:
                    input:
                      type: docker
                      containers.ids:
                        - ${data.kubernetes.container.id}
            - condition.equals:
                kubernetes.labels.app: mysql
              config:
                - module: mysql
                  error:
                    input:
                      type: docker
                      containers.ids:
                        - ${data.kubernetes.container.id}
                  slowlog:
                    input:
                      type: docker
                      containers.ids:
                        - ${data.kubernetes.container.id}

    processors:
      - drop_event:
          when.or:
              - and:
                  - regexp:
                      message: '^\d+\.\d+\.\d+\.\d+ '
                  - equals:
                      fileset.name: error
              - and:
                  - not:
                      regexp:
                          message: '^\d+\.\d+\.\d+\.\d+ '
                  - equals:
                      fileset.name: access
      - add_cloud_metadata:
      - add_kubernetes_metadata:
      - add_docker_metadata:

    cloud.auth: ${ELASTIC_CLOUD_AUTH}
    cloud.id: ${ELASTIC_CLOUD_ID}

    output.elasticsearch:
      hosts: ${ELASTICSEARCH_HOSTS}
      username: ${ELASTICSEARCH_USERNAME}
      password: ${ELASTICSEARCH_PASSWORD}
      ssl.certificate_authorities:
        - /etc/certificate/ca.pem
    setup.kibana:
      host: ${KIBANA_HOST}
      ssl.certificate_authorities:
        - /etc/certificate/ca.pem
    
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: filebeat-dynamic
  namespace: kube-system
  labels:
    k8s-app: filebeat-dynamic
    kubernetes.io/cluster-service: "true"
spec:
  template:
    metadata:
      labels:
        k8s-app: filebeat-dynamic
        kubernetes.io/cluster-service: "true"
    spec:
      serviceAccountName: filebeat-dynamic
      terminationGracePeriodSeconds: 30
      containers:
      - name: filebeat-dynamic
        image: docker.elastic.co/beats/filebeat:7.2.0
        imagePullPolicy: Always
        args: [
          "-c", "/etc/filebeat.yml",
          "-e",
        ]
        env:
        - name: ELASTIC_CLOUD_ID
          valueFrom:
            secretKeyRef:
              name: dynamic-logging
              key: ELASTIC_CLOUD_ID
              optional: true
        - name: ELASTIC_CLOUD_AUTH
          valueFrom:
            secretKeyRef:
              name: dynamic-logging
              key: ELASTIC_CLOUD_AUTH
              optional: true
        - name: ELASTICSEARCH_HOSTS
          valueFrom:
            secretKeyRef:
              name: dynamic-logging
              key: ELASTICSEARCH_HOSTS
              optional: true
        - name: KIBANA_HOST
          valueFrom:
            secretKeyRef:
              name: dynamic-logging
              key: KIBANA_HOST
              optional: true
        - name: ELASTICSEARCH_USERNAME
          valueFrom:
            secretKeyRef:
              name: dynamic-logging
              key: ELASTICSEARCH_USERNAME
              optional: true
        - name: ELASTICSEARCH_PASSWORD
          valueFrom:
            secretKeyRef:
              name: dynamic-logging
              key: ELASTICSEARCH_PASSWORD
              optional: true
        securityContext:
          runAsUser: 0
        volumeMounts:
        - name: config
          mountPath: /etc/filebeat.yml
          readOnly: true
          subPath: filebeat.yml
        - name: data
          mountPath: /usr/share/filebeat/data
        - name: varlog
          mountPath: /var/log
          readOnly: true
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: dockersock
          mountPath: /var/run/docker.sock
        - name: certificate
          mountPath: /etc/certificate/ca.pem
          readOnly: true
          subPath: ca.pem
      volumes:
      - name: config
        configMap:
          defaultMode: 0600
          name: filebeat-dynamic-config
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: dockersock
        hostPath:
          path: /var/run/docker.sock
      - name: data
        emptyDir: {}
      - name: certificate
        configMap:
          defaultMode: 0400
          name: certificate
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: filebeat-dynamic
subjects:
- kind: ServiceAccount
  name: filebeat-dynamic
  namespace: kube-system
roleRef:
  kind: ClusterRole
  name: filebeat-dynamic
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: filebeat-dynamic
  labels:
    k8s-app: filebeat-dynamic
rules:
- apiGroups: [""] # "" indicates the core API group
  resources:
  - namespaces
  - pods
  verbs:
  - get
  - watch
  - list
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: filebeat-dynamic
  namespace: kube-system
  labels:
    k8s-app: filebeat-dynamic