fluentd daemon set problem while sending to splunk http connector

[Oğuz Yarımtepe]

Below is the yaml files to fluentd setup to send http connector of my splunk cluster

apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd
  namespace: logging
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluentd
rules:
- apiGroups: [""]
  resources: ["pods", "namespaces"]
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: fluentd
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: fluentd
subjects:
  - kind: ServiceAccount
    name: fluentd
    namespace: logging

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: fluentd
  namespace: logging
  labels:
    app: fluentd
    version: v1
spec:
  template:
    metadata:
      labels:
        app: fluentd
        version: v1
    spec:
      containers:
      - name: fluentd
        image: docker-registry-prod.app.foo.net:5000/fluentd
        env:
          - name: KUBE_NODE
            valueFrom:
              fieldRef:
                fieldPath: spec.nodeName
        resources:
          limits:
            cpu: 300m
            memory: 250Mi
          requests:
            cpu: 100m
            memory: 200Mi
        volumeMounts:
        - name: config
          mountPath: /fluentd/etc
        - name: varlog
          mountPath: /var/log
        - name: journal
          mountPath: /run/log/journal
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: dockersock
          mountPath: /var/run/docker.sock
          readOnly: true
      terminationGracePeriodSeconds: 30
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: dockersock
        hostPath:
          path: /var/run/docker.sock
      - name: journal
        hostPath:
          path: /run/log/journal
      - name: config
        configMap:
          name: fluentd

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd
  namespace: logging
data:
  fluent.conf: |
    @include kubernetes.conf
    <match *.kubernetes.journal.container.fluentd>
       @type splunk-http-eventcollector
       server "oslspk01.host.foo.net:8088"
       token "some_token_here"
       protocol https
       index foo
       sourcetype ${tag}
       source fluentd
       host "#{ENV['KUBE_NODE']}"
       all_items true


       buffer_chunk_limit 700k
       batch_size_limit 1000000
       buffer_type file
       buffer_path /fluentd/log/buffer-infrastructure-fluentd
       flush_interval 10s
    </match>
  kubernetes.conf: |


    #<match fluent.**>
    #  @type null
    #</match>


    # Example:
        # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
        #   path /var/log/containers/*.log
    <source>
       @type tail
       path /var/log/containers/*.log
       pos_file /fluentd/log/fluentd-containers.log.pos
       time_format %Y-%m-%dT%H:%M:%S.%NZ
       tag kubernetes.*
       format json
       read_from_head true
     </source>




    <source>
      @type tail
      format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<msg>.*)$/
      time_format %Y-%m-%d %H:%M:%S
      path /var/log/salt/minion
      pos_file /fluentd/log/fluentd-salt.pos
      tag salt
    </source>


    <source>
      @type tail
      format syslog
      path /var/log/startupscript.log
      pos_file /fluentd/log/fluentd-startupscript.log.pos
      tag startupscript
    </source>


    <source>
      @type tail
      format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<msg>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
      path /var/log/docker.log
      pos_file /fluentd/log/fluentd-docker.log.pos
      tag docker
    </source>


    # Example:
    # I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
    <source>
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/kubelet.log
      pos_file /fluentd/log/fluentd-kubelet.log.pos
      tag kubelet
    </source>


    # Example:
    # # I1118 21:26:53.975789       6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed
    <source>
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/kube-proxy.log
      pos_file /fluentd/log/fluentd-kube-proxy.log.pos
      tag kube-proxy
    </source>


    # Example:
    # I0204 07:00:19.604280       5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
    <source>
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/kube-apiserver.log
      pos_file /fluentd/log/fluentd-kube-apiserver.log.pos
      tag kube-apiserver
    </source>




    # Example:
    # I0204 06:55:31.872680       5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
    <source>
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/kube-controller-manager.log
      pos_file /fluentd/log/fluentd-kube-controller-manager.log.pos
      tag kube-controller-manager
    </source>


    # Example:
    # W0204 06:49:18.239674       7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
    <source>
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/kube-scheduler.log
      pos_file /fluentd/log/fluentd-kube-scheduler.log.pos
      tag kube-scheduler
    </source>


    # Example:
    # I1104 10:36:20.242766       5 rescheduler.go:73] Running Rescheduler
    <source>
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/rescheduler.log
      pos_file /fluentd/log/fluentd-rescheduler.log.pos
      tag rescheduler
    </source>


    # Example:
    # I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
    <source>
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/cluster-autoscaler.log
      pos_file /fluentd/log/fluentd-cluster-autoscaler.log.pos
      tag cluster-autoscaler
    </source>


    <source>
      @type tail
      format none
      path /var/log/etcd.log
      pos_file /fluentd/log/fluentd-etcd.log.pos
      tag etcd
    </source>


    <filter kubernetes.**>
      @type kubernetes_metadata
    </filter>

docker image is the one created from fluentd official dockerfile just added required gem files to sent logs to http connector

When i look at the fluentd logs i see Fluentd log is filled up with warnings about "log unreadable. It is excluded and would be examined next time."

I can see that /var/log/containers are full with symlinks to the logs files. I tried tailing them by entering one of the fluentd shells, it worked.

Any idea what is missing?