Index Template

 
{
    "template" : "log-*",
    "settings" : {
      "index" : {
        "mapping" : {
          "total_fields" : {
            "limit" : "100"
          }
        },
        "refresh_interval" : "5s",
        "number_of_shards" : "3",
        "number_of_replicas" : "0"
      }
    },
    "mappings" : {
      "dynamic_templates" : [
        {
          "strings_as_keyword" : {
            "mapping" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "match_mapping_type" : "string"
          }
        }
      ],
      "date_detection" : false,
      "properties" : {
        "exception" : {
          "properties" : {
            "stacktrace" : {
              "norms" : false,
              "type" : "text"
            },
            "exception_class" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "exception_message" : {
              "norms" : false,
              "type" : "text"
            }
          }
        },
        "akkaSource" : {
          "ignore_above" : 1024,
          "index" : false,
          "type" : "keyword"
        },
        "x_upstream" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "read_timestamp" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "task_id" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "source" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "error" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "type" : {
          "ignore_above" : 1024,
          "index" : false,
          "type" : "keyword"
        },
        "x_tenant_id" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "file" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "sourceActorSystem" : {
          "ignore_above" : 1024,
          "index" : false,
          "type" : "keyword"
        },
        "line_number" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "@version" : {
          "index" : false,
          "type" : "long"
        },
        "beat" : {
          "properties" : {
            "hostname" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "name" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "version" : {
              "index" : false,
              "type" : "keyword"
            }
          }
        },
        "logger_name" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "class" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "source_host" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "offset" : {
          "index" : false,
          "type" : "long"
        },
        "method" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "level" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "input_type" : {
          "ignore_above" : 1024,
          "index" : false,
          "type" : "keyword"
        },
        "x_request_id" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "message" : {
          "norms" : false,
          "type" : "text"
        },
        "env" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "mdc" : {
          "properties" : {
            "replica_id" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "x_request_id" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "task_id" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "env" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "x_tenant_id" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            }
          }
        },
        "tags" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "oozie_job_id" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "@timestamp" : {
          "type" : "date"
        },
        "level_value" : {
          "index" : false,
          "type" : "long"
        },
        "service" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "thread_name" : {
          "ignore_above" : 1024,
          "type" : "keyword"
        },
        "stack_trace" : {
          "norms" : false,
          "type" : "text"
        },
        "akkaTimestamp" : {
          "ignore_above" : 1024,
          "index" : false,
          "type" : "keyword"
        }
      }
    },
    "aliases" : {
           "total-log" : { }
    }
  }

K8S Filebeat Resource

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: logging
  labels:
    k8s-app: filebeat
data:
  filebeat.yml: |-
    filebeat.inputs:
    - type: log
      enabled: true
      paths:
        - /opt/log/stash/*/*.log
      fields_under_root: true
      fields:
        k8s.node: ${NODE_NAME}
        pod.ip: ${POD_IP}
      json.keys_under_root: true
      json.overwrite_keys: false
  
    processors:
    - drop_fields:
        fields: ["beat","level_value","@version","input_type","agent.ephemeral_id","ecs.version"]
  
    output.elasticsearch:
      hosts: ["xxxx:9200"]
      username: "elastic"
      password: "xxxxxxxxx"
      worker: 3
      index: "log-环境名-%{+yyyy.MM.dd}"
  
    setup.ilm.enabled: false
    setup.template.enabled: false
  
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: filebeat
  namespace: logging
  labels:
    k8s-app: filebeat
spec:
  selector:
    matchLabels:
      k8s-app: filebeat
  template:
    metadata:
      labels:
        k8s-app: filebeat
    spec:
      terminationGracePeriodSeconds: 30
      containers:
        - name: filebeat
          image: registry.cn-hangzhou.aliyuncs.com/clab-docker/filebeat:7.5.1
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
          securityContext:
            runAsUser: 0
          resources:
            requests:
              cpu: 100m
              memory: 100Mi
            limits:
              memory: 1024Mi
              ephemeral-storage: "10Gi"
          volumeMounts:
            - name: config
              mountPath: /usr/share/filebeat/filebeat.yml
              readOnly: true
              subPath: filebeat.yml
            - name: registry
              mountPath: /usr/share/filebeat/data
            - name: log
              mountPath: /opt/log/stash
              readOnly: true
      volumes:
        - name: config
          configMap:
            defaultMode: 0600
            name: filebeat-config
        - name: log
          hostPath:
            path: /opt/log/stash
        - name: registry
          hostPath:
            path: /opt/filebeat/data
            type: DirectoryOrCreate