将日志写入es集群

将日志写入es

vi /app/filebeat/config/filestream.yaml
filebeat.inputs:
- type: filestream
  enabled: true
  paths:
    - /tmp/ccc/shopping.json
  parsers:
    - multiline:
        type: count
        count_lines: 7
    - ndjson:
       add_error_key: true
       overwrite_keys: true

# 将日志输出到ES集群
output.elasticsearch:
  # 指定ES集群地址
  hosts: 
  - "http://10.0.0.101:9200"
  - "http://10.0.0.102:9200"
  - "http://10.0.0.103:9200"
  # 指定索引
  index: "log001-%{+yyyy.MM.dd}"

# 禁用索引声明管理周期，若不禁用则自动忽略自定义索引名称
setup.ilm.enabled: false
# 设置索引模板的名称
setup.template.name: "log001"
# 指定索引模板的匹配模式
setup.template.pattern: "log00*"
# 是否覆盖原有的索引模板
setup.template.overwrite: true
# 设置索引模板
setup.template.settings:
  # 指定分片数量为8
  index.number_of_shards: 8
  # 指定副本数量为0
  index.number_of_replicas: 0

指定多个索引

filebeat.inputs:
- type: filestream
  enabled: true
  tags: "docker"
  paths:
    - /tmp/app/docker.json
  parsers:
    - ndjson:
       add_error_key: true
       #overwrite_keys: true


- type: filestream
  enabled: true
  tags: "linux"
  paths:
    - /tmp/app/linux.log
  parsers:
    - multiline:
        type: count
        count_lines: 3


- type: filestream
  enabled: true
  tags: "demo"
  paths:
    - /tmp/app/demo.log
  parsers:
    - multiline:
        type: count
        count_lines: 4
    - ndjson:
       add_error_key: true
       overwrite_keys: true
       target: demo


output.elasticsearch:
  hosts: 
  - "http://10.0.0.101:9200"
  - "http://10.0.0.102:9200"
  - "http://10.0.0.103:9200"
  # index: "shopping-%{+yyyy.MM.dd}"
  indices:
     - index: "docker-%{+yyyy.MM.dd}"
       # 检索tag是docker 
       when.contains:
         tags: "docker"
     - index: "linux-%{+yyyy.MM.dd}"
      # 检索tag 是linux
       when.contains:
         tags: "linux"
     - index: "demo-%{+yyyy.MM.dd}"
      # 检索tag 是demo
       when.contains:
         tags: "demo"

setup.ilm.enabled: false
setup.template.name: "oldboyedu-jiaoshi07"
setup.template.pattern: "oldboyedu-jiaoshi07-*"
setup.template.overwrite: true
setup.template.settings:
  index.number_of_shards: 3
  index.number_of_replicas: 0