Appearance
使用
docker 采集日志
配置文件
yaml
# ====================== 【全局采集配置】 ======================
filebeat.inputs:
- type: filestream
enabled: true
paths:
- /app-logs/*.log
parsers:
# 第一步：先合并多行异常堆栈
- multiline:
type: pattern
pattern: '^\{"level"'
negate: true
match: after
max_lines: 1000 # 单次最多合并1000行堆栈，防止内存溢出
timeout: 5s # 5秒无新行就强制提交本条日志
# 第二步：再解析单行JSON日志
- ndjson:
overwrite_keys: true
add_error_key: true
ignore_decoding_error: false
# 全局自定义标签，平铺到日志根字段
fields:
service_name: backend-api
env: local
project: base
fields_under_root: true
# 文件扫描&回收策略
scan_frequency: 1s
backoff.init: 1s
backoff.max: 10s
close_inactive: 5m
ignore_older: 72h
# ====================== 输出至ES配置 ======================
output.elasticsearch:
# 和docker-compose中ES的service名称保持一致，你的compose写es就填es
hosts: ['http://elasticsearch:9200']
username: elastic
password: NQjwVTUbVtJTUgDV
index: 'backend-api-log-%{+YYYY.MM.dd}'
bulk_max_size: 1000
flush.min_events: 200
flush.timeout: 1s
# 可选配置
setup.template.enabled: false
setup.template.overwrite: falsedocker启动
yaml
elasticsearch:
image: elasticsearch:8.19.19
container_name: elasticsearch
restart: always
ports:
- "9200:9200"
environment:
- TZ=Asia/Shanghai
# 容器启动自动设置elastic密码
- ELASTIC_PASSWORD=NQjwVTUbVtJTUgDV
# 单节点模式，不开启集群发现
- discovery.type=single-node
# 开启ES日志记录，方便排查问题
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
# 关键：关闭http的ssl认证，允许http访问（开发必备）
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=false
# 节点间通信ssl关闭（单节点）
- xpack.security.transport.ssl.enabled=false
volumes:
- ./volumes/elasticsearch/data:/usr/share/elasticsearch/data
kibana:
image: kibana:8.19.19
container_name: kibana
restart: always
ports:
- "5601:5601"
environment:
- TZ=Asia/Shanghai
- I18N_LOCALE=zh-CN # 固定中文
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200
# # 配置访问ES的账号密码
- ELASTICSEARCH_USERNAME=kibana
- ELASTICSEARCH_PASSWORD=ykrh7FiNPn4CXqYbcrQ=
# 关闭kibana内部ssl
- XPACK_SECURITY_ENCRYPTIONKEYS_ENCRYPTEDKEYS=development
depends_on:
- elasticsearch
filebeat:
image: elastic/filebeat:8.19.19
container_name: filebeat
user: root
environment:
- TZ=Asia/Shanghai
volumes:
# 挂载filebeat配置文件（宿主机新建filebeat.yml）
- ./volumes/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
# 挂载你的go-zero项目日志目录（修改为你真实项目绝对路径）
- /app/backend/api/logs:/app-logs:ro
depends_on:
- elasticsearchk8s 采集日志
- 需要k8s相关知识
namespace
yaml
apiVersion: v1
kind: Namespace
metadata:
name: loggingconfigMap
yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: logging
labels:
app: filebeat
data:
filebeat.yml: |
filebeat.inputs:
# 类型：容器日志采集
- type: container
# K8s标准容器日志路径
paths:
- /var/log/containers/*article-rpc*.log # 采集文章服务日志
- /var/log/containers/*backend-api*.log # 采集后端服务日志
# 1.先做多行合并（异常堆栈合并）
multiline.type: pattern
multiline.pattern: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'
multiline.negate: true
multiline.match: after
multiline.max_lines: 300
# 2. 解决多行多个JSON解析报错
parsers:
- ndjson:
overwrite_keys: true
add_error_key: true
# 一行多个json自动切割拆分
split_message: true
# 输入内处理器：注入k8s元数据（修正缩进）
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/log/containers/"
ignore_older: 72h
scan_frequency: 10s
# ===================== 全局处理器流水线（清洗、过滤、打标） =====================
processors:
# 1、解析容器外层JSON结构
- decode_json_fields:
fields: [ "message" ]
target: ""
overwrite_keys: true
add_error_key: true
# 2、剔除无用字段（独立一条）
- drop_fields:
fields: [ "agent", "input", "prospector", "ecs" ]
ignore_missing: true
# 3、添加自定义环境标签（独立一条）
- add_fields:
fields:
env: "test"
log_source: "k8s_stdout"
#==================== 采集偏移持久化（断点续传） ====================
filebeat.registry.path: /var/lib/filebeat/registry
filebeat.registry.flush: 1s
filebeat.shutdown_timeout: 10s
#==================== 输出ES ====================
output.elasticsearch:
hosts:
# 方便测试，用本机的，正式环境要改啊
- "http://192.168.0.104:9200"
index: "k8s-node-logs-%{+YYYY.MM.dd}"
username: elastic
password: "NQjwVTUbVtJTUgDV"
bulk_max_size: 1024
compression_level: 5
timeout: 90s
# ES索引模板自动创建
setup.template.enabled: true
setup.template.name: "k8s-logs-template"
setup.template.pattern: "k8s-logs-*"
# 对外开放5066监控接口（探针、stats查看必备）
http.enabled: true
http.host: 0.0.0.0
http.port: 5066DaemonSet
yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: logging
labels:
app: filebeat-stdout
spec:
selector:
matchLabels:
app: filebeat-stdout
template:
metadata:
labels:
app: filebeat-stdout
spec:
serviceAccountName: filebeat
securityContext:
runAsUser: 0
# 容忍master节点污点，采集所有节点日志
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
volumes:
- name: config-volume
configMap:
name: filebeat-config
# 宿主机stdout日志目录
- name: var-log-containers
hostPath:
path: /var/log/containers
type: Directory
- name: var-log-pods
hostPath:
path: /var/log/pods
type: Directory
# offset持久化目录
- name: data-volume
hostPath:
path: /var/lib/filebeat-registry
type: DirectoryOrCreate
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:8.19.19
imagePullPolicy: IfNotPresent
args:
- '-c'
- '/etc/filebeat/filebeat.yml'
- '-e' # 打印filebeat自身运行日志，方便排查问题
env:
# 注入当前节点名称，配置中${NODE_NAME}使用
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
# 资源限制，根据集群节点配置调整
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
volumeMounts:
# 挂载filebeat配置文件
- name: config-volume
mountPath: /etc/filebeat
readOnly: true
# 挂载stdout日志软链目录
- name: var-log-containers
mountPath: /var/log/containers
readOnly: true
# 补上pod目录挂载
- name: var-log-pods
mountPath: /var/log/pods
readOnly: true
# 持久化采集偏移数据，防止offset丢失
- name: data-volume
mountPath: /var/lib/filebeat
# 新增探针（必须filebeat.yml开启5066端口才生效）
livenessProbe:
httpGet:
path: /stats
port: 5066
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 5
readinessProbe:
httpGet:
path: /stats
port: 5066
initialDelaySeconds: 5
periodSeconds: 5
timeoutSeconds: 3