网站网站开发需要多少钱,广州建设网站的公司哪家好,汉阳网站建设哪家便宜,网站开发书籍推荐ELK日志管理实现的3种常见方法
1. 日志收集方法
1.1 使用DaemonSet方式日志收集
通过将node节点的/var/log/pods目录挂载给以DaemonSet方式部署的logstash来读取容器日志,并将日志吐给kafka并分布写入Zookeeper数据库.再使用logstash将Zookeeper中的数据写入ES,并通过kibana…ELK日志管理实现的3种常见方法
1. 日志收集方法
1.1 使用DaemonSet方式日志收集
通过将node节点的/var/log/pods目录挂载给以DaemonSet方式部署的logstash来读取容器日志,并将日志吐给kafka并分布写入Zookeeper数据库.再使用logstash将Zookeeper中的数据写入ES,并通过kibana将数据进行展示.
标准日志和错误日志:
标准日志 --/dev/stdout
错误日志 ---- /dev/stderr
1.2 使用Logstash SideCar日志收集
pod中两个容器,1个是业务容器,另一个是日志收集容器,通过emptydir实现文件共享
1.3 容器镜像中filebeat进程日志收集
对业务容器镜像修改,容器中启动filebeat
3种方式的对比:
daemonset资源占用更少sidecar和filebeat可以更多的定制,但sidecar资源占用会更多
2. ElasticSearch集群部署
2.1 ElasticSearch器安装
下载deb包
清华源下载elasticsearch-7.12.1-amd64.deb
https://mirrors.tuna.tsinghua.edu.cn/elasticstack/7.x/apt/pool/main/e/elasticsearch/elasticsearch-7.12.1-amd64.deb
3台ES服务器安装
dpkg -i elasticsearch-7.12.1-amd64.deb 修改配置文件
vi /etc/elasticsearch/elasticsearch.yml cluster.name: k8s-els # 保证一样
node.name: es-01 # 3台保证不同
#bootstrap.memory_lock: true # 启动占用内存,如果打开需要修改/etc/elasticsearch/jvm.options# 在/etc/elasticsearch/jvm.options中打开以下选项确保内存占用是连续的## -Xms4g## -Xmx4g
# 监听地址和端口
network.host: 192.168.31.101 # 也可以写成0.0.0.0
http.port: 9200
# 集群中有哪些服务器
discovery.seed_hosts: [192.168.31.101, 192.168.31.102,192.168.31.103]
# 哪些服务器可以作为master
cluster.initial_master_nodes: [192.168.31.101, 192.168.31.102,192.168.31.103]
# 删除数据不允许模糊匹配
action.destructive_requires_name: true启动elasticsearch
systemctl enable --now elasticsearch.service 确认服务启动完成
systemctl status elasticsearch.service 2.2 Kibana安装
下载
清华源下载kibana-7.12.1-amd64.deb
https://mirrors.tuna.tsinghua.edu.cn/elasticstack/7.x/apt/pool/main/k/kibana/kibana-7.12.1-amd64.deb
安装
dpkg -i kibana-7.12.1-amd64.deb修改配置
vi /etc/kibana/kibana.yml 修改内容
server.port: 5601
server.host: 192.168.31.101 # 也可以写成0.0.0.0
elasticsearch.hosts: [http://192.168.31.101:9200] # 任意一个节点即可
i18n.locale: zh-CN启动服务
systemctl enable --now kibana确认服务
systemctl status kibana3. Zookeeper集群部署
3.1 Zookeeper安装
下载
官网下载zookeeper3.6.4(https://zookeeper.apache.org/) https://archive.apache.org/dist/zookeeper/zookeeper-3.6.4/apache-zookeeper-3.6.4-bin.tar.gz
安装
zookeeper依赖jdk8,先安装jdk8
apt install openjdk-8-jdk -y解压缩zookeeper
mkdir /apps
cd /apps
tar xf apache-zookeeper-3.6.4-bin.tar.gz
ln -sf /apps/apache-zookeeper-3.6.4-bin /apps/zookeeper配置修改
cd /apps/zookeeper/conf/
cp zoo_sample.cfg zoo.cfg修改配置文件
vi /apps/zookeeper/conf/zoo.cfg# 检查时间间隔
tickTime2000
# 初始化次数
initLimit10
# 存活检查次数
syncLimit5
# 数据目录
dataDir/data/zookeeper
# 客户端端口
clientPort2181
# 集群配置 2888数据同步,3888集群选举
server.1192.168.31.111:2888:3888
server.2192.168.31.112:2888:3888
server.3192.168.31.113:2888:3888创建数据id
mkdir -p /data/zookeeper
echo 1 /data/zookeeper/myid # 其他节点依次为2和3启动服务
/apps/zookeeper/bin/zkServer.sh start5.确认
/apps/zookeeper/bin/zkServer.sh status确认状态是leader或者是follower 3.2 Kafka安装
1.下载
官网下载kafka(https://zookeeper.apache.org/)
https://dlcdn.apache.org/kafka/3.7.0/kafka_2.13-3.7.0.tgz
安装
解压kafka包
tar xf kafka_2.13-3.7.0.tgz
ln -sf /apps/kafka_2.13-3.7.0 /apps/kafka配置修改
cd /apps/kafka/config/
vi server.properties修改内容
# 节点id保证不重复
broker.id111
# 本机ip
listenersPLAINTEXT://192.168.31.111:9092 # 确保每台服务器定义自己的ip
# 日志目录
log.dirs/data/kafka-logs
# 数据保留时间 默认7天
log.retention.hours168
# zookeeper集群连接配置
zookeeper.connect192.168.31.111:2181,192.168.31.112:2181,192.168.31.113:2181启动服务 3台服务器上,以daemon方式启动服务
/apps/kafka/bin/kafka-server-start.sh -daemon /apps/kafka/config/server.properties 确认 启动后会监听在9092端口
ss -ntlp|grep 9092通过offset Explorer 4. Logstash安装
下载
logstash-7.12.1-amd64.deb
https://mirrors.tuna.tsinghua.edu.cn/elasticstack/7.x/apt/pool/main/l/logstash/logstash-7.12.1-amd64.deb
安装
apt install openjdk-8-jdk -y
dpkg -i logstash-7.12.1-amd64.deb 配置修改
vi /etc/logstash/conf.d/daemonset-log-to-es.conf
input {kafka {bootstrap_servers 192.168.31.111:9092,192.168.31.112:9092,192.168.31.113:9092topics [jsonfile-log-topic]codec json}
}output {#if [fields][type] app1-access-log {if [type] jsonfile-daemonset-applog {elasticsearch {hosts [192.168.31.101:9200,192.168.31.102:9200]index jsonfile-daemonset-applog-%{YYYY.MM.dd}}}if [type] jsonfile-daemonset-syslog {elasticsearch {hosts [192.168.31.101:9200,192.168.31.102:9200]index jsonfile-daemonset-syslog-%{YYYY.MM.dd}}}}启动
systemctl enable --now logstash.service 测试 systemctl status logstash.service 5. DaemonSet 5.1 构建镜像
Dockerfile
FROM logstash:7.12.1
USER root
WORKDIR /usr/share/logstash
#RUN rm -rf config/logstash-sample.conf
ADD logstash.yml /usr/share/logstash/config/logstash.yml
ADD logstash.conf /usr/share/logstash/pipeline/logstash.conf logstash.conf
input {file {#path /var/lib/docker/containers/*/*-json.log #dockerpath /var/log/pods/*/*/*.logstart_position beginningtype jsonfile-daemonset-applog}file {path /var/log/*.logstart_position beginningtype jsonfile-daemonset-syslog}
}output {if [type] jsonfile-daemonset-applog {kafka {bootstrap_servers ${KAFKA_SERVER}topic_id ${TOPIC_ID}batch_size 16384 #logstash每次向ES传输的数据量大小,单位为字节codec ${CODEC} } }if [type] jsonfile-daemonset-syslog {kafka {bootstrap_servers ${KAFKA_SERVER}topic_id ${TOPIC_ID}batch_size 16384codec ${CODEC} #系统日志不是json格式}}
}logstash.yml
http.host: 0.0.0.0
#xpack.monitoring.elasticsearch.hosts: [ http://elasticsearch:9200 ]构建镜像
nerdctl build -t harbor.panasonic.cn/baseimages/logstash:v7.12.1-json-file-log-v2 .
nerdctl push harbor.panasonic.cn/baseimages/logstash:v7.12.1-json-file-log-v25.2 DaemonSet
DaemonSet yaml文件
apiVersion: apps/v1
kind: DaemonSet
metadata:name: logstash-elasticsearchnamespace: kube-systemlabels:k8s-app: logstash-logging
spec:selector:matchLabels:name: logstash-elasticsearchtemplate:metadata:labels:name: logstash-elasticsearchspec:tolerations:# this toleration is to have the daemonset runnable on master nodes# remove it if your masters cant run pods- key: node-role.kubernetes.io/masteroperator: Existseffect: NoSchedulecontainers:- name: logstash-elasticsearchimage: harbor.panasonic.cn/baseimages/logstash:v7.12.1-json-file-log-v1env:- name: KAFKA_SERVERvalue: 192.168.31.111:9092,192.168.31.112:9092,192.168.31.113:9092- name: TOPIC_IDvalue: jsonfile-log-topic- name: CODECvalue: json
# resources:
# limits:
# cpu: 1000m
# memory: 1024Mi
# requests:
# cpu: 500m
# memory: 1024MivolumeMounts:- name: varlog #定义宿主机系统日志挂载路径mountPath: /var/log #宿主机系统日志挂载点- name: varlibdockercontainers #定义容器日志挂载路径,和logstash配置文件中的收集路径保持一直#mountPath: /var/lib/docker/containers #docker挂载路径mountPath: /var/log/pods #containerd挂载路径,此路径与logstash的日志收集路径必须一致readOnly: falseterminationGracePeriodSeconds: 30volumes:- name: varloghostPath:path: /var/log #宿主机系统日志- name: varlibdockercontainershostPath:path: /var/lib/docker/containers #docker的宿主机日志路径path: /var/log/pods #containerd的宿主机日志路径部署
kubectl apply daemonset.yaml此时在Elasticsearch的dashboard上已经可以看到applog和syslog 配置logstash服务器将日志从kafka抽到es上
vi /etc/logstash/conf.d/daemonset-log-to-es.conf配置kafka地址和es地址
input {kafka {bootstrap_servers 192.168.31.111:9092,192.168.31.112:9092,192.168.31.113:9092topics [jsonfile-log-topic]codec json}
}output {#if [fields][type] app1-access-log {if [type] jsonfile-daemonset-applog {elasticsearch {hosts [192.168.31.101:9200,192.168.31.102:9200]index jsonfile-daemonset-applog-%{YYYY.MM.dd}}}if [type] jsonfile-daemonset-syslog {elasticsearch {hosts [192.168.31.101:9200,192.168.31.102:9200]index jsonfile-daemonset-syslog-%{YYYY.MM.dd}}}}重启服务后可以在es服务器上看到相关数据
相关内容也符合我们的预期 手动加入一段日志
rootk8s-master01# echo test-20240312-14:13 /var/log/dpkg.log日志也出现在els中 es服务器上创建syslog索引 日志前缀加*匹配日志 选择timestramp 同样,再次创建applog 6. SideCar 6.1 构建镜像
dockerfile
FROM logstash:7.12.1USER root
WORKDIR /usr/share/logstash
#RUN rm -rf config/logstash-sample.conf
ADD logstash.yml /usr/share/logstash/config/logstash.yml
ADD logstash.conf /usr/share/logstash/pipeline/logstash.conf logstash.conf
input {file {path /var/log/applog/catalina.outstart_position beginningtype app1-sidecar-catalina-log}file {path /var/log/applog/localhost_access_log.*.txtstart_position beginningtype app1-sidecar-access-log}
}output {if [type] app1-sidecar-catalina-log {kafka {bootstrap_servers ${KAFKA_SERVER}topic_id ${TOPIC_ID}batch_size 16384 #logstash每次向ES传输的数据量大小,单位为字节codec ${CODEC} } }if [type] app1-sidecar-access-log {kafka {bootstrap_servers ${KAFKA_SERVER}topic_id ${TOPIC_ID}batch_size 16384codec ${CODEC}} }
}logstash.yml
http.host: 0.0.0.0
#xpack.monitoring.elasticsearch.hosts: [ http://elasticsearch:9200 build-commond.sh
#!/bin/bash#docker build -t harbor.magedu.local/baseimages/logstash:v7.12.1-sidecar .#docker push harbor.magedu.local/baseimages/logstash:v7.12.1-sidecar
nerdctl build -t harbor.panasonic.cn/baseimages/logstash:v7.12.1-sidecar .
nerdctl push harbor.panasonic.cn/baseimages/logstash:v7.12.1-sidecar6.2 SideCar
tomcat-app1.yaml
kind: Deployment
apiVersion: apps/v1
metadata:labels:app: pana-tomcat-app1-deployment-labelname: pana-tomcat-app1-deployment #当前版本的deployment 名称namespace: pana
spec:replicas: 3selector:matchLabels:app: pana-tomcat-app1-selectortemplate:metadata:labels:app: pana-tomcat-app1-selectorspec:containers:- name: sidecar-containerimage: harbor.panasonic.cn/baseimages/logstash:v7.12.1-sidecarimagePullPolicy: IfNotPresent#imagePullPolicy: Alwaysenv:- name: KAFKA_SERVERvalue: 192.168.31.111:9092,192.168.31.112:9092,192.168.31.113:9092- name: TOPIC_IDvalue: tomcat-app1-topic- name: CODECvalue: jsonvolumeMounts:- name: applogsmountPath: /var/log/applog- name: pana-tomcat-app1-containerimage: registry.cn-hangzhou.aliyuncs.com/zhangshijie/tomcat-app1:v1imagePullPolicy: IfNotPresent#imagePullPolicy: Alwaysports:- containerPort: 8080protocol: TCPname: httpenv:- name: passwordvalue: 123456- name: agevalue: 18resources:limits:cpu: 1memory: 512Mirequests:cpu: 500mmemory: 512MivolumeMounts:- name: applogsmountPath: /apps/tomcat/logsstartupProbe:httpGet:path: /myapp/index.htmlport: 8080initialDelaySeconds: 5 #首次检测延迟5sfailureThreshold: 3 #从成功转为失败的次数periodSeconds: 3 #探测间隔周期readinessProbe:httpGet:#path: /monitor/monitor.htmlpath: /myapp/index.htmlport: 8080initialDelaySeconds: 5periodSeconds: 3timeoutSeconds: 5successThreshold: 1failureThreshold: 3livenessProbe:httpGet:#path: /monitor/monitor.htmlpath: /myapp/index.htmlport: 8080initialDelaySeconds: 5periodSeconds: 3timeoutSeconds: 5successThreshold: 1failureThreshold: 3volumes:- name: applogs #定义通过emptyDir实现业务容器与sidecar容器的日志共享以让sidecar收集业务容器中的日志emptyDir: {}tomcat-service.yaml
---
kind: Service
apiVersion: v1
metadata:labels:app: pana-tomcat-app1-service-labelname: pana-tomcat-app1-servicenamespace: pana
spec:type: NodePortports:- name: httpport: 80protocol: TCPtargetPort: 8080nodePort: 40080selector:app: pana-tomcat-app1-selectorsidecar.conf
input {kafka {bootstrap_servers 192.168.31.111:9092,192.168.31.112:9092,192.168.31.113:9092topics [tomcat-app1-topic]codec json}
}output {#if [fields][type] app1-access-log {if [type] app1-sidecar-access-log {elasticsearch {hosts [192.168.31.101:9200,192.168.31.102:9200]index sidecar-app1-accesslog-%{YYYY.MM.dd}}}#if [fields][type] app1-catalina-log {if [type] app1-sidecar-catalina-log {elasticsearch {hosts [192.168.31.101:9200,192.168.31.102:9200]index sidecar-app1-catalinalog-%{YYYY.MM.dd}}}
# stdout {
# codec rubydebug
# }
}7. 容器镜像中安装filebeat 7.1 镜像制作
1.Dockerfile
filebeat-7.12.1-amd64.deb 从清华镜像源获取 https://mirrors.tuna.tsinghua.edu.cn/elasticstack/7.x/apt/pool/main/f/filebeat/
#tomcat web1
FROM tomcat:8.5.99-jdk8ADD filebeat-7.12.1-amd64.deb /tmp/
RUN dpkg -i /tmp/filebeat-7.12.1-amd64.deb rm -f /tmp/filebeat-7.12.1-amd64.deb
ADD catalina.sh /usr/local/tomcat/bin/catalina.sh
ADD server.xml /usr/local/tomcat/conf/server.xml
ADD myapp.tar.gz /usr/local/tomcat/webapps/myapp/
ADD run_tomcat.sh /usr/local/tomcat/bin/run_tomcat.sh
ADD filebeat.yml /etc/filebeat/filebeat.yml
ADD sources.list /etc/apt/sources.listEXPOSE 8080 8443CMD [/usr/local/tomcat/bin/run_tomcat.sh]run_tomcat.sh
#!/bin/bash
/usr/share/filebeat/bin/filebeat -e -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat
/usr/local/tomcat/bin/catalina.sh start
tail -f /etc/hostsserver.xml
Host namelocalhost appBase/usr/local/tomcat/webapps unpackWARsfalse autoDeployfalse镜像制作 7.2 服务创建
serviceaccount
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:name: filebeat-serviceaccount-clusterrolelabels:k8s-app: filebeat-serviceaccount-clusterrole
rules:
- apiGroups: [] # indicates the core API groupresources:- namespaces- pods- nodesverbs:- get- watch- list---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:name: filebeat-serviceaccount-clusterrolebinding
subjects:
- kind: ServiceAccountname: defaultnamespace: pana
roleRef:kind: ClusterRolename: filebeat-serviceaccount-clusterroleapiGroup: rbac.authorization.k8s.iodeployment
kind: Deployment
apiVersion: apps/v1
metadata:labels:app: pana-tomcat-app1-filebeat-deployment-labelname: pana-tomcat-app1-filebeat-deploymentnamespace: pana
spec:replicas: 2selector:matchLabels:app: pana-tomcat-app1-filebeat-selectortemplate:metadata:labels:app: pana-tomcat-app1-filebeat-selectorspec:containers:- name: pana-tomcat-app1-filebeat-containerimage: harbor.panasonic.cn/tomcat/tomcat-app1:v11imagePullPolicy: IfNotPresentports:- containerPort: 8080protocol: TCPname: httpenv:- name: passwordvalue: 123456- name: agevalue: 18resources:limits:cpu: 1memory: 512Mirequests:cpu: 500mmemory: 512Miservice
---
kind: Service
apiVersion: v1
metadata:labels:app: pana-tomcat-app1-filebeat-service-labelname: pana-tomcat-app1-filebeat-servicenamespace: pana
spec:type: NodePortports:- name: httpport: 80protocol: TCPtargetPort: 8080nodePort: 30092selector:app: pana-tomcat-app1-filebeat-selectorkubectl apply -f *.yaml7.3 logstash配置
input {kafka {bootstrap_servers 192.168.31.111:9092,192.168.31.112:9092,192.168.31.113:9092topics [filebeat-tomcat-app1]codec json}
}output {if [fields][type] filebeat-tomcat-catalina {elasticsearch {hosts [192.168.31.101:9200,192.168.31.102:9200]index filebeat-tomcat-catalina-%{YYYY.MM.dd}}}if [fields][type] filebeat-tomcat-accesslog {elasticsearch {hosts [192.168.31.101:9200,192.168.31.102:9200]index filebeat-tomcat-accesslog-%{YYYY.MM.dd}}}
}重启logstash服务
systemctl restart logstash7.4 sls日志查询
