部署elk7.2.0

说明:

12	1 单台k8s,本机目录挂载(未配置cephfs)2 如果replicas大于1, 就会出现多个es挂载同一个目录,会出现报错(uuid block)

1. es配置本地挂载 k8s-es-7.2.0.yml

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128

apiVersion: v1kind: ServiceAccountmetadata: labels:   app: elasticsearch name: elasticsearch7-admin namespace: ns-elastic7apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRoleBindingmetadata: name: elasticsearch7-admin labels:   app: elasticsearchroleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-adminsubjects: - kind: ServiceAccount   name: elasticsearch7-admin   namespace: ns-elastic7apiVersion: apps/v1kind: StatefulSetmetadata:  labels:    app: elasticsearch    role: master  name: elasticsearch-master  namespace: ns-elastic7spec:  replicas: 1  serviceName: elasticsearch-master  selector:    matchLabels:      app: elasticsearch      role: master  template:    metadata:      labels:        app: elasticsearch        role: master    spec:      serviceAccountName: elasticsearch7-admin      restartPolicy: Always      securityContext:        fsGroup: 1000      containers:        - name: elasticsearch-master          image: hub.boqii.com/bq/elasticsearch:7.2.0          command: ["bash", "-c", "ulimit -l unlimited && sysctl -w vm.max_map_count=262144 && chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/data && exec su elasticsearch docker-entrypoint.sh"]          imagePullPolicy: IfNotPresent          securityContext:            privileged: true          ports:            - containerPort: 9200              protocol: TCP            - containerPort: 9300              protocol: TCP          resources:            requests:              cpu: "50m"            limits:              cpu: "800m"          env:            - name: cluster.name              value: "es_cluster"            - name: node.master              value: "true"            - name: node.data              value: "true"            - name: cluster.initial_master_nodes              value: "elasticsearch-master-0"             - name: discovery.zen.ping_timeout              value: "5s"            - name: node.ingest              value: "false"            - name: ES_JAVA_OPTS              value: "-Xms1g -Xmx1g"            - name: "discovery.zen.ping.unicast.hosts"              value: "elasticsearch-discovery" # Disvocery Service            - name: "http.cors.enabled"              value: "true"            - name: "http.cors.allow-origin"              value: "*"          volumeMounts:            - name: elasticsearch-data-volume              mountPath: /usr/share/elasticsearch/data      volumes:        - name: elasticsearch-data-volume          hostPath:            path: /data/k8s-container/elk-7.2.0/es-7.2.0/dataapiVersion: v1kind: Servicemetadata:  labels:    app: elasticsearch  name: elasticsearch-discovery  namespace: ns-elastic7spec:  publishNotReadyAddresses: true  ports:  - name: transport    port: 9300    targetPort: 9300  selector:    app: elasticsearch    role: masterkind: ServiceapiVersion: v1metadata: labels:   app: elasticsearch name: elasticsearch-service namespace: ns-elastic7spec: type: NodePort ports:   - port: 9200     targetPort: 9200     nodePort: 19230     protocol: TCP selector:   app: elasticsearch

2. es配置nfs动态挂载 k8s-es-7.2.0-nfs.yml

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129

---apiVersion: v1kind: ServiceAccountmetadata: labels:   app: elasticsearch name: elasticsearch-admin namespace: ns-elastic---apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRoleBindingmetadata: name: elasticsearch-admin labels:   app: elasticsearchroleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-adminsubjects: - kind: ServiceAccount   name: elasticsearch-admin   namespace: ns-elastic---apiVersion: apps/v1kind: StatefulSetmetadata:  labels:    app: elasticsearch    role: master  name: elasticsearch-master  namespace: ns-elasticspec:  replicas: 2  volumeClaimTemplates:  - metadata:      name: elasticsearch-data-nfs      annotations:        volume.beta.kubernetes.io/storage-class: "nfs"    spec:      accessModes: [ "ReadWriteOnce" ]      resources:        requests:          storage: 2Gi  serviceName: elasticsearch-master  selector:    matchLabels:      app: elasticsearch      role: master  template:    metadata:      labels:        app: elasticsearch        role: master    spec:      serviceAccountName: elasticsearch-admin      restartPolicy: Always      securityContext:        fsGroup: 1000      containers:        - name: elasticsearch-master          image: elasticsearch:7.2.0          command: ["bash", "-c", "ulimit -l unlimited && sysctl -w vm.max_map_count=262144 && chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/data && exec su elasticsearch docker-entrypoint.sh"]          imagePullPolicy: IfNotPresent          volumeMounts:          - name: elasticsearch-data-nfs            mountPath: /usr/share/elasticsearch/data          securityContext:            privileged: true          ports:            - containerPort: 9200              protocol: TCP            - containerPort: 9300              protocol: TCP          env:            - name: cluster.name              value: "es_cluster"            - name: node.master              value: "true"            - name: node.data              value: "true"            - name: cluster.initial_master_nodes              value: "elasticsearch-master-0,elasticsearch-master-1"             - name: discovery.zen.ping_timeout              value: "5s"            - name: node.ingest              value: "false"            - name: ES_JAVA_OPTS              value: "-Xms1g -Xmx1g"            - name: "discovery.zen.ping.unicast.hosts"              value: "elasticsearch-discovery" # Disvocery Service            - name: "http.cors.enabled"              value: "true"            - name: "http.cors.allow-origin"              value: "*"---apiVersion: v1kind: Servicemetadata:  labels:    app: elasticsearch  name: elasticsearch-discovery  namespace: ns-elasticspec:  publishNotReadyAddresses: true  ports:  - name: transport    port: 9300    targetPort: 9300  selector:    app: elasticsearch    role: master---kind: ServiceapiVersion: v1metadata: labels:   app: elasticsearch name: elasticsearch-service namespace: ns-elasticspec: type: NodePort ports:   - port: 9200     targetPort: 9200     nodePort: 19220     protocol: TCP selector:   app: elasticsearch

3. kibana配置k8s-kibana-7.2.0.yml

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485

apiVersion: v1kind: ConfigMapmetadata:  name: kibana-config  namespace: ns-elastic7  labels:    elastic-app: kibanadata:  kibana.yml: |    server.name: kibana    server.host: "0"    elasticsearch.hosts: [ "http://elasticsearch-service:9200" ]    xpack.monitoring.ui.container.elasticsearch.enabled: true---kind: DeploymentapiVersion: apps/v1beta2metadata:  labels:    elastic-app: kibana  name: kibana  namespace: ns-elastic7spec:  replicas: 1  revisionHistoryLimit: 10  selector:    matchLabels:      elastic-app: kibana  template:    metadata:      labels:        elastic-app: kibana    spec:      containers:        - name: kibana          image: hub.boqii.com/bq/kibana:7.2.0          ports:            - containerPort: 5601              protocol: TCP          resources:            requests:              cpu: "50m"            limits:              cpu: "800m"          volumeMounts:            - name: kibana-config              mountPath: /usr/share/kibana/config      volumes:        - name: kibana-config          configMap:            name: kibana-config      tolerations:        - key: node-role.kubernetes.io/master          effect: NoSchedule---kind: ServiceapiVersion: v1metadata:  labels:    elastic-app: kibana  name: kibana-service  namespace: ns-elastic7spec:  ports:    - port: 5601      targetPort: 5601  selector:    elastic-app: kibana  type: NodePort---apiVersion: extensions/v1beta1kind: Ingressmetadata: labels:   elastic-app: kibana name: kibana-ingress namespace: ns-elastic7spec: rules:   - host: elk-kibana-dev.boqii.com     http:       paths:         - backend:             serviceName: kibana-service             servicePort: 5601

4. logstash配置本地挂载 k8s-logstash-7.2.0.yml

4.1 config/pipelines.yml

12	- pipeline.id: main path.config: "/usr/share/logstash/config/pipeline/*.conf"

4.2 首先配置grok规则 config/pipeline/logstash.conf

12345678910111213141516171819202122

input {    udp {        port => "10000"        }    } filter {      grok {          match => {            "message" => "{"id":"(?<id>(.)*)","tag":"(?<tag>(.)*)","title":"%{GREEDYDATA:title}(?<title>(.|r|n)*)","value":"%{GREEDYDATA:value}(?<value>(.|r|n)*)","createdAt":"(?<createdAt>S+ S+)","Telephone":"(?<Telephone>(.)*)","uid":"(?<uid>(.)*)","updateTime":"(?<updateTime>(.)*)","appVersion":"(?<appVersion>(.)*)","mobileModel":"(?<mobileModel>(.)*)","osVersion":"(?<osVersion>(.)*)","channel":"(?<channel>(.)*)","UDID":"(?<UDID>(.)*)"}"              }            } }output {    elasticsearch {        hosts =>  [ "http://elasticsearch-service:9200" ]        index => "k8s2-dev-%{+YYYY.MM.dd}"        }    }

4.3 配置文件 k8s-logstash-7.2.0.yml

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455

---kind: DeploymentapiVersion: apps/v1beta2metadata:  labels:    elastic-app: logstash  name: logstash  namespace: ns-elasticspec:  replicas: 1  revisionHistoryLimit: 10  selector:    matchLabels:      elastic-app: logstash  template:    metadata:      labels:        elastic-app: logstash    spec:      containers:        - name: logstash          image: hub.boqii.com/bq/logstash:7.2.0          ports:            - containerPort: 10000              protocol: UDP          volumeMounts:            - name: logstash-config              mountPath: /usr/share/logstash/config      volumes:        - name: logstash-config          hostPath:            path: /data/k8s-pod/elk-7.2.0/logstash-7.2.0/config      tolerations:        - key: node-role.kubernetes.io/master          effect: NoSchedule---kind: ServiceapiVersion: v1metadata:  labels:    elastic-app: logstash  name: logstash-service  namespace: ns-elasticspec:  type: NodePort  ports:    - port: 10000      targetPort: 10000      nodePort: 10000      protocol: UDP  selector:    elastic-app: logstash  type: NodePort---

来源：https://www.cnblogs.com/liuzhongrong/p/12361907.html

标签

ElasticSearch