一、二进制部署 k8s集群
1）参考文章
博客: https://blog.qikqiak.com 文章: https://www.qikqiak.com/post/manual-install-high-available-kubernetes-cluster/
2）环境架构
master:
192.168.10.12
192.168.10.22
etcd：类似于数据库，尽量使用高可用
192.168.10.12(etcd01)
192.168.10.22(etcd01)
二、创建证书
1）hosts 文件修改
[root@master01 ~]# cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.10.12 k8s-api.virtual.local
k8s-api.virtual.local 为后期设计的高可用的访问地址。现在临时设置
2）环境变量定义
[root@master01 ~]# head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 0b340751863956f119cbc624465db92b [root@master01 ~]# cat env.sh BOOTSTRAP_TOKEN="0b340751863956f119cbc624465db92b" SERVICE_CIDR="10.254.0.0/16" CLUSTER_CIDR="172.30.0.0/16" NODE_PORT_RANGE="30000-32766" ETCD_ENDPOINTS="https://192.168.10.12:2379,https://192.168.10.22:2379" FLANNEL_ETCD_PREFIX="/kubernetes/network" CLUSTER_KUBERNETES_SVC_IP="10.254.0.1" CLUSTER_DNS_SVC_IP="10.254.0.2" CLUSTER_DNS_DOMAIN="cluster.local." MASTER_URL="k8s-api.virtual.local" [root@master01 ~]# mkdir -p /usr/k8s/bin [root@master01 ~]# mv env.sh /usr/k8s/bin/
head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成token值，每次都不一样
3）创建CA 证书和密钥
3.1）下载创建证书的命令
[root@master01 ~]# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 [root@master01 ~]# chmod +x cfssl_linux-amd64 [root@master01 ~]# mv cfssl_linux-amd64 /usr/k8s/bin/cfssl [root@master01 ~]# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 [root@master01 ~]# chmod +x cfssljson_linux-amd64 [root@master01 ~]# mv cfssljson_linux-amd64 /usr/k8s/bin/cfssljson [root@master01 ~]# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 [root@master01 ~]# chmod +x cfssl-certinfo_linux-amd64 [root@master01 ~]# mv cfssl-certinfo_linux-amd64 /usr/k8s/bin/cfssl-certinfo
[root@master01 ~]# chmod +x /usr/k8s/bin/cfssl*
3.2）为了方便使用命令，添加环境变量
[root@master01 bin]# ls /usr/k8s/bin/ cfssl cfssl-certinfo cfssljson env.sh [root@master01 ~]# export PATH=/usr/k8s/bin/:$PATH [root@master01 ~]# echo $PATH /usr/k8s/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
环境变量写入配置文件永久生效
[root@master01 ~]# cat .bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
PATH=$PATH:$HOME/bin
export PATH=/usr/k8s/bin/:$PATH # 新增
source /usr/k8s/bin/env.sh # 新增，后面需要
export PATH
3.3 ）创建默认证书文件
[root@master01 ~]# mkdir ssl [root@master01 ~]# cd ssl/ [root@master01 ssl]# cfssl print-defaults config > config.json [root@master01 ssl]# cfssl print-defaults csr > csr.json [root@master01 ssl]# ls config.json csr.json
3.5）改为需要的证书文件。ca-csr.json 和 ca-config.json
[root@master01 ssl]# cp config.json config.json.bak
[root@master01 ssl]# mv config.json ca-config.json
[root@master01 ssl]# cat ca-config.json # 修改为需要的
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
[root@master01 ssl]# cp csr.json ca-csr.json
[root@master01 ssl]# mv csr.json csr.json.bak
[root@master01 ssl]# cat ca-csr.json # 修改为需要的
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
[root@master01 ssl]# ls
ca-config.json ca-csr.json config.json.bak csr.json.bak
3.6）根据证书生成证书秘钥对
[root@master01 ssl]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca
2019/05/22 14:42:24 [INFO] generating a new CA key and certificate from CSR
2019/05/22 14:42:24 [INFO] generate received request
2019/05/22 14:42:24 [INFO] received CSR
2019/05/22 14:42:24 [INFO] generating key: rsa-2048
2019/05/22 14:42:24 [INFO] encoded CSR
2019/05/22 14:42:24 [INFO] signed certificate with serial number 205407174390394697284979809186701593413605316370
[root@master01 ssl]# ls ca*
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem
备注：ca.pem 为私钥
ca-key.pem 为公钥
3.7）将证书拷贝到 所有的节点的 k8s的指定目录
[root@master01 ssl]# mkdir -p /etc/kubernetes/ssl [root@master01 ssl]# cp ca* /etc/kubernetes/ssl [root@master01 ssl]# ls /etc/kubernetes/ssl ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem
强调：所有 k8s 节点
二、创建 etcd 集群
1）环境变量生效
export NODE_NAME=etcd01 export NODE_IP=192.168.10.12 export NODE_IPS="192.168.10.12 192.168.10.22" export ETCD_NODES=etcd01=https://192.168.10.12:2380,etcd02=https://192.168.10.22:2380
执行过程
[root@master01 ssl]# source /usr/k8s/bin/env.sh [root@master01 ssl]# echo $ETCD_ENDPOINTS # 检验变量 https://192.168.10.12:2379,https://192.168.10.22:2379 [root@master01 ssl]# export NODE_NAME=etcd01 [root@master01 ssl]# export NODE_IP=192.168.10.12 [root@master01 ssl]# export NODE_IPS="192.168.10.12 192.168.10.22" [root@master01 ssl]# export ETCD_NODES=etcd01=https://192.168.10.12:2380,etcd02=https://192.168.10.22:2380 [root@master01 ssl]# echo $NODE_NAME etcd01
2）导入etcd的命令。从github下载
https://github.com/coreos/etcd 找 releases 包 https://github.com/etcd-io/etcd/releases [root@master01 ~]# wget https://github.com/etcd-io/etcd/releases/download/v3.3.13/etcd-v3.3.13-linux-amd64.tar.gz [root@master01 ~]# tar xf etcd-v3.3.13-linux-amd64.tar.gz [root@master01 ~]# ls etcd-v3.3.13-linux-amd64 Documentation etcd etcdctl README-etcdctl.md README.md READMEv2-etcdctl.md [root@master01 ~]# cp etcd-v3.3.13-linux-amd64/etcd* /usr/k8s/bin/
3）创建etcd需要的json文件
[root@master01 ~]# cd /root/ssl/
[root@master01 ~]# cat > etcd-csr.json <<EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"${NODE_IP}"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
注意："${NODE_IP}" 根据环境变量替换为当前ip
[root@master01 ssl]# cat etcd-csr.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.10.12"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
4）创建etcd秘钥文件
过程简略
$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \ -ca-key=/etc/kubernetes/ssl/ca-key.pem \ -config=/etc/kubernetes/ssl/ca-config.json \ -profile=kubernetes etcd-csr.json | cfssljson -bare etcd $ ls etcd* etcd.csr etcd-csr.json etcd-key.pem etcd.pem $ sudo mkdir -p /etc/etcd/ssl $ sudo mv etcd*.pem /etc/etcd/ssl/
执行过程
[root@master01 ssl]# cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
> -ca-key=/etc/kubernetes/ssl/ca-key.pem \
> -config=/etc/kubernetes/ssl/ca-config.json \
> -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
2019/05/22 17:52:42 [INFO] generate received request
2019/05/22 17:52:42 [INFO] received CSR
2019/05/22 17:52:42 [INFO] generating key: rsa-2048
2019/05/22 17:52:42 [INFO] encoded CSR
2019/05/22 17:52:42 [INFO] signed certificate with serial number 142898298242549096204648651997326366332634729441
2019/05/22 17:52:42 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@master01 ssl]# ls etcd*
etcd.csr etcd-csr.json etcd-key.pem etcd.pem
备注：etcd-key.pem 私钥文件
etcd.pem 证书文件
[root@master01 ssl]# mkdir -p /etc/etcd/ssl
[root@master01 ssl]# mv etcd*.pem /etc/etcd/ssl/
[root@master01 ssl]# ll /etc/etcd/ssl/
总用量 8
-rw------- 1 root root 1679 5月 22 17:52 etcd-key.pem
-rw-r--r-- 1 root root 1419 5月 22 17:52 etcd.pem
5）创建etcd 的systemd unit 文件
$ sudo mkdir -p /var/lib/etcd # 必须要先创建工作目录
$ cat > etcd.service <<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/k8s/bin/etcd \\
--name=${NODE_NAME} \\
--cert-file=/etc/etcd/ssl/etcd.pem \\
--key-file=/etc/etcd/ssl/etcd-key.pem \\
--peer-cert-file=/etc/etcd/ssl/etcd.pem \\
--peer-key-file=/etc/etcd/ssl/etcd-key.pem \\
--trusted-ca-file=/etc/kubernetes/ssl/ca.pem \\
--peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \\
--initial-advertise-peer-urls=https://${NODE_IP}:2380 \\
--listen-peer-urls=https://${NODE_IP}:2380 \\
--listen-client-urls=https://${NODE_IP}:2379,http://127.0.0.1:2379 \\
--advertise-client-urls=https://${NODE_IP}:2379 \\
--initial-cluster-token=etcd-cluster-0 \\
--initial-cluster=${ETCD_NODES} \\
--initial-cluster-state=new \\
--data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
6.1）将创建的etcd.service文件放到指定位置，启动
$ sudo mv etcd.service /etc/systemd/system $ sudo systemctl daemon-reload $ sudo systemctl enable etcd $ sudo systemctl start etcd $ sudo systemctl status etcd
6.2）启动报错处理
[root@master01 ssl]# systemctl start etcd Job for etcd.service failed because a timeout was exceeded. See "systemctl status etcd.service" and "journalctl -xe" for details.
处理错误过程
[root@master01 ssl]# systemctl status etcd.service -l ........ 5月 22 18:08:33 master01 etcd[9567]: health check for peer 21e3841b92f796be could not connect: dial tcp 192.168.10.22:2380: connect: connection refused (prober "ROUND_TRIPPER_RAFT_MESSAGE") 5月 22 18:08:34 master01 etcd[9567]: 1af68d968c7e3f22 is starting a new election at term 199 5月 22 18:08:34 master01 etcd[9567]: 1af68d968c7e3f22 became candidate at term 200 5月 22 18:08:34 master01 etcd[9567]: 1af68d968c7e3f22 received MsgVoteResp from 1af68d968c7e3f22 at term 200 5月 22 18:08:34 master01 etcd[9567]: 1af68d968c7e3f22 [logterm: 1, index: 2] sent MsgVote request to 21e3841b92f796be at term 200 5月 22 18:08:36 master01 etcd[9567]: 1af68d968c7e3f22 is starting a new election at term 200 5月 22 18:08:36 master01 etcd[9567]: 1af68d968c7e3f22 became candidate at term 201 5月 22 18:08:36 master01 etcd[9567]: 1af68d968c7e3f22 received MsgVoteResp from 1af68d968c7e3f22 at term 201 5月 22 18:08:36 master01 etcd[9567]: 1af68d968c7e3f22 [logterm: 1, index: 2] sent MsgVote request to 21e3841b92f796be at term 201 5月 22 18:08:36 master01 etcd[9567]: publish error: etcdserver: request timed out
原因。需要在其他节点安装 etcd，并启动etcd。如果启动了，该错误就直接消失了
最先启动的etcd 进程会卡住一段时间，等待其他节点启动加入集群，在所有的etcd 节点重复上面的步骤，直到所有的机器etcd 服务都已经启动。
7）在其他节点安装etcd
7.1）拷贝文件到其他节点
[root@master01 usr]# zip -r k8s.zip k8s/
[root@master01 usr]# scp k8s.zip root@master02:/usr/
[root@master01 etc]# zip -r kubernetes.zip kubernetes/
[root@master01 etc]# scp kubernetes.zip root@master02:/etc/
去master02解压该文件
[root@master02 etc]# tree /usr/k8s/
/usr/k8s/
└── bin
├── cfssl
├── cfssl-certinfo
├── cfssljson
├── env.sh
├── etcd
└── etcdctl
[root@master02 etc]# tree /etc/kubernetes/
/etc/kubernetes/
└── ssl
├── ca-config.json
├── ca.csr
├── ca-csr.json
├── ca-key.pem
└── ca.pem
7.2）其他服务器的节点变量生效
export NODE_NAME=etcd02 export NODE_IP=192.168.10.22 export NODE_IPS="192.168.10.12 192.168.10.22" export ETCD_NODES=etcd01=https://192.168.10.12:2380,etcd02=https://192.168.10.22:2380 source /usr/k8s/bin/env.sh
export PATH=/usr/k8s/bin/:$PATH
7.3）给其他节点创建TLS的证书请求
[root@master02 ~]# mkdir ssl
[root@master02 ~]# cd ssl/
cat > etcd-csr.json <<EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"${NODE_IP}"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
生成etcd证书和私钥
$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \ -ca-key=/etc/kubernetes/ssl/ca-key.pem \ -config=/etc/kubernetes/ssl/ca-config.json \ -profile=kubernetes etcd-csr.json | cfssljson -bare etcd $ ls etcd* etcd.csr etcd-csr.json etcd-key.pem etcd.pem $ sudo mkdir -p /etc/etcd/ssl $ sudo mv etcd*.pem /etc/etcd/ssl/
[root@master02 ssl]# ls /etc/etcd/ssl/
etcd-key.pem etcd.pem
7.4）创建etcd的systemd unit文件
$ sudo mkdir -p /var/lib/etcd # 必须要先创建工作目录
$ cat > etcd.service <<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/k8s/bin/etcd \\
--name=${NODE_NAME} \\
--cert-file=/etc/etcd/ssl/etcd.pem \\
--key-file=/etc/etcd/ssl/etcd-key.pem \\
--peer-cert-file=/etc/etcd/ssl/etcd.pem \\
--peer-key-file=/etc/etcd/ssl/etcd-key.pem \\
--trusted-ca-file=/etc/kubernetes/ssl/ca.pem \\
--peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \\
--initial-advertise-peer-urls=https://${NODE_IP}:2380 \\
--listen-peer-urls=https://${NODE_IP}:2380 \\
--listen-client-urls=https://${NODE_IP}:2379,http://127.0.0.1:2379 \\
--advertise-client-urls=https://${NODE_IP}:2379 \\
--initial-cluster-token=etcd-cluster-0 \\
--initial-cluster=${ETCD_NODES} \\
--initial-cluster-state=new \\
--data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
7.5）启动etcd服务
[root@master02 ssl]# mv etcd.service /etc/systemd/system systemctl daemon-reload systemctl enable etcd systemctl start etcd systemctl status etcd
现在再次进入master01，执行systemctl status etcd 查看状态
8）验证etcd集群，在任意集群的节点执行下面命令
for ip in ${NODE_IPS}; do
ETCDCTL_API=3 /usr/k8s/bin/etcdctl \
--endpoints=https://${ip}:2379 \
--cacert=/etc/kubernetes/ssl/ca.pem \
--cert=/etc/etcd/ssl/etcd.pem \
--key=/etc/etcd/ssl/etcd-key.pem \
endpoint health; done
输出
https://192.168.10.12:2379 is healthy: successfully committed proposal: took = 1.298547ms https://192.168.10.22:2379 is healthy: successfully committed proposal: took = 2.740962ms
三、搭建master集群
1）下载文件
https://github.com/kubernetes/kubernetes/ https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.9.md wget https://dl.k8s.io/v1.9.10/kubernetes-server-linux-amd64.tar.gz 下载该包较大，且国内无法访问
2）拷贝kubernetes的命令到 k8s的bin目录下
[root@master01 ~]# tar xf kubernetes-server-linux-amd64.tar.gz
[root@master01 ~]# cd kubernetes
[root@master01 kubernetes]# cp -r server/bin/{kube-apiserver,kube-controller-manager,kube-scheduler} /usr/k8s/bin/
[root@master01 kubernetes]# ls /usr/k8s/bin/kube-*
/usr/k8s/bin/kube-apiserver /usr/k8s/bin/kube-controller-manager /usr/k8s/bin/kube-scheduler
3）创建kubernetes 证书
cat > kubernetes-csr.json <<EOF
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"${NODE_IP}",
"${MASTER_URL}",
"${CLUSTER_KUBERNETES_SVC_IP}",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
------------
[root@master01 kubernetes]# cat kubernetes-csr.json
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.10.12",
"k8s-api.virtual.local",
"10.254.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
mv kubernetes-csr.json ../ssl/
4）生成kubernetes 证书和私钥
$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \ -ca-key=/etc/kubernetes/ssl/ca-key.pem \ -config=/etc/kubernetes/ssl/ca-config.json \ -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes $ ls kubernetes* kubernetes.csr kubernetes-csr.json kubernetes-key.pem kubernetes.pem $ sudo mkdir -p /etc/kubernetes/ssl/ $ sudo mv kubernetes*.pem /etc/kubernetes/ssl/
4.1）操作过程
[root@master01 ssl]# cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
> -ca-key=/etc/kubernetes/ssl/ca-key.pem \
> -config=/etc/kubernetes/ssl/ca-config.json \
> -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
2019/05/27 19:57:35 [INFO] generate received request
2019/05/27 19:57:35 [INFO] received CSR
2019/05/27 19:57:35 [INFO] generating key: rsa-2048
2019/05/27 19:57:36 [INFO] encoded CSR
2019/05/27 19:57:36 [INFO] signed certificate with serial number 700523371489172612405920435814032644060474436709
2019/05/27 19:57:36 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@master01 ssl]# ls kubernetes*
kubernetes.csr kubernetes-csr.json kubernetes-key.pem kubernetes.pem
[root@master01 ssl]# ls /etc/kubernetes/ssl/
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem
[root@master01 ssl]# mv kubernetes*.pem /etc/kubernetes/ssl/
[root@master01 ssl]# ls /etc/kubernetes/ssl/
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem kubernetes-key.pem kubernetes.pem
5） 配置和启动kube-apiserver
5.1）创建kube-apiserver 使用的客户端token 文件
kubelet 首次启动时向kube-apiserver 发送TLS Bootstrapping 请求，kube-apiserver 验证请求中的token 是否与它配置的token.csv 一致，如果一致则自动为kubelet 生成证书和密钥。
$ # 导入的 environment.sh 文件定义了 BOOTSTRAP_TOKEN 变量
$ cat > token.csv <<EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
$ sudo mv token.csv /etc/kubernetes/
操作流程
[root@master01 ssl]# cat > token.csv <<EOF
> ${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
> EOF
[root@master01 ssl]# cat token.csv
0b340751863956f119cbc624465db92b,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
[root@master01 ssl]#
[root@master01 ssl]# mv token.csv /etc/kubernetes/
5.2）审查日志策略文件内容如下：（/etc/kubernetes/audit-policy.yaml）
apiVersion: audit.k8s.io/v1beta1 # This is required.
kind: Policy
# Don't generate audit events for all requests in RequestReceived stage.
omitStages:
- "RequestReceived"
rules:
# Log pod changes at RequestResponse level
- level: RequestResponse
resources:
- group: ""
# Resource "pods" doesn't match requests to any subresource of pods,
# which is consistent with the RBAC policy.
resources: ["pods"]
# Log "pods/log", "pods/status" at Metadata level
- level: Metadata
resources:
- group: ""
resources: ["pods/log", "pods/status"]
# Don't log requests to a configmap called "controller-leader"
- level: None
resources:
- group: ""
resources: ["configmaps"]
resourceNames: ["controller-leader"]
# Don't log watch requests by the "system:kube-proxy" on endpoints or services
- level: None
users: ["system:kube-proxy"]
verbs: ["watch"]
resources:
- group: "" # core API group
resources: ["endpoints", "services"]
# Don't log authenticated requests to certain non-resource URL paths.
- level: None
userGroups: ["system:authenticated"]
nonResourceURLs:
- "/api*" # Wildcard matching.
- "/version"
# Log the request body of configmap changes in kube-system.
- level: Request
resources:
- group: "" # core API group
resources: ["configmaps"]
# This rule only applies to resources in the "kube-system" namespace.
# The empty string "" can be used to select non-namespaced resources.
namespaces: ["kube-system"]
# Log configmap and secret changes in all other namespaces at the Metadata level.
- level: Metadata
resources:
- group: "" # core API group
resources: ["secrets", "configmaps"]
# Log all other resources in core and extensions at the Request level.
- level: Request
resources:
- group: "" # core API group
- group: "extensions" # Version of group should NOT be included.
# A catch-all rule to log all other requests at the Metadata level.
- level: Metadata
# Long-running requests like watches that fall under this rule will not
# generate an audit event in RequestReceived.
omitStages:
5.3）命令行启动测试
/usr/k8s/bin/kube-apiserver \
--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \
--advertise-address=${NODE_IP} \
--bind-address=0.0.0.0 \
--insecure-bind-address=${NODE_IP} \
--authorization-mode=Node,RBAC \
--runtime-config=rbac.authorization.k8s.io/v1alpha1 \
--kubelet-https=true \
--enable-bootstrap-token-auth \
--token-auth-file=/etc/kubernetes/token.csv \
--service-cluster-ip-range=${SERVICE_CIDR} \
--service-node-port-range=${NODE_PORT_RANGE} \
--tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/etc/kubernetes/ssl/ca.pem \
--etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \
--etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \
--etcd-servers=${ETCD_ENDPOINTS} \
--enable-swagger-ui=true \
--allow-privileged=true \
--apiserver-count=2 \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/lib/audit.log \
--audit-policy-file=/etc/kubernetes/audit-policy.yaml \
--event-ttl=1h \
--logtostderr=true \
--v=6
5.4）创建kube-apiserver 的systemd unit文件。将上面命令行执行成功的文件换到下面的文件内
cat > kube-apiserver.service <<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
ExecStart=/usr/k8s/bin/kube-apiserver \\
--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \\
--advertise-address=${NODE_IP} \\
--bind-address=0.0.0.0 \\
--insecure-bind-address=${NODE_IP} \\
--authorization-mode=Node,RBAC \\
--runtime-config=rbac.authorization.k8s.io/v1alpha1 \\
--kubelet-https=true \\
--enable-bootstrap-token-auth \\
--token-auth-file=/etc/kubernetes/token.csv \\
--service-cluster-ip-range=${SERVICE_CIDR} \\
--service-node-port-range=${NODE_PORT_RANGE} \\
--tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \\
--tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \\
--client-ca-file=/etc/kubernetes/ssl/ca.pem \\
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \\
--etcd-cafile=/etc/kubernetes/ssl/ca.pem \\
--etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \\
--etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \\
--etcd-servers=${ETCD_ENDPOINTS} \\
--enable-swagger-ui=true \\
--allow-privileged=true \\
--apiserver-count=2 \\
--audit-log-maxage=30 \\
--audit-log-maxbackup=3 \\
--audit-log-maxsize=100 \\
--audit-log-path=/var/lib/audit.log \\
--audit-policy-file=/etc/kubernetes/audit-policy.yaml \\
--event-ttl=1h \\
--logtostderr=true \\
--v=6
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
查看文件
[root@master01 ssl]# cat /root/ssl/kube-apiserver.service [Unit] Description=Kubernetes API Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=network.target [Service] ExecStart=/usr/k8s/bin/kube-apiserver \ --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \ --advertise-address=192.168.10.12 \ --bind-address=0.0.0.0 \ --insecure-bind-address=192.168.10.12 \ --authorization-mode=Node,RBAC \ --runtime-config=rbac.authorization.k8s.io/v1alpha1 \ --kubelet-https=true \ --enable-bootstrap-token-auth \ --token-auth-file=/etc/kubernetes/token.csv \ --service-cluster-ip-range=10.254.0.0/16 \ --service-node-port-range=30000-32766 \ --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \ --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \ --client-ca-file=/etc/kubernetes/ssl/ca.pem \ --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \ --etcd-cafile=/etc/kubernetes/ssl/ca.pem \ --etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \ --etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \ --etcd-servers=https://192.168.10.12:2379,https://192.168.10.22:2379 \ --enable-swagger-ui=true \ --allow-privileged=true \ --apiserver-count=2 \ --audit-log-maxage=30 \ --audit-log-maxbackup=3 \ --audit-log-maxsize=100 \ --audit-log-path=/var/lib/audit.log \ --audit-policy-file=/etc/kubernetes/audit-policy.yaml \ --event-ttl=1h \ --logtostderr=true \ --v=6 Restart=on-failure RestartSec=5 Type=notify LimitNOFILE=65536 [Install] WantedBy=multi-user.target
5.5) 设置api_server开机自启动。并启动服务
[root@master01 ssl]# mv kube-apiserver.service /etc/systemd/system/
[root@master01 ssl]# systemctl daemon-reload
[root@master01 ssl]# systemctl enable kube-apiserver
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /etc/systemd/system/kube-apiserver.service.
[root@master01 ssl]# systemctl start kube-apiserver
[root@master01 ssl]# systemctl status kube-apiserver
● kube-apiserver.service - Kubernetes API Server
Loaded: loaded (/etc/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled)
Active: active (running) since 一 2019-05-27 20:52:20 CST; 15s ago
Docs: https://github.com/GoogleCloudPlatform/kubernetes
Main PID: 37700 (kube-apiserver)
CGroup: /system.slice/kube-apiserver.service
└─37700 /usr/k8s/bin/kube-apiserver --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota --advertise-address=192.168.10.12 --bind-address=0....
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.558316 37700 handler.go:160] kube-aggregator: GET "/api/v1/namespaces/default/services/kubernetes" satisfied by nonGoRestful
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.558345 37700 pathrecorder.go:247] kube-aggregator: "/api/v1/namespaces/default/services/kubernetes" satisfied by prefix /api/
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.558363 37700 handler.go:150] kube-apiserver: GET "/api/v1/namespaces/default/services/kubernetes" satisfied by gorestfu...rvice /api/v1
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.563809 37700 wrap.go:42] GET /api/v1/namespaces/default/services/kubernetes: (5.592002ms) 200 [[kube-apiserver/v1.9.10 ....0.0.1:42872]
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.564282 37700 round_trippers.go:436] GET https://127.0.0.1:6443/api/v1/namespaces/default/services/kubernetes 200 OK in 6 milliseconds
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.565420 37700 handler.go:160] kube-aggregator: GET "/api/v1/namespaces/default/endpoints/kubernetes" satisfied by nonGoRestful
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.565545 37700 pathrecorder.go:247] kube-aggregator: "/api/v1/namespaces/default/endpoints/kubernetes" satisfied by prefix /api/
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.565631 37700 handler.go:150] kube-apiserver: GET "/api/v1/namespaces/default/endpoints/kubernetes" satisfied by gorestf...rvice /api/v1
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.572355 37700 wrap.go:42] GET /api/v1/namespaces/default/endpoints/kubernetes: (7.170178ms) 200 [[kube-apiserver/v1.9.10....0.0.1:42872]
5月 27 20:52:30 master01 kube-apiserver[37700]: I0527 20:52:30.572846 37700 round_trippers.go:436] GET https://127.0.0.1:6443/api/v1/namespaces/default/endpoints/kubernetes 200 OK in 8 milliseconds
[root@master01 ssl]# systemctl status kube-apiserver -l
6）配置和启动kube-controller-manager
6.1）先命令行测试启动
/usr/k8s/bin/kube-controller-manager \
--address=127.0.0.1 \
--master=http://${MASTER_URL}:8080 \
--allocate-node-cidrs=true \
--service-cluster-ip-range=${SERVICE_CIDR} \
--cluster-cidr=${CLUSTER_CIDR} \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \
--root-ca-file=/etc/kubernetes/ssl/ca.pem \
--leader-elect=true \
--v=2
执行没有错误，创建文件
$ cat > kube-controller-manager.service <<EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
ExecStart=/usr/k8s/bin/kube-controller-manager \\
--address=127.0.0.1 \\
--master=http://${MASTER_URL}:8080 \\
--allocate-node-cidrs=true \\
--service-cluster-ip-range=${SERVICE_CIDR} \\
--cluster-cidr=${CLUSTER_CIDR} \\
--cluster-name=kubernetes \\
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \\
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \\
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \\
--root-ca-file=/etc/kubernetes/ssl/ca.pem \\
--leader-elect=true \\
--v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
查看该文件
[root@master01 ssl]# cat kube-controller-manager.service [Unit] Description=Kubernetes Controller Manager Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service] ExecStart=/usr/k8s/bin/kube-controller-manager \ --address=127.0.0.1 \ --master=http://k8s-api.virtual.local:8080 \ --allocate-node-cidrs=true \ --service-cluster-ip-range=10.254.0.0/16 \ --cluster-cidr=172.30.0.0/16 \ --cluster-name=kubernetes \ --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \ --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \ --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \ --root-ca-file=/etc/kubernetes/ssl/ca.pem \ --leader-elect=true \ --v=2 Restart=on-failure RestartSec=5 [Install] WantedBy=multi-user.target
6.2）设置kube-controller-manager.service 开机自启动
[root@master01 ssl]# mv kube-controller-manager.service /etc/systemd/system
[root@master01 ssl]# systemctl daemon-reload
[root@master01 ssl]# systemctl enable kube-controller-manager.service
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /etc/systemd/system/kube-controller-manager.service.
[root@master01 ssl]# systemctl start kube-controller-manager.service
[root@master01 ssl]# systemctl status kube-controller-manager.service
● kube-controller-manager.service - Kubernetes Controller Manager
Loaded: loaded (/etc/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled)
Active: active (running) since 一 2019-05-27 21:04:35 CST; 13s ago
Docs: https://github.com/GoogleCloudPlatform/kubernetes
Main PID: 37783 (kube-controller)
CGroup: /system.slice/kube-controller-manager.service
└─37783 /usr/k8s/bin/kube-controller-manager --address=127.0.0.1 --master=http://k8s-api.virtual.local:8080 --allocate-node-cidrs=true --service-cluster-ip-range=10.254.0.0/16 --cluster-c...
5月 27 21:04:47 master01 kube-controller-manager[37783]: I0527 21:04:47.333573 37783 controller_utils.go:1026] Caches are synced for cidrallocator controller
5月 27 21:04:47 master01 kube-controller-manager[37783]: I0527 21:04:47.353933 37783 controller_utils.go:1026] Caches are synced for certificate controller
5月 27 21:04:47 master01 kube-controller-manager[37783]: I0527 21:04:47.407102 37783 controller_utils.go:1026] Caches are synced for certificate controller
5月 27 21:04:47 master01 kube-controller-manager[37783]: I0527 21:04:47.424111 37783 controller_utils.go:1026] Caches are synced for job controller
5月 27 21:04:47 master01 kube-controller-manager[37783]: I0527 21:04:47.490109 37783 controller_utils.go:1026] Caches are synced for resource quota controller
5月 27 21:04:47 master01 kube-controller-manager[37783]: I0527 21:04:47.506266 37783 controller_utils.go:1026] Caches are synced for garbage collector controller
5月 27 21:04:47 master01 kube-controller-manager[37783]: I0527 21:04:47.506311 37783 garbagecollector.go:144] Garbage collector: all resource monitors have synced. Proceeding to collect garbage
5月 27 21:04:47 master01 kube-controller-manager[37783]: I0527 21:04:47.524898 37783 controller_utils.go:1026] Caches are synced for resource quota controller
5月 27 21:04:47 master01 kube-controller-manager[37783]: I0527 21:04:47.525473 37783 controller_utils.go:1026] Caches are synced for persistent volume controller
5月 27 21:04:48 master01 kube-controller-manager[37783]: I0527 21:04:48.444031 37783 garbagecollector.go:190] syncing garbage collector with updated resources from discovery: map[{ v1...:{} {rbac.au
7）配置和启动kube-scheduler
7.1）命令行测试命令是否正常
/usr/k8s/bin/kube-scheduler \
--address=127.0.0.1 \
--master=http://${MASTER_URL}:8080 \
--leader-elect=true \
--v=2
正常则创建kube-scheduler.service文件
$ cat > kube-scheduler.service <<EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
ExecStart=/usr/k8s/bin/kube-scheduler \\
--address=127.0.0.1 \\
--master=http://${MASTER_URL}:8080 \\
--leader-elect=true \\
--v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
查看文件
[root@master01 ssl]# cat kube-scheduler.service [Unit] Description=Kubernetes Scheduler Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service] ExecStart=/usr/k8s/bin/kube-scheduler \ --address=127.0.0.1 \ --master=http://k8s-api.virtual.local:8080 \ --leader-elect=true \ --v=2 Restart=on-failure RestartSec=5 [Install] WantedBy=multi-user.target
7.2）设置kube-scheduler的开机启动
[root@master01 ssl]# mv kube-scheduler.service /etc/systemd/system
[root@master01 ssl]# systemctl daemon-reload
[root@master01 ssl]# systemctl enable kube-scheduler
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /etc/systemd/system/kube-scheduler.service.
[root@master01 ssl]# systemctl start kube-scheduler
[root@master01 ssl]# systemctl status kube-scheduler
● kube-scheduler.service - Kubernetes Scheduler
Loaded: loaded (/etc/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled)
Active: active (running) since 一 2019-05-27 21:13:42 CST; 17s ago
Docs: https://github.com/GoogleCloudPlatform/kubernetes
Main PID: 37850 (kube-scheduler)
CGroup: /system.slice/kube-scheduler.service
└─37850 /usr/k8s/bin/kube-scheduler --address=127.0.0.1 --master=http://k8s-api.virtual.local:8080 --leader-elect=true --v=2
5月 27 21:13:42 master01 systemd[1]: Starting Kubernetes Scheduler...
5月 27 21:13:42 master01 kube-scheduler[37850]: W0527 21:13:42.436689 37850 server.go:162] WARNING: all flags than --config are deprecated. Please begin using a config file ASAP.
5月 27 21:13:42 master01 kube-scheduler[37850]: I0527 21:13:42.438090 37850 server.go:554] Version: v1.9.10
5月 27 21:13:42 master01 kube-scheduler[37850]: I0527 21:13:42.438222 37850 factory.go:837] Creating scheduler from algorithm provider 'DefaultProvider'
5月 27 21:13:42 master01 kube-scheduler[37850]: I0527 21:13:42.438231 37850 factory.go:898] Creating scheduler with fit predicates 'map[MaxEBSVolumeCount:{} MaxGCEPDVolumeCount:{} Ge... MatchInterPo
5月 27 21:13:42 master01 kube-scheduler[37850]: I0527 21:13:42.438431 37850 server.go:573] starting healthz server on 127.0.0.1:10251
5月 27 21:13:43 master01 kube-scheduler[37850]: I0527 21:13:43.243364 37850 controller_utils.go:1019] Waiting for caches to sync for scheduler controller
5月 27 21:13:43 master01 kube-scheduler[37850]: I0527 21:13:43.345039 37850 controller_utils.go:1026] Caches are synced for scheduler controller
5月 27 21:13:43 master01 kube-scheduler[37850]: I0527 21:13:43.345105 37850 leaderelection.go:174] attempting to acquire leader lease...
5月 27 21:13:43 master01 kube-scheduler[37850]: I0527 21:13:43.359717 37850 leaderelection.go:184] successfully acquired lease kube-system/kube-scheduler
Hint: Some lines were ellipsized, use -l to show in full.
8）配置kubectl 命令行工具
8.1）配置环境变量
$ export KUBE_APISERVER="https://${MASTER_URL}:6443"
环境变量定义在配置文件
[root@master01 ssl]# cat ~/.bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
PATH=$PATH:$HOME/bin
export PATH=/usr/k8s/bin/:$PATH
export NODE_IP=192.168.10.12
source /usr/k8s/bin/env.sh
export KUBE_APISERVER="https://${MASTER_URL}:6443"
export PATH
8.2) 配置 kubectl命令
[root@master01 ~]# ls kubernetes/server/bin/kubectl
kubernetes/server/bin/kubectl
[root@master01 ~]# cp kubernetes/server/bin/kubectl /usr/k8s/bin/
[root@master01 ~]# kubectl version
Client Version: version.Info{Major:"1", Minor:"9", GitVersion:"v1.9.10", GitCommit:"098570796b32895c38a9a1c9286425fb1ececa18", GitTreeState:"clean", BuildDate:"2018-08-02T17:19:54Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"}
The connection to the server localhost:8080 was refused - did you specify the right host or port?
8.3）配置admin证书
$ cat > admin-csr.json <<EOF
{
"CN": "admin",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "system:masters",
"OU": "System"
}
]
}
EOF
备注
后续kube-apiserver使用RBAC 对客户端(如kubelet、kube-proxy、Pod)请求进行授权 kube-apiserver 预定义了一些RBAC 使用的RoleBindings，如cluster-admin 将Group system:masters与Role cluster-admin绑定，该Role 授予了调用kube-apiserver所有API 的权限 O 指定了该证书的Group 为system:masters，kubectl使用该证书访问kube-apiserver时，由于证书被CA 签名，所以认证通过，同时由于证书用户组为经过预授权的system:masters，所以被授予访问所有API 的劝降 hosts 属性值为空列表
8.4）生成admin 证书和私钥
$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \ -ca-key=/etc/kubernetes/ssl/ca-key.pem \ -config=/etc/kubernetes/ssl/ca-config.json \ -profile=kubernetes admin-csr.json | cfssljson -bare admin $ ls admin admin.csr admin-csr.json admin-key.pem admin.pem $ sudo mv admin*.pem /etc/kubernetes/ssl/
8.5）创建kubectl kubeconfig 文件
# 设置集群参数
$ kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER}
# 设置客户端认证参数
$ kubectl config set-credentials admin \
--client-certificate=/etc/kubernetes/ssl/admin.pem \
--embed-certs=true \
--client-key=/etc/kubernetes/ssl/admin-key.pem \
--token=${BOOTSTRAP_TOKEN}
# 设置上下文参数
$ kubectl config set-context kubernetes \
--cluster=kubernetes \
--user=admin
# 设置默认上下文
$ kubectl config use-context kubernetes
执行过程
[root@master01 ssl]# echo $KUBE_APISERVER
https://k8s-api.virtual.local:6443
[root@master01 ssl]# kubectl config set-cluster kubernetes \
> --certificate-authority=/etc/kubernetes/ssl/ca.pem \
> --embed-certs=true \
> --server=${KUBE_APISERVER}
Cluster "kubernetes" set.
[root@master01 ssl]#
[root@master01 ssl]# echo $BOOTSTRAP_TOKEN
0b340751863956f119cbc624465db92b
[root@master01 ssl]# kubectl config set-credentials admin \
> --client-certificate=/etc/kubernetes/ssl/admin.pem \
> --embed-certs=true \
> --client-key=/etc/kubernetes/ssl/admin-key.pem \
> --token=${BOOTSTRAP_TOKEN}
User "admin" set.
[root@master01 ssl]# kubectl config set-context kubernetes \
> --cluster=kubernetes \
> --user=admin
Context "kubernetes" created.
[root@master01 ssl]# kubectl config use-context kubernetes
Switched to context "kubernetes".
8.6）此时验证kubect是否可以使用
[root@master01 ssl]# ls ~/.kube/
config
[root@master01 ssl]# ls ~/.kube/config
/root/.kube/config
[root@master01 ssl]# kubectl version
Client Version: version.Info{Major:"1", Minor:"9", GitVersion:"v1.9.10", GitCommit:"098570796b32895c38a9a1c9286425fb1ececa18", GitTreeState:"clean", BuildDate:"2018-08-02T17:19:54Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"9", GitVersion:"v1.9.10", GitCommit:"098570796b32895c38a9a1c9286425fb1ececa18", GitTreeState:"clean", BuildDate:"2018-08-02T17:11:51Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"}
[root@master01 ssl]# kubectl get cs
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy {"health":"true"}
etcd-1 Healthy {"health":"true"}
四、部署flannel网络（所有节点都需要安装）
1） 给node节点 创建环境变量
将master01的/usr/k8s/bin/env.sh 拷贝过来
[root@k8s01-node01 ~]# export NODE_IP=192.168.10.23
[root@k8s01-node01 ~]# cat ~/.bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
PATH=$PATH:$HOME/bin
PATH=/usr/k8s/bin:$PATH
source /usr/k8s/bin/env.sh
export NODE_IP=192.168.10.23
export PATH
2）从master01导入ca相关证书
[root@k8s01-node01 ~]# mkdir -p /etc/kubernetes/ssl [root@master01 ssl]# ll ca* # 将master01节点的ca证书拷贝到node01中 -rw-r--r-- 1 root root 387 5月 22 14:45 ca-config.json -rw-r--r-- 1 root root 1001 5月 22 14:45 ca.csr -rw-r--r-- 1 root root 264 5月 22 14:45 ca-csr.json -rw------- 1 root root 1679 5月 22 14:45 ca-key.pem -rw-r--r-- 1 root root 1359 5月 22 14:45 ca.pem [root@master01 ssl]# scp ca* root@n1:/etc/kubernetes/ssl/ ca-config.json 100% 387 135.6KB/s 00:00 ca.csr 100% 1001 822.8KB/s 00:00 ca-csr.json 100% 264 81.2KB/s 00:00 ca-key.pem 100% 1679 36.6KB/s 00:00 ca.pem [root@k8s01-node01 ~]# ll /etc/kubernetes/ssl 查看nodes的ca证书 总用量 20 -rw-r--r-- 1 root root 387 5月 28 10:42 ca-config.json -rw-r--r-- 1 root root 1001 5月 28 10:42 ca.csr -rw-r--r-- 1 root root 264 5月 28 10:42 ca-csr.json -rw------- 1 root root 1679 5月 28 10:42 ca-key.pem -rw-r--r-- 1 root root 1359 5月 28 10:42 ca.pem
3）创建flanneld 证书签名请求
cat > flanneld-csr.json <<EOF
{
"CN": "flanneld",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
3.1）生成flanneld 证书和私钥
$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \ -ca-key=/etc/kubernetes/ssl/ca-key.pem \ -config=/etc/kubernetes/ssl/ca-config.json \ -profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld $ ls flanneld* flanneld.csr flanneld-csr.json flanneld-key.pem flanneld.pem $ sudo mkdir -p /etc/flanneld/ssl $ sudo mv flanneld*.pem /etc/flanneld/ssl
3.2）实际操作（证书在master生成，发布给nodes）
node01操作
[root@k8s01-node01 ~]# mkdir -p /etc/flanneld/ssl
master01
创建flanneld 证书签名请求
$ cat > flanneld-csr.json <<EOF
{
"CN": "flanneld",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
生成flanneld 证书和私钥：
[root@master01 ssl]# cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
> -ca-key=/etc/kubernetes/ssl/ca-key.pem \
> -config=/etc/kubernetes/ssl/ca-config.json \
> -profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld
2019/05/28 10:48:11 [INFO] generate received request
2019/05/28 10:48:11 [INFO] received CSR
2019/05/28 10:48:11 [INFO] generating key: rsa-2048
2019/05/28 10:48:11 [INFO] encoded CSR
2019/05/28 10:48:11 [INFO] signed certificate with serial number 110687193783617600207260591820939534389315422637
2019/05/28 10:48:11 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@master01 ssl]# ls flanneld*
flanneld.csr flanneld-csr.json flanneld-key.pem flanneld.pem
[root@master01 ssl]# scp flanneld* root@n1:/etc/flanneld/ssl/
flanneld.csr 100% 997 370.5KB/s 00:00
flanneld-csr.json 100% 221 223.1KB/s 00:00
flanneld-key.pem 100% 1679 2.4MB/s 00:00
flanneld.pem 100% 1391 2.4MB/s 00:00
[root@master01 ssl]#
node01查看
[root@k8s01-node01 ~]# ll /etc/flanneld/ssl
总用量 16
-rw-r--r-- 1 root root 997 5月 28 10:49 flanneld.csr
-rw-r--r-- 1 root root 221 5月 28 10:49 flanneld-csr.json
-rw------- 1 root root 1679 5月 28 10:49 flanneld-key.pem
-rw-r--r-- 1 root root 1391 5月 28 10:49 flanneld.pem
4）向etcd 写入集群Pod 网段信息。该步骤只需在第一次部署Flannel 网络时执行，后续在其他节点上部署Flanneld 时无需再写入该信息
[root@master01 ssl]# mkdir -p /etc/flanneld/ssl
[root@master01 ssl]# mv flanneld*.pem /etc/flanneld/ssl/
$ etcdctl \
--endpoints=${ETCD_ENDPOINTS} \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--cert-file=/etc/flanneld/ssl/flanneld.pem \
--key-file=/etc/flanneld/ssl/flanneld-key.pem \
set ${FLANNEL_ETCD_PREFIX}/config '{"Network":"'${CLUSTER_CIDR}'", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}'
# 得到如下反馈信息
{"Network":"172.30.0.0/16", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}
写入的 Pod 网段(${CLUSTER_CIDR}，172.30.0.0/16) 必须与kube-controller-manager 的 --cluster-cidr 选项值一致；
4.1）实际操作
[root@master01 ssl]# mkdir -p /etc/flanneld/ssl
[root@master01 ssl]# mv flanneld*.pem /etc/flanneld/ssl/
[root@master01 ssl]# ll /etc/flanneld/ssl/
总用量 8
-rw------- 1 root root 1679 5月 28 10:48 flanneld-key.pem
-rw-r--r-- 1 root root 1391 5月 28 10:48 flanneld.pem
[root@master01 ssl]# echo $FLANNEL_ETCD_PREFIX
/kubernetes/network
[root@master01 ssl]# echo $CLUSTER_CIDR
172.30.0.0/16
[root@master01 ssl]# echo $ETCD_ENDPOINTS
https://192.168.10.12:2379,https://192.168.10.22:2379
[root@master01 ssl]# etcdctl \
> --endpoints=${ETCD_ENDPOINTS} \
> --ca-file=/etc/kubernetes/ssl/ca.pem \
> --cert-file=/etc/flanneld/ssl/flanneld.pem \
> --key-file=/etc/flanneld/ssl/flanneld-key.pem \
> set ${FLANNEL_ETCD_PREFIX}/config '{"Network":"'${CLUSTER_CIDR}'", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}'
{"Network":"172.30.0.0/16", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}
[root@master01 ssl]# cat /etc/systemd/system/kube-controller-manager.service |grep "cluster-cidr"
--cluster-cidr=172.30.0.0/16 \
5）下载最新版的flanneld 二进制文件。
https://github.com/coreos/flannel/releases wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz [root@k8s01-node01 ~]# ls flannel-v0.11.0-linux-amd64.tar.gz flannel-v0.11.0-linux-amd64.tar.gz [root@k8s01-node01 ~]# tar xf flannel-v0.11.0-linux-amd64.tar.gz [root@k8s01-node01 ~]# cp flanneld /usr/k8s/bin/ [root@k8s01-node01 ~]# cp mk-docker-opts.sh /usr/k8s/bin/ [root@k8s01-node01 ~]# ls /usr/k8s/bin/ env.sh flanneld mk-docker-opts.sh
6）创建flanneld的systemd unit 文件
6.1）测试命令是否能正常执行（node01操作）
/usr/k8s/bin/flanneld \
-etcd-cafile=/etc/kubernetes/ssl/ca.pem \
-etcd-certfile=/etc/flanneld/ssl/flanneld.pem \
-etcd-keyfile=/etc/flanneld/ssl/flanneld-key.pem \
-etcd-endpoints=${ETCD_ENDPOINTS} \
-etcd-prefix=${FLANNEL_ETCD_PREFIX}
6.2）node01创建flanneld.service文件
$ cat > flanneld.service << EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service
[Service]
Type=notify
ExecStart=/usr/k8s/bin/flanneld \\
-etcd-cafile=/etc/kubernetes/ssl/ca.pem \\
-etcd-certfile=/etc/flanneld/ssl/flanneld.pem \\
-etcd-keyfile=/etc/flanneld/ssl/flanneld-key.pem \\
-etcd-endpoints=${ETCD_ENDPOINTS} \\
-etcd-prefix=${FLANNEL_ETCD_PREFIX}
ExecStartPost=/usr/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=on-failure
[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF
6.3) 设置 flanneld 的开机启动
[root@k8s01-node01 ~]# mv flanneld.service /etc/systemd/system
[root@k8s01-node01 ~]# systemctl daemon-reload
[root@k8s01-node01 ~]# systemctl enable flanneld
Created symlink from /etc/systemd/system/multi-user.target.wants/flanneld.service to /etc/systemd/system/flanneld.service.
Created symlink from /etc/systemd/system/docker.service.requires/flanneld.service to /etc/systemd/system/flanneld.service.
[root@k8s01-node01 ~]# systemctl start flanneld
[root@k8s01-node01 ~]# systemctl status flanneld
● flanneld.service - Flanneld overlay address etcd agent
Loaded: loaded (/etc/systemd/system/flanneld.service; enabled; vendor preset: disabled)
Active: active (running) since 二 2019-05-28 13:37:18 CST; 21s ago
Process: 28063 ExecStartPost=/usr/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker (code=exited, status=0/SUCCESS)
Main PID: 28051 (flanneld)
Memory: 10.3M
CGroup: /system.slice/flanneld.service
└─28051 /usr/k8s/bin/flanneld -etcd-cafile=/etc/kubernetes/ssl/ca.pem -etcd-certfile=/etc/flanneld/ssl/flanneld.pem -etcd-keyfile=/etc/flanneld/ssl/flanneld-key.pem -etcd-endpoints=https:...
5月 28 13:37:18 k8s01-node01 flanneld[28051]: I0528 13:37:18.747119 28051 main.go:244] Created subnet manager: Etcd Local Manager with Previous Subnet: 172.30.23.0/24
5月 28 13:37:18 k8s01-node01 flanneld[28051]: I0528 13:37:18.747123 28051 main.go:247] Installing signal handlers
5月 28 13:37:18 k8s01-node01 flanneld[28051]: I0528 13:37:18.819160 28051 main.go:386] Found network config - Backend type: vxlan
5月 28 13:37:18 k8s01-node01 flanneld[28051]: I0528 13:37:18.819293 28051 vxlan.go:120] VXLAN config: VNI=1 Port=0 GBP=false DirectRouting=false
5月 28 13:37:18 k8s01-node01 flanneld[28051]: I0528 13:37:18.830870 28051 local_manager.go:147] Found lease (172.30.23.0/24) for current IP (192.168.10.23), reusing
5月 28 13:37:18 k8s01-node01 flanneld[28051]: I0528 13:37:18.844660 28051 main.go:317] Wrote subnet file to /run/flannel/subnet.env
5月 28 13:37:18 k8s01-node01 flanneld[28051]: I0528 13:37:18.844681 28051 main.go:321] Running backend.
5月 28 13:37:18 k8s01-node01 flanneld[28051]: I0528 13:37:18.844739 28051 vxlan_network.go:60] watching for new subnet leases
5月 28 13:37:18 k8s01-node01 flanneld[28051]: I0528 13:37:18.851989 28051 main.go:429] Waiting for 22h59m59.998588039s to renew lease
5月 28 13:37:18 k8s01-node01 systemd[1]: Started Flanneld overlay address etcd agent.
查看网络
[root@k8s01-node01 ~]# ifconfig flannel.1
flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet 172.30.23.0 netmask 255.255.255.255 broadcast 0.0.0.0
inet6 fe80::b4e5:d4ff:fe53:16da prefixlen 64 scopeid 0x20<link>
ether b6:e5:d4:53:16:da txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 8 overruns 0 carrier 0 collisions 0
6.4）其他node节点，同样的方法安装flanneld网络
7）master节点查看nodes网络节点信息
$ etcdctl \
--endpoints=${ETCD_ENDPOINTS} \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--cert-file=/etc/flanneld/ssl/flanneld.pem \
--key-file=/etc/flanneld/ssl/flanneld-key.pem \
ls ${FLANNEL_ETCD_PREFIX}/subnets
/kubernetes/network/subnets/172.30.23.0-24 # 为node01节点的网络。可在node01使用ifconfig flannel.1查看
五、部署node节点
1）配置环境变量，已经hosts文件
[root@k8s01-node01 ~]# export KUBE_APISERVER="https://${MASTER_URL}:6443"
[root@k8s01-node01 ~]# echo $KUBE_APISERVER
https://k8s-api.virtual.local:6443
[root@k8s01-node01 ~]# cat .bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
PATH=$PATH:$HOME/bin
PATH=/usr/k8s/bin:$PATH
source /usr/k8s/bin/env.sh
export NODE_IP=192.168.10.23
export KUBE_APISERVER="https://${MASTER_URL}:6443"
export PATH
[root@k8s01-node01 ~]# cat /etc/hosts|grep k8s-api.virtual.local
192.168.10.12 k8s-api.virtual.local
2）开启路由转发
net.ipv4.ip_forward=1
[root@k8s01-node01 ~]# cat /etc/sysctl.conf # sysctl settings are defined through files in # /usr/lib/sysctl.d/, /run/sysctl.d/, and /etc/sysctl.d/. # # Vendors settings live in /usr/lib/sysctl.d/. # To override a whole file, create a new file with the same in # /etc/sysctl.d/ and put new settings there. To override # only specific settings, add a file with a lexically later # name in /etc/sysctl.d/ and put new settings there. # # For more information, see sysctl.conf(5) and sysctl.d(5). net.ipv4.ip_forward=1
[root@k8s01-node01 ~]# sysctl -p
net.ipv4.ip_forward = 1
3）根据官网进行docker的安装
https://docs.docker.com/install/linux/docker-ce/centos # docker 官网安装
先安装仓库
yum install -y yum-utils \
device-mapper-persistent-data \
lvm2
安装源
yum-config-manager \
--add-repo \
https://download.docker.com/linux/centos/docker-ce.repo
安装docker
yum install docker-ce docker-ce-cli containerd.io -y
[root@k8s01-node01 ~]# docker version
Client:
Version: 18.09.6
API version: 1.39
Go version: go1.10.8
Git commit: 481bc77156
Built: Sat May 4 02:34:58 2019
OS/Arch: linux/amd64
Experimental: false
Server: Docker Engine - Community
Engine:
Version: 18.09.6
API version: 1.39 (minimum version 1.12)
Go version: go1.10.8
Git commit: 481bc77
Built: Sat May 4 02:02:43 2019
OS/Arch: linux/amd64
Experimental: false
4）修改docker 的systemd unit 文件
vim /usr/lib/systemd/system/docker.service ..... [Service] Type=notify # the default is not to use systemd for cgroups because the delegate issues still # exists and systemd currently does not support the cgroup feature set required # for containers run by docker EnvironmentFile=-/run/flannel/docker # 新增 ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --log-level=info $DOCKER_NETWORK_OPTIONS # 新增 --log-level=info $DOCKER_NETWORK_OPTIONS ExecReload=/bin/kill -s HUP $MAINPID TimeoutSec=0 RestartSec=2 Restart=always .....................
查看文件内容
[Unit] Description=Docker Application Container Engine Documentation=https://docs.docker.com BindsTo=containerd.service After=network-online.target firewalld.service containerd.service Wants=network-online.target Requires=docker.socket [Service] Type=notify # the default is not to use systemd for cgroups because the delegate issues still # exists and systemd currently does not support the cgroup feature set required # for containers run by docker EnvironmentFile=-/run/flannel/docker ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --log-level=info $DOCKER_NETWORK_OPTIONS ExecReload=/bin/kill -s HUP $MAINPID TimeoutSec=0 RestartSec=2 Restart=always # Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. # Both the old, and new location are accepted by systemd 229 and up, so using the old location # to make them work for either version of systemd. StartLimitBurst=3 # Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. # Both the old, and new name are accepted by systemd 230 and up, so using the old name to make # this option work for either version of systemd. StartLimitInterval=60s # Having non-zero Limit*s causes performance problems due to accounting overhead # in the kernel. We recommend using cgroups to do container-local accounting. LimitNOFILE=infinity LimitNPROC=infinity LimitCORE=infinity # Comment TasksMax if your systemd version does not supports it. # Only systemd 226 and above support this option. TasksMax=infinity # set delegate yes so that systemd does not reset the cgroups of docker containers Delegate=yes # kill only the docker process, not all processes in the cgroup KillMode=process [Install] WantedBy=multi-user.target
5）为了加快 pull image 的速度，可以使用国内的仓库镜像服务器，同时增加下载的并发数。(如果 dockerd 已经运行，则需要重启 dockerd 生效。)
[root@k8s01-node01 ~]# cat /etc/docker/daemon.json
{
"max-concurrent-downloads": 10
}
6）启动docker
[root@k8s01-node01 ~]# systemctl daemon-reload [root@k8s01-node01 ~]# systemctl enable docker [root@k8s01-node01 ~]# systemctl start docker [root@k8s01-node01 ~]# systemctl status docker
内核优化
[root@k8s01-node01 ~]# cat /etc/sysctl.conf # sysctl settings are defined through files in # /usr/lib/sysctl.d/, /run/sysctl.d/, and /etc/sysctl.d/. # # Vendors settings live in /usr/lib/sysctl.d/. # To override a whole file, create a new file with the same in # /etc/sysctl.d/ and put new settings there. To override # only specific settings, add a file with a lexically later # name in /etc/sysctl.d/ and put new settings there. # # For more information, see sysctl.conf(5) and sysctl.d(5). net.ipv4.ip_forward=1 net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1
sysctl -p
[root@k8s01-node01 ~]# sysctl -p net.ipv4.ip_forward = 1 net.bridge.bridge-nf-call-iptables = 1 net.bridge.bridge-nf-call-ip6tables = 1
[root@k8s01-node01 ~]# systemctl restart docker
7）防火墙问题
$ sudo systemctl daemon-reload $ sudo systemctl stop firewalld $ sudo systemctl disable firewalld $ sudo iptables -F && sudo iptables -X && sudo iptables -F -t nat && sudo iptables -X -t nat #清空以前的防火墙 $ sudo systemctl enable docker $ sudo systemctl start docker
查看防火墙
[root@k8s01-node01 ~]# iptables -L -n Chain INPUT (policy ACCEPT) target prot opt source destination Chain FORWARD (policy ACCEPT) target prot opt source destination DOCKER-USER all -- 0.0.0.0/0 0.0.0.0/0 DOCKER-ISOLATION-STAGE-1 all -- 0.0.0.0/0 0.0.0.0/0 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED DOCKER all -- 0.0.0.0/0 0.0.0.0/0 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 ACCEPT all -- 172.30.0.0/16 0.0.0.0/0 ACCEPT all -- 0.0.0.0/0 172.30.0.0/16 Chain OUTPUT (policy ACCEPT) target prot opt source destination Chain DOCKER (1 references) target prot opt source destination Chain DOCKER-ISOLATION-STAGE-1 (1 references) target prot opt source destination DOCKER-ISOLATION-STAGE-2 all -- 0.0.0.0/0 0.0.0.0/0 RETURN all -- 0.0.0.0/0 0.0.0.0/0 Chain DOCKER-ISOLATION-STAGE-2 (1 references) target prot opt source destination DROP all -- 0.0.0.0/0 0.0.0.0/0 RETURN all -- 0.0.0.0/0 0.0.0.0/0 Chain DOCKER-USER (1 references) target prot opt source destination RETURN all -- 0.0.0.0/0 0.0.0.0/0
六、安装和配置kubelet
1）node1节点安装kubelet命令
node01执行
[root@k8s01-node01 ~]# ls /usr/k8s/bin/ # 查看kubectl是否有
env.sh flanneld mk-docker-opts.sh
[root@k8s01-node01 ~]# mkdir .kube
master01传输文件
[root@master01 ~]# scp /usr/k8s/bin/kubectl root@n1:/usr/k8s/bin/
[root@master01 ~]# scp .kube/config root@n1:/root/.kube/
node01查看
[root@k8s01-node01 ~]# ls /usr/k8s/bin/
env.sh flanneld kubectl mk-docker-opts.sh
[root@k8s01-node01 ~]# ls .kube/
config
[root@k8s01-node01 ~]# kubectl version
Client Version: version.Info{Major:"1", Minor:"14", GitVersion:"v1.14.2", GitCommit:"66049e3b21efe110454d67df4fa62b08ea79a19b", GitTreeState:"clean", BuildDate:"2019-05-16T16:23:09Z", GoVersion:"go1.12.5", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"9", GitVersion:"v1.9.10", GitCommit:"098570796b32895c38a9a1c9286425fb1ececa18", GitTreeState:"clean", BuildDate:"2018-08-02T17:11:51Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"}
2）给特殊用户赋予规则
kubelet 启动时向kube-apiserver 发送TLS bootstrapping 请求，需要先将bootstrap token 文件中的kubelet-bootstrap 用户赋予system:node-bootstrapper 角色，然后kubelet 才有权限创建认证请求(certificatesigningrequests)： [root@k8s01-node01 ~]# kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap clusterrolebinding.rbac.authorization.k8s.io/kubelet-bootstrap created 另外1.8 版本中还需要为Node 请求创建一个RBAC 授权规则： [root@k8s01-node01 ~]# kubectl create clusterrolebinding kubelet-nodes --clusterrole=system:node --group=system:nodes clusterrolebinding.rbac.authorization.k8s.io/kubelet-nodes created
3）下载最新的kubelet 和kube-proxy 二进制文件（前面下载kubernetes 目录下面其实也有）
$ wget https://dl.k8s.io/v1.8.2/kubernetes-server-linux-amd64.tar.gz
$ tar -xzvf kubernetes-server-linux-amd64.tar.gz
$ cd kubernetes
$ tar -xzvf kubernetes-src.tar.gz
$ sudo cp -r ./server/bin/{kube-proxy,kubelet} /usr/k8s/bin/
方法二（实际操作）
[root@master01 ~]# ls kubernetes/server/bin/kubelet kubernetes/server/bin/kubelet [root@master01 ~]# ls kubernetes/server/bin/kube-proxy kubernetes/server/bin/kube-proxy [root@master01 ~]# scp kubernetes/server/bin/kube-proxy root@n1:/usr/k8s/bin/ [root@master01 ~]# scp kubernetes/server/bin/kubelet root@n1:/usr/k8s/bin/
4）创建kubelet bootstapping kubeconfig 文件
$ # 设置集群参数
$ kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=bootstrap.kubeconfig
$ # 设置客户端认证参数
$ kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} \
--kubeconfig=bootstrap.kubeconfig
$ # 设置上下文参数
$ kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=bootstrap.kubeconfig
$ # 设置默认上下文
$ kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
$ mv bootstrap.kubeconfig /etc/kubernetes/
操作过程
[root@k8s01-node01 ~]# echo $KUBE_APISERVER
https://k8s-api.virtual.local:6443
[root@k8s01-node01 ~]# kubectl config set-cluster kubernetes \
> --certificate-authority=/etc/kubernetes/ssl/ca.pem \
> --embed-certs=true \
> --server=${KUBE_APISERVER} \
> --kubeconfig=bootstrap.kubeconfig
Cluster "kubernetes" set.
[root@k8s01-node01 ~]# echo $BOOTSTRAP_TOKEN
0b340751863956f119cbc624465db92b
[root@k8s01-node01 ~]# kubectl config set-credentials kubelet-bootstrap \
> --token=${BOOTSTRAP_TOKEN} \
> --kubeconfig=bootstrap.kubeconfig
User "kubelet-bootstrap" set.
[root@k8s01-node01 ~]# kubectl config set-context default \
> --cluster=kubernetes \
> --user=kubelet-bootstrap \
> --kubeconfig=bootstrap.kubeconfig
Context "default" created.
[root@k8s01-node01 ~]# kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
Switched to context "default".
[root@k8s01-node01 ~]# ls bootstrap.kubeconfig
bootstrap.kubeconfig
[root@k8s01-node01 ~]# mv bootstrap.kubeconfig /etc/kubernetes/
5）创建kubelet 的systemd unit 文件
5.1）命令行测试是否通过
/usr/k8s/bin/kubelet \
--fail-swap-on=false \
--cgroup-driver=cgroupfs \
--address=${NODE_IP} \
--hostname-override=${NODE_IP} \
--experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
--require-kubeconfig \
--cert-dir=/etc/kubernetes/ssl \
--cluster-dns=${CLUSTER_DNS_SVC_IP} \
--cluster-domain=${CLUSTER_DNS_DOMAIN} \
--hairpin-mode promiscuous-bridge \
--allow-privileged=true \
--serialize-image-pulls=false \
--logtostderr=true \
--v=2
5.2）通过则创建文件
$ sudo mkdir /var/lib/kubelet # 必须先创建工作目录
$ cat > kubelet.service <<EOF
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service
[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/k8s/bin/kubelet \\
--fail-swap-on=false \\
--cgroup-driver=cgroupfs \\
--address=${NODE_IP} \\
--hostname-override=${NODE_IP} \\
--experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \\
--require-kubeconfig \\
--cert-dir=/etc/kubernetes/ssl \\
--cluster-dns=${CLUSTER_DNS_SVC_IP} \\
--cluster-domain=${CLUSTER_DNS_DOMAIN} \\
--hairpin-mode promiscuous-bridge \\
--allow-privileged=true \\
--serialize-image-pulls=false \\
--logtostderr=true \\
--v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
5.3）开机自启动
[root@k8s01-node01 ~]# mv kubelet.service /etc/systemd/system
[root@k8s01-node01 ~]# systemctl daemon-reload
[root@k8s01-node01 ~]# systemctl restart kubelet
[root@k8s01-node01 ~]# systemctl status kubelet
[root@k8s01-node01 ~]# systemctl status kubelet -l 详细情况查看
● kubelet.service - Kubernetes Kubelet
Loaded: loaded (/etc/systemd/system/kubelet.service; enabled; vendor preset: disabled)
Active: active (running) since 二 2019-05-28 19:48:40 CST; 14s ago
Docs: https://github.com/GoogleCloudPlatform/kubernetes
[root@k8s01-node01 ~]# systemctl enable kubelet # 问题
Failed to execute operation: File exists
6) 通过kubelet 的TLS 证书请求
6.1) 查看未通过时的状态
[root@k8s01-node01 ~]# kubectl get csr NAME AGE REQUESTOR CONDITION node-csr-mTYmEsL6Eh5rKSJOgfO8trHBq8LHI1SX7QDr2OqJ2Zg 29m kubelet-bootstrap Pending [root@k8s01-node01 ~]# kubectl get nodes No resources found.
6.2）通过CSR请求
[root@k8s01-node01 ~]# kubectl get csr NAME AGE REQUESTOR CONDITION node-csr-mTYmEsL6Eh5rKSJOgfO8trHBq8LHI1SX7QDr2OqJ2Zg 31m kubelet-bootstrap Pending [root@k8s01-node01 ~]# kubectl certificate approve node-csr-mTYmEsL6Eh5rKSJOgfO8trHBq8LHI1SX7QDr2OqJ2Zg # 认证通过的命令 certificatesigningrequest.certificates.k8s.io/node-csr-mTYmEsL6Eh5rKSJOgfO8trHBq8LHI1SX7QDr2OqJ2Zg approved [root@k8s01-node01 ~]# kubectl get csr NAME AGE REQUESTOR CONDITION node-csr-mTYmEsL6Eh5rKSJOgfO8trHBq8LHI1SX7QDr2OqJ2Zg 32m kubelet-bootstrap Approved,Issued [root@k8s01-node01 ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION 192.168.10.23 Ready <none> 3m15s v1.9.10
6.3）此时会生成一些秘钥
[root@k8s01-node01 ~]# ll /etc/kubernetes/ssl/kubelet* -rw-r--r-- 1 root root 1046 5月 28 19:57 /etc/kubernetes/ssl/kubelet-client.crt -rw------- 1 root root 227 5月 28 19:25 /etc/kubernetes/ssl/kubelet-client.key -rw-r--r-- 1 root root 1115 5月 28 19:25 /etc/kubernetes/ssl/kubelet.crt -rw------- 1 root root 1679 5月 28 19:25 /etc/kubernetes/ssl/kubelet.key
七、配置kube-proxy
1）创建kube-proxy 证书签名请求（master操作）
$ cat > kube-proxy-csr.json <<EOF
{
"CN": "system:kube-proxy",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
2）生成kube-proxy 客户端证书和私钥（master操作）
$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \ -ca-key=/etc/kubernetes/ssl/ca-key.pem \ -config=/etc/kubernetes/ssl/ca-config.json \ -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy $ ls kube-proxy* kube-proxy.csr kube-proxy-csr.json kube-proxy-key.pem kube-proxy.pem $ sudo mv kube-proxy*.pem /etc/kubernetes/ssl/
2.1）实际操作过程。将生成的秘钥发送给了node01
[root@master01 ssl]# cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
> -ca-key=/etc/kubernetes/ssl/ca-key.pem \
> -config=/etc/kubernetes/ssl/ca-config.json \
> -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
2019/05/28 20:14:58 [INFO] generate received request
2019/05/28 20:14:58 [INFO] received CSR
2019/05/28 20:14:58 [INFO] generating key: rsa-2048
2019/05/28 20:14:58 [INFO] encoded CSR
2019/05/28 20:14:58 [INFO] signed certificate with serial number 680983213307794519299992995799571930531865790014
2019/05/28 20:14:58 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@master01 ssl]# ls kube-proxy*
kube-proxy.csr kube-proxy-csr.json kube-proxy-key.pem kube-proxy.pem
[root@master01 ssl]# ls kube-proxy*.pem
kube-proxy-key.pem kube-proxy.pem
[root@master01 ssl]# scp kube-proxy*.pem root@n1:/etc/kubernetes/ssl/
kube-proxy-key.pem 100% 1675 931.9KB/s 00:00
kube-proxy.pem
在node01查看
[root@k8s01-node01 ~]# ll /etc/kubernetes/ssl/kube-proxy*.pem -rw------- 1 root root 1675 5月 28 20:27 /etc/kubernetes/ssl/kube-proxy-key.pem -rw-r--r-- 1 root root 1403 5月 28 20:27 /etc/kubernetes/ssl/kube-proxy.pem
3）创建kube-proxy kubeconfig 文件 （node01操作）
$ # 设置集群参数
$ kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-proxy.kubeconfig
$ # 设置客户端认证参数
$ kubectl config set-credentials kube-proxy \
--client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
--client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig
$ # 设置上下文参数
$ kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig
$ # 设置默认上下文
$ kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
$ mv kube-proxy.kubeconfig /etc/kubernetes/
实际操作过程
[root@k8s01-node01 ~]# kubectl config set-cluster kubernetes \
> --certificate-authority=/etc/kubernetes/ssl/ca.pem \
> --embed-certs=true \
> --server=${KUBE_APISERVER} \
> --kubeconfig=kube-proxy.kubeconfig
Cluster "kubernetes" set.
[root@k8s01-node01 ~]# kubectl config set-credentials kube-proxy \
> --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
> --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
> --embed-certs=true \
> --kubeconfig=kube-proxy.kubeconfig
User "kube-proxy" set.
[root@k8s01-node01 ~]# kubectl config set-context default \
> --cluster=kubernetes \
> --user=kube-proxy \
> --kubeconfig=kube-proxy.kubeconfig
Context "default" created.
[root@k8s01-node01 ~]# kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
Switched to context "default".
[root@k8s01-node01 ~]# ls kube-proxy.kubeconfig
kube-proxy.kubeconfig
[root@k8s01-node01 ~]# mv kube-proxy.kubeconfig /etc/kubernetes/
4）创建kube-proxy 的systemd unit 文件
4.1）命令行执行判断是否执行成功
/usr/k8s/bin/kube-proxy \
--bind-address=${NODE_IP} \
--hostname-override=${NODE_IP} \
--cluster-cidr=${SERVICE_CIDR} \
--kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \
--logtostderr=true \
--v=2
4.2）创建该文件
$ sudo mkdir -p /var/lib/kube-proxy # 必须先创建工作目录
$ cat > kube-proxy.service <<EOF
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/k8s/bin/kube-proxy \\
--bind-address=${NODE_IP} \\
--hostname-override=${NODE_IP} \\
--cluster-cidr=${SERVICE_CIDR} \\
--kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \\
--logtostderr=true \\
--v=2
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
5）设置开机自启动
[root@k8s01-node01 ~]# mv kube-proxy.service /etc/systemd/system
[root@k8s01-node01 ~]# systemctl daemon-reload
[root@k8s01-node01 ~]# systemctl enable kube-proxy
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /etc/systemd/system/kube-proxy.service.
[root@k8s01-node01 ~]# systemctl start kube-proxy
[root@k8s01-node01 ~]# systemctl status kube-proxy
● kube-proxy.service - Kubernetes Kube-Proxy Server
Loaded: loaded (/etc/systemd/system/kube-proxy.service; enabled; vendor preset: disabled)
Active: active (running) since 二 2019-05-28 20:42:37 CST; 12s ago
Docs: https://github.com/GoogleCloudPlatform/kubernetes
6）此时防火墙写入了新的规则
[root@k8s01-node01 ~]# iptables -L -n Chain INPUT (policy ACCEPT) target prot opt source destination KUBE-SERVICES all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes service portals */ KUBE-FIREWALL all -- 0.0.0.0/0 0.0.0.0/0 Chain FORWARD (policy ACCEPT) target prot opt source destination KUBE-FORWARD all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes forward rules */ DOCKER-USER all -- 0.0.0.0/0 0.0.0.0/0 DOCKER-ISOLATION-STAGE-1 all -- 0.0.0.0/0 0.0.0.0/0 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED DOCKER all -- 0.0.0.0/0 0.0.0.0/0 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 ACCEPT all -- 172.30.0.0/16 0.0.0.0/0 ACCEPT all -- 0.0.0.0/0 172.30.0.0/16 Chain OUTPUT (policy ACCEPT) target prot opt source destination KUBE-SERVICES all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes service portals */ KUBE-FIREWALL all -- 0.0.0.0/0 0.0.0.0/0 Chain DOCKER (1 references) target prot opt source destination Chain DOCKER-ISOLATION-STAGE-1 (1 references) target prot opt source destination DOCKER-ISOLATION-STAGE-2 all -- 0.0.0.0/0 0.0.0.0/0 RETURN all -- 0.0.0.0/0 0.0.0.0/0 Chain DOCKER-ISOLATION-STAGE-2 (1 references) target prot opt source destination DROP all -- 0.0.0.0/0 0.0.0.0/0 RETURN all -- 0.0.0.0/0 0.0.0.0/0 Chain DOCKER-USER (1 references) target prot opt source destination RETURN all -- 0.0.0.0/0 0.0.0.0/0 Chain KUBE-FIREWALL (2 references) target prot opt source destination DROP all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes firewall for dropping marked packets */ mark match 0x8000/0x8000 Chain KUBE-FORWARD (1 references) target prot opt source destination ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes forwarding rules */ mark match 0x4000/0x4000 ACCEPT all -- 10.254.0.0/16 0.0.0.0/0 /* kubernetes forwarding conntrack pod source rule */ ctstate RELATED,ESTABLISHED ACCEPT all -- 0.0.0.0/0 10.254.0.0/16 /* kubernetes forwarding conntrack pod destination rule */ ctstate RELATED,ESTABLISHED Chain KUBE-SERVICES (2 references) target prot opt source destination
7）对节点是否显示主机名的问题
[root@k8s01-node01 ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION 192.168.10.23 Ready <none> 50m v1.9.10 如果这里的名字想显示为主机名时： [root@k8s01-node01 ~]# cat /etc/systemd/system/kubelet.service |grep hostname --hostname-override=192.168.10.23 去掉该参数 [root@k8s01-node01 ~]# cat /etc/systemd/system/kube-proxy.service |grep hostname --hostname-override=192.168.10.23 去掉该参数
八、验证节点功能
1）定义yaml 文件：（将下面内容保存为：nginx-ds.yaml）
apiVersion: v1
kind: Service
metadata:
name: nginx-ds
labels:
app: nginx-ds
spec:
type: NodePort
selector:
app: nginx-ds
ports:
- name: http
port: 80
targetPort: 80
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: nginx-ds
labels:
addonmanager.kubernetes.io/mode: Reconcile
spec:
template:
metadata:
labels:
app: nginx-ds
spec:
containers:
- name: my-nginx
image: nginx:1.7.9
ports:
- containerPort: 80
2）创建该pods
[root@k8s01-node01 ~]# kubectl create -f nginx-ds.yaml service/nginx-ds created daemonset.extensions/nginx-ds created [root@k8s01-node01 ~]# kubectl get pods NAME READY STATUS RESTARTS AGE nginx-ds-txxnb 0/1 ContainerCreating 0 2m35s
kind: DaemonSet 参数是在每个节点启动一个pod
[root@k8s01-node01 ~]# kubectl get pods NAME READY STATUS RESTARTS AGE nginx-ds-txxnb 0/1 ContainerCreating 0 2m35s [root@k8s01-node01 ~]# kubectl describe pod nginx-ds-txxnb ....... Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal SuccessfulMountVolume 7m25s kubelet, 192.168.10.23 MountVolume.SetUp succeeded for volume "default-token-7l55q" Warning FailedCreatePodSandBox 15s (x16 over 7m10s) kubelet, 192.168.10.23 Failed create pod sandbox. [root@k8s01-node01 ~]# journalctl -u kubelet -f 更详细的查看报错 5月 28 21:05:29 k8s01-node01 kubelet[39148]: E0528 21:05:29.924991 39148 kuberuntime_manager.go:647] createPodSandbox for pod "nginx-ds-txxnb_default(004fdd27-8148-11e9-8809-000c29a2d5b5)" failed: rpc error: code = Unknown desc = failed pulling image "gcr.io/google_containers/pause-amd64:3.0": Error response from daemon: Get https://gcr.io/v2/: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
3）原因，从国外源无法拉取镜像。更改镜像源
原因，从国外源无法拉取镜像 [root@k8s01-node01 ~]# vim /etc/systemd/system/kubelet.service 修改镜像地址 ...... [Service] WorkingDirectory=/var/lib/kubelet ExecStart=/usr/k8s/bin/kubelet \ --fail-swap-on=false \ --cgroup-driver=cgroupfs \ --address=192.168.10.23 \ --hostname-override=192.168.10.23 \ --pod-infra-container-image=cnych/pause-amd64:3.0 \ # 新增内容 [root@k8s01-node01 ~]# systemctl daemon-reload [root@k8s01-node01 ~]# systemctl restart kubelet [root@k8s01-node01 ~]# docker images REPOSITORY TAG IMAGE ID CREATED SIZE cnych/pause-amd64 3.0 99e59f495ffa 3 years ago 747kB nginx 1.7.9 84581e99d807 4 years ago 91.7MB [root@k8s01-node01 ~]# kubectl get pods NAME READY STATUS RESTARTS AGE nginx-ds-txxnb 1/1 Running 0 14h
4）访问
[root@k8s01-node01 ~]# kubectl get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES nginx-ds-txxnb 1/1 Running 0 14h 172.30.23.2 192.168.10.23 <none> <none> [root@k8s01-node01 ~]# curl 172.30.23.2 [root@k8s01-node01 ~]# kubectl get svc NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes ClusterIP 10.254.0.1 <none> 443/TCP 38h nginx-ds NodePort 10.254.66.68 <none> 80:31717/TCP 14h [root@k8s01-node01 ~]# netstat -lntup|grep 31717 tcp6 0 0 :::31717 :::* LISTEN 41250/kube-proxy [root@k8s01-node01 ~]# curl 192.168.10.23:31717
九、部署kubedns 插件
1）下载，修改文件
wget https://raw.githubusercontent.com/kubernetes/kubernetes/v1.9.3/cluster/addons/dns/kube-dns.yaml.base cp kube-dns.yaml.base kube-dns.yaml
改好的文件
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Should keep target in cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml
# in sync with this file.
# __MACHINE_GENERATED_WARNING__
apiVersion: v1
kind: Service
metadata:
name: kube-dns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/name: "KubeDNS"
spec:
selector:
k8s-app: kube-dns
clusterIP: 10.254.0.2
ports:
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
protocol: TCP
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kube-dns
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kube-dns
namespace: kube-system
labels:
addonmanager.kubernetes.io/mode: EnsureExists
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: kube-dns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
# replicas: not specified here:
# 1. In order to make Addon Manager do not reconcile this replicas parameter.
# 2. Default is 1.
# 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
strategy:
rollingUpdate:
maxSurge: 10%
maxUnavailable: 0
selector:
matchLabels:
k8s-app: kube-dns
template:
metadata:
labels:
k8s-app: kube-dns
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
volumes:
- name: kube-dns-config
configMap:
name: kube-dns
optional: true
containers:
- name: kubedns
image: cnych/k8s-dns-kube-dns-amd64:1.14.8
resources:
# TODO: Set memory limits when we've profiled the container for large
# clusters, then set request = limit to keep this container in
# guaranteed class. Currently, this container falls into the
# "burstable" category so the kubelet doesn't backoff from restarting it.
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
livenessProbe:
httpGet:
path: /healthcheck/kubedns
port: 10054
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
httpGet:
path: /readiness
port: 8081
scheme: HTTP
# we poll on pod startup for the Kubernetes master service and
# only setup the /readiness HTTP server once that's available.
initialDelaySeconds: 3
timeoutSeconds: 5
args:
- --domain=cluster.local.
- --dns-port=10053
- --config-dir=/kube-dns-config
- --v=2
env:
- name: PROMETHEUS_PORT
value: "10055"
ports:
- containerPort: 10053
name: dns-local
protocol: UDP
- containerPort: 10053
name: dns-tcp-local
protocol: TCP
- containerPort: 10055
name: metrics
protocol: TCP
volumeMounts:
- name: kube-dns-config
mountPath: /kube-dns-config
- name: dnsmasq
image: cnych/k8s-dns-dnsmasq-nanny-amd64:1.14.8
livenessProbe:
httpGet:
path: /healthcheck/dnsmasq
port: 10054
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
args:
- -v=2
- -logtostderr
- -configDir=/etc/k8s/dns/dnsmasq-nanny
- -restartDnsmasq=true
- --
- -k
- --cache-size=1000
- --no-negcache
- --log-facility=-
- --server=/cluster.local/127.0.0.1#10053
- --server=/in-addr.arpa/127.0.0.1#10053
- --server=/ip6.arpa/127.0.0.1#10053
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
# see: https://github.com/kubernetes/kubernetes/issues/29055 for details
resources:
requests:
cpu: 150m
memory: 20Mi
volumeMounts:
- name: kube-dns-config
mountPath: /etc/k8s/dns/dnsmasq-nanny
- name: sidecar
image: cnych/k8s-dns-sidecar-amd64:1.14.8
livenessProbe:
httpGet:
path: /metrics
port: 10054
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
args:
- --v=2
- --logtostderr
- --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local.,5,SRV
- --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local.,5,SRV
ports:
- containerPort: 10054
name: metrics
protocol: TCP
resources:
requests:
memory: 20Mi
cpu: 10m
dnsPolicy: Default # Don't use cluster DNS.
serviceAccountName: kube-dns
对比区别
[root@k8s01-node01 ~]# diff kube-dns.yaml kube-dns.yaml.base 33c33 < clusterIP: 10.254.0.2 --- > clusterIP: __PILLAR__DNS__SERVER__ 97c97 < image: cnych/k8s-dns-kube-dns-amd64:1.14.8 --- > image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.8 127c127 < - --domain=cluster.local. --- > - --domain=__PILLAR__DNS__DOMAIN__. 148c148 < image: cnych/k8s-dns-dnsmasq-nanny-amd64:1.14.8 --- > image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.8 168c168 < - --server=/cluster.local/127.0.0.1#10053 --- > - --server=/__PILLAR__DNS__DOMAIN__/127.0.0.1#10053 187c187 < image: cnych/k8s-dns-sidecar-amd64:1.14.8 --- > image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.8 200,201c200,201 < - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local.,5,SRV < - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local.,5,SRV --- > - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.__PILLAR__DNS__DOMAIN__,5,SRV > - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.__PILLAR__DNS__DOMAIN__,5,SRV
2）启动查看运行状态
[root@k8s01-node01 ~]# kubectl create -f kube-dns.yaml service/kube-dns created serviceaccount/kube-dns created configmap/kube-dns created deployment.extensions/kube-dns created [root@k8s01-node01 ~]# kubectl get pods -n kube-system NAME READY STATUS RESTARTS AGE kube-dns-77d877c467-j5q2h 3/3 Running 0 44s [root@k8s01-node01 ~]# kubectl describe pod kube-dns-77d877c467-j5q2h -n kube-system [root@k8s01-node01 ~]# kubectl get svc -n kube-system NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kube-dns ClusterIP 10.254.0.2 <none> 53/UDP,53/TCP 2m21s
3）验证kubedns功能
3.1）新建一个Deployment
cat > my-nginx.yaml<<EOF
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: my-nginx
spec:
replicas: 2
template:
metadata:
labels:
run: my-nginx
spec:
containers:
- name: my-nginx
image: nginx:1.7.9
ports:
- containerPort: 80
EOF
启动服务
[root@k8s01-node01 ~]# kubectl create -f my-nginx.yaml deployment.extensions/my-nginx created [root@k8s01-node01 ~]# kubectl get pods NAME READY STATUS RESTARTS AGE my-nginx-56b48db847-945bx 1/1 Running 0 13s my-nginx-56b48db847-tdlrd 1/1 Running 0 13s
Expose 该Deployment，生成my-nginx 服务
[root@k8s01-node01 ~]# kubectl expose deploy my-nginx service/my-nginx exposed [root@k8s01-node01 ~]# kubectl get svc|grep my-nginx my-nginx ClusterIP 10.254.57.19 <none> 80/TCP 40s
3.2）创建另外一个Pod
然后创建另外一个Pod，查看/etc/resolv.conf是否包含kubelet配置的--cluster-dns 和--cluster-domain， 是否能够将服务my-nginx 解析到上面显示的CLUSTER-IP 10.254.57.19上
pod内容
$ cat > pod-nginx.yaml<<EOF
apiVersion: v1
kind: Pod
metadata:
name: nginx
spec:
containers:
- name: nginx
image: nginx:1.7.9
ports:
- containerPort: 80
EOF
创建pod，并验证dns服务
[root@k8s01-node01 ~]# kubectl create -f pod-nginx.yaml pod/nginx created [root@k8s01-node01 ~]# kubectl get pods NAME READY STATUS RESTARTS AGE my-nginx-56b48db847-945bx 1/1 Running 0 3m31s my-nginx-56b48db847-tdlrd 1/1 Running 0 3m31s nginx 1/1 Running 0 35s nginx-ds-txxnb 1/1 Running 0 37h [root@k8s01-node01 ~]# kubectl exec nginx -i -t -- /bin/bash root@nginx:/# cat /etc/resolv.conf nameserver 10.254.0.2 search default.svc.cluster.local. svc.cluster.local. cluster.local. options ndots:5
十、部署Dashboard 插件
1）下载kubernetes-dashboard.yaml文件，修改文件
wget https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml
修改的文件
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------- Dashboard Secret ------------------- #
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-certs
namespace: kube-system
type: Opaque
---
# ------------------- Dashboard Service Account ------------------- #
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
---
# ------------------- Dashboard Role & Role Binding ------------------- #
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kubernetes-dashboard-minimal
namespace: kube-system
rules:
# Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret.
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create"]
# Allow Dashboard to create 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["create"]
# Allow Dashboard to get, update and delete Dashboard exclusive secrets.
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"]
verbs: ["get", "update", "delete"]
# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
resourceNames: ["kubernetes-dashboard-settings"]
verbs: ["get", "update"]
# Allow Dashboard to get metrics from heapster.
- apiGroups: [""]
resources: ["services"]
resourceNames: ["heapster"]
verbs: ["proxy"]
- apiGroups: [""]
resources: ["services/proxy"]
resourceNames: ["heapster", "http:heapster:", "https:heapster:"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kubernetes-dashboard-minimal
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubernetes-dashboard-minimal
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kube-system
---
# ------------------- Dashboard Deployment ------------------- #
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
spec:
containers:
- name: kubernetes-dashboard
image: cnych/kubernetes-dashboard-amd64:v1.8.3
ports:
- containerPort: 8443
protocol: TCP
args:
- --auto-generate-certificates
# Uncomment the following line to manually specify Kubernetes API server Host
# If not specified, Dashboard will attempt to auto discover the API server and connect
# to it. Uncomment only if the default does not work.
# - --apiserver-host=http://my-address:port
volumeMounts:
- name: kubernetes-dashboard-certs
mountPath: /certs
# Create on-disk volume to store exec logs
- mountPath: /tmp
name: tmp-volume
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
volumes:
- name: kubernetes-dashboard-certs
secret:
secretName: kubernetes-dashboard-certs
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
---
# ------------------- Dashboard Service ------------------- #
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
type: NodePort
ports:
- port: 443
targetPort: 8443
selector:
k8s-app: kubernetes-dashboard
对比修改的内容
[root@k8s01-node01 ~]# diff kubernetes-dashboard.yaml kubernetes-dashboard.yaml.bak 112c112 < image: cnych/kubernetes-dashboard-amd64:v1.8.3 --- > image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1 158d157 < type: NodePort
2）启动服务
[root@k8s01-node01 ~]# kubectl create -f kubernetes-dashboard.yaml secret/kubernetes-dashboard-certs created serviceaccount/kubernetes-dashboard created role.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created deployment.apps/kubernetes-dashboard created service/kubernetes-dashboard created [root@k8s01-node01 ~]# kubectl get pods -n kube-system NAME READY STATUS RESTARTS AGE kube-dns-77d877c467-j5q2h 3/3 Running 0 27m kubernetes-dashboard-6857cb46c4-bd4s5 1/1 Running 0 35s [root@k8s01-node01 ~]# kubectl get svc -n kube-system|grep kubernetes-dashboard kubernetes-dashboard NodePort 10.254.207.13 <none> 443:31569/TCP 78s
3）火狐浏览器访问 https://192.168.10.23:31569/
4）创建身份认证文件。https://www.qikqiak.com/post/update-kubernetes-dashboard-more-secure/
[root@k8s01-node01 ~]# vim admin-sa.yaml
[root@k8s01-node01 ~]# cat admin-sa.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: admin
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: admin
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
创建该yaml文件
[root@k8s01-node01 ~]# kubectl create -f admin-sa.yaml clusterrolebinding.rbac.authorization.k8s.io/admin created serviceaccount/admin created
5）上面的admin用户创建完成后我们就可以获取到该用户对应的token了，如下命令：
$ kubectl get secret -n kube-system|grep admin-token
admin-token-d5jsg kubernetes.io/service-account-token 3 1d
$ kubectl get secret admin-token-d5jsg -o jsonpath={.data.token} -n kube-system |base64 -d
# 会生成一串很长的base64后的字符串
示例
[root@k8s01-node01 ~]# kubectl get secret -n kube-system|grep admin-token
admin-token-8nfs6 kubernetes.io/service-account-token 3 9m36s
[root@k8s01-node01 ~]# kubectl get secret admin-token-8nfs6 -o jsonpath={.data.token} -n kube-system |base64 -d
eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi10b2tlbi04bmZzNiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJhZG1pbiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjQ3Y2FlNzg2LTgyODctMTFlOS04ODA5LTAwMGMyOWEyZDViNSIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTphZG1pbiJ9.wLxu87Kn5B4cD3obq8u8kB0cNT49l6om2zWqP14S3KcCigqbzMsvQkf07EG6cots1l9xYomgZQhrNuKR3_xiaksoRaLeREs_JLAbCqXeSPg10XfqHvVVEHIgL6d3r5ujBZza2dOGjN8L4V6q38Xkt2rIfykqX1gB8x9EkVeseAc25IN3h5l7l0SWpuEE_nrY4FZc0FjXxPu5zczjelLFQsUHLf3MwCw1VPpyiQadv_oy9yUCyWXvveCEhHKY3scGHkEr8YZ0zsZYerTe16_ncQn2AKutkwrJWNdBFjF8O53izq6LJ7a2E5a-hVohLB9XJtXJYqlQj3BIPBYJVd_6zQ[root@k8s01-node01 ~]#
拷贝很长的字符串秘钥
6）部署Heapster插件 。测试存在问题
https://github.com/kubernetes-retired/heapster/releases wget https://github.com/kubernetes-retired/heapster/archive/v1.5.4.tar.gz
教学版本测试
wget https://raw.githubusercontent.com/kubernetes/dashboard/v1.8.2/src/deploy/recommended/kubernetes-dashboard.yaml
wget https://github.com/kubernetes-retired/heapster/archive/v1.5.1.tar.gz
1）修改镜像
[root@master ~]# cat heapster-1.5.1/deploy/kube-config/influxdb/grafana.yaml |grep image
image: cnych/heapster-grafana-amd64:v4.4.3
[root@master ~]# cat heapster-1.5.1/deploy/kube-config/influxdb/heapster.yaml |grep image
image: cnych/heapster-amd64:v1.5.1
imagePullPolicy: IfNotPresent
[root@master ~]# cat heapster-1.5.1/deploy/kube-config/influxdb/influxdb.yaml |grep image
image: cnych/heapster-influxdb-amd64:v1.3.3
2）调试
vim heapster-1.5.1/deploy/kube-config/influxdb/grafana.yaml
spec:
# In a production setup, we recommend accessing Grafana through an external Loadbalancer
# or through a public IP.
# type: LoadBalancer
# You could also use NodePort to expose the service at a randomly-generated port
type: NodePort # 去掉注释
[root@master dashboard]# vim kubernetes-dashboard.yaml
args:
- --auto-generate-certificates
- --heapster-host=http://heapster # 新增，前提是dashboard和heapster 在同一个名称空间
- --heapster-host=http://heapster.kube-system # 如果在其他名称空间
6.1）修改镜像地址，启动
[root@k8s01-node01 ~]# tar xf v1.5.4.tar.gz [root@k8s01-node01 kube-config]# pwd /root/heapster-1.5.4/deploy/kube-config [root@k8s01-node01 kube-config]# kubectl create -f rbac/heapster-rbac.yaml clusterrolebinding "heapster" created [root@k8s01-node01 kube-config]# kubectl apply -f influxdb/ # 需要修改镜像地址 deployment "monitoring-grafana" created service "monitoring-grafana" created serviceaccount "heapster" created deployment "heapster" created service "heapster" created deployment "monitoring-influxdb" created service "monitoring-influxdb" created
6.2）查看是否运行正常
[root@k8s01-node01 kube-config]# kubectl get deployments -n kube-system NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE heapster 1 1 1 0 1m kube-dns 1 1 1 1 8h kubernetes-dashboard 1 1 1 1 8h monitoring-grafana 1 1 1 0 1m monitoring-influxdb 1 1 1 0 1m [root@k8s01-node01 kube-config]# kubectl get pods -n kube-system NAME READY STATUS RESTARTS AGE heapster-b559f89b5-wfcz5 1/1 Running 0 38s kube-dns-77d877c467-j5q2h 3/3 Running 3 8h kubernetes-dashboard-6857cb46c4-bd4s5 1/1 Running 1 8h monitoring-grafana-5d48df8df5-hgcz9 1/1 Running 0 38s monitoring-influxdb-6d88965c6f-t7tlj 1/1 Running 0 38s
6.3）修改kubernetes-dashboard.yaml 配合Heapster
[root@k8s01-node01 ~]# vim kubernetes-dashboard.yaml .......... args: - --auto-generate-certificates - --heapster-host=http://heapster # 添加 [root@k8s01-node01 ~]# kubectl apply -f kubernetes-dashboard.yaml
查看效果
7）grafana改为type: NodePort
[root@k8s01-node01 kube-config]# vim influxdb/grafana.yaml
...........
spec:
# In a production setup, we recommend accessing Grafana through an external Loadbalancer
# or through a public IP.
# type: LoadBalancer
# You could also use NodePort to expose the service at a randomly-generated port
type: NodePort # 去掉注释
ports:
- port: 80
targetPort: 3000
7.1）查看默认生成的端口
[root@k8s01-node01 kube-config]# kubectl get svc -n kube-system NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE heapster ClusterIP 10.254.230.254 <none> 80/TCP 27m kube-dns ClusterIP 10.254.0.2 <none> 53/UDP,53/TCP 9h kubernetes-dashboard NodePort 10.254.20.231 <none> 443:30201/TCP 17m monitoring-grafana NodePort 10.254.87.101 <none> 80:31703/TCP 27m monitoring-influxdb ClusterIP 10.254.58.122 <none> 8086/TCP 27m
访问：http://192.168.10.23:31703
十一、部署traefik
1）简介
Traefik是一款开源的反向代理与负载均衡工具。它最大的优点是能够与常见的微服务系统直接整合，可以实现自动化动态配置。目前支持Docker、Swarm、Mesos/Marathon、 Mesos、Kubernetes、Consul、Etcd、Zookeeper、BoltDB、Rest API等等后端模型。
2）编辑文件
apiVersion: v1
kind: ServiceAccount
metadata:
name: ingress
namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: ingress
subjects:
- kind: ServiceAccount
name: ingress
namespace: kube-system
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
---
kind: ConfigMap
apiVersion: v1
metadata:
name: traefik-conf
namespace: kube-system
data:
traefik-config: |-
defaultEntryPoints = ["http"]
[entryPoints]
[entryPoints.http]
address = ":80"
---
kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
name: traefik-ingress
namespace: kube-system
labels:
k8s-app: traefik-ingress
spec:
template:
metadata:
labels:
k8s-app: traefik-ingress
name: traefik-ingress
spec:
terminationGracePeriodSeconds: 60
restartPolicy: Always
serviceAccountName: ingress
containers:
- image: traefik:1.5.3
name: traefik-ingress
ports:
- name: http
containerPort: 80
hostPort: 80
- name: https
containerPort: 443
hostPort: 443
- name: admin
containerPort: 8080
args:
- --configFile=/etc/traefik/traefik.toml
- -d
- --web
- --kubernetes
- --logLevel=DEBUG
volumeMounts:
- name: traefik-config-volume
mountPath: /etc/traefik
volumes:
- name: traefik-config-volume
configMap:
name: traefik-conf
items:
- key: traefik-config
path: traefik.toml
3）启动服务
[root@k8s01-node01 ~]# kubectl create -f traefik-daemonset.yaml serviceaccount "ingress" created clusterrolebinding "ingress" created configmap "traefik-conf" created daemonset "traefik-ingress" created [root@k8s01-node01 ~]# kubectl get pods -n kube-system|grep traefik traefik-ingress-frtnn 1/1 Running 0 37s
来源：https://www.cnblogs.com/linu/p/10905379.html