k8s集群搭建

资源规划

主机名	IP地址	网关/DNS	CPU/内存	磁盘	系统
k8s-master	172.199.0.201/10.0.0.201	172.199.0.1	2U/4G	100G	OracleLinux-R9
k8s-node1	172.199.0.202/10.0.0.202	172.199.0.1	2U/4G	100G	OracleLinux-R9
k8s-node2	172.199.0.203/10.0.0.203	172.199.0.1	2U/4G	100G	OracleLinux-R9
k8s-node3	172.199.0.204/10.0.0.204	172.199.0.1	2U/4G	100G	OracleLinux-R9
k8s-node4	172.199.0.205/10.0.0.205	172.199.0.1	2U/4G	100G	OracleLinux-R9

关闭防火墙

systemctl stop firewalld

systemctl disable firewalld

关闭selinux

sed -i 's/enforcing/disabled/' /etc/selinux/config  # 永久

setenforce 0  # 临时

关闭swap

swapoff -a  # 临时

sed -ri 's/.*swap.*/#&/' /etc/fstab    # 永久

根据规划设置主机名

# 在172.199.0.201执行
hostnamectl set-hostname k8s-master
# 在172.199.0.202执行
hostnamectl set-hostname k8s-node1
# 在172.199.0.203执行
hostnamectl set-hostname k8s-node2
# 在172.199.0.204执行
hostnamectl set-hostname k8s-node3
# 在172.199.0.205执行
hostnamectl set-hostname k8s-node4

添加hosts

cat >> /etc/hosts << EOF
172.199.0.201 k8s-master
172.199.0.202 k8s-node1
172.199.0.203 k8s-node2
172.199.0.204 k8s-node3
172.199.0.205 k8s-node4

10.0.0.201 k8s-master
10.0.0.202 k8s-node1
10.0.0.203 k8s-node2
10.0.0.204 k8s-node3
10.0.0.205 k8s-node4
EOF

将桥接的IPv4流量传递到iptables的链

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

# 设置所需的 sysctl 参数，参数在重新启动后保持不变
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

# 应用 sysctl 参数而不重新启动
sudo sysctl --system

lsmod | grep br_netfilter
lsmod | grep overlay

sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward

# 如果init时仍提示iptables错误请执行
echo "1">/proc/sys/net/bridge/bridge-nf-call-iptables
echo "1">/proc/sys/net/ipv4/ip_forward

时间同步

systemctl start chronyd.service && systemctl enable chronyd.service

安装Docker

# https://download.docker.com/linux/centos/docker-ce.repo
# 官方源可能都下不了，改成国内源
wget https://mirrors.aliyun.com/docker-ce/linux/rhel/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo

sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

systemctl enable docker && systemctl start docker

systemctl status docker

安装cri-dockerd

Github地址：https://github.com/Mirantis/cri-dockerd/releases

wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.14/cri-dockerd-0.3.14-3.el7.x86_64.rpm # 通过wget下载最新版cri-dockerd
wget https://yum.oracle.com/repo/OracleLinux/OL8/baseos/latest/x86_64/getPackage/libcgroup-0.41-19.el8.x86_64.rpm

安装rpm文件

rpm -ivh libcgroup-0.41-19.el8.x86_64.rpm cri-dockerd-0.3.14-3.el7.x86_64.rpm

重载系统守护进程→设置cri-dockerd自启动→启动cri-dockerd

systemctl daemon-reload # 重载系统守护进程
systemctl enable cri-docker.socket cri-docker # 设置cri-dockerd自启动
systemctl start cri-docker.socket cri-docker # 启动cri-dockerd
systemctl status cri-docker.socket
systemctl status cri-docker

安装Kubernetes

安装kubectl

通过命令行下载。

curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"

下载校验文件，验证 kubectl 的可执行文件。

curl -LO "https://dl.k8s.io/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl.sha256"

安装 kubectl。

sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

测试安装是否成功和安装的版本，安装成功会返回一个WARNING，这是正常的，只要版本输出是正确的就说明kubectl安装成功。

kubectl version --client

安装kubeadm

安装 kubeadm、kubelet 和 kubectl，配置yum文件

cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=1
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
        http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
exclude=kubelet kubeadm kubectl
EOF

执行安装

sudo yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes

设置kubelet自启动

sudo systemctl enable --now kubelet

搭建Kubernetes集群

安装runc，这是Kubernetes必须要的运行环境。这里下载之后上传到server。 下载：https://github.com/opencontainers/runc/releases/download/v1.2.0-rc.3/runc.amd64

install -m 755 runc.amd64 /usr/local/bin/runc # 安装runc
runc -v # 检查是否安装成功

Docker和cri-dockerd设置国内镜像加速。

sudo tee /etc/docker/daemon.json <<-'EOF'
{
    "registry-mirrors":[
    "https://docker.m.daocloud.io",
    "https://huecker.io",
    "https://dockerhub.timeweb.cloud",
    "https://noohub.ru",
    "https://b6ce57c867d045bob163be8658fd1438.mirror.swr.myhuaweicloud.com",
    "https://f2kfz0k0.mirror.aliyuncs.com",
    "https://registry.docker-cn.com",
    "http://hub-mirror.c.163.com",
    "https://docker.mirrors.ustc.edu.cn"
    ]
}
EOF

vim /usr/lib/systemd/system/cri-docker.service 

找到第10行ExecStart= 修改为

ExecStart=/usr/bin/cri-dockerd --network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.7

重启服务

systemctl daemon-reload && systemctl restart docker cri-docker.socket cri-docker # 重启Docker组件
systemctl status docker cir-docker.socket cri-docker # 检查Docker组件状态

初始化控制平面节点/Master

Master 初始化

--image-repository=registry.aliyuncs.com/google_containers # 将下载容器镜像源替换为阿里云，否则因为网络原因会导致镜像拉不下来，一定会执行不成功。
--cri-socket=unix:///var/run/cri-dockerd.sock # 这是指定容器运行时，因为containerd也是Docker的组件之一，下载Docker会一并将containerd下载下来，在执行初始化时当Kubernetes检测到有多个容器运行时环境，就必须要手动选择一个。这里也可以看出containerd实际上比Docker更轻量得多。
--apiserver-advertise-address=172.199.0.201 # 为API server设置广播地址，这里选择本机的ipv4地址，这里不希望API SERVER设置在其他node上的话就不要改为其他地址。
--pod-network-cidr=10.244.0.0/16 # 指明 pod 网络可以使用的 IP 地址段，暂时不清楚的可以先不管就用这个值。
--service-cidr=10.96.0.0/12 # 为服务的虚拟 IP 地址另外指定 IP 地址段，暂时不清楚的可以先不管就用这个值。

kubeadm init --node-name=k8s-master --image-repository=registry.aliyuncs.com/google_containers --cri-socket=unix:///var/run/cri-dockerd.sock --apiserver-advertise-address=172.199.0.201 --pod-network-cidr=10.244.0.0/16 --service-cidr=10.96.0.0/12

会输出：

W0927 09:07:47.727803  135512 version.go:104] could not fetch a Kubernetes version from the internet: unable to get URL "https://dl.k8s.io/release/stable-1.txt": Get "https://dl.k8s.io/release/stable-1.txt": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
W0927 09:07:47.727863  135512 version.go:105] falling back to the local client version: v1.28.2
[init] Using Kubernetes version: v1.28.2
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
W0927 09:08:11.994386  135512 checks.go:835] detected that the sandbox image "registry.aliyuncs.com/google_containers/pause:3.7" of the container runtime is inconsistent with that used by kubeadm. It is recommended that using "registry.aliyuncs.com/google_containers/pause:3.9" as the CRI sandbox image.
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [k8s-master kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 172.199.0.201]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [k8s-master localhost] and IPs [172.199.0.201 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [k8s-master localhost] and IPs [172.199.0.201 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 7.501235 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node k8s-master as control-plane by adding the labels: [node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node k8s-master as control-plane by adding the taints [node-role.kubernetes.io/control-plane:NoSchedule]
[bootstrap-token] Using token: 923psv.dushswc2z05xq03d
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 172.199.0.201:6443 --token 923psv.dushswc2z05xq03d \
        --discovery-token-ca-cert-hash sha256:ef46adb3e9cde714f30829fa6c69930abbcf1d80cf880de03e44c441ae155e1d

# 非root用户请执行
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# root用户直接执行
export KUBECONFIG=/etc/kubernetes/admin.conf # 临时生效，重启后失效，不推荐。
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >>  ~/.bash_profile # 永久生效，执行kubeadm reset后再次init也无需再次执行这条命令
source ~/.bash_profile # 执行永久生效命令之后需要source一下使其生效
echo $KUBECONFIG # 检测配置是否生效

安装配置网络插件——flannel

可以从Github地址下载：https://github.com/flannel-io/flannel/releases/tag/v0.22.0

vim kube-flannel.yml

apiVersion: v1
kind: Namespace
metadata:
  labels:
    k8s-app: flannel
    pod-security.kubernetes.io/enforce: privileged
  name: kube-flannel
---
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: flannel
  name: flannel
  namespace: kube-flannel
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    k8s-app: flannel
  name: flannel
rules:
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
- apiGroups:
  - networking.k8s.io
  resources:
  - clustercidrs
  verbs:
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:
    k8s-app: flannel
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-flannel
---
apiVersion: v1
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }
kind: ConfigMap
metadata:
  labels:
    app: flannel
    k8s-app: flannel
    tier: node
  name: kube-flannel-cfg
  namespace: kube-flannel
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  labels:
    app: flannel
    k8s-app: flannel
    tier: node
  name: kube-flannel-ds
  namespace: kube-flannel
spec:
  selector:
    matchLabels:
      app: flannel
      k8s-app: flannel
  template:
    metadata:
      labels:
        app: flannel
        k8s-app: flannel
        tier: node
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      containers:
      - args:
        - --ip-masq
        - --kube-subnet-mgr
        - --iface=ens18 # 双网卡需修改为指定网卡
        command:
        - /opt/bin/flanneld
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: EVENT_QUEUE_DEPTH
          value: "5000"
        image: docker.io/flannel/flannel:v0.22.0
        name: kube-flannel
        resources:
          requests:
            cpu: 100m
            memory: 50Mi
        securityContext:
          capabilities:
            add:
            - NET_ADMIN
            - NET_RAW
          privileged: false
        volumeMounts:
        - mountPath: /run/flannel
          name: run
        - mountPath: /etc/kube-flannel/
          name: flannel-cfg
        - mountPath: /run/xtables.lock
          name: xtables-lock
      hostNetwork: true
      initContainers:
      - args:
        - -f
        - /flannel
        - /opt/cni/bin/flannel
        command:
        - cp
        image: docker.io/flannel/flannel-cni-plugin:v1.1.2
        name: install-cni-plugin
        volumeMounts:
        - mountPath: /opt/cni/bin
          name: cni-plugin
      - args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        command:
        - cp
        image: docker.io/flannel/flannel:v0.22.0
        name: install-cni
        volumeMounts:
        - mountPath: /etc/cni/net.d
          name: cni
        - mountPath: /etc/kube-flannel/
          name: flannel-cfg
      priorityClassName: system-node-critical
      serviceAccountName: flannel
      tolerations:
      - effect: NoSchedule
        operator: Exists
      volumes:
      - hostPath:
          path: /run/flannel
        name: run
      - hostPath:
          path: /opt/cni/bin
        name: cni-plugin
      - hostPath:
          path: /etc/cni/net.d
        name: cni
      - configMap:
          name: kube-flannel-cfg
        name: flannel-cfg
      - hostPath:
          path: /run/xtables.lock
          type: FileOrCreate
        name: xtables-lock

最后apply kube-flannel.yml文件。

kubectl apply -f kube-flannel.yml # 为Kubernetes配置flannel网络插件

cat /run/flannel/subnet.env 

没有这个文件或文件夹的话则需要手动创建，内容同下

cat <<EOF | sudo tee /run/flannel/subnet.env 
FLANNEL_NETWORK=10.244.0.0/16
FLANNEL_SUBNET=10.244.0.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true
EOF

Node节点加入Master

在执行join指令前还需要将master节点上的/etc/kubernetes/admin.conf拷贝到node节点上，在局域网状态下，可以使用scp命令快速高效地完成两个节点之间的文件传输。

# 格式：scp ${current_server_path}/file_name target_server_ip:${target_server_path} 
scp /etc/kubernetes/admin.conf 172.199.0.202:/etc/kubernetes/
scp /etc/kubernetes/admin.conf 172.199.0.203:/etc/kubernetes/
scp /etc/kubernetes/admin.conf 172.199.0.204:/etc/kubernetes/
scp /etc/kubernetes/admin.conf 172.199.0.205:/etc/kubernetes/

# 到node节点检查admin.conf文件是否传输完成
cd /etc/kubernetes/
ls
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >>  ~/.bash_profile # 不要忘记将admin.conf加入环境变量，这里直接使用永久生效。
source ~/.bash_profile

kubeadm join命令加入集群

kubeadm join 172.199.0.201:6443 --token 923psv.dushswc2z05xq03d \
        --discovery-token-ca-cert-hash sha256:ef46adb3e9cde714f30829fa6c69930abbcf1d80cf880de03e44c441ae155e1d --cri-socket=unix:///var/run/cri-dockerd.sock

查看是否全部为Ready

kubectl get nodes
kubectl get pod -A