K8s部署前期准备 - 所有节点初始化系统hostscat/etc/hostsEOF 10.0.0.250 harbor.qltang.com 10.0.0.201 master201 10.0.0.202 master202 10.0.0.203 master203 10.0.0.204 worker204 EOF内核参数cat/etc/modules-load.d/k8s.confEOF overlay br_netfilter EOFmodprobe overlay modprobe br_netfiltercat/etc/security/limits.confEOF * soft nofile 655360 # 打开文件描述符的最大数量软限制655360 是一个较高的值确保容器和进程不会因为 fd 不足而被限制 * hard nofile 655360 # 打开文件描述符的最大数量硬限制硬限制 软限制两者设为相同保持配置一致性 * soft nproc 655350 # 最大进程/线程数软限制 * hard nproc 655350 # 最大进程/线程数硬限制 * soft memlock unlimited # 锁定内存大小限制软限制 unlimited 允许进程锁定任意大小的内存如 JVM、数据库等需要 * hard memlock unlimited # 锁定内存大小限制硬限制 EOFcat/etc/sysctl.d/k8s.confEOF #K8s必要 net.bridge.bridge-nf-call-iptables 1 # 允许 iptables 处理桥接网络的流量K8s 网络策略和 Service 转发所必需 net.bridge.bridge-nf-call-ip6tables 1 # 允许 ip6tables 处理桥接网络的 IPv6 流量 net.ipv4.ip_forward 1 # 开启 IPv4 数据包转发Pod 跨节点通信和 Service 访问所必需 fs.may_detach_mounts 1 # 允许在挂载点被占用时仍然可以分离挂载systemd 和容器运行时需要 #文件句柄与 inotify fs.file-max 52706963 # 系统级最大文件句柄数 fs.nr_open 52706963 # 单个进程最大文件句柄数 fs.inotify.max_user_watches 89100 # inotify 最大用户监控文件数容器化环境中监控大量文件时需要调大 # 网络调优 net.netfilter.nf_conntrack_max 2310720 # 连接跟踪表的最大条目数高并发节点需要调大否则新连接会被丢弃 net.core.somaxconn 16384 # 监听队列的最大长度处理大量突发连接避免 SYN 包被丢弃 net.ipv4.tcp_max_syn_backlog 16384 # SYN 队列的最大长度同上配合 somaxconn 使用 net.ipv4.tcp_max_tw_buckets 36000 # TIME_WAIT 状态的最大 socket 数量减少高并发短连接场景下的资源消耗 net.ipv4.tcp_tw_reuse 1 # 允许复用 TIME_WAIT 状态的 socket提升高并发下客户端的连接复用效率 net.ipv4.tcp_max_orphans 327680 # 最大孤儿 TCP 连接数防止内存被大量孤立连接耗尽 net.ipv4.tcp_orphan_retries 3 # 孤儿连接重试次数减少孤儿连接对资源的占用 net.ipv4.tcp_syncookies 1 # 开启 TCP SYN Cookie 防护防止 SYN Flood 攻击 net.ipv4.tcp_keepalive_time 600 # TCP 保活探测间隔600 秒检测一次空闲连接及时回收死连接 net.ipv4.tcp_keepalive_probes 3 # TCP 保活探测次数连续 3 次探测无响应则判定连接失效 net.ipv4.tcp_keepalive_intvl 15 # TCP 保活探测间隔每次探测间隔 15 秒 #内存与 OOM vm.overcommit_memory 0 # 内存超额分配策略0 表示内核根据当前负载智能决策1 表示始终允许过量分配 vm.panic_on_oom 0 # 内存耗尽时内核是否触发 panic0 表示不 panic启用 OOM Killer 处理 EOFsysctl--systemipvsadm# 1. 安装工具包apt-yinstallipvsadm ipset sysstat conntrack# 2. 配置开机自动加载模块cat/etc/modules-load.d/ipvs.confEOF ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_sed ip_vs_ftp nf_conntrack ip_tables ip_set xt_set ipt_set ipt_rpfilter ipt_REJECT ipip EOF# 3. 手动加载模块modprobe ip_vs modprobe ip_vs_wrr modprobe nf_conntrack modprobe ip_tables modprobe ip_set modprobe xt_set modprobe ipt_set modprobe ipip# 4. 验证加载结果lsmod|grep--colorauto-eip_vs-enf_conntrack-eipipcontainerd安装#安装必要系统工具apt-get-yinstallapt-transport-https ca-certificatescurlsoftware-properties-common#安装GPG证书curl-fsSLhttps://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg|apt-keyadd-#写入软件源信息add-apt-repositorydeb [archamd64] https://mirrors.aliyun.com/docker-ce/linux/ubuntu$(lsb_release-cs)stable#更新软件源apt-getupdate#安装containerd组件apt-cachemadison containerd.io#查看可用版本apt-getinstall-ycontainerd.io2.2.6-1~ubuntu.22.04~jammy#安装指定版本apt-mark hold containerd.io#锁定containerd版本#其他命令apt-mark unhold containerd.io#解除版本锁定apt-mark showhold#查看被锁定的软件#生成containerd配置文件containerd config default/etc/containerd/config.toml#将 containerd 的 cgroup 驱动从默认的 cgroupfs 改为 systemdsed-is/SystemdCgroup false/SystemdCgroup true//etc/containerd/config.tomlgrepSystemdCgroup /etc/containerd/config.toml SystemdCgrouptrue#修改pause的基础镜像名称sed-is#registry.k8s.io/pause:3.10.1#registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.10.1#/etc/containerd/config.tomlgrepsandbox /etc/containerd/config.toml#启动containerd服务systemctl daemon-reload systemctlenable--nowcontainerd systemctl status containerd ctr version#配置crictl客户端连接的运行时位置cat/etc/crictl.yamlEOF runtime-endpoint: unix:///run/containerd/containerd.sock image-endpoint: unix:///run/containerd/containerd.sock timeout: 10 debug: false EOF#crictl安装wgethttps://github.com/kubernetes-sigs/cri-tools/releases/download/v1.31.0/crictl-v1.31.0-linux-amd64.tar.gztarzxvf crictl-v1.31.0-linux-amd64.tar.gz-C/usr/local/bin crictl version#镜像加速rootmaster201:~# cat /etc/containerd/config.toml | grep -B 5 config_path[plugins.io.containerd.cri.v1.images.pinned_images]sandboxregistry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.10.1[plugins.io.containerd.cri.v1.images.registry]config_path/etc/containerd/certs.dmkdir-p/etc/containerd/certs.d/docker.iocat/etc/containerd/certs.d/docker.io/hosts.toml serverhttps://registry-1.docker.io[host.https://docker.1ms.run]capabilities[pull,resolve][host.https://dockerproxy.net]capabilities[pull,resolve][host.https://proxy.vvvv.ee]capabilities[pull,resolve][host.https://dockerproxy.link]capabilities[pull,resolve]#私有harbormkdir-p/etc/containerd/certs.d/harbor.qltang.com rootmaster201:~# cat /etc/containerd/certs.d/harbor.qltang.com/hosts.tomlserverhttps://harbor.qltang.com[host.https://harbor.qltang.com]capabilities[pull,resolve,push]skip_verifytruesystemctl restart containerd#测试crictl pull docker.io/library/redis:7-alpine Image is up todateforsha256:2a51817f79255c8b69f86a974459c2e0359aff81417d80158f2b9e541e6f4b33 crictl pull harbor.qltang.com/linux/alpine:3.20.2 Image is up todateforsha256:324bc02ae1231fd9255658c128086395d3fa0aedd5a41ab6b034fd649d1a9260 crictl imageslsIMAGE TAG IMAGE ID SIZE docker.io/library/redis7-alpine 2a51817f7925516.3MB harbor.qltang.com/linux/alpine3.20.2 324bc02ae12313.62MBk8s组件安装#添加密钥mkdir-p-m755/etc/apt/keyringscurl-fsSLhttps://mirrors.aliyun.com/kubernetes-new/core/stable/v1.35/deb/Release.key|gpg--dearmor-o/etc/apt/keyrings/kubernetes-apt-keyring.gpg# 写入 v1.35 源echodeb [signed-by/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.35/deb/ /|tee/etc/apt/sources.list.d/kubernetes.listapt-getupdateVERSION1.35.0-1.1apt-getinstall-ykubelet${VERSION}kubeadm${VERSION}kubectl${VERSION}# 锁定版本apt-mark hold kubelet kubeadm kubectl systemctlenable--nowkubelethaproxykeepaliveapt-getinstall-yhaproxy keepalived#haproxycat/etc/haproxy/haproxy.cfgEOF global maxconn 2000 ulimit-n 16384 log 127.0.0.1 local0 err stats timeout 30s defaults log global mode tcp option tcplog timeout connect 5000ms timeout client 50000ms timeout server 50000ms # HAProxy 监控与探活接口 (端口 9999) frontend monitor-haproxy bind *:9999 mode http option httplog monitor-uri /ruok # K8s API Server 代理入口 (VIP:8443 - Master:6443) frontend k8s-master-frontend bind 0.0.0.0:8443 mode tcp option tcplog tcp-request inspect-delay 5s default_backend k8s-master-backend # 后端三台 Master API Server backend k8s-master-backend mode tcp option tcplog option tcp-check balance roundrobin default-server inter 3s downinter 2s rise 2 fall 3 maxconn 250 maxqueue 256 weight 100 server master201 10.0.0.201:6443 check server master202 10.0.0.202:6443 check server master203 10.0.0.203:6443 check EOFsystemctl restart haproxy systemctlenablehaproxy#keepalive#master201cat/etc/keepalived/keepalived.confEOF global_defs { router_id master201 } vrrp_script chk_haproxy { script /etc/keepalived/check_port.sh 8443 interval 2 weight -20 } vrrp_instance VI_1 { state MASTER interface eth0 #本机网卡名 virtual_router_id 51 priority 100 # 主节点最高优先级 advert_int 1 mcast_src_ip 10.0.0.201 #真实ip authentication { auth_type PASS auth_pass 123456 } track_script { chk_haproxy } virtual_ipaddress { 10.0.0.200/24 # 集群 VIP } } EOF#master202cat/etc/keepalived/keepalived.confEOF global_defs { router_id master202 } vrrp_script chk_haproxy { script /etc/keepalived/check_port.sh 8443 interval 2 weight -20 } vrrp_instance VI_1 { state BACKUP interface eth0 virtual_router_id 51 priority 99 advert_int 1 mcast_src_ip 10.0.0.202 authentication { auth_type PASS auth_pass 123456 } track_script { chk_haproxy } virtual_ipaddress { 10.0.0.200/24 } } EOF#master203cat/etc/keepalived/keepalived.confEOF global_defs { router_id master203 } vrrp_script chk_haproxy { script /etc/keepalived/check_port.sh 8443 interval 2 weight -20 } vrrp_instance VI_1 { state BACKUP interface eth0 virtual_router_id 51 priority 98 advert_int 1 mcast_src_ip 10.0.0.203 authentication { auth_type PASS auth_pass 123456 } track_script { chk_haproxy } virtual_ipaddress { 10.0.0.200/24 } } EOFcat/etc/keepalived/check_port.shEOF #!/bin/bash CHK_PORT$1 if [ -n $CHK_PORT ]; then #匹配监听端口 PORT_PROCESS$(ss -ltn | grep -E :$CHK_PORT\b | wc -l) if [ $PORT_PROCESS -eq 0 ]; then echo $CHK_PORT异常 exit 1 # 端口异常 else exit 0 # 端口正常 fi else echo 端口不能为空 exit 1 fi EOFchmodx /etc/keepalived/check_port.sh systemctl--nowenablekeepalivedcurlhttp://10.0.0.200:9999/ruokhtmlbodyh1200OK/h1Service ready./body/html初始化集群catkubeadm-config.yamlEOF apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration kubernetesVersion: v1.35.0 controlPlaneEndpoint: 10.0.0.200:8443 #设置阿里云镜像仓库 imageRepository: registry.aliyuncs.com/google_containers networking: podSubnet: 10.244.0.0/16 serviceSubnet: 10.96.0.0/12 EOFkubeadm config images list--configkubeadm-config.yaml#查看需要拉去的镜像kubeadm config images pull--configkubeadm-config.yaml#预先拉取镜像kubeadm init--configkubeadm-config.yaml --upload-certs#初始化集群... To start using your cluster, you need to run the following as a regular user:mkdir-p$HOME/.kubesudocp-i/etc/kubernetes/admin.conf$HOME/.kube/configsudochown$(id-u):$(id-g)$HOME/.kube/config Alternatively,ifyou are the root user, you can run:exportKUBECONFIG/etc/kubernetes/admin.conf You should now deploy a pod network to the cluster. Runkubectl apply -f [podnetwork].yamlwith one of the options listed at: https://kubernetes.io/docs/concepts/cluster-administration/addons/ You can nowjoinany number of control-plane nodes running the followingcommandon each as root:#master节点kubeadmjoin10.0.0.200:8443--token58ou8b.wsq5w636x8fnyz7d\--discovery-token-ca-cert-hash sha256:a45a627d01bd609db7303ab74f50446fd7b202184649986d6e0a923d8fe3c969\--control-plane --certificate-key 812c0917d59165ff3cbd93128a41cccf0b2d93e82b08121609f1acc136c0eba7 Please note that the certificate-key gives access to cluster sensitive data, keep it secret!As a safeguard, uploaded-certs will be deletedintwo hours;If necessary, you can usekubeadm init phase upload-certs --upload-certsto reload certs afterward. Then you canjoinany number of worker nodes by running the following on each as root:#worker节点kubeadmjoin10.0.0.200:8443--token58ou8b.wsq5w636x8fnyz7d\--discovery-token-ca-cert-hash sha256:a45a627d01bd609db7303ab74f50446fd7b202184649986d6e0a923d8fe3c969#master201mkdir-p$HOME/.kubesudocp-i/etc/kubernetes/admin.conf$HOME/.kube/configsudochown$(id-u):$(id-g)$HOME/.kube/config#master202和master203kubeadmjoin10.0.0.200:8443--token58ou8b.wsq5w636x8fnyz7d\--discovery-token-ca-cert-hash sha256:a45a627d01bd609db7303ab74f50446fd7b202184649986d6e0a923d8fe3c969\--control-plane --certificate-key 812c0917d59165ff3cbd93128a41cccf0b2d93e82b08121609f1acc136c0eba7mkdir-p$HOME/.kubesudocp-i/etc/kubernetes/admin.conf$HOME/.kube/configsudochown$(id-u):$(id-g)$HOME/.kube/config#worker204kubeadmjoin10.0.0.200:8443--token58ou8b.wsq5w636x8fnyz7d\--discovery-token-ca-cert-hash sha256:a45a627d01bd609db7303ab74f50446fd7b202184649986d6e0a923d8fe3c969 rootmaster201:~# kubectl get nodes -o wideNAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME master201 NotReady control-plane 8m26s v1.35.010.0.0.201noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 master202 NotReady control-plane 2m30s v1.35.010.0.0.202noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 master203 NotReady control-plane 43s v1.35.010.0.0.203noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 worker204 NotReadynone26s v1.35.010.0.0.204noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6calicokubectl apply-fhttps://ghfast.top/https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/calico.yaml kubectl get pods-nkube-system-w#等待如果有镜像拉取问题手动拉镜像rootmaster201:~# kubectl get nodes -o wideNAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME master201 Ready control-plane 17m v1.35.010.0.0.201noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 master202 Ready control-plane 11m v1.35.010.0.0.202noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 master203 Ready control-plane 9m21s v1.35.010.0.0.203noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 worker204 Readynone9m4s v1.35.010.0.0.204noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6#测试dns解析kubectl run-it--rmdebug--imageharbor.qltang.com/linux/busybox:qltangv1--restartNever --nslookupkube-dns.kube-system.svc.cluster.local Server:10.96.0.10 Address:10.96.0.10:53 Name: kube-dns.kube-system.svc.cluster.local Address:10.96.0.10 poddebugdeleted from default namespace kubectl run-it--rmdebug--imageharbor.qltang.com/linux/busybox:qltangv1--restartNever --nslookupbaidu.com Server:10.96.0.10 Address:10.96.0.10:53 Non-authoritative answer: Name: baidu.com Address:111.63.65.103 Name: baidu.com Address:111.63.65.247 Name: baidu.com Address:110.242.74.102 Name: baidu.com Address:124.237.177.164 Non-authoritative answer: poddebugdeleted from default namespace#测试高可用rootmaster201:~# systemctl stop haproxy.service #停止haprootmaster201:~# ip a...2: eth0:BROADCAST,MULTICAST,UP,LOWER_UPmtu1500qdisc fq_codel state UP group default qlen1000link/ether 00:0c:29:09:59:e0 brd ff:ff:ff:ff:ff:ff altname enp2s1 altname ens33 inet10.0.0.201/24 brd10.0.0.255 scope global eth0#节点已漂移valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe09:59e0/64 scopelinkvalid_lft forever preferred_lft forevercurlhttp://10.0.0.200:9999/ruokhtmlbodyh1200OK/h1Service ready./body/htmlrootmaster202:~# ip a...2: eth0:BROADCAST,MULTICAST,UP,LOWER_UPmtu1500qdisc fq_codel state UP group default qlen1000link/ether 00:0c:29:9c:01:51 brd ff:ff:ff:ff:ff:ff altname enp2s1 altname ens33 inet10.0.0.202/24 brd10.0.0.255 scope global eth0 valid_lft forever preferred_lft forever inet10.0.0.200/24 scope global secondary eth0 valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe9c:151/64 scopelinkvalid_lft forever preferred_lft forever rootmaster201:~# kubectl get nodes -o wideNAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME master201 Ready control-plane 27m v1.35.010.0.0.201noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 master202 Ready control-plane 21m v1.35.010.0.0.202noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 master203 Ready control-plane 19m v1.35.010.0.0.203noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 worker204 Readynone19m v1.35.010.0.0.204noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6helmcurlhttps://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3|HELM_INSTALL_DIR/usr/local/binbashhelm version version.BuildInfo{Version:v3.21.3, GitCommit:1ad6e68924fdf6fb0c7dcef8e9e1dfc0f36eaed6, GitTreeState:clean, GoVersion:go1.26.5}helm repoaddprometheus-community https://prometheus-community.github.io/helm-charts#得挂梯子helm repoaddaliyun https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts helm repo update rootmaster201:~# helm repo listNAME URL aliyun https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts prometheus-community https://prometheus-community.github.io/helm-chartsMetalLB#修改ipvs模式kubectl edit configmap-nkube-system kube-proxy... ipvs:... strictARP:truemode:kubectl rollout restart daemonset kube-proxy-nkube-system kubectl rollout status daemonset kube-proxy-nkube-system rootmaster201:~# ipvsadm -lnIP Virtual Server version1.2.1(size4096)Prot LocalAddress:Port Scheduler Flags -RemoteAddress:Port Forward Weight ActiveConn InActConn TCP10.96.0.1:443 rr -10.0.0.201:6443 Masq100-10.0.0.202:6443 Masq100-10.0.0.203:6443 Masq110TCP10.96.0.10:53 rr -10.244.61.70:53 Masq100-10.244.61.72:53 Masq100TCP10.96.0.10:9153 rr -10.244.61.70:9153 Masq100-10.244.61.72:9153 Masq100UDP10.96.0.10:53 rr -10.244.61.70:53 Masq100-10.244.61.72:53 Masq100kubectl apply-fhttps://raw.githubusercontent.com/metallb/metallb/v0.14.8/config/manifests/metallb-native.yaml kubectl get pods-nmetallb-system-wcatmetallb-pool.yamlEOF apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: name: first-pool namespace: metallb-system spec: addresses: - 10.0.0.50-10.0.0.150 --- apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: name: l2-advertisement namespace: metallb-system spec: ipAddressPools: - first-pool EOFcatweb.htmlEOF apiVersion: apps/v1 kind: Deployment metadata: name: web labels: app: web spec: replicas: 3 selector: matchLabels: app: web template: metadata: labels: app: web spec: # imagePullSecrets: # - name: registry-pull-secret containers: - name: web image: harbor.qltang.com/study/web:20 imagePullPolicy: Always ports: - containerPort: 80 livenessProbe: httpGet: path: /health port: 80 initialDelaySeconds: 5 periodSeconds: 10 readinessProbe: httpGet: path: /health port: 80 initialDelaySeconds: 3 periodSeconds: 5 --- apiVersion: v1 kind: Service metadata: name: web-svc labels: app: web spec: type: LoadBalancer ports: - port: 80 targetPort: 80 selector: app: web EOFkubectl apply-fweb.html rootmaster201:~/k8s/web# kubectl get pods,svc -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES pod/web-8456f58b9c-6nbfn1/1 Running056s10.244.61.74 worker204nonenonepod/web-8456f58b9c-8kc5g1/1 Running056s10.244.61.76 worker204nonenonepod/web-8456f58b9c-w26jf1/1 Running056s10.244.61.75 worker204nonenoneNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S)AGE SELECTOR service/kubernetes ClusterIP10.96.0.1none443/TCP 117mnoneservice/web-svc LoadBalancer10.102.198.5010.0.0.5080:32758/TCP 56sappwebingresshelm repoaddingress-nginx https://kubernetes.github.io/ingress-nginx helm repo update helminstallingress-nginx ingress-nginx/ingress-nginx\--namespaceingress-nginx --create-namespace\--setcontroller.service.typeLoadBalancer rootmaster201:~/k8s/web# kubectl get svc -n ingress-nginx.NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S)AGE ingress-nginx-controller LoadBalancer10.99.184.4510.0.0.5080:31185/TCP,443:32033/TCP 2m5s ingress-nginx-controller-admission ClusterIP10.96.15.111none443/TCP 2m5scatweb-ingress.htmlEOF apiVersion: apps/v1 kind: Deployment metadata: namespace: default name: web labels: app: web spec: replicas: 3 selector: matchLabels: app: web template: metadata: labels: app: web spec: # imagePullSecrets: # - name: registry-pull-secret containers: - name: web image: harbor.qltang.com/study/web:20 imagePullPolicy: Always ports: - containerPort: 80 livenessProbe: httpGet: path: /health port: 80 initialDelaySeconds: 5 periodSeconds: 10 readinessProbe: httpGet: path: /health port: 80 initialDelaySeconds: 3 periodSeconds: 5 --- apiVersion: v1 kind: Service metadata: name: web-svc namespace: default labels: app: web spec: type: ClusterIP ports: - port: 80 targetPort: 80 selector: app: web --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: web-ingress namespace: default annotations: kubernetes.io/ingress.class: nginx spec: ingressClassName: nginx rules: - host: web.qltang.com http: paths: - path: / pathType: Prefix backend: service: name: web-svc port: number: 80 EOFrootmaster201:~/k8s/web# kubectl get pods,svc,ingress -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES pod/web-8456f58b9c-kb84r1/1 Running04m31s10.244.61.82 worker204nonenonepod/web-8456f58b9c-l2sdg1/1 Running04m31s10.244.61.84 worker204nonenonepod/web-8456f58b9c-n4zh21/1 Running04m31s10.244.61.83 worker204nonenoneNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S)AGE SELECTOR service/kubernetes ClusterIP10.96.0.1none443/TCP 142mnoneservice/web-svc ClusterIP10.110.120.176none80/TCP 4m31sappweb NAME CLASS HOSTS ADDRESS PORTS AGE ingress.networking.k8s.io/web-ingress nginx web.qltang.com10.0.0.50804m31scurl-I-s-HHost: web.qltang.comhttp://10.0.0.50 HTTP/1.1200OK Content-Length:29627Accept-Ranges: bytes Connection: keep-alive Content-Type: text/html Date: Sat, 08 Aug202613:41:42 GMT Etag:6a618408-73bbKeep-Alive:timeout4Last-Modified: Thu,23Jul202603:01:28 GMT Proxy-Connection: keep-alivecurl-I-s-HHost: web.qltang.comhttp://10.0.0.50/health HTTP/1.1200OK Content-Length:2Connection: keep-alive Content-Type: text/plain Date: Sat, 08 Aug202613:47:42 GMT Keep-Alive:timeout4Proxy-Connection: keep-aliveprometheus-operator - 未完待续部署kubectl create namespace monitoring helm pull prometheus-community/kube-prometheus-stack--untar#下载并解压chart包 基于高可用k8s的kube-prometheus监控
K8s部署前期准备 - 所有节点初始化系统hostscat/etc/hostsEOF 10.0.0.250 harbor.qltang.com 10.0.0.201 master201 10.0.0.202 master202 10.0.0.203 master203 10.0.0.204 worker204 EOF内核参数cat/etc/modules-load.d/k8s.confEOF overlay br_netfilter EOFmodprobe overlay modprobe br_netfiltercat/etc/security/limits.confEOF * soft nofile 655360 # 打开文件描述符的最大数量软限制655360 是一个较高的值确保容器和进程不会因为 fd 不足而被限制 * hard nofile 655360 # 打开文件描述符的最大数量硬限制硬限制 软限制两者设为相同保持配置一致性 * soft nproc 655350 # 最大进程/线程数软限制 * hard nproc 655350 # 最大进程/线程数硬限制 * soft memlock unlimited # 锁定内存大小限制软限制 unlimited 允许进程锁定任意大小的内存如 JVM、数据库等需要 * hard memlock unlimited # 锁定内存大小限制硬限制 EOFcat/etc/sysctl.d/k8s.confEOF #K8s必要 net.bridge.bridge-nf-call-iptables 1 # 允许 iptables 处理桥接网络的流量K8s 网络策略和 Service 转发所必需 net.bridge.bridge-nf-call-ip6tables 1 # 允许 ip6tables 处理桥接网络的 IPv6 流量 net.ipv4.ip_forward 1 # 开启 IPv4 数据包转发Pod 跨节点通信和 Service 访问所必需 fs.may_detach_mounts 1 # 允许在挂载点被占用时仍然可以分离挂载systemd 和容器运行时需要 #文件句柄与 inotify fs.file-max 52706963 # 系统级最大文件句柄数 fs.nr_open 52706963 # 单个进程最大文件句柄数 fs.inotify.max_user_watches 89100 # inotify 最大用户监控文件数容器化环境中监控大量文件时需要调大 # 网络调优 net.netfilter.nf_conntrack_max 2310720 # 连接跟踪表的最大条目数高并发节点需要调大否则新连接会被丢弃 net.core.somaxconn 16384 # 监听队列的最大长度处理大量突发连接避免 SYN 包被丢弃 net.ipv4.tcp_max_syn_backlog 16384 # SYN 队列的最大长度同上配合 somaxconn 使用 net.ipv4.tcp_max_tw_buckets 36000 # TIME_WAIT 状态的最大 socket 数量减少高并发短连接场景下的资源消耗 net.ipv4.tcp_tw_reuse 1 # 允许复用 TIME_WAIT 状态的 socket提升高并发下客户端的连接复用效率 net.ipv4.tcp_max_orphans 327680 # 最大孤儿 TCP 连接数防止内存被大量孤立连接耗尽 net.ipv4.tcp_orphan_retries 3 # 孤儿连接重试次数减少孤儿连接对资源的占用 net.ipv4.tcp_syncookies 1 # 开启 TCP SYN Cookie 防护防止 SYN Flood 攻击 net.ipv4.tcp_keepalive_time 600 # TCP 保活探测间隔600 秒检测一次空闲连接及时回收死连接 net.ipv4.tcp_keepalive_probes 3 # TCP 保活探测次数连续 3 次探测无响应则判定连接失效 net.ipv4.tcp_keepalive_intvl 15 # TCP 保活探测间隔每次探测间隔 15 秒 #内存与 OOM vm.overcommit_memory 0 # 内存超额分配策略0 表示内核根据当前负载智能决策1 表示始终允许过量分配 vm.panic_on_oom 0 # 内存耗尽时内核是否触发 panic0 表示不 panic启用 OOM Killer 处理 EOFsysctl--systemipvsadm# 1. 安装工具包apt-yinstallipvsadm ipset sysstat conntrack# 2. 配置开机自动加载模块cat/etc/modules-load.d/ipvs.confEOF ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_sed ip_vs_ftp nf_conntrack ip_tables ip_set xt_set ipt_set ipt_rpfilter ipt_REJECT ipip EOF# 3. 手动加载模块modprobe ip_vs modprobe ip_vs_wrr modprobe nf_conntrack modprobe ip_tables modprobe ip_set modprobe xt_set modprobe ipt_set modprobe ipip# 4. 验证加载结果lsmod|grep--colorauto-eip_vs-enf_conntrack-eipipcontainerd安装#安装必要系统工具apt-get-yinstallapt-transport-https ca-certificatescurlsoftware-properties-common#安装GPG证书curl-fsSLhttps://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg|apt-keyadd-#写入软件源信息add-apt-repositorydeb [archamd64] https://mirrors.aliyun.com/docker-ce/linux/ubuntu$(lsb_release-cs)stable#更新软件源apt-getupdate#安装containerd组件apt-cachemadison containerd.io#查看可用版本apt-getinstall-ycontainerd.io2.2.6-1~ubuntu.22.04~jammy#安装指定版本apt-mark hold containerd.io#锁定containerd版本#其他命令apt-mark unhold containerd.io#解除版本锁定apt-mark showhold#查看被锁定的软件#生成containerd配置文件containerd config default/etc/containerd/config.toml#将 containerd 的 cgroup 驱动从默认的 cgroupfs 改为 systemdsed-is/SystemdCgroup false/SystemdCgroup true//etc/containerd/config.tomlgrepSystemdCgroup /etc/containerd/config.toml SystemdCgrouptrue#修改pause的基础镜像名称sed-is#registry.k8s.io/pause:3.10.1#registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.10.1#/etc/containerd/config.tomlgrepsandbox /etc/containerd/config.toml#启动containerd服务systemctl daemon-reload systemctlenable--nowcontainerd systemctl status containerd ctr version#配置crictl客户端连接的运行时位置cat/etc/crictl.yamlEOF runtime-endpoint: unix:///run/containerd/containerd.sock image-endpoint: unix:///run/containerd/containerd.sock timeout: 10 debug: false EOF#crictl安装wgethttps://github.com/kubernetes-sigs/cri-tools/releases/download/v1.31.0/crictl-v1.31.0-linux-amd64.tar.gztarzxvf crictl-v1.31.0-linux-amd64.tar.gz-C/usr/local/bin crictl version#镜像加速rootmaster201:~# cat /etc/containerd/config.toml | grep -B 5 config_path[plugins.io.containerd.cri.v1.images.pinned_images]sandboxregistry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.10.1[plugins.io.containerd.cri.v1.images.registry]config_path/etc/containerd/certs.dmkdir-p/etc/containerd/certs.d/docker.iocat/etc/containerd/certs.d/docker.io/hosts.toml serverhttps://registry-1.docker.io[host.https://docker.1ms.run]capabilities[pull,resolve][host.https://dockerproxy.net]capabilities[pull,resolve][host.https://proxy.vvvv.ee]capabilities[pull,resolve][host.https://dockerproxy.link]capabilities[pull,resolve]#私有harbormkdir-p/etc/containerd/certs.d/harbor.qltang.com rootmaster201:~# cat /etc/containerd/certs.d/harbor.qltang.com/hosts.tomlserverhttps://harbor.qltang.com[host.https://harbor.qltang.com]capabilities[pull,resolve,push]skip_verifytruesystemctl restart containerd#测试crictl pull docker.io/library/redis:7-alpine Image is up todateforsha256:2a51817f79255c8b69f86a974459c2e0359aff81417d80158f2b9e541e6f4b33 crictl pull harbor.qltang.com/linux/alpine:3.20.2 Image is up todateforsha256:324bc02ae1231fd9255658c128086395d3fa0aedd5a41ab6b034fd649d1a9260 crictl imageslsIMAGE TAG IMAGE ID SIZE docker.io/library/redis7-alpine 2a51817f7925516.3MB harbor.qltang.com/linux/alpine3.20.2 324bc02ae12313.62MBk8s组件安装#添加密钥mkdir-p-m755/etc/apt/keyringscurl-fsSLhttps://mirrors.aliyun.com/kubernetes-new/core/stable/v1.35/deb/Release.key|gpg--dearmor-o/etc/apt/keyrings/kubernetes-apt-keyring.gpg# 写入 v1.35 源echodeb [signed-by/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.35/deb/ /|tee/etc/apt/sources.list.d/kubernetes.listapt-getupdateVERSION1.35.0-1.1apt-getinstall-ykubelet${VERSION}kubeadm${VERSION}kubectl${VERSION}# 锁定版本apt-mark hold kubelet kubeadm kubectl systemctlenable--nowkubelethaproxykeepaliveapt-getinstall-yhaproxy keepalived#haproxycat/etc/haproxy/haproxy.cfgEOF global maxconn 2000 ulimit-n 16384 log 127.0.0.1 local0 err stats timeout 30s defaults log global mode tcp option tcplog timeout connect 5000ms timeout client 50000ms timeout server 50000ms # HAProxy 监控与探活接口 (端口 9999) frontend monitor-haproxy bind *:9999 mode http option httplog monitor-uri /ruok # K8s API Server 代理入口 (VIP:8443 - Master:6443) frontend k8s-master-frontend bind 0.0.0.0:8443 mode tcp option tcplog tcp-request inspect-delay 5s default_backend k8s-master-backend # 后端三台 Master API Server backend k8s-master-backend mode tcp option tcplog option tcp-check balance roundrobin default-server inter 3s downinter 2s rise 2 fall 3 maxconn 250 maxqueue 256 weight 100 server master201 10.0.0.201:6443 check server master202 10.0.0.202:6443 check server master203 10.0.0.203:6443 check EOFsystemctl restart haproxy systemctlenablehaproxy#keepalive#master201cat/etc/keepalived/keepalived.confEOF global_defs { router_id master201 } vrrp_script chk_haproxy { script /etc/keepalived/check_port.sh 8443 interval 2 weight -20 } vrrp_instance VI_1 { state MASTER interface eth0 #本机网卡名 virtual_router_id 51 priority 100 # 主节点最高优先级 advert_int 1 mcast_src_ip 10.0.0.201 #真实ip authentication { auth_type PASS auth_pass 123456 } track_script { chk_haproxy } virtual_ipaddress { 10.0.0.200/24 # 集群 VIP } } EOF#master202cat/etc/keepalived/keepalived.confEOF global_defs { router_id master202 } vrrp_script chk_haproxy { script /etc/keepalived/check_port.sh 8443 interval 2 weight -20 } vrrp_instance VI_1 { state BACKUP interface eth0 virtual_router_id 51 priority 99 advert_int 1 mcast_src_ip 10.0.0.202 authentication { auth_type PASS auth_pass 123456 } track_script { chk_haproxy } virtual_ipaddress { 10.0.0.200/24 } } EOF#master203cat/etc/keepalived/keepalived.confEOF global_defs { router_id master203 } vrrp_script chk_haproxy { script /etc/keepalived/check_port.sh 8443 interval 2 weight -20 } vrrp_instance VI_1 { state BACKUP interface eth0 virtual_router_id 51 priority 98 advert_int 1 mcast_src_ip 10.0.0.203 authentication { auth_type PASS auth_pass 123456 } track_script { chk_haproxy } virtual_ipaddress { 10.0.0.200/24 } } EOFcat/etc/keepalived/check_port.shEOF #!/bin/bash CHK_PORT$1 if [ -n $CHK_PORT ]; then #匹配监听端口 PORT_PROCESS$(ss -ltn | grep -E :$CHK_PORT\b | wc -l) if [ $PORT_PROCESS -eq 0 ]; then echo $CHK_PORT异常 exit 1 # 端口异常 else exit 0 # 端口正常 fi else echo 端口不能为空 exit 1 fi EOFchmodx /etc/keepalived/check_port.sh systemctl--nowenablekeepalivedcurlhttp://10.0.0.200:9999/ruokhtmlbodyh1200OK/h1Service ready./body/html初始化集群catkubeadm-config.yamlEOF apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration kubernetesVersion: v1.35.0 controlPlaneEndpoint: 10.0.0.200:8443 #设置阿里云镜像仓库 imageRepository: registry.aliyuncs.com/google_containers networking: podSubnet: 10.244.0.0/16 serviceSubnet: 10.96.0.0/12 EOFkubeadm config images list--configkubeadm-config.yaml#查看需要拉去的镜像kubeadm config images pull--configkubeadm-config.yaml#预先拉取镜像kubeadm init--configkubeadm-config.yaml --upload-certs#初始化集群... To start using your cluster, you need to run the following as a regular user:mkdir-p$HOME/.kubesudocp-i/etc/kubernetes/admin.conf$HOME/.kube/configsudochown$(id-u):$(id-g)$HOME/.kube/config Alternatively,ifyou are the root user, you can run:exportKUBECONFIG/etc/kubernetes/admin.conf You should now deploy a pod network to the cluster. Runkubectl apply -f [podnetwork].yamlwith one of the options listed at: https://kubernetes.io/docs/concepts/cluster-administration/addons/ You can nowjoinany number of control-plane nodes running the followingcommandon each as root:#master节点kubeadmjoin10.0.0.200:8443--token58ou8b.wsq5w636x8fnyz7d\--discovery-token-ca-cert-hash sha256:a45a627d01bd609db7303ab74f50446fd7b202184649986d6e0a923d8fe3c969\--control-plane --certificate-key 812c0917d59165ff3cbd93128a41cccf0b2d93e82b08121609f1acc136c0eba7 Please note that the certificate-key gives access to cluster sensitive data, keep it secret!As a safeguard, uploaded-certs will be deletedintwo hours;If necessary, you can usekubeadm init phase upload-certs --upload-certsto reload certs afterward. Then you canjoinany number of worker nodes by running the following on each as root:#worker节点kubeadmjoin10.0.0.200:8443--token58ou8b.wsq5w636x8fnyz7d\--discovery-token-ca-cert-hash sha256:a45a627d01bd609db7303ab74f50446fd7b202184649986d6e0a923d8fe3c969#master201mkdir-p$HOME/.kubesudocp-i/etc/kubernetes/admin.conf$HOME/.kube/configsudochown$(id-u):$(id-g)$HOME/.kube/config#master202和master203kubeadmjoin10.0.0.200:8443--token58ou8b.wsq5w636x8fnyz7d\--discovery-token-ca-cert-hash sha256:a45a627d01bd609db7303ab74f50446fd7b202184649986d6e0a923d8fe3c969\--control-plane --certificate-key 812c0917d59165ff3cbd93128a41cccf0b2d93e82b08121609f1acc136c0eba7mkdir-p$HOME/.kubesudocp-i/etc/kubernetes/admin.conf$HOME/.kube/configsudochown$(id-u):$(id-g)$HOME/.kube/config#worker204kubeadmjoin10.0.0.200:8443--token58ou8b.wsq5w636x8fnyz7d\--discovery-token-ca-cert-hash sha256:a45a627d01bd609db7303ab74f50446fd7b202184649986d6e0a923d8fe3c969 rootmaster201:~# kubectl get nodes -o wideNAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME master201 NotReady control-plane 8m26s v1.35.010.0.0.201noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 master202 NotReady control-plane 2m30s v1.35.010.0.0.202noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 master203 NotReady control-plane 43s v1.35.010.0.0.203noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 worker204 NotReadynone26s v1.35.010.0.0.204noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6calicokubectl apply-fhttps://ghfast.top/https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/calico.yaml kubectl get pods-nkube-system-w#等待如果有镜像拉取问题手动拉镜像rootmaster201:~# kubectl get nodes -o wideNAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME master201 Ready control-plane 17m v1.35.010.0.0.201noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 master202 Ready control-plane 11m v1.35.010.0.0.202noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 master203 Ready control-plane 9m21s v1.35.010.0.0.203noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 worker204 Readynone9m4s v1.35.010.0.0.204noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6#测试dns解析kubectl run-it--rmdebug--imageharbor.qltang.com/linux/busybox:qltangv1--restartNever --nslookupkube-dns.kube-system.svc.cluster.local Server:10.96.0.10 Address:10.96.0.10:53 Name: kube-dns.kube-system.svc.cluster.local Address:10.96.0.10 poddebugdeleted from default namespace kubectl run-it--rmdebug--imageharbor.qltang.com/linux/busybox:qltangv1--restartNever --nslookupbaidu.com Server:10.96.0.10 Address:10.96.0.10:53 Non-authoritative answer: Name: baidu.com Address:111.63.65.103 Name: baidu.com Address:111.63.65.247 Name: baidu.com Address:110.242.74.102 Name: baidu.com Address:124.237.177.164 Non-authoritative answer: poddebugdeleted from default namespace#测试高可用rootmaster201:~# systemctl stop haproxy.service #停止haprootmaster201:~# ip a...2: eth0:BROADCAST,MULTICAST,UP,LOWER_UPmtu1500qdisc fq_codel state UP group default qlen1000link/ether 00:0c:29:09:59:e0 brd ff:ff:ff:ff:ff:ff altname enp2s1 altname ens33 inet10.0.0.201/24 brd10.0.0.255 scope global eth0#节点已漂移valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe09:59e0/64 scopelinkvalid_lft forever preferred_lft forevercurlhttp://10.0.0.200:9999/ruokhtmlbodyh1200OK/h1Service ready./body/htmlrootmaster202:~# ip a...2: eth0:BROADCAST,MULTICAST,UP,LOWER_UPmtu1500qdisc fq_codel state UP group default qlen1000link/ether 00:0c:29:9c:01:51 brd ff:ff:ff:ff:ff:ff altname enp2s1 altname ens33 inet10.0.0.202/24 brd10.0.0.255 scope global eth0 valid_lft forever preferred_lft forever inet10.0.0.200/24 scope global secondary eth0 valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe9c:151/64 scopelinkvalid_lft forever preferred_lft forever rootmaster201:~# kubectl get nodes -o wideNAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME master201 Ready control-plane 27m v1.35.010.0.0.201noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 master202 Ready control-plane 21m v1.35.010.0.0.202noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 master203 Ready control-plane 19m v1.35.010.0.0.203noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6 worker204 Readynone19m v1.35.010.0.0.204noneUbuntu22.04.5 LTS5.15.0-179-generic containerd://2.2.6helmcurlhttps://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3|HELM_INSTALL_DIR/usr/local/binbashhelm version version.BuildInfo{Version:v3.21.3, GitCommit:1ad6e68924fdf6fb0c7dcef8e9e1dfc0f36eaed6, GitTreeState:clean, GoVersion:go1.26.5}helm repoaddprometheus-community https://prometheus-community.github.io/helm-charts#得挂梯子helm repoaddaliyun https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts helm repo update rootmaster201:~# helm repo listNAME URL aliyun https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts prometheus-community https://prometheus-community.github.io/helm-chartsMetalLB#修改ipvs模式kubectl edit configmap-nkube-system kube-proxy... ipvs:... strictARP:truemode:kubectl rollout restart daemonset kube-proxy-nkube-system kubectl rollout status daemonset kube-proxy-nkube-system rootmaster201:~# ipvsadm -lnIP Virtual Server version1.2.1(size4096)Prot LocalAddress:Port Scheduler Flags -RemoteAddress:Port Forward Weight ActiveConn InActConn TCP10.96.0.1:443 rr -10.0.0.201:6443 Masq100-10.0.0.202:6443 Masq100-10.0.0.203:6443 Masq110TCP10.96.0.10:53 rr -10.244.61.70:53 Masq100-10.244.61.72:53 Masq100TCP10.96.0.10:9153 rr -10.244.61.70:9153 Masq100-10.244.61.72:9153 Masq100UDP10.96.0.10:53 rr -10.244.61.70:53 Masq100-10.244.61.72:53 Masq100kubectl apply-fhttps://raw.githubusercontent.com/metallb/metallb/v0.14.8/config/manifests/metallb-native.yaml kubectl get pods-nmetallb-system-wcatmetallb-pool.yamlEOF apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: name: first-pool namespace: metallb-system spec: addresses: - 10.0.0.50-10.0.0.150 --- apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: name: l2-advertisement namespace: metallb-system spec: ipAddressPools: - first-pool EOFcatweb.htmlEOF apiVersion: apps/v1 kind: Deployment metadata: name: web labels: app: web spec: replicas: 3 selector: matchLabels: app: web template: metadata: labels: app: web spec: # imagePullSecrets: # - name: registry-pull-secret containers: - name: web image: harbor.qltang.com/study/web:20 imagePullPolicy: Always ports: - containerPort: 80 livenessProbe: httpGet: path: /health port: 80 initialDelaySeconds: 5 periodSeconds: 10 readinessProbe: httpGet: path: /health port: 80 initialDelaySeconds: 3 periodSeconds: 5 --- apiVersion: v1 kind: Service metadata: name: web-svc labels: app: web spec: type: LoadBalancer ports: - port: 80 targetPort: 80 selector: app: web EOFkubectl apply-fweb.html rootmaster201:~/k8s/web# kubectl get pods,svc -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES pod/web-8456f58b9c-6nbfn1/1 Running056s10.244.61.74 worker204nonenonepod/web-8456f58b9c-8kc5g1/1 Running056s10.244.61.76 worker204nonenonepod/web-8456f58b9c-w26jf1/1 Running056s10.244.61.75 worker204nonenoneNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S)AGE SELECTOR service/kubernetes ClusterIP10.96.0.1none443/TCP 117mnoneservice/web-svc LoadBalancer10.102.198.5010.0.0.5080:32758/TCP 56sappwebingresshelm repoaddingress-nginx https://kubernetes.github.io/ingress-nginx helm repo update helminstallingress-nginx ingress-nginx/ingress-nginx\--namespaceingress-nginx --create-namespace\--setcontroller.service.typeLoadBalancer rootmaster201:~/k8s/web# kubectl get svc -n ingress-nginx.NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S)AGE ingress-nginx-controller LoadBalancer10.99.184.4510.0.0.5080:31185/TCP,443:32033/TCP 2m5s ingress-nginx-controller-admission ClusterIP10.96.15.111none443/TCP 2m5scatweb-ingress.htmlEOF apiVersion: apps/v1 kind: Deployment metadata: namespace: default name: web labels: app: web spec: replicas: 3 selector: matchLabels: app: web template: metadata: labels: app: web spec: # imagePullSecrets: # - name: registry-pull-secret containers: - name: web image: harbor.qltang.com/study/web:20 imagePullPolicy: Always ports: - containerPort: 80 livenessProbe: httpGet: path: /health port: 80 initialDelaySeconds: 5 periodSeconds: 10 readinessProbe: httpGet: path: /health port: 80 initialDelaySeconds: 3 periodSeconds: 5 --- apiVersion: v1 kind: Service metadata: name: web-svc namespace: default labels: app: web spec: type: ClusterIP ports: - port: 80 targetPort: 80 selector: app: web --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: web-ingress namespace: default annotations: kubernetes.io/ingress.class: nginx spec: ingressClassName: nginx rules: - host: web.qltang.com http: paths: - path: / pathType: Prefix backend: service: name: web-svc port: number: 80 EOFrootmaster201:~/k8s/web# kubectl get pods,svc,ingress -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES pod/web-8456f58b9c-kb84r1/1 Running04m31s10.244.61.82 worker204nonenonepod/web-8456f58b9c-l2sdg1/1 Running04m31s10.244.61.84 worker204nonenonepod/web-8456f58b9c-n4zh21/1 Running04m31s10.244.61.83 worker204nonenoneNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S)AGE SELECTOR service/kubernetes ClusterIP10.96.0.1none443/TCP 142mnoneservice/web-svc ClusterIP10.110.120.176none80/TCP 4m31sappweb NAME CLASS HOSTS ADDRESS PORTS AGE ingress.networking.k8s.io/web-ingress nginx web.qltang.com10.0.0.50804m31scurl-I-s-HHost: web.qltang.comhttp://10.0.0.50 HTTP/1.1200OK Content-Length:29627Accept-Ranges: bytes Connection: keep-alive Content-Type: text/html Date: Sat, 08 Aug202613:41:42 GMT Etag:6a618408-73bbKeep-Alive:timeout4Last-Modified: Thu,23Jul202603:01:28 GMT Proxy-Connection: keep-alivecurl-I-s-HHost: web.qltang.comhttp://10.0.0.50/health HTTP/1.1200OK Content-Length:2Connection: keep-alive Content-Type: text/plain Date: Sat, 08 Aug202613:47:42 GMT Keep-Alive:timeout4Proxy-Connection: keep-aliveprometheus-operator - 未完待续部署kubectl create namespace monitoring helm pull prometheus-community/kube-prometheus-stack--untar#下载并解压chart包