Kubernetes

Kubernetes 分布式部署（6）Kubernetes 1.30.x Node配置

By jiutingqiu

1、kubelet配置

首先加入MASTER1

复制kubelet，kube-proxy

kubelet 负责接收master的命令

Kube-proxy 负责网络通讯

root@k8s-master-u2404-4-20-101:~# mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs}
root@k8s-master-u2404-4-20-101:~# cd kubernetes_install/kubernetes/server/bin/
root@k8s-master-u2404-4-20-101:~/kubernetes_install/kubernetes/server/bin# cp kubelet kube-proxy /opt/kubernetes/bin/

kubelet.conf

root@k8s-master-u2404-4-20-101:~# vim /opt/kubernetes/cfg/kubelet.conf 
KUBELET_OPTS=" \
--v=2 \
--hostname-override=k8s-master-u2404-4-20-101 \
--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \
--bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \
--config=/opt/kubernetes/cfg/kubelet-config.yml \
--cert-dir=/opt/kubernetes/ssl \
--runtime-request-timeout=15m  \
--container-runtime-endpoint=unix:///run/cri-dockerd.sock \
--cgroup-driver=systemd \
--node-labels=node.kubernetes.io/node=''"

kubelet-config.yml

里面有些参数可能，不太正确后续需要测试一下优化后的模版

至于用不用swap就看使用场景了，不过如果资源上比较拮据，那么还是更推荐负载循环

root@k8s-master-u2404-4-20-101:~# vim /opt/kubernetes/cfg/kubelet-config.yml 
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS:
- 10.0.0.2
clusterDomain: cluster.local 
failSwapOn: false
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true
  x509:
    clientCAFile: /opt/kubernetes/ssl/ca.pem 
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
evictionHard:
  imagefs.available: 15%
  memory.available: 100Mi
  nodefs.available: 10%
  nodefs.inodesFree: 5%
maxOpenFiles: 1000000
maxPods: 110

###优化版本
apiVersion: kubelet.config.k8s.io/v1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 0
cgroupDriver: systemd 
clusterDNS:
  - 10.0.0.2
clusterDomain: cluster.local
failSwapOn: true   
authentication:
  anonymous:
    enabled: false
  webhook:
    enabled: true
    cacheTTL: 2m0s
  x509:
    clientCAFile: /opt/kubernetes/ssl/ca.pem
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
evictionHard:
  imagefs.available: 15%
  memory.available: 100Mi
  nodefs.available: 10%
  nodefs.inodesFree: 5%
maxOpenFiles: 100000
maxPods: 110

bootstrap.kubeconfig

节点首次加入集群时用于 TLS 自举，详情回归第五章的配置和说明

root@k8s-master-u2404-4-20-101:~# cd TLS/k8s/
root@k8s-master-u2404-4-20-101:~/TLS/k8s# vim bootstrap-kubeconfig
KUBE_CONFIG="/opt/kubernetes/cfg/bootstrap.kubeconfig"
KUBE_APISERVER="https://172.16.101.101:6443" # apiserver IP:PORT
TOKEN="af0d25ea1e38abacfecc5270450ea3c3" # 与token.csv里保持一致

# 生成 kubelet bootstrap kubeconfig 配置文件
kubectl config set-cluster kubernetes \
  --certificate-authority=/opt/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials "kubelet-bootstrap" \
  --token=${TOKEN} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user="kubelet-bootstrap" \
  --kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

root@k8s-master-u2404-4-20-101:~/TLS/k8s# source bootstrap-kubeconfig 
Cluster "kubernetes" set.
User "kubelet-bootstrap" set.
Context "default" created.
Switched to context "default".
root@k8s-master-u2404-4-20-101:~/TLS/k8s# cat /opt/kubernetes/cfg/bootstrap.kubeconfig
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURvakNDQW9xZ0F3SUJBZ0lVUWZhNXRzVzJEWmFZSk1rSEZPRTdiclhWc2VRd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2FURUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbAphV3BwYm1jeEREQUtCZ05WQkFvVEEyMTVZekVRTUE0R0ExVUVDeE1IYjNCbGJuTnpiREVXTUJRR0ExVUVBeE1OCmEzVmlaWEp1WlhSbGN5QkRRVEFlRncweU5UQTNNakV3TXpFME1EQmFGdzB6TURBM01qQXdNekUwTURCYU1Ha3gKQ3pBSkJnTlZCQVlUQWtOT01SQXdEZ1lEVlFRSUV3ZENaV2xxYVc1bk1SQXdEZ1lEVlFRSEV3ZENaV2xxYVc1bgpNUXd3Q2dZRFZRUUtFd050ZVdNeEVEQU9CZ05WQkFzVEIyOXdaVzV6YzJ3eEZqQVVCZ05WQkFNVERXdDFZbVZ5CmJtVjBaWE1nUTBFd2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUNjWWNteHNQSTcKOWFRNFhqYi9NcjVUUjJ1NlROaTlmSWtxRGRYaG5kK25Ib2FTVWZzQ3hzMUFXc2k0bmJhWGRxZWZsc0J5TTd0TQpDUlpTeWpkNkIyTzlGSWdaNjQ0RXU4NEx4S2I0V0E5bWM1MkZudytsbXNIakN0cmUwS2VaUWxDZ013bGRqSTdkClptN2M1WG9NcmxXSHB6aCsraU1ha2p6YUd6ZkdRdEdlSjg4ZUxqSjQ1WGloMGluYm41MFVjTzNLYUs5b2gvVDIKNVdRN1BqR1dqV1VpWVFuVWJKdWJVV2xrQ0d0OTNHSUJwOG9lc2VoZzJCaDhrM1Q4ZjlVSmx1Z0FPR0x6NmtsNQpsWjVQL1ppazVscnVwdU5NUm9xblh4VU5JWUlrc2ExMU8wZ2xYN3lxUmN5YVNGTUQzZml5V2hNVms0dUUxbzJvClB0SFpTZzlEMkVTZkFnTUJBQUdqUWpCQU1BNEdBMVVkRHdFQi93UUVBd0lCQmpBUEJnTlZIUk1CQWY4RUJUQUQKQVFIL01CMEdBMVVkRGdRV0JCUkIvMGd3SGdpMGY1aHVORFkybFM4aGV2NnB4ekFOQmdrcWhraUc5dzBCQVFzRgpBQU9DQVFFQUt5UHJXVnZKQmFuVmM4ZGFtMktiMVBVOCtRamxGMGwyek9hN25BZTlGWHY1YXQ2MHg3SzB0bG9KCmVKbWtJZ08vbnVhWjI4ak9TWWUrZ1p5YVhtQWdsbkNDTWQ3b29pNlVoMUtGbXJaUmovS0FqUW5VZUh6L01WYkQKbWxaWFRkRzcrZEx2MjIyUHZMRndHN3BQdHNnNmtmUWtUUlgwNUZxa0NuZUtCMS9zRS9Qaks4OWhSMW9iRUxtQwowYTNab3VpbVF6THBKN0ZIRVJobWpmVENWNmMvMktlaEMxWmdidmlJMVp5dGtkQllMcW5JUjZSaUNmZTQ4aDlLCjZ3SC8wbWp5ZG5oKzhYbEtIRWV5SU52WTVJTkJscmRiNmJ2OENlNnBVRUZMdERjZlVDUDI0YzlqelpnR3BrZ24KeE55NGlpY1ZyK2tFNmpPTytGdDN6Q2wySEcwUEZ3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
    server: https://172.16.101.101:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: kubelet-bootstrap
  name: default
current-context: default
kind: Config
preferences: {}
users:
- name: kubelet-bootstrap
  user:
    token: af0d25ea1e38abacfecc5270450ea3c3

kubelet.service

root@k8s-master-u2404-4-20-101:~# vim /usr/lib/systemd/system/kubelet.service 
[Unit]
Description=Kubernetes Kubelet
After=docker.service

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kubelet.conf
ExecStart=/opt/kubernetes/bin/kubelet $KUBELET_OPTS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

root@k8s-master-u2404-4-20-101:~# systemctl daemon-reload
root@k8s-master-u2404-4-20-101:~# systemctl enable --now kubelet

授权加入集群

master授权 加入集群 加入后会在/opt/kubernetes/ssl/

看到自动获取的证书和密钥

root@k8s-master-u2404-4-20-101:~# ls /opt/kubernetes/ssl/kubelet-client* /opt/kubernetes/ssl/kubelet-client-2025-07-21-15-58-23.pem

/opt/kubernetes/ssl/kubelet-client-current.pem

root@k8s-master-u2404-4-20-101:~# kubectl get csr
NAME                                                   AGE   SIGNERNAME                                    REQUESTOR           REQUESTEDDURATION   CONDITION
node-csr-kMcZhciKRAvd8R1dbG7A8lhomWqQ8zjvZ9krpLk_y6s   52s   kubernetes.io/kube-apiserver-client-kubelet   kubelet-bootstrap   <none>              Pending

root@k8s-master-u2404-4-20-101:~# kubectl certificate approve node-csr-kMcZhciKRAvd8R1dbG7A8lhomWqQ8zjvZ9krpLk_y6s
certificatesigningrequest.certificates.k8s.io/node-csr-kMcZhciKRAvd8R1dbG7A8lhomWqQ8zjvZ9krpLk_y6s approved

root@k8s-master-u2404-4-20-101:~# kubectl get nodes
NAME                        STATUS     ROLES    AGE   VERSION
k8s-master-u2404-4-20-101   NotReady   <none>   3s    v1.30.2

错误记录

别使用优化版本的yml
Nov 12 16:01:26 node1 kubelet[3431]: E1112 16:01:26.506963 3431 run.go:74] "command failed" err="failed to loa>

Nov 12 16:01:26 node1 kubelet[3431]: Flag --runtime-request-timeout has been deprecated, This parameter should be>
Nov 12 16:01:26 node1 kubelet[3431]: Flag --container-runtime-endpoint has been deprecated, This parameter should>
Nov 12 16:01:26 node1 kubelet[3431]: Flag --cgroup-driver has been deprecated, This parameter should be set via t>
Nov 12 16:01:26 node1 kubelet[3431]: E1112 16:01:26.506963    3431 run.go:74] "command failed" err="failed to loa>
Nov 12 16:01:26 node1 systemd[1]: kubelet.service: Main process exited, code=exited, status=1/FAILURE
Nov 12 16:01:26 node1 systemd[1]: kubelet.service: Failed with result 'exit-code'.
Nov 12 16:01:26 node1 systemd[1]: kubelet.service: Scheduled restart job, restart counter is at 5.
Nov 12 16:01:26 node1 systemd[1]: kubelet.service: Start request repeated too quickly.
Nov 12 16:01:26 node1 systemd[1]: kubelet.service: Failed with result 'exit-code'.
Nov 12 16:01:26 node1 systemd[1]: Failed to start kubelet.service - Kubernetes Kubelet.
Kubeconfig exists and is valid, skipping bootstrap
3937 feature_gate.go:254] feature gates: {map[]} 8 3937 bootstrap.go:116] "Kubeconfig exists and is valid, skipping bootstrap" path="/opt/kubernetes/cfg/kubele> 6 3937 dynamic_cafile_content.go:119] "Loaded a new CA Bundle and Verifier" name="client-ca-bundle::/opt/kuber> 5 3937 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/opt/kubernetes/ssl/ca.pem" 2 3937 logging.go:59] [core] [Channel #1 SubChannel #2] grpc: addrConn.createTransport failed to connect to {A> 3 3937 run.go:74] "command failed" err="failed to run Kubelet: validate service connection: validate CRI v1 ru> rocess exited, code=exited, status=1/FAILURE with result 'exit-code'. led restart job, restart counter is at 5. request repeated too quickly. with result 'exit-code'. .service - Kubernetes Kubelet.
这个很复杂

首先看 kubelet
然后看 docker

root@node1:~/TLS/k8s# modprobe overlay modprobe: ERROR: ../libkmod/libkmod-index.c:790 index_mm_open() magic check fail: b0881000 instead of b007f457 modprobe: FATAL: Module overlay not found in directory /lib/modules/6.8.0-87-generic root@node1:~/TLS/k8s# modprobe br_netfilter modprobe: ERROR: ../libkmod/libkmod-index.c:790 index_mm_open() magic check fail: b0881000 instead of b007f457 modprobe: FATAL: Module br_netfilter not found in directory /lib/modules/6.8.0-87-generic

root@node1:~/TLS/k8s# cat /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "registry-mirrors": [
    "https://docker.mirrors.ustc.edu.cn",
    "http://hub-mirror.c.163.com"
  ],
  "storage-driver": "vfs", ### add
  "max-concurrent-downloads": 10,
  "log-driver": "json-file",
  "log-level": "warn",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
  },
  "data-root": "/var/lib/docker"
}

2、kube-proxy配置

kube-proxy 网络代理，负责server的服务发现和对应的负载均衡，四层负载

kube-proxy.conf

root@k8s-master-u2404-4-20-101:~# vim /opt/kubernetes/cfg/kube-proxy.conf
KUBE_PROXY_OPTS=" \
--v=2 \
--config=/opt/kubernetes/cfg/kube-proxy-config.yml"

kube-proxy-config.yml

使用ipvs 轮询rr

如果用的不是ipvs，请自行修改配置

root@k8s-master-u2404-4-20-101:~# vim /opt/kubernetes/cfg/kube-proxy-config.yml
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
metricsBindAddress: 0.0.0.0:10249
clientConnection:
  kubeconfig: /opt/kubernetes/cfg/kube-proxy.kubeconfig
hostnameOverride: "k8s-master-u2404-4-20-101"
clusterCIDR: 10.244.0.0/16
mode: ipvs
ipvs:
  scheduler: "rr"
iptables:
  masqueradeAll: true

kube-proxy证书配置

root@k8s-master-u2404-4-20-101:~/TLS/k8s# vim kube-proxy-csr.json
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

root@k8s-master-u2404-4-20-101:~/TLS/k8s# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
2025/07/21 16:07:07 [INFO] generate received request
2025/07/21 16:07:07 [INFO] received CSR
2025/07/21 16:07:07 [INFO] generating key: rsa-2048
2025/07/21 16:07:07 [INFO] encoded CSR
2025/07/21 16:07:07 [INFO] signed certificate with serial number 584823920913523607565839504753323947523543819448
2025/07/21 16:07:07 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

root@k8s-master-u2404-4-20-101:~/TLS/k8s# ls kube-proxy*
kube-proxy.csr  kube-proxy-csr.json  kube-proxy-key.pem  kube-proxy.pem

kube-proxy-kubeconfig

root@k8s-master-u2404-4-20-101:~/TLS/k8s# vim kube-proxy-kubeconfig
KUBE_CONFIG="/opt/kubernetes/cfg/kube-proxy.kubeconfig"
KUBE_APISERVER="https://172.16.101.101:6443"

kubectl config set-cluster kubernetes \
  --certificate-authority=/opt/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials kube-proxy \
  --client-certificate=./kube-proxy.pem \
  --client-key=./kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-proxy \
  --kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

root@k8s-master-u2404-4-20-101:~/TLS/k8s# source kube-proxy-kubeconfig 
Cluster "kubernetes" set.
User "kube-proxy" set.
Context "default" created.
Switched to context "default".
root@k8s-master-u2404-4-20-101:~/TLS/k8s# cat /opt/kubernetes/cfg/kube-proxy.kubeconfig
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURvakNDQW9xZ0F3SUJBZ0lVUWZhNXRzVzJEWmFZSk1rSEZPRTdiclhWc2VRd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2FURUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbAphV3BwYm1jeEREQUtCZ05WQkFvVEEyMTVZekVRTUE0R0ExVUVDeE1IYjNCbGJuTnpiREVXTUJRR0ExVUVBeE1OCmEzVmlaWEp1WlhSbGN5QkRRVEFlRncweU5UQTNNakV3TXpFME1EQmFGdzB6TURBM01qQXdNekUwTURCYU1Ha3gKQ3pBSkJnTlZCQVlUQWtOT01SQXdEZ1lEVlFRSUV3ZENaV2xxYVc1bk1SQXdEZ1lEVlFRSEV3ZENaV2xxYVc1bgpNUXd3Q2dZRFZRUUtFd050ZVdNeEVEQU9CZ05WQkFzVEIyOXdaVzV6YzJ3eEZqQVVCZ05WQkFNVERXdDFZbVZ5CmJtVjBaWE1nUTBFd2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUNjWWNteHNQSTcKOWFRNFhqYi9NcjVUUjJ1NlROaTlmSWtxRGRYaG5kK25Ib2FTVWZzQ3hzMUFXc2k0bmJhWGRxZWZsc0J5TTd0TQpDUlpTeWpkNkIyTzlGSWdaNjQ0RXU4NEx4S2I0V0E5bWM1MkZudytsbXNIakN0cmUwS2VaUWxDZ013bGRqSTdkClptN2M1WG9NcmxXSHB6aCsraU1ha2p6YUd6ZkdRdEdlSjg4ZUxqSjQ1WGloMGluYm41MFVjTzNLYUs5b2gvVDIKNVdRN1BqR1dqV1VpWVFuVWJKdWJVV2xrQ0d0OTNHSUJwOG9lc2VoZzJCaDhrM1Q4ZjlVSmx1Z0FPR0x6NmtsNQpsWjVQL1ppazVscnVwdU5NUm9xblh4VU5JWUlrc2ExMU8wZ2xYN3lxUmN5YVNGTUQzZml5V2hNVms0dUUxbzJvClB0SFpTZzlEMkVTZkFnTUJBQUdqUWpCQU1BNEdBMVVkRHdFQi93UUVBd0lCQmpBUEJnTlZIUk1CQWY4RUJUQUQKQVFIL01CMEdBMVVkRGdRV0JCUkIvMGd3SGdpMGY1aHVORFkybFM4aGV2NnB4ekFOQmdrcWhraUc5dzBCQVFzRgpBQU9DQVFFQUt5UHJXVnZKQmFuVmM4ZGFtMktiMVBVOCtRamxGMGwyek9hN25BZTlGWHY1YXQ2MHg3SzB0bG9KCmVKbWtJZ08vbnVhWjI4ak9TWWUrZ1p5YVhtQWdsbkNDTWQ3b29pNlVoMUtGbXJaUmovS0FqUW5VZUh6L01WYkQKbWxaWFRkRzcrZEx2MjIyUHZMRndHN3BQdHNnNmtmUWtUUlgwNUZxa0NuZUtCMS9zRS9Qaks4OWhSMW9iRUxtQwowYTNab3VpbVF6THBKN0ZIRVJobWpmVENWNmMvMktlaEMxWmdidmlJMVp5dGtkQllMcW5JUjZSaUNmZTQ4aDlLCjZ3SC8wbWp5ZG5oKzhYbEtIRWV5SU52WTVJTkJscmRiNmJ2OENlNnBVRUZMdERjZlVDUDI0YzlqelpnR3BrZ24KeE55NGlpY1ZyK2tFNmpPTytGdDN6Q2wySEcwUEZ3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
    server: https://172.16.101.101:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: kube-proxy
  name: default
current-context: default
kind: Config
preferences: {}
users:
- name: kube-proxy
  user:
    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ0akNDQXNxZ0F3SUJBZ0lVWm5CcFhpWWl4aWFZMllRRUJSSkpyb0hXWkxnd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2FURUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbAphV3BwYm1jeEREQUtCZ05WQkFvVEEyMTVZekVRTUE0R0ExVUVDeE1IYjNCbGJuTnpiREVXTUJRR0ExVUVBeE1OCmEzVmlaWEp1WlhSbGN5QkRRVEFlRncweU5UQTNNakV3T0RBeU1EQmFGdzB6TlRBM01Ua3dPREF5TURCYU1Hd3gKQ3pBSkJnTlZCQVlUQWtOT01SQXdEZ1lEVlFRSUV3ZENaV2xLYVc1bk1SQXdEZ1lEVlFRSEV3ZENaV2xLYVc1bgpNUXd3Q2dZRFZRUUtFd05yT0hNeER6QU5CZ05WQkFzVEJsTjVjM1JsYlRFYU1CZ0dBMVVFQXhNUmMzbHpkR1Z0Ck9tdDFZbVV0Y0hKdmVIa3dnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFERFRUZVQKUzBzN2d3SDNXcEpCY2F5Y09JL29xek1kcXRHUkNnWXhpNTJPcmRKRUJTbURpNEY2M3dRUGJvRHkwVWhydjlaSwpZZ1FJL2YxMEVXWSt4cEhJY21qN0hvRU1tQW04d0lqZFZPUmtYVCtKM2drWjloUUpncGJ1bFZBUWRBc2tGTlduCkE1VVJYUi9qY3Z3RlM3cUhkNnMzcnpyNVFYUHc5L05TcUtQNUFRSXlLanlFVGZrNU53b29HR2hBbDFBVTdOQmQKVWsyWGhsQ3lzZGc3ejBSN0Z3YlcrUHA1TDN0b09WOHY1RkFCZjcxbDEwTHNoTmVnd1ljWC95RXQyMUc5R1NpUQpKMHJPVERZM1JubDFmV2NjQW43amFSYnNIcW5GdjJSU25RcVNaVWNaMno1NU0vaDl3dXFwQTBLdXR6WGVEd1c0ClR5c0Jyd2RTQytZZjZTNEhBZ01CQUFHamZ6QjlNQTRHQTFVZER3RUIvd1FFQXdJRm9EQWRCZ05WSFNVRUZqQVUKQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0RBWURWUjBUQVFIL0JBSXdBREFkQmdOVkhRNEVGZ1FVQWlILwpSVFpxdldHS0ZVNjFGM0JqUTVNSC9Wd3dId1lEVlIwakJCZ3dGb0FVUWY5SU1CNEl0SCtZYmpRMk5wVXZJWHIrCnFjY3dEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBQUhQaytyc2JQRDg2cmRLYVZseWxTa3pkZ3hmNmRlbDZYOXIKcGJ2VzJGS0cveDFnWXFCZTlzWFBrZzRPZTdZSGR4elVWYWNYcDlIMlBXVnUrdWJ2WUpFaVlVbDF0V1ZRRHN1WAp5RFcxL29UYzl4YWpRS1djWlNOcjVrK0ZYS2d5ZnZORFpLcXM3anhtaS9GRWFtc3Bzck5wb2kycGQ3K2RiZDJTCjBhK0JYYW9GaWJrcG5tbjFMcFBHZkRvMGxPdUFmSlJvaGdJc2VzSGhVT1pLUC96dVkwemZrczdXY25scUZaVWMKa2NpaGcvNGFaYUc2ZnlVRmFZcHVQdkhobjQrZVdaNFdwV2tOTmRPSFM1YlVGdDdtMnZyczAyWDQ4VWkzL2NCOApCNFMxb0J3K2wrUEdkL3lubDFRUmNPbyt5bjY5Z2pUdTRXVnhJOTgvaCt5NXFsYUVWVkU9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdzAwM2swdExPNE1COTFxU1FYR3NuRGlQNktzekhhclJrUW9HTVl1ZGpxM1NSQVVwCmc0dUJldDhFRDI2QTh0RklhNy9XU21JRUNQMzlkQkZtUHNhUnlISm8reDZCREpnSnZNQ0kzVlRrWkYwL2lkNEoKR2ZZVUNZS1c3cFZRRUhRTEpCVFZwd09WRVYwZjQzTDhCVXU2aDNlck42ODYrVUZ6OFBmelVxaWorUUVDTWlvOApoRTM1T1RjS0tCaG9RSmRRRk96UVhWSk5sNFpRc3JIWU84OUVleGNHMXZqNmVTOTdhRGxmTCtSUUFYKzlaZGRDCjdJVFhvTUdIRi84aExkdFJ2Umtva0NkS3prdzJOMFo1ZFgxbkhBSis0MmtXN0I2cHhiOWtVcDBLa21WSEdkcysKZVRQNGZjTHFxUU5DcnJjMTNnOEZ1RThyQWE4SFVndm1IK2t1QndJREFRQUJBb0lCQVFEQ2JuUmVxbjM2Z0dHTAp2TnFRVnJnT1N1MVNLR1phS2I0N0UreTRsSk5HQVZ5c3BTc0FML20ydHFIWEx4OG5IaDFvVC9McDJJekYydjdoCjk1Ui8rTFlZQlhib1VpVU53YXNvQTlIOTcwNzFjRFFhY0RkRlprdnFiOTBOSnc4dllpZlpRdmkxTGdiUTQ1SWsKb1V1dmhnWHNhY0hHVFNoOEQ2SjZpbzd1Z3IvRDIzUjFIZTJUMlU3ek5NNzUvQmtJNTRLS1BrUGo2a1FjU3FjZApuLytoek9iTXFGRGZXVVVmemR3MVhmUStZalllQVorajFWeW5SaXB2dHp5azAzY3daNCtLYTlKYUNZSDBWREN3CnpEbmRIV3NTaG9CeVJtcmNhWVE1bmN6ZTMwbmVZayttS3FPQzltekRWWStFV1N5OW4yNVNaVWQxakpCVjNDUWIKaGY1U3pyOFJBb0dCQU9xcUZoRURrNHBQVHZtZjFxb1M3SzNvMDRHZG9RRS9qUzFmODhXMWVCcWpKRWdOWHB2dQoxdWhqeUZsazhwZlo3Zkt1b01CdEs5YXJFL2owendNM1QzdmV2UjE2anBqSlhabkxSMGU2R2dEL3Qzbk9NeVkyCmNJQi9EaEZCRlN5b3ZyR1dmcW5DMG9Lei9QM240UlZVSWxYQVRSMVVoSnZId1MraDZnU3lPMHhmQW9HQkFOVU8KOGhjUnVWM25EaENMZWxSN2hKVjU2ZlRLUVNYVnBEbGtqVE8zUlQvaGNHc2MwdE05N2JtTUF2Y1Q0U3ZYMjNESgppRlVGQkNERTlIOXJaUllvempkaE9HN1ZINHhSUFRTK0o2c1QrOU13Vnp2KzJxblJSQ1Fmcy9xWlN3Z0UxMC9lCnVDUWN5TzJ0U3pwdWhaUkJ3WHZnL1RyMzFORUFSZE83K1QxaVovOVpBb0dBYW9VOHNRcHpqY1FFY0hHcTZ0YmUKVXU4dUcxY25QNzdPWWZ0OVY1RFJiUjRER1NYZ2ZWNEVkY2JkcWg2VnRISjdmYjdoUDVnTzMwb1hMZHBLb2dZWQpjWDBoVVdGUm0yOUVxc3hJbGZtWFZqNjkxVy9SYmhZbUtWNERUN2ZBV2Q1YmVaZDc5YWwzRDFlT2p0OFVJQzlwCnlvMFpOQ2dxYmVCV0JENDJVVjFxWjNjQ2dZQitGNFJMQnVCUHBLajhQYSsrUkFPcGV5ZHRDaTRZT0NJakhWRTcKNUNFRnQyUVM0RklkVnE5VWVucGV3OU1sWVhDcmhYbzc1dVlVaHJ1MGxWaVVEcjVNcENzRDNQbjR6ek9ML1dYNApwN0VId2JvKzlZdHZxVHZNclQ2VTl4Z2l2TXE0cURVT21rS0g5MkZvU1hTZzAzR3c0T3k5YzJMdDlsMWMvQWg3CnZidVNzUUtCZ0FFN1F6ZEJidnNlVXFrOGUrS1ZiOWk2ZXJ2by90MW81ZW9BZlcwOC83eUg1c2Q5TXMzTVVmWk4KelJTTHJvR3R5bTRCNzk3TytCSUdqdjcza1I5RnBmeHU4VkRjSWZMYzAwemxpdGszSEFmQXhaNzJxV1dycnRFdApJay9RMVNNdFBnN2ZZQklEUmw0amZua0hhWExoMWE1aWhXTUtiejVyU0ZZQ1RDYWZkekNtCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==

kube-proxy.service

root@k8s-master-u2404-4-20-101:~/TLS/k8s# vim /usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Proxy
After=network.target

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-proxy.conf
ExecStart=/opt/kubernetes/bin/kube-proxy $KUBE_PROXY_OPTS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

root@k8s-master-u2404-4-20-101:~/TLS/k8s# systemctl daemon-reload
root@k8s-master-u2404-4-20-101:~/TLS/k8s# systemctl enable --now kube-proxy
Created symlink /etc/systemd/system/multi-user.target.wants/kube-proxy.service → /usr/lib/systemd/system/kube-proxy.service.
错误记录
缺少ipset
root@k8s-master-u2404-4-20-101:~/TLS/k8s# journalctl -u kube-proxy
Jul 21 16:09:57 k8s-master-u2404-4-20-101 systemd[1]: Started kube-proxy.service - Kubernetes Proxy.
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393404    5505 flags.go:64] FLAG: --bind-address="0.0.0.0"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393444    5505 flags.go:64] FLAG: --bind-address-hard-fail="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393447    5505 flags.go:64] FLAG: --boot-id-file="/proc/sys/kernel/random/boot_id"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393449    5505 flags.go:64] FLAG: --cleanup="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393460    5505 flags.go:64] FLAG: --cluster-cidr=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393461    5505 flags.go:64] FLAG: --config="/opt/kubernetes/cfg/kube-proxy-config.yml"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393464    5505 flags.go:64] FLAG: --config-sync-period="15m0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393466    5505 flags.go:64] FLAG: --conntrack-max-per-core="32768"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393467    5505 flags.go:64] FLAG: --conntrack-min="131072"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393469    5505 flags.go:64] FLAG: --conntrack-tcp-be-liberal="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393470    5505 flags.go:64] FLAG: --conntrack-tcp-timeout-close-wait="1h0m0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393472    5505 flags.go:64] FLAG: --conntrack-tcp-timeout-established="24h0m0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393473    5505 flags.go:64] FLAG: --conntrack-udp-timeout="0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393474    5505 flags.go:64] FLAG: --conntrack-udp-timeout-stream="0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393477    5505 flags.go:64] FLAG: --detect-local-mode=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393478    5505 flags.go:64] FLAG: --feature-gates=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393481    5505 flags.go:64] FLAG: --healthz-bind-address="0.0.0.0:10256"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393484    5505 flags.go:64] FLAG: --healthz-port="10256"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393486    5505 flags.go:64] FLAG: --help="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393487    5505 flags.go:64] FLAG: --hostname-override=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393488    5505 flags.go:64] FLAG: --init-only="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393490    5505 flags.go:64] FLAG: --iptables-localhost-nodeports="true"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393492    5505 flags.go:64] FLAG: --iptables-masquerade-bit="14"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393493    5505 flags.go:64] FLAG: --iptables-min-sync-period="1s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393494    5505 flags.go:64] FLAG: --iptables-sync-period="30s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393495    5505 flags.go:64] FLAG: --ipvs-exclude-cidrs="[]"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393497    5505 flags.go:64] FLAG: --ipvs-min-sync-period="0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393499    5505 flags.go:64] FLAG: --ipvs-scheduler=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393500    5505 flags.go:64] FLAG: --ipvs-strict-arp="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393501    5505 flags.go:64] FLAG: --ipvs-sync-period="30s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393502    5505 flags.go:64] FLAG: --ipvs-tcp-timeout="0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393504    5505 flags.go:64] FLAG: --ipvs-tcpfin-timeout="0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393505    5505 flags.go:64] FLAG: --ipvs-udp-timeout="0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393506    5505 flags.go:64] FLAG: --kube-api-burst="10"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393507    5505 flags.go:64] FLAG: --kube-api-content-type="application/vnd.kubernetes.protobuf"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393508    5505 flags.go:64] FLAG: --kube-api-qps="5"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393509    5505 flags.go:64] FLAG: --kubeconfig=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393510    5505 flags.go:64] FLAG: --log-flush-frequency="5s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393511    5505 flags.go:64] FLAG: --log-json-info-buffer-size="0"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393514    5505 flags.go:64] FLAG: --log-json-split-stream="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393515    5505 flags.go:64] FLAG: --log-text-info-buffer-size="0"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393517    5505 flags.go:64] FLAG: --log-text-split-stream="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393518    5505 flags.go:64] FLAG: --logging-format="text"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393519    5505 flags.go:64] FLAG: --machine-id-file="/etc/machine-id,/var/lib/dbus/machine-id"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393520    5505 flags.go:64] FLAG: --masquerade-all="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393521    5505 flags.go:64] FLAG: --master=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393522    5505 flags.go:64] FLAG: --metrics-bind-address="127.0.0.1:10249"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393524    5505 flags.go:64] FLAG: --metrics-port="10249"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393525    5505 flags.go:64] FLAG: --nodeport-addresses="[]"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393545    5505 flags.go:64] FLAG: --oom-score-adj="-999"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393546    5505 flags.go:64] FLAG: --pod-bridge-interface=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393547    5505 flags.go:64] FLAG: --pod-interface-name-prefix=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393550    5505 flags.go:64] FLAG: --profiling="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393551    5505 flags.go:64] FLAG: --proxy-mode=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393552    5505 flags.go:64] FLAG: --proxy-port-range=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393555    5505 flags.go:64] FLAG: --show-hidden-metrics-for-version=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393557    5505 flags.go:64] FLAG: --v="2"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393559    5505 flags.go:64] FLAG: --version="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393561    5505 flags.go:64] FLAG: --vmodule=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393562    5505 flags.go:64] FLAG: --write-config-to=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.393643    5505 feature_gate.go:254] feature gates: {map[]}
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.400741    5505 server.go:1062] "Successfully retrieved node IP(s)" IPs=["172.16.101.101"]
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.400927    5505 sysinfo.go:242] Found node without cache information, nodeDir: /sys/devices/system/node/node0
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.401049    5505 conntrack.go:119] "Set sysctl" entry="net/netfilter/nf_conntrack_max" value=131072
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.401070    5505 conntrack.go:59] "Setting nf_conntrack_max" nfConntrackMax=131072
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.401119    5505 conntrack.go:119] "Set sysctl" entry="net/netfilter/nf_conntrack_tcp_timeout_established" value=86400
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.401139    5505 conntrack.go:119] "Set sysctl" entry="net/netfilter/nf_conntrack_tcp_timeout_close_wait" value=3600
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: I0721 16:09:57.402942    5505 server.go:659] "kube-proxy running in dual-stack mode" primary ipFamily="IPv4"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: E0721 16:09:57.403713    5505 server.go:558] "Error running ProxyServer" err="can't use the IPVS proxier: error getting ipset version, error: executable file not found in $PATH"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5505]: E0721 16:09:57.403737    5505 run.go:74] "command failed" err="can't use the IPVS proxier: error getting ipset version, error: executable file not found in $PATH"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 16:09:57 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Failed with result 'exit-code'.
Jul 21 16:09:57 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Scheduled restart job, restart counter is at 1.
Jul 21 16:09:57 k8s-master-u2404-4-20-101 systemd[1]: Started kube-proxy.service - Kubernetes Proxy.
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773663    5518 flags.go:64] FLAG: --bind-address="0.0.0.0"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773707    5518 flags.go:64] FLAG: --bind-address-hard-fail="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773709    5518 flags.go:64] FLAG: --boot-id-file="/proc/sys/kernel/random/boot_id"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773712    5518 flags.go:64] FLAG: --cleanup="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773714    5518 flags.go:64] FLAG: --cluster-cidr=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773716    5518 flags.go:64] FLAG: --config="/opt/kubernetes/cfg/kube-proxy-config.yml"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773718    5518 flags.go:64] FLAG: --config-sync-period="15m0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773721    5518 flags.go:64] FLAG: --conntrack-max-per-core="32768"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773723    5518 flags.go:64] FLAG: --conntrack-min="131072"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773724    5518 flags.go:64] FLAG: --conntrack-tcp-be-liberal="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773726    5518 flags.go:64] FLAG: --conntrack-tcp-timeout-close-wait="1h0m0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773727    5518 flags.go:64] FLAG: --conntrack-tcp-timeout-established="24h0m0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773728    5518 flags.go:64] FLAG: --conntrack-udp-timeout="0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773730    5518 flags.go:64] FLAG: --conntrack-udp-timeout-stream="0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773731    5518 flags.go:64] FLAG: --detect-local-mode=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773733    5518 flags.go:64] FLAG: --feature-gates=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773736    5518 flags.go:64] FLAG: --healthz-bind-address="0.0.0.0:10256"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773738    5518 flags.go:64] FLAG: --healthz-port="10256"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773741    5518 flags.go:64] FLAG: --help="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773742    5518 flags.go:64] FLAG: --hostname-override=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773743    5518 flags.go:64] FLAG: --init-only="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773744    5518 flags.go:64] FLAG: --iptables-localhost-nodeports="true"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773746    5518 flags.go:64] FLAG: --iptables-masquerade-bit="14"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773747    5518 flags.go:64] FLAG: --iptables-min-sync-period="1s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773749    5518 flags.go:64] FLAG: --iptables-sync-period="30s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773750    5518 flags.go:64] FLAG: --ipvs-exclude-cidrs="[]"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773756    5518 flags.go:64] FLAG: --ipvs-min-sync-period="0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773757    5518 flags.go:64] FLAG: --ipvs-scheduler=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773759    5518 flags.go:64] FLAG: --ipvs-strict-arp="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773760    5518 flags.go:64] FLAG: --ipvs-sync-period="30s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773761    5518 flags.go:64] FLAG: --ipvs-tcp-timeout="0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773763    5518 flags.go:64] FLAG: --ipvs-tcpfin-timeout="0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773764    5518 flags.go:64] FLAG: --ipvs-udp-timeout="0s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773765    5518 flags.go:64] FLAG: --kube-api-burst="10"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773767    5518 flags.go:64] FLAG: --kube-api-content-type="application/vnd.kubernetes.protobuf"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773768    5518 flags.go:64] FLAG: --kube-api-qps="5"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773770    5518 flags.go:64] FLAG: --kubeconfig=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773772    5518 flags.go:64] FLAG: --log-flush-frequency="5s"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773773    5518 flags.go:64] FLAG: --log-json-info-buffer-size="0"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773776    5518 flags.go:64] FLAG: --log-json-split-stream="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773777    5518 flags.go:64] FLAG: --log-text-info-buffer-size="0"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773779    5518 flags.go:64] FLAG: --log-text-split-stream="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773781    5518 flags.go:64] FLAG: --logging-format="text"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773782    5518 flags.go:64] FLAG: --machine-id-file="/etc/machine-id,/var/lib/dbus/machine-id"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773784    5518 flags.go:64] FLAG: --masquerade-all="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773785    5518 flags.go:64] FLAG: --master=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773786    5518 flags.go:64] FLAG: --metrics-bind-address="127.0.0.1:10249"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773788    5518 flags.go:64] FLAG: --metrics-port="10249"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773789    5518 flags.go:64] FLAG: --nodeport-addresses="[]"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773791    5518 flags.go:64] FLAG: --oom-score-adj="-999"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773792    5518 flags.go:64] FLAG: --pod-bridge-interface=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773793    5518 flags.go:64] FLAG: --pod-interface-name-prefix=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773794    5518 flags.go:64] FLAG: --profiling="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773796    5518 flags.go:64] FLAG: --proxy-mode=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773797    5518 flags.go:64] FLAG: --proxy-port-range=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773799    5518 flags.go:64] FLAG: --show-hidden-metrics-for-version=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773801    5518 flags.go:64] FLAG: --v="2"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773802    5518 flags.go:64] FLAG: --version="false"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773805    5518 flags.go:64] FLAG: --vmodule=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773806    5518 flags.go:64] FLAG: --write-config-to=""
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.773842    5518 feature_gate.go:254] feature gates: {map[]}
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.781366    5518 server.go:1062] "Successfully retrieved node IP(s)" IPs=["172.16.101.101"]
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.781622    5518 sysinfo.go:242] Found node without cache information, nodeDir: /sys/devices/system/node/node0
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.781742    5518 conntrack.go:59] "Setting nf_conntrack_max" nfConntrackMax=131072
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: I0721 16:09:57.784141    5518 server.go:659] "kube-proxy running in dual-stack mode" primary ipFamily="IPv4"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: E0721 16:09:57.785084    5518 server.go:558] "Error running ProxyServer" err="can't use the IPVS proxier: error getting ipset version, error: executable file not found in $PATH"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 kube-proxy[5518]: E0721 16:09:57.785144    5518 run.go:74] "command failed" err="can't use the IPVS proxier: error getting ipset version, error: executable file not found in $PATH"
Jul 21 16:09:57 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 16:09:57 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Failed with result 'exit-code'.
Jul 21 16:09:57 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Scheduled restart job, restart counter is at 2.
Jul 21 16:09:57 k8s-master-u2404-4-20-101 systemd[1]: Started kube-proxy.service - Kubernetes Proxy.
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012101    5531 flags.go:64] FLAG: --bind-address="0.0.0.0"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012133    5531 flags.go:64] FLAG: --bind-address-hard-fail="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012136    5531 flags.go:64] FLAG: --boot-id-file="/proc/sys/kernel/random/boot_id"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012138    5531 flags.go:64] FLAG: --cleanup="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012140    5531 flags.go:64] FLAG: --cluster-cidr=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012143    5531 flags.go:64] FLAG: --config="/opt/kubernetes/cfg/kube-proxy-config.yml"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012144    5531 flags.go:64] FLAG: --config-sync-period="15m0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012149    5531 flags.go:64] FLAG: --conntrack-max-per-core="32768"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012151    5531 flags.go:64] FLAG: --conntrack-min="131072"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012152    5531 flags.go:64] FLAG: --conntrack-tcp-be-liberal="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012153    5531 flags.go:64] FLAG: --conntrack-tcp-timeout-close-wait="1h0m0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012155    5531 flags.go:64] FLAG: --conntrack-tcp-timeout-established="24h0m0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012157    5531 flags.go:64] FLAG: --conntrack-udp-timeout="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012159    5531 flags.go:64] FLAG: --conntrack-udp-timeout-stream="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012160    5531 flags.go:64] FLAG: --detect-local-mode=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012162    5531 flags.go:64] FLAG: --feature-gates=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012164    5531 flags.go:64] FLAG: --healthz-bind-address="0.0.0.0:10256"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012165    5531 flags.go:64] FLAG: --healthz-port="10256"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012167    5531 flags.go:64] FLAG: --help="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012168    5531 flags.go:64] FLAG: --hostname-override=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012170    5531 flags.go:64] FLAG: --init-only="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012171    5531 flags.go:64] FLAG: --iptables-localhost-nodeports="true"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012172    5531 flags.go:64] FLAG: --iptables-masquerade-bit="14"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012173    5531 flags.go:64] FLAG: --iptables-min-sync-period="1s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012174    5531 flags.go:64] FLAG: --iptables-sync-period="30s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012176    5531 flags.go:64] FLAG: --ipvs-exclude-cidrs="[]"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012179    5531 flags.go:64] FLAG: --ipvs-min-sync-period="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012181    5531 flags.go:64] FLAG: --ipvs-scheduler=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012182    5531 flags.go:64] FLAG: --ipvs-strict-arp="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012183    5531 flags.go:64] FLAG: --ipvs-sync-period="30s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012184    5531 flags.go:64] FLAG: --ipvs-tcp-timeout="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012185    5531 flags.go:64] FLAG: --ipvs-tcpfin-timeout="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012187    5531 flags.go:64] FLAG: --ipvs-udp-timeout="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012188    5531 flags.go:64] FLAG: --kube-api-burst="10"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012189    5531 flags.go:64] FLAG: --kube-api-content-type="application/vnd.kubernetes.protobuf"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012191    5531 flags.go:64] FLAG: --kube-api-qps="5"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012192    5531 flags.go:64] FLAG: --kubeconfig=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012194    5531 flags.go:64] FLAG: --log-flush-frequency="5s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012195    5531 flags.go:64] FLAG: --log-json-info-buffer-size="0"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012197    5531 flags.go:64] FLAG: --log-json-split-stream="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012199    5531 flags.go:64] FLAG: --log-text-info-buffer-size="0"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012200    5531 flags.go:64] FLAG: --log-text-split-stream="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012202    5531 flags.go:64] FLAG: --logging-format="text"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012204    5531 flags.go:64] FLAG: --machine-id-file="/etc/machine-id,/var/lib/dbus/machine-id"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012205    5531 flags.go:64] FLAG: --masquerade-all="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012207    5531 flags.go:64] FLAG: --master=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012208    5531 flags.go:64] FLAG: --metrics-bind-address="127.0.0.1:10249"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012210    5531 flags.go:64] FLAG: --metrics-port="10249"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012211    5531 flags.go:64] FLAG: --nodeport-addresses="[]"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012214    5531 flags.go:64] FLAG: --oom-score-adj="-999"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012216    5531 flags.go:64] FLAG: --pod-bridge-interface=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012217    5531 flags.go:64] FLAG: --pod-interface-name-prefix=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012218    5531 flags.go:64] FLAG: --profiling="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012219    5531 flags.go:64] FLAG: --proxy-mode=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012222    5531 flags.go:64] FLAG: --proxy-port-range=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012224    5531 flags.go:64] FLAG: --show-hidden-metrics-for-version=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012226    5531 flags.go:64] FLAG: --v="2"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012228    5531 flags.go:64] FLAG: --version="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012232    5531 flags.go:64] FLAG: --vmodule=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012234    5531 flags.go:64] FLAG: --write-config-to=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.012290    5531 feature_gate.go:254] feature gates: {map[]}
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.019023    5531 server.go:1062] "Successfully retrieved node IP(s)" IPs=["172.16.101.101"]
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.019419    5531 sysinfo.go:242] Found node without cache information, nodeDir: /sys/devices/system/node/node0
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.019534    5531 conntrack.go:59] "Setting nf_conntrack_max" nfConntrackMax=131072
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: I0721 16:09:58.022299    5531 server.go:659] "kube-proxy running in dual-stack mode" primary ipFamily="IPv4"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: E0721 16:09:58.023172    5531 server.go:558] "Error running ProxyServer" err="can't use the IPVS proxier: error getting ipset version, error: executable file not found in $PATH"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5531]: E0721 16:09:58.023188    5531 run.go:74] "command failed" err="can't use the IPVS proxier: error getting ipset version, error: executable file not found in $PATH"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Failed with result 'exit-code'.
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Scheduled restart job, restart counter is at 3.
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: Started kube-proxy.service - Kubernetes Proxy.
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.254949    5543 flags.go:64] FLAG: --bind-address="0.0.0.0"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.254986    5543 flags.go:64] FLAG: --bind-address-hard-fail="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.254990    5543 flags.go:64] FLAG: --boot-id-file="/proc/sys/kernel/random/boot_id"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.254993    5543 flags.go:64] FLAG: --cleanup="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.254995    5543 flags.go:64] FLAG: --cluster-cidr=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.254998    5543 flags.go:64] FLAG: --config="/opt/kubernetes/cfg/kube-proxy-config.yml"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255001    5543 flags.go:64] FLAG: --config-sync-period="15m0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255003    5543 flags.go:64] FLAG: --conntrack-max-per-core="32768"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255006    5543 flags.go:64] FLAG: --conntrack-min="131072"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255008    5543 flags.go:64] FLAG: --conntrack-tcp-be-liberal="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255010    5543 flags.go:64] FLAG: --conntrack-tcp-timeout-close-wait="1h0m0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255012    5543 flags.go:64] FLAG: --conntrack-tcp-timeout-established="24h0m0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255014    5543 flags.go:64] FLAG: --conntrack-udp-timeout="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255016    5543 flags.go:64] FLAG: --conntrack-udp-timeout-stream="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255018    5543 flags.go:64] FLAG: --detect-local-mode=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255021    5543 flags.go:64] FLAG: --feature-gates=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255024    5543 flags.go:64] FLAG: --healthz-bind-address="0.0.0.0:10256"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255027    5543 flags.go:64] FLAG: --healthz-port="10256"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255029    5543 flags.go:64] FLAG: --help="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255030    5543 flags.go:64] FLAG: --hostname-override=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255032    5543 flags.go:64] FLAG: --init-only="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255034    5543 flags.go:64] FLAG: --iptables-localhost-nodeports="true"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255036    5543 flags.go:64] FLAG: --iptables-masquerade-bit="14"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255038    5543 flags.go:64] FLAG: --iptables-min-sync-period="1s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255039    5543 flags.go:64] FLAG: --iptables-sync-period="30s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255042    5543 flags.go:64] FLAG: --ipvs-exclude-cidrs="[]"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255047    5543 flags.go:64] FLAG: --ipvs-min-sync-period="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255051    5543 flags.go:64] FLAG: --ipvs-scheduler=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255053    5543 flags.go:64] FLAG: --ipvs-strict-arp="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255054    5543 flags.go:64] FLAG: --ipvs-sync-period="30s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255056    5543 flags.go:64] FLAG: --ipvs-tcp-timeout="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255058    5543 flags.go:64] FLAG: --ipvs-tcpfin-timeout="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255060    5543 flags.go:64] FLAG: --ipvs-udp-timeout="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255062    5543 flags.go:64] FLAG: --kube-api-burst="10"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255063    5543 flags.go:64] FLAG: --kube-api-content-type="application/vnd.kubernetes.protobuf"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255066    5543 flags.go:64] FLAG: --kube-api-qps="5"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255069    5543 flags.go:64] FLAG: --kubeconfig=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255070    5543 flags.go:64] FLAG: --log-flush-frequency="5s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255072    5543 flags.go:64] FLAG: --log-json-info-buffer-size="0"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255075    5543 flags.go:64] FLAG: --log-json-split-stream="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255077    5543 flags.go:64] FLAG: --log-text-info-buffer-size="0"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255079    5543 flags.go:64] FLAG: --log-text-split-stream="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255083    5543 flags.go:64] FLAG: --logging-format="text"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255085    5543 flags.go:64] FLAG: --machine-id-file="/etc/machine-id,/var/lib/dbus/machine-id"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255087    5543 flags.go:64] FLAG: --masquerade-all="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255089    5543 flags.go:64] FLAG: --master=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255091    5543 flags.go:64] FLAG: --metrics-bind-address="127.0.0.1:10249"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255093    5543 flags.go:64] FLAG: --metrics-port="10249"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255095    5543 flags.go:64] FLAG: --nodeport-addresses="[]"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255098    5543 flags.go:64] FLAG: --oom-score-adj="-999"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255101    5543 flags.go:64] FLAG: --pod-bridge-interface=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255102    5543 flags.go:64] FLAG: --pod-interface-name-prefix=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255104    5543 flags.go:64] FLAG: --profiling="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255106    5543 flags.go:64] FLAG: --proxy-mode=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255108    5543 flags.go:64] FLAG: --proxy-port-range=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255110    5543 flags.go:64] FLAG: --show-hidden-metrics-for-version=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255112    5543 flags.go:64] FLAG: --v="2"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255115    5543 flags.go:64] FLAG: --version="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255117    5543 flags.go:64] FLAG: --vmodule=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255120    5543 flags.go:64] FLAG: --write-config-to=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.255164    5543 feature_gate.go:254] feature gates: {map[]}
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.261862    5543 server.go:1062] "Successfully retrieved node IP(s)" IPs=["172.16.101.101"]
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.262085    5543 sysinfo.go:242] Found node without cache information, nodeDir: /sys/devices/system/node/node0
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.262238    5543 conntrack.go:59] "Setting nf_conntrack_max" nfConntrackMax=131072
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: I0721 16:09:58.265177    5543 server.go:659] "kube-proxy running in dual-stack mode" primary ipFamily="IPv4"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: E0721 16:09:58.266224    5543 server.go:558] "Error running ProxyServer" err="can't use the IPVS proxier: error getting ipset version, error: executable file not found in $PATH"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5543]: E0721 16:09:58.266243    5543 run.go:74] "command failed" err="can't use the IPVS proxier: error getting ipset version, error: executable file not found in $PATH"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Failed with result 'exit-code'.
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Scheduled restart job, restart counter is at 4.
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: Started kube-proxy.service - Kubernetes Proxy.
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510750    5555 flags.go:64] FLAG: --bind-address="0.0.0.0"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510814    5555 flags.go:64] FLAG: --bind-address-hard-fail="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510817    5555 flags.go:64] FLAG: --boot-id-file="/proc/sys/kernel/random/boot_id"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510819    5555 flags.go:64] FLAG: --cleanup="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510821    5555 flags.go:64] FLAG: --cluster-cidr=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510823    5555 flags.go:64] FLAG: --config="/opt/kubernetes/cfg/kube-proxy-config.yml"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510825    5555 flags.go:64] FLAG: --config-sync-period="15m0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510827    5555 flags.go:64] FLAG: --conntrack-max-per-core="32768"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510829    5555 flags.go:64] FLAG: --conntrack-min="131072"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510831    5555 flags.go:64] FLAG: --conntrack-tcp-be-liberal="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510833    5555 flags.go:64] FLAG: --conntrack-tcp-timeout-close-wait="1h0m0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510840    5555 flags.go:64] FLAG: --conntrack-tcp-timeout-established="24h0m0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510842    5555 flags.go:64] FLAG: --conntrack-udp-timeout="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510844    5555 flags.go:64] FLAG: --conntrack-udp-timeout-stream="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510846    5555 flags.go:64] FLAG: --detect-local-mode=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510850    5555 flags.go:64] FLAG: --feature-gates=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510852    5555 flags.go:64] FLAG: --healthz-bind-address="0.0.0.0:10256"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510854    5555 flags.go:64] FLAG: --healthz-port="10256"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510856    5555 flags.go:64] FLAG: --help="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510857    5555 flags.go:64] FLAG: --hostname-override=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510858    5555 flags.go:64] FLAG: --init-only="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510859    5555 flags.go:64] FLAG: --iptables-localhost-nodeports="true"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510860    5555 flags.go:64] FLAG: --iptables-masquerade-bit="14"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510862    5555 flags.go:64] FLAG: --iptables-min-sync-period="1s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510863    5555 flags.go:64] FLAG: --iptables-sync-period="30s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510865    5555 flags.go:64] FLAG: --ipvs-exclude-cidrs="[]"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510867    5555 flags.go:64] FLAG: --ipvs-min-sync-period="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510868    5555 flags.go:64] FLAG: --ipvs-scheduler=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510870    5555 flags.go:64] FLAG: --ipvs-strict-arp="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510871    5555 flags.go:64] FLAG: --ipvs-sync-period="30s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510873    5555 flags.go:64] FLAG: --ipvs-tcp-timeout="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510875    5555 flags.go:64] FLAG: --ipvs-tcpfin-timeout="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510876    5555 flags.go:64] FLAG: --ipvs-udp-timeout="0s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510877    5555 flags.go:64] FLAG: --kube-api-burst="10"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510879    5555 flags.go:64] FLAG: --kube-api-content-type="application/vnd.kubernetes.protobuf"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510880    5555 flags.go:64] FLAG: --kube-api-qps="5"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510882    5555 flags.go:64] FLAG: --kubeconfig=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510883    5555 flags.go:64] FLAG: --log-flush-frequency="5s"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510885    5555 flags.go:64] FLAG: --log-json-info-buffer-size="0"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510887    5555 flags.go:64] FLAG: --log-json-split-stream="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510889    5555 flags.go:64] FLAG: --log-text-info-buffer-size="0"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510890    5555 flags.go:64] FLAG: --log-text-split-stream="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510894    5555 flags.go:64] FLAG: --logging-format="text"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510896    5555 flags.go:64] FLAG: --machine-id-file="/etc/machine-id,/var/lib/dbus/machine-id"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510900    5555 flags.go:64] FLAG: --masquerade-all="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510901    5555 flags.go:64] FLAG: --master=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510902    5555 flags.go:64] FLAG: --metrics-bind-address="127.0.0.1:10249"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510904    5555 flags.go:64] FLAG: --metrics-port="10249"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510905    5555 flags.go:64] FLAG: --nodeport-addresses="[]"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510907    5555 flags.go:64] FLAG: --oom-score-adj="-999"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510909    5555 flags.go:64] FLAG: --pod-bridge-interface=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510910    5555 flags.go:64] FLAG: --pod-interface-name-prefix=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510912    5555 flags.go:64] FLAG: --profiling="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510913    5555 flags.go:64] FLAG: --proxy-mode=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510914    5555 flags.go:64] FLAG: --proxy-port-range=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510916    5555 flags.go:64] FLAG: --show-hidden-metrics-for-version=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510918    5555 flags.go:64] FLAG: --v="2"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510920    5555 flags.go:64] FLAG: --version="false"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510922    5555 flags.go:64] FLAG: --vmodule=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510923    5555 flags.go:64] FLAG: --write-config-to=""
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.510956    5555 feature_gate.go:254] feature gates: {map[]}
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.517767    5555 server.go:1062] "Successfully retrieved node IP(s)" IPs=["172.16.101.101"]
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.518019    5555 sysinfo.go:242] Found node without cache information, nodeDir: /sys/devices/system/node/node0
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.518123    5555 conntrack.go:59] "Setting nf_conntrack_max" nfConntrackMax=131072
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: I0721 16:09:58.520555    5555 server.go:659] "kube-proxy running in dual-stack mode" primary ipFamily="IPv4"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: E0721 16:09:58.521484    5555 server.go:558] "Error running ProxyServer" err="can't use the IPVS proxier: error getting ipset version, error: executable file not found in $PATH"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 kube-proxy[5555]: E0721 16:09:58.521499    5555 run.go:74] "command failed" err="can't use the IPVS proxier: error getting ipset version, error: executable file not found in $PATH"
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Failed with result 'exit-code'.
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Scheduled restart job, restart counter is at 5.
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Start request repeated too quickly.
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: kube-proxy.service: Failed with result 'exit-code'.
Jul 21 16:09:58 k8s-master-u2404-4-20-101 systemd[1]: Failed to start kube-proxy.service - Kubernetes Proxy.

碎碎念

这块的会出现问题的点大致集中在证书问题，只要证书签的对，同时路径和命令都在话基本没啥太大问题

对了证书错误，是无法使 kube-proxy 直接报错的，注意查看 kube-proxy 日志

后面在安装网络插件的时候也可能涉及到这边的一些重启

By jiutingqiu

Related Post

Kubernetes

Kubernetes 分布式部署（5）Kubernetes 1.30.x Master配置

jiutingqiu
Kubernetes

Kubernetes 分布式部署（4）ETCD 部署配置

jiutingqiu
Kubernetes

Kubernetes 分布式部署（3）证书配置

jiutingqiu

发表回复

您的电子邮箱地址不会被公开。 必填项已用*标注

You Missed

Kubernetes

Kubernetes 分布式部署（6）Kubernetes 1.30.x Node配置

Kubernetes

Kubernetes 分布式部署（5）Kubernetes 1.30.x Master配置

Kubernetes

Kubernetes 分布式部署（4）ETCD 部署配置

Kubernetes

Kubernetes 分布式部署（3）证书配置