diff --git a/cmd/kubenest/node-agent/app/serve/serve.go b/cmd/kubenest/node-agent/app/serve/serve.go index b7b4045f0..aed88f45a 100644 --- a/cmd/kubenest/node-agent/app/serve/serve.go +++ b/cmd/kubenest/node-agent/app/serve/serve.go @@ -69,6 +69,11 @@ func Start(addr, certFile, keyFile, user, password string) error { passwordHash := sha256.Sum256([]byte(password)) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/healthz" || r.URL.Path == "/readyz" { + w.WriteHeader(http.StatusOK) + return + } + auth := r.Header.Get("Authorization") if auth == "" { http.Error(w, "Unauthorized", http.StatusUnauthorized) diff --git a/cmd/kubenest/operator/app/operator.go b/cmd/kubenest/operator/app/operator.go index 35d38067c..494432d00 100644 --- a/cmd/kubenest/operator/app/operator.go +++ b/cmd/kubenest/operator/app/operator.go @@ -15,6 +15,7 @@ import ( cliflag "k8s.io/component-base/cli/flag" "k8s.io/klog/v2" controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/config" @@ -223,11 +224,24 @@ func run(ctx context.Context, config *config.Config) error { LeaderElection: config.LeaderElection.LeaderElect, LeaderElectionID: config.LeaderElection.ResourceName, LeaderElectionNamespace: config.LeaderElection.ResourceNamespace, + LivenessEndpointName: "/healthz", + ReadinessEndpointName: "/readyz", + HealthProbeBindAddress: ":8081", }) if err != nil { return fmt.Errorf("failed to build controller manager: %v", err) } + err = mgr.AddHealthzCheck("healthz", healthz.Ping) + if err != nil { + return fmt.Errorf("failed to build healthz: %v", err) + } + + err = mgr.AddReadyzCheck("readyz", healthz.Ping) + if err != nil { + return fmt.Errorf("failed to build readyz: %v", err) + } + hostKubeClient, err := kubernetes.NewForConfig(config.RestConfig) if err != nil { return fmt.Errorf("could not create clientset: %v", err) diff --git a/deploy/virtual-cluster-operator.yml b/deploy/virtual-cluster-operator.yml index cd8a6de39..77c6a542a 100644 --- a/deploy/virtual-cluster-operator.yml +++ b/deploy/virtual-cluster-operator.yml @@ -103,11 +103,37 @@ spec: values: - virtual-cluster-operator topologyKey: kubernetes.io/hostname + initContainers: + - name: init-container + image: ghcr.io/kosmos-io/virtual-cluster-operator:__VERSION__ + command: + - sh + - "-c" + - | + mkdir -p /kosmos && \ + tar -xzf /config/manifest-use.tar -C /kosmos + volumeMounts: + - name: manifest-volume + mountPath: /kosmos + - name: manifest-tar + mountPath: /config containers: - name: virtual-cluster-operator # Change a valid image address image: ghcr.io/kosmos-io/virtual-cluster-operator:__VERSION__ imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 env: - name: IMAGE_REPOSITIRY value: ghcr.io/kosmos-io @@ -150,8 +176,8 @@ spec: - name: shellscript mountPath: /etc/vc-node-dir/config.yaml subPath: config.yaml - - mountPath: /kosmos/manifest - name: components-manifest + - mountPath: /kosmos + name: manifest-volume command: - virtual-cluster-operator - --kubeconfig=/etc/virtual-cluster-operator/kubeconfig @@ -163,10 +189,11 @@ spec: - name: shellscript configMap: name: virtual-cluster-operator - - hostPath: - path: /home/vc-operator/manifest - type: DirectoryOrCreate - name: components-manifest + - name: manifest-tar + configMap: + name: manifest-tar + - name: manifest-volume + emptyDir: {} --- apiVersion: apps/v1 kind: DaemonSet @@ -216,6 +243,20 @@ spec: containers: - name: install-agent image: cis-hub-huabei-3.cmecloud.cn/node-agent/node-agent:latest + livenessProbe: + httpGet: + path: /healthz + port: 5678 + scheme: HTTPS + initialDelaySeconds: 15 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /readyz + port: 5678 + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 10 securityContext: privileged: true # container privileged command: diff --git a/hack/k8s-in-k8s/g.env.sh b/hack/k8s-in-k8s/g.env.sh deleted file mode 100644 index fd1f0d64e..000000000 --- a/hack/k8s-in-k8s/g.env.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -# ##### -# Generate by script generate_env.sh -# ##### - -SCRIPT_VERSION=0.0.1 -# tmp dir of kosmos -PATH_FILE_TMP=/apps/conf/kosmos/tmp -################################################## -# path for kubeadm config -PATH_KUBEADM_CONFIG=/etc/kubeadm -################################################## -# path for kubernetes, from kubelet args --config -PATH_KUBERNETES=/etc/kubernetes -PATH_KUBERNETES_PKI=/etc/kubernetes/pki -# name for kubelet kubeconfig file -KUBELET_KUBE_CONFIG_NAME= -################################################## -# path for kubelet -PATH_KUBELET_LIB=/var/lib/kubelet -# path for kubelet -PATH_KUBELET_CONF=. -# name for config file of kubelet -KUBELET_CONFIG_NAME= -HOST_CORE_DNS=10.96.0.10 -# kubeadm switch -USE_KUBEADM=false -# Generate kubelet.conf TIMEOUT -KUBELET_CONF_TIMEOUT=30 - -function GenerateKubeadmConfig() { - echo "--- -apiVersion: kubeadm.k8s.io/v1beta2 -caCertPath: /etc/kubernetes/pki/ca.crt -discovery: - bootstrapToken: - apiServerEndpoint: apiserver.cluster.local:6443 - token: $1 - unsafeSkipCAVerification: true -kind: JoinConfiguration -nodeRegistration: - criSocket: /run/containerd/containerd.sock - kubeletExtraArgs: - container-runtime: remote - container-runtime-endpoint: unix:///run/containerd/containerd.sock - taints: null" > $2/kubeadm.cfg.current -} - -function GenerateStaticNginxProxy() { - echo "apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: nginx-proxy - namespace: kube-system -spec: - containers: - - image: registry.paas/cmss/nginx:1.21.4 - imagePullPolicy: IfNotPresent - name: nginx-proxy - resources: - limits: - cpu: 300m - memory: 512M - requests: - cpu: 25m - memory: 32M - securityContext: - privileged: true - volumeMounts: - - mountPath: /etc/nginx - name: etc-nginx - readOnly: true - hostNetwork: true - priorityClassName: system-node-critical - volumes: - - hostPath: - path: /apps/conf/nginx - type: - name: etc-nginx -status: {}" > /etc/kubernetes/manifests/nginx-proxy.yaml -} - -