From d583c73fc53de1b7e3f248326ee7631dfcea4bde Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Mon, 16 Oct 2023 10:29:42 -0600 Subject: [PATCH 1/8] implement 'IsHelmVM' function based on presence of configmap if embedded-cluster-config exists in kube-system it is helmvm --- pkg/helmvm/util.go | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/pkg/helmvm/util.go b/pkg/helmvm/util.go index 7d2817f93e..3c2e4fbe66 100644 --- a/pkg/helmvm/util.go +++ b/pkg/helmvm/util.go @@ -1,11 +1,43 @@ package helmvm import ( + "context" + "fmt" + corev1 "k8s.io/api/core/v1" + + kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) +const configMapName = "embedded-cluster-config" +const configMapNamespace = "kube-system" + +// ReadConfigMap will read the Kurl config from a configmap +func ReadConfigMap(client kubernetes.Interface) (*corev1.ConfigMap, error) { + return client.CoreV1().ConfigMaps(configMapNamespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) +} + func IsHelmVM(clientset kubernetes.Interface) (bool, error) { - return false, nil + if clientset == nil { + return false, fmt.Errorf("clientset is nil") + } + + configMapExists := false + _, err := ReadConfigMap(clientset) + if err == nil { + configMapExists = true + } else if kuberneteserrors.IsNotFound(err) { + configMapExists = false + } else if kuberneteserrors.IsUnauthorized(err) { + configMapExists = false + } else if kuberneteserrors.IsForbidden(err) { + configMapExists = false + } else if err != nil { + return false, fmt.Errorf("failed to get embedded cluster configmap: %w", err) + } + + return configMapExists, nil } func IsHA(clientset kubernetes.Interface) (bool, error) { From 9bfc65162ba097b065a3768ff4d3d18a9c7109c4 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Oct 2023 11:08:03 -0600 Subject: [PATCH 2/8] wip --- Makefile | 4 ++++ deploy/okteto/okteto.Dockerfile | 12 ++++++------ node_modules/.yarn-integrity | 10 ++++++++++ pkg/helmvm/node_join.go | 7 ++++++- pkg/helmvm/util.go | 2 ++ 5 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 node_modules/.yarn-integrity diff --git a/Makefile b/Makefile index c4cb1e1106..a3cd5960ce 100644 --- a/Makefile +++ b/Makefile @@ -131,6 +131,10 @@ all-ttl.sh: build-ttl.sh docker tag rqlite/rqlite:${RQLITE_TAG} ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} docker push ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} +api-ttl.sh: + docker build -f ./deploy/okteto/okteto.Dockerfile -t ttl.sh/${CURRENT_USER}/kotsadm-api:24h . + docker push ttl.sh/${CURRENT_USER}/kotsadm-api:24h + .PHONY: build-alpha build-alpha: docker build --pull -f deploy/Dockerfile --build-arg version=${GIT_TAG} -t kotsadm/kotsadm:alpha . diff --git a/deploy/okteto/okteto.Dockerfile b/deploy/okteto/okteto.Dockerfile index c98a583c61..a626708f29 100644 --- a/deploy/okteto/okteto.Dockerfile +++ b/deploy/okteto/okteto.Dockerfile @@ -167,11 +167,11 @@ RUN --mount=target=$GOMODCACHE,id=kots-gomodcache,type=cache \ mv ./bin/kotsadm /kotsadm && \ mv ./bin/kots /kots -RUN --mount=target=/tmp/.cache/gocache,id=kots-gocache,type=cache \ - --mount=target=/tmp/.cache/gomodcache,id=kots-gomodcache,type=cache \ - mkdir -p $GOCACHE \ - && cp -r /tmp/.cache/gocache/* $GOCACHE \ - && mkdir -p $GOMODCACHE \ - && cp -r /tmp/.cache/gomodcache/* $GOMODCACHE +#RUN --mount=target=/tmp/.cache/gocache,id=kots-gocache,type=cache \ +# --mount=target=/tmp/.cache/gomodcache,id=kots-gomodcache,type=cache \ +# mkdir -p $GOCACHE \ +# && cp -r /tmp/.cache/gocache/* $GOCACHE \ +# && mkdir -p $GOMODCACHE \ +# && cp -r /tmp/.cache/gomodcache/* $GOMODCACHE ENTRYPOINT [ "/kotsadm", "api"] diff --git a/node_modules/.yarn-integrity b/node_modules/.yarn-integrity new file mode 100644 index 0000000000..3c4955ff32 --- /dev/null +++ b/node_modules/.yarn-integrity @@ -0,0 +1,10 @@ +{ + "systemParams": "darwin-x64-111", + "modulesFolders": [], + "flags": [], + "linkedModules": [], + "topLevelPatterns": [], + "lockfileEntries": {}, + "files": [], + "artifacts": {} +} \ No newline at end of file diff --git a/pkg/helmvm/node_join.go b/pkg/helmvm/node_join.go index 6aad6255a9..4f47a5128f 100644 --- a/pkg/helmvm/node_join.go +++ b/pkg/helmvm/node_join.go @@ -8,5 +8,10 @@ import ( // GenerateAddNodeCommand will generate the HelmVM node add command for a primary or secondary node func GenerateAddNodeCommand(client kubernetes.Interface, primary bool) ([]string, *time.Time, error) { - return nil, nil, nil + tomorrow := time.Now().Add(time.Hour * 24) + if primary { + return []string{"this is a primary join command string", "that can be multiple strings"}, &tomorrow, nil + } else { + return []string{"this is a secondary join command string", "that can be multiple strings"}, &tomorrow, nil + } } diff --git a/pkg/helmvm/util.go b/pkg/helmvm/util.go index 3c2e4fbe66..1b1d7b93e4 100644 --- a/pkg/helmvm/util.go +++ b/pkg/helmvm/util.go @@ -37,6 +37,8 @@ func IsHelmVM(clientset kubernetes.Interface) (bool, error) { return false, fmt.Errorf("failed to get embedded cluster configmap: %w", err) } + fmt.Printf("Is Embedded Cluster Config found: %v\n", configMapExists) + return configMapExists, nil } From 05be39862d3a4142a902a6747007a372dea20a83 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Oct 2023 11:48:26 -0600 Subject: [PATCH 3/8] node metrics --- pkg/handlers/helmvm_get.go | 2 +- pkg/helmvm/helmvm_nodes.go | 49 ++++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/pkg/handlers/helmvm_get.go b/pkg/handlers/helmvm_get.go index cd440d116f..91133a03b6 100644 --- a/pkg/handlers/helmvm_get.go +++ b/pkg/handlers/helmvm_get.go @@ -16,7 +16,7 @@ func (h *Handler) GetHelmVMNodes(w http.ResponseWriter, r *http.Request) { return } - nodes, err := helmvm.GetNodes(client) + nodes, err := helmvm.GetNodes(r.Context(), client) if err != nil { logger.Error(err) w.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/helmvm/helmvm_nodes.go b/pkg/helmvm/helmvm_nodes.go index e00dca2108..ccf570c9eb 100644 --- a/pkg/helmvm/helmvm_nodes.go +++ b/pkg/helmvm/helmvm_nodes.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "encoding/json" "fmt" + "github.com/replicatedhq/kots/pkg/k8sutil" "io" "math" "net/http" @@ -14,20 +15,30 @@ import ( "github.com/pkg/errors" "github.com/replicatedhq/kots/pkg/helmvm/types" - "github.com/replicatedhq/kots/pkg/logger" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" statsv1alpha1 "k8s.io/kubelet/pkg/apis/stats/v1alpha1" + metricsv "k8s.io/metrics/pkg/client/clientset/versioned" ) // GetNodes will get a list of nodes with stats -func GetNodes(client kubernetes.Interface) (*types.HelmVMNodes, error) { - nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) +func GetNodes(ctx context.Context, client kubernetes.Interface) (*types.HelmVMNodes, error) { + nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { return nil, errors.Wrap(err, "list nodes") } + clientConfig, err := k8sutil.GetClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get cluster config") + } + + metricsClient, err := metricsv.NewForConfig(clientConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to create metrics client") + } + toReturn := types.HelmVMNodes{} for _, node := range nodes.Items { @@ -44,32 +55,24 @@ func GetNodes(client kubernetes.Interface) (*types.HelmVMNodes, error) { podCapacity.Capacity = float64(node.Status.Capacity.Pods().Value()) - nodeIP := "" - for _, address := range node.Status.Addresses { - if address.Type == corev1.NodeInternalIP { - nodeIP = address.Address - } + nodeMetrics, err := metricsClient.MetricsV1beta1().NodeMetricses().Get(ctx, node.Name, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrap(err, "list pod metrics") } - if nodeIP == "" { - logger.Infof("Did not find address for node %s, %+v", node.Name, node.Status.Addresses) - } else { - nodeMetrics, err := getNodeMetrics(nodeIP) - if err != nil { - logger.Infof("Got error retrieving stats for node %q: %v", node.Name, err) - } else { - if nodeMetrics.Node.Memory != nil && nodeMetrics.Node.Memory.AvailableBytes != nil { - memoryCapacity.Available = float64(*nodeMetrics.Node.Memory.AvailableBytes) / math.Pow(2, 30) - } + str, _ := json.Marshal(nodeMetrics) + fmt.Printf("node %s metrics: %s\n", node.Name, str) - if nodeMetrics.Node.CPU != nil && nodeMetrics.Node.CPU.UsageNanoCores != nil { - cpuCapacity.Available = cpuCapacity.Capacity - (float64(*nodeMetrics.Node.CPU.UsageNanoCores) / math.Pow(10, 9)) - } + if nodeMetrics.Usage.Memory() != nil { + memoryCapacity.Available = float64(nodeMetrics.Usage.Memory().Value()) / math.Pow(2, 30) + } - podCapacity.Available = podCapacity.Capacity - float64(len(nodeMetrics.Pods)) - } + if nodeMetrics.Usage.Cpu() != nil { + cpuCapacity.Available = cpuCapacity.Capacity - float64(nodeMetrics.Usage.Cpu().Value()) } + podCapacity.Available = podCapacity.Capacity - float64(nodeMetrics.Usage.Pods().Value()) + nodeLabelArray := []string{} for k, v := range node.Labels { nodeLabelArray = append(nodeLabelArray, fmt.Sprintf("%s:%s", k, v)) From 6768100abd3eabf71cfabdf64594a804d0d87330 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Oct 2023 12:40:21 -0600 Subject: [PATCH 4/8] add node pod capacity and list of pods --- pkg/helmvm/helmvm_nodes.go | 79 +++++++++++++++----------------------- pkg/helmvm/types/types.go | 3 ++ pkg/helmvm/util.go | 2 +- 3 files changed, 35 insertions(+), 49 deletions(-) diff --git a/pkg/helmvm/helmvm_nodes.go b/pkg/helmvm/helmvm_nodes.go index ccf570c9eb..c09bff582e 100644 --- a/pkg/helmvm/helmvm_nodes.go +++ b/pkg/helmvm/helmvm_nodes.go @@ -2,23 +2,16 @@ package helmvm import ( "context" - "crypto/tls" - "encoding/json" "fmt" - "github.com/replicatedhq/kots/pkg/k8sutil" - "io" "math" - "net/http" - "os" "strconv" - "time" "github.com/pkg/errors" "github.com/replicatedhq/kots/pkg/helmvm/types" + "github.com/replicatedhq/kots/pkg/k8sutil" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - statsv1alpha1 "k8s.io/kubelet/pkg/apis/stats/v1alpha1" metricsv "k8s.io/metrics/pkg/client/clientset/versioned" ) @@ -41,6 +34,11 @@ func GetNodes(ctx context.Context, client kubernetes.Interface) (*types.HelmVMNo toReturn := types.HelmVMNodes{} + nodePods, err := podsPerNode(ctx, client) + if err != nil { + return nil, errors.Wrap(err, "pods per node") + } + for _, node := range nodes.Items { cpuCapacity := types.CapacityAvailable{} memoryCapacity := types.CapacityAvailable{} @@ -60,18 +58,15 @@ func GetNodes(ctx context.Context, client kubernetes.Interface) (*types.HelmVMNo return nil, errors.Wrap(err, "list pod metrics") } - str, _ := json.Marshal(nodeMetrics) - fmt.Printf("node %s metrics: %s\n", node.Name, str) - if nodeMetrics.Usage.Memory() != nil { - memoryCapacity.Available = float64(nodeMetrics.Usage.Memory().Value()) / math.Pow(2, 30) + memoryCapacity.Available = memoryCapacity.Capacity - float64(nodeMetrics.Usage.Memory().Value())/math.Pow(2, 30) } if nodeMetrics.Usage.Cpu() != nil { - cpuCapacity.Available = cpuCapacity.Capacity - float64(nodeMetrics.Usage.Cpu().Value()) + cpuCapacity.Available = cpuCapacity.Capacity - nodeMetrics.Usage.Cpu().AsApproximateFloat64() } - podCapacity.Available = podCapacity.Capacity - float64(nodeMetrics.Usage.Pods().Value()) + podCapacity.Available = podCapacity.Capacity - float64(len(nodePods[node.Name])) nodeLabelArray := []string{} for k, v := range node.Labels { @@ -90,6 +85,7 @@ func GetNodes(ctx context.Context, client kubernetes.Interface) (*types.HelmVMNo Pods: podCapacity, Labels: nodeLabelArray, Conditions: findNodeConditions(node.Status.Conditions), + PodList: nodePods[node.Name], }) } @@ -127,49 +123,36 @@ func findNodeConditions(conditions []corev1.NodeCondition) types.NodeConditions return discoveredConditions } -// get kubelet PKI info from /etc/kubernetes/pki/kubelet, use it to hit metrics server at `http://${nodeIP}:10255/stats/summary` -func getNodeMetrics(nodeIP string) (*statsv1alpha1.Summary, error) { - client := http.Client{ - Timeout: time.Second, +// podsPerNode returns a map of node names to pods, across all namespaces +func podsPerNode(ctx context.Context, client kubernetes.Interface) (map[string][]corev1.Pod, error) { + namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, errors.Wrap(err, "list namespaces") } - port := 10255 - // only use mutual TLS if client cert exists - _, err := os.ReadFile("/etc/kubernetes/pki/kubelet/client.crt") - if err == nil { - cert, err := tls.LoadX509KeyPair("/etc/kubernetes/pki/kubelet/client.crt", "/etc/kubernetes/pki/kubelet/client.key") - if err != nil { - return nil, errors.Wrap(err, "get client keypair") - } + toReturn := map[string][]corev1.Pod{} - // this will leak memory - client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{ - Certificates: []tls.Certificate{cert}, - InsecureSkipVerify: true, - }, + for _, ns := range namespaces.Items { + nsPods, err := client.CoreV1().Pods(ns.Name).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "list pods in namespace %s", ns.Name) } - port = 10250 - } - r, err := client.Get(fmt.Sprintf("https://%s:%d/stats/summary", nodeIP, port)) - if err != nil { - return nil, errors.Wrapf(err, "get node %s stats", nodeIP) - } - defer r.Body.Close() + for _, pod := range nsPods.Items { + pod := pod + if pod.Spec.NodeName == "" { + continue + } - body, err := io.ReadAll(r.Body) - if err != nil { - return nil, errors.Wrapf(err, "read node %s stats response", nodeIP) - } + if _, ok := toReturn[pod.Spec.NodeName]; !ok { + toReturn[pod.Spec.NodeName] = []corev1.Pod{} + } - summary := statsv1alpha1.Summary{} - err = json.Unmarshal(body, &summary) - if err != nil { - return nil, errors.Wrapf(err, "parse node %s stats response", nodeIP) + toReturn[pod.Spec.NodeName] = append(toReturn[pod.Spec.NodeName], pod) + } } - return &summary, nil + return toReturn, nil } func isConnected(node corev1.Node) bool { diff --git a/pkg/helmvm/types/types.go b/pkg/helmvm/types/types.go index c298dfbd93..78ea23f248 100644 --- a/pkg/helmvm/types/types.go +++ b/pkg/helmvm/types/types.go @@ -1,5 +1,7 @@ package types +import corev1 "k8s.io/api/core/v1" + type HelmVMNodes struct { Nodes []Node `json:"nodes"` HA bool `json:"ha"` @@ -18,6 +20,7 @@ type Node struct { Pods CapacityAvailable `json:"pods"` Labels []string `json:"labels"` Conditions NodeConditions `json:"conditions"` + PodList []corev1.Pod `json:"podList"` } type CapacityAvailable struct { diff --git a/pkg/helmvm/util.go b/pkg/helmvm/util.go index 1b1d7b93e4..3267996896 100644 --- a/pkg/helmvm/util.go +++ b/pkg/helmvm/util.go @@ -43,5 +43,5 @@ func IsHelmVM(clientset kubernetes.Interface) (bool, error) { } func IsHA(clientset kubernetes.Interface) (bool, error) { - return false, nil + return true, nil } From 75b4ed90586ae4ad3c86f6fa74c9f458a2ea1068 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Oct 2023 12:59:42 -0600 Subject: [PATCH 5/8] remove debug log --- pkg/helmvm/util.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/helmvm/util.go b/pkg/helmvm/util.go index 3267996896..5dd2cdc11f 100644 --- a/pkg/helmvm/util.go +++ b/pkg/helmvm/util.go @@ -37,8 +37,6 @@ func IsHelmVM(clientset kubernetes.Interface) (bool, error) { return false, fmt.Errorf("failed to get embedded cluster configmap: %w", err) } - fmt.Printf("Is Embedded Cluster Config found: %v\n", configMapExists) - return configMapExists, nil } From 53d4a5bf8d07ae4002a0dab62895baa9b24e13c0 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Oct 2023 13:32:15 -0600 Subject: [PATCH 6/8] implement per-node metrics endpoint with podlist --- pkg/handlers/handlers.go | 2 + pkg/handlers/helmvm_get.go | 19 +++++++ pkg/handlers/interface.go | 1 + pkg/handlers/mock/mock.go | 12 +++++ pkg/helmvm/helmvm_node.go | 105 +++++++++++++++++++++++++++++++++++++ pkg/helmvm/helmvm_nodes.go | 22 +++----- 6 files changed, 145 insertions(+), 16 deletions(-) create mode 100644 pkg/helmvm/helmvm_node.go diff --git a/pkg/handlers/handlers.go b/pkg/handlers/handlers.go index aa711b7b89..aca714d3b2 100644 --- a/pkg/handlers/handlers.go +++ b/pkg/handlers/handlers.go @@ -287,6 +287,8 @@ func RegisterSessionAuthRoutes(r *mux.Router, kotsStore store.Store, handler KOT HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.DeleteHelmVMNode)) r.Name("GetHelmVMNodes").Path("/api/v1/helmvm/nodes").Methods("GET"). HandlerFunc(middleware.EnforceAccess(policy.ClusterRead, handler.GetHelmVMNodes)) + r.Name("GetHelmVMNode").Path("/api/v1/helmvm/node/{nodeName}").Methods("GET"). + HandlerFunc(middleware.EnforceAccess(policy.ClusterRead, handler.GetHelmVMNode)) // Prometheus r.Name("SetPrometheusAddress").Path("/api/v1/prometheus").Methods("POST"). diff --git a/pkg/handlers/helmvm_get.go b/pkg/handlers/helmvm_get.go index 91133a03b6..6aa4fa715b 100644 --- a/pkg/handlers/helmvm_get.go +++ b/pkg/handlers/helmvm_get.go @@ -1,6 +1,7 @@ package handlers import ( + "github.com/gorilla/mux" "net/http" "github.com/replicatedhq/kots/pkg/helmvm" @@ -24,3 +25,21 @@ func (h *Handler) GetHelmVMNodes(w http.ResponseWriter, r *http.Request) { } JSON(w, http.StatusOK, nodes) } + +func (h *Handler) GetHelmVMNode(w http.ResponseWriter, r *http.Request) { + client, err := k8sutil.GetClientset() + if err != nil { + logger.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + nodeName := mux.Vars(r)["nodeName"] + node, err := helmvm.GetNode(r.Context(), client, nodeName) + if err != nil { + logger.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + JSON(w, http.StatusOK, node) +} diff --git a/pkg/handlers/interface.go b/pkg/handlers/interface.go index c6cb2a00db..b69ae161fd 100644 --- a/pkg/handlers/interface.go +++ b/pkg/handlers/interface.go @@ -144,6 +144,7 @@ type KOTSHandler interface { DrainHelmVMNode(w http.ResponseWriter, r *http.Request) DeleteHelmVMNode(w http.ResponseWriter, r *http.Request) GetHelmVMNodes(w http.ResponseWriter, r *http.Request) + GetHelmVMNode(w http.ResponseWriter, r *http.Request) // Prometheus SetPrometheusAddress(w http.ResponseWriter, r *http.Request) diff --git a/pkg/handlers/mock/mock.go b/pkg/handlers/mock/mock.go index cf9fe09ede..f496224c74 100644 --- a/pkg/handlers/mock/mock.go +++ b/pkg/handlers/mock/mock.go @@ -778,6 +778,18 @@ func (mr *MockKOTSHandlerMockRecorder) GetHelmVMNodes(w, r interface{}) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHelmVMNodes", reflect.TypeOf((*MockKOTSHandler)(nil).GetHelmVMNodes), w, r) } +// GetHelmVMNodes mocks base method. +func (m *MockKOTSHandler) GetHelmVMNode(w http.ResponseWriter, r *http.Request) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "GetHelmVMNode", w, r) +} + +// GetHelmVMNode indicates an expected call of GetHelmVMNode. +func (mr *MockKOTSHandlerMockRecorder) GetHelmVMNode(w, r interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHelmVMNode", reflect.TypeOf((*MockKOTSHandler)(nil).GetHelmVMNode), w, r) +} + // GetIdentityServiceConfig mocks base method. func (m *MockKOTSHandler) GetIdentityServiceConfig(w http.ResponseWriter, r *http.Request) { m.ctrl.T.Helper() diff --git a/pkg/helmvm/helmvm_node.go b/pkg/helmvm/helmvm_node.go new file mode 100644 index 0000000000..7805400c1d --- /dev/null +++ b/pkg/helmvm/helmvm_node.go @@ -0,0 +1,105 @@ +package helmvm + +import ( + "context" + "fmt" + "math" + "strconv" + + "github.com/replicatedhq/kots/pkg/helmvm/types" + "github.com/replicatedhq/kots/pkg/k8sutil" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + metricsv "k8s.io/metrics/pkg/client/clientset/versioned" +) + +// GetNode will get a node with stats and podlist +func GetNode(ctx context.Context, client kubernetes.Interface, nodeName string) (*types.Node, error) { + node, err := client.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get node %s: %w", nodeName, err) + } + + clientConfig, err := k8sutil.GetClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to get cluster config: %w", err) + } + + metricsClient, err := metricsv.NewForConfig(clientConfig) + if err != nil { + return nil, fmt.Errorf("failed to create metrics client: %w", err) + } + + nodePods, err := podsOnNode(ctx, client, nodeName) + if err != nil { + return nil, fmt.Errorf("pods per node: %w", err) + } + + cpuCapacity := types.CapacityAvailable{} + memoryCapacity := types.CapacityAvailable{} + podCapacity := types.CapacityAvailable{} + + memoryCapacity.Capacity = float64(node.Status.Capacity.Memory().Value()) / math.Pow(2, 30) // capacity in GB + + cpuCapacity.Capacity, err = strconv.ParseFloat(node.Status.Capacity.Cpu().String(), 64) + if err != nil { + return nil, fmt.Errorf("parse CPU capacity %q for node %s: %w", node.Status.Capacity.Cpu().String(), node.Name, err) + } + + podCapacity.Capacity = float64(node.Status.Capacity.Pods().Value()) + + nodeMetrics, err := metricsClient.MetricsV1beta1().NodeMetricses().Get(ctx, node.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("list pod metrics: %w", err) + } + + if nodeMetrics.Usage.Memory() != nil { + memoryCapacity.Available = memoryCapacity.Capacity - float64(nodeMetrics.Usage.Memory().Value())/math.Pow(2, 30) + } + + if nodeMetrics.Usage.Cpu() != nil { + cpuCapacity.Available = cpuCapacity.Capacity - nodeMetrics.Usage.Cpu().AsApproximateFloat64() + } + + podCapacity.Available = podCapacity.Capacity - float64(len(nodePods)) + + nodeLabelArray := []string{} + for k, v := range node.Labels { + nodeLabelArray = append(nodeLabelArray, fmt.Sprintf("%s:%s", k, v)) + } + + return &types.Node{ + Name: node.Name, + IsConnected: isConnected(*node), + IsReady: isReady(*node), + IsPrimaryNode: isPrimary(*node), + CanDelete: node.Spec.Unschedulable && !isConnected(*node), + KubeletVersion: node.Status.NodeInfo.KubeletVersion, + CPU: cpuCapacity, + Memory: memoryCapacity, + Pods: podCapacity, + Labels: nodeLabelArray, + Conditions: findNodeConditions(node.Status.Conditions), + PodList: nodePods, + }, nil +} + +func podsOnNode(ctx context.Context, client kubernetes.Interface, nodeName string) ([]corev1.Pod, error) { + namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("list namespaces: %w", err) + } + + toReturn := []corev1.Pod{} + + for _, ns := range namespaces.Items { + nsPods, err := client.CoreV1().Pods(ns.Name).List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("spec.nodeName=%s", nodeName)}) + if err != nil { + return nil, fmt.Errorf("list pods on %s in namespace %s: %w", nodeName, ns.Name, err) + } + + toReturn = append(toReturn, nsPods.Items...) + } + return toReturn, nil +} diff --git a/pkg/helmvm/helmvm_nodes.go b/pkg/helmvm/helmvm_nodes.go index c09bff582e..f8bfefff4b 100644 --- a/pkg/helmvm/helmvm_nodes.go +++ b/pkg/helmvm/helmvm_nodes.go @@ -66,7 +66,7 @@ func GetNodes(ctx context.Context, client kubernetes.Interface) (*types.HelmVMNo cpuCapacity.Available = cpuCapacity.Capacity - nodeMetrics.Usage.Cpu().AsApproximateFloat64() } - podCapacity.Available = podCapacity.Capacity - float64(len(nodePods[node.Name])) + podCapacity.Available = podCapacity.Capacity - float64(nodePods[node.Name]) nodeLabelArray := []string{} for k, v := range node.Labels { @@ -85,7 +85,6 @@ func GetNodes(ctx context.Context, client kubernetes.Interface) (*types.HelmVMNo Pods: podCapacity, Labels: nodeLabelArray, Conditions: findNodeConditions(node.Status.Conditions), - PodList: nodePods[node.Name], }) } @@ -123,14 +122,14 @@ func findNodeConditions(conditions []corev1.NodeCondition) types.NodeConditions return discoveredConditions } -// podsPerNode returns a map of node names to pods, across all namespaces -func podsPerNode(ctx context.Context, client kubernetes.Interface) (map[string][]corev1.Pod, error) { +// podsPerNode returns a map of node names to the number of pods, across all namespaces +func podsPerNode(ctx context.Context, client kubernetes.Interface) (map[string]int, error) { namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { return nil, errors.Wrap(err, "list namespaces") } - toReturn := map[string][]corev1.Pod{} + toReturn := map[string]int{} for _, ns := range namespaces.Items { nsPods, err := client.CoreV1().Pods(ns.Name).List(ctx, metav1.ListOptions{}) @@ -145,10 +144,10 @@ func podsPerNode(ctx context.Context, client kubernetes.Interface) (map[string][ } if _, ok := toReturn[pod.Spec.NodeName]; !ok { - toReturn[pod.Spec.NodeName] = []corev1.Pod{} + toReturn[pod.Spec.NodeName] = 0 } - toReturn[pod.Spec.NodeName] = append(toReturn[pod.Spec.NodeName], pod) + toReturn[pod.Spec.NodeName]++ } } @@ -187,12 +186,3 @@ func isPrimary(node corev1.Node) bool { return false } - -func internalIP(node corev1.Node) string { - for _, address := range node.Status.Addresses { - if address.Type == corev1.NodeInternalIP { - return address.Address - } - } - return "" -} From 83caec74f9cec89b60a2fd9aafeda676a77eb28e Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Oct 2023 13:36:55 -0600 Subject: [PATCH 7/8] make mergable --- Makefile | 4 ---- deploy/okteto/okteto.Dockerfile | 12 ++++++------ node_modules/.yarn-integrity | 10 ---------- 3 files changed, 6 insertions(+), 20 deletions(-) delete mode 100644 node_modules/.yarn-integrity diff --git a/Makefile b/Makefile index a3cd5960ce..c4cb1e1106 100644 --- a/Makefile +++ b/Makefile @@ -131,10 +131,6 @@ all-ttl.sh: build-ttl.sh docker tag rqlite/rqlite:${RQLITE_TAG} ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} docker push ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} -api-ttl.sh: - docker build -f ./deploy/okteto/okteto.Dockerfile -t ttl.sh/${CURRENT_USER}/kotsadm-api:24h . - docker push ttl.sh/${CURRENT_USER}/kotsadm-api:24h - .PHONY: build-alpha build-alpha: docker build --pull -f deploy/Dockerfile --build-arg version=${GIT_TAG} -t kotsadm/kotsadm:alpha . diff --git a/deploy/okteto/okteto.Dockerfile b/deploy/okteto/okteto.Dockerfile index a626708f29..c98a583c61 100644 --- a/deploy/okteto/okteto.Dockerfile +++ b/deploy/okteto/okteto.Dockerfile @@ -167,11 +167,11 @@ RUN --mount=target=$GOMODCACHE,id=kots-gomodcache,type=cache \ mv ./bin/kotsadm /kotsadm && \ mv ./bin/kots /kots -#RUN --mount=target=/tmp/.cache/gocache,id=kots-gocache,type=cache \ -# --mount=target=/tmp/.cache/gomodcache,id=kots-gomodcache,type=cache \ -# mkdir -p $GOCACHE \ -# && cp -r /tmp/.cache/gocache/* $GOCACHE \ -# && mkdir -p $GOMODCACHE \ -# && cp -r /tmp/.cache/gomodcache/* $GOMODCACHE +RUN --mount=target=/tmp/.cache/gocache,id=kots-gocache,type=cache \ + --mount=target=/tmp/.cache/gomodcache,id=kots-gomodcache,type=cache \ + mkdir -p $GOCACHE \ + && cp -r /tmp/.cache/gocache/* $GOCACHE \ + && mkdir -p $GOMODCACHE \ + && cp -r /tmp/.cache/gomodcache/* $GOMODCACHE ENTRYPOINT [ "/kotsadm", "api"] diff --git a/node_modules/.yarn-integrity b/node_modules/.yarn-integrity deleted file mode 100644 index 3c4955ff32..0000000000 --- a/node_modules/.yarn-integrity +++ /dev/null @@ -1,10 +0,0 @@ -{ - "systemParams": "darwin-x64-111", - "modulesFolders": [], - "flags": [], - "linkedModules": [], - "topLevelPatterns": [], - "lockfileEntries": {}, - "files": [], - "artifacts": {} -} \ No newline at end of file From 66e0a7f70e09c21ec9543a594d89f8f66cb4c646 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Oct 2023 13:39:27 -0600 Subject: [PATCH 8/8] order --- pkg/handlers/helmvm_get.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/handlers/helmvm_get.go b/pkg/handlers/helmvm_get.go index 6aa4fa715b..4f736996ed 100644 --- a/pkg/handlers/helmvm_get.go +++ b/pkg/handlers/helmvm_get.go @@ -1,9 +1,9 @@ package handlers import ( - "github.com/gorilla/mux" "net/http" + "github.com/gorilla/mux" "github.com/replicatedhq/kots/pkg/helmvm" "github.com/replicatedhq/kots/pkg/k8sutil" "github.com/replicatedhq/kots/pkg/logger"