From eb7536cfe31d049bbc781cc1b4aa16724d90ab60 Mon Sep 17 00:00:00 2001 From: Mohammad Yosefpor Date: Mon, 16 Oct 2023 12:43:40 +0330 Subject: [PATCH] feat: add support to detect unused CRDs --- .gitignore | 1 + README.md | 8 ++-- cmd/kor/all.go | 6 ++- cmd/kor/crds.go | 32 +++++++++++++++ cmd/kor/exporter.go | 4 +- go.mod | 1 + go.sum | 2 + pkg/kor/all.go | 25 +++++++++++- pkg/kor/crds.go | 99 +++++++++++++++++++++++++++++++++++++++++++++ pkg/kor/exporter.go | 10 +++-- pkg/kor/kor.go | 74 +++++++++++++++++++++++++++++++++ 11 files changed, 250 insertions(+), 12 deletions(-) create mode 100644 cmd/kor/crds.go create mode 100644 pkg/kor/crds.go diff --git a/.gitignore b/.gitignore index 0cc2124b..7af92aa7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.iml .idea/ +kor diff --git a/README.md b/README.md index cdc8e755..5b595e94 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ ![Kor Logo](/images/kor_logo.png) Kor is a tool to discover unused Kubernetes resources. Currently, Kor can identify and list unused: -- ConfigMaps + +- ConfigMaps - Secrets - Services - ServiceAccounts @@ -19,6 +20,7 @@ Kor is a tool to discover unused Kubernetes resources. Currently, Kor can identi - PVCs - Ingresses - PDBs +- CRDs ![Kor Screenshot](/images/screenshot.png) @@ -64,7 +66,7 @@ helm upgrade -i kor \ ``` -For more information see [in cluster usage](#in-cluster-usage) +For more information see [in cluster usage](#in-cluster-usage) ## Usage @@ -82,6 +84,7 @@ Kor provides various subcommands to identify and list unused resources. The avai - `pvc` - Gets unused PVCs for the specified namespace or all namespaces. - `ingress` - Gets unused Ingresses for the specified namespace or all namespaces. - `pdb` - Gets unused PDBs for the specified namespace or all namespaces. +- `crd` - Gets unused CRDs in the cluster - `exporter` - Export Prometheus metrics. ### Supported Flags @@ -198,4 +201,3 @@ Contributions are welcome! If you encounter any bugs or have suggestions for imp ## License This open-source project is available under the [MIT License](LICENSE). Feel free to use, modify, and distribute it as per the terms of the license. - diff --git a/cmd/kor/all.go b/cmd/kor/all.go index 3aaf7e51..96c5f7dd 100644 --- a/cmd/kor/all.go +++ b/cmd/kor/all.go @@ -13,14 +13,16 @@ var allCmd = &cobra.Command{ Args: cobra.ExactArgs(0), Run: func(cmd *cobra.Command, args []string) { clientset := kor.GetKubeClient(kubeconfig) + apiExtClient := kor.GetAPIExtensionsClient(kubeconfig) + dynamicClient := kor.GetDynamicClient(kubeconfig) if outputFormat == "json" || outputFormat == "yaml" { - if response, err := kor.GetUnusedAllStructured(includeExcludeLists, clientset, outputFormat); err != nil { + if response, err := kor.GetUnusedAllStructured(includeExcludeLists, clientset, apiExtClient, dynamicClient, outputFormat); err != nil { fmt.Println(err) } else { fmt.Println(response) } } else { - kor.GetUnusedAll(includeExcludeLists, clientset, slackOpts) + kor.GetUnusedAll(includeExcludeLists, clientset, apiExtClient, dynamicClient, slackOpts) } }, diff --git a/cmd/kor/crds.go b/cmd/kor/crds.go new file mode 100644 index 00000000..b14be8c1 --- /dev/null +++ b/cmd/kor/crds.go @@ -0,0 +1,32 @@ +package kor + +import ( + "fmt" + + "github.com/spf13/cobra" + "github.com/yonahd/kor/pkg/kor" +) + +var crdCmd = &cobra.Command{ + Use: "customresourcedefinition", + Aliases: []string{"crd", "customresourcedefinitions"}, + Short: "Gets unused crds", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + apiExtClient := kor.GetAPIExtensionsClient(kubeconfig) + dynamicClient := kor.GetDynamicClient(kubeconfig) + if outputFormat == "json" || outputFormat == "yaml" { + if response, err := kor.GetUnusedCrdsStructured(apiExtClient, dynamicClient, outputFormat); err != nil { + fmt.Println(err) + } else { + fmt.Println(response) + } + } else { + kor.GetUnusedCrds(apiExtClient, dynamicClient, slackOpts) + } + }, +} + +func init() { + rootCmd.AddCommand(crdCmd) +} diff --git a/cmd/kor/exporter.go b/cmd/kor/exporter.go index 5f8a8003..88561eac 100644 --- a/cmd/kor/exporter.go +++ b/cmd/kor/exporter.go @@ -11,7 +11,9 @@ var exporterCmd = &cobra.Command{ Args: cobra.ExactArgs(0), Run: func(cmd *cobra.Command, args []string) { clientset := kor.GetKubeClient(kubeconfig) - kor.Exporter(includeExcludeLists, clientset, "json") + apiExtClient := kor.GetAPIExtensionsClient(kubeconfig) + dynamicClient := kor.GetDynamicClient(kubeconfig) + kor.Exporter(includeExcludeLists, clientset, apiExtClient, dynamicClient, "json") }, } diff --git a/go.mod b/go.mod index e36c7406..e7f33743 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/prometheus/client_golang v1.17.0 github.com/spf13/cobra v1.7.0 k8s.io/api v0.28.2 + k8s.io/apiextensions-apiserver v0.28.2 k8s.io/apimachinery v0.28.2 k8s.io/client-go v0.28.2 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 diff --git a/go.sum b/go.sum index e0bbb7e8..7b500dc7 100644 --- a/go.sum +++ b/go.sum @@ -177,6 +177,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= +k8s.io/apiextensions-apiserver v0.28.2 h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU= +k8s.io/apiextensions-apiserver v0.28.2/go.mod h1:5tnkxLGa9nefefYzWuAlWZ7RZYuN/765Au8cWLA6SRg= k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= diff --git a/pkg/kor/all.go b/pkg/kor/all.go index abbd0842..5e0af495 100644 --- a/pkg/kor/all.go +++ b/pkg/kor/all.go @@ -6,6 +6,8 @@ import ( "fmt" "os" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "sigs.k8s.io/yaml" ) @@ -119,7 +121,16 @@ func getUnusedPdbs(clientset kubernetes.Interface, namespace string) ResourceDif return namespacePdbDiff } -func GetUnusedAll(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, slackOpts SlackOpts) { +func getUnusedCrds(apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface) ResourceDiff { + crdDiff, err := processCrds(apiExtClient, dynamicClient) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to get crds: %v\n", err) + } + namespaceCrdDiff := ResourceDiff{"Crd", crdDiff} // This is only named namespaceCrd for inconsistency + return namespaceCrdDiff +} + +func GetUnusedAll(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface, slackOpts SlackOpts) { namespaces := SetNamespaceList(includeExcludeLists, clientset) var outputBuffer bytes.Buffer @@ -155,6 +166,16 @@ func GetUnusedAll(includeExcludeLists IncludeExcludeLists, clientset kubernetes. outputBuffer.WriteString("\n") } + // cluster scope diffs + var allDiffs []ResourceDiff + crdDiff := getUnusedCrds(apiExtClient, dynamicClient) + allDiffs = append(allDiffs, crdDiff) + + output := FormatOutputAll("", allDiffs) + + outputBuffer.WriteString(output) + outputBuffer.WriteString("\n") + if slackOpts != (SlackOpts{}) { if err := SendToSlack(SlackMessage{}, slackOpts, outputBuffer.String()); err != nil { fmt.Fprintf(os.Stderr, "Failed to send message to slack: %v\n", err) @@ -165,7 +186,7 @@ func GetUnusedAll(includeExcludeLists IncludeExcludeLists, clientset kubernetes. } } -func GetUnusedAllStructured(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, outputFormat string) (string, error) { +func GetUnusedAllStructured(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface, outputFormat string) (string, error) { namespaces := SetNamespaceList(includeExcludeLists, clientset) // Create the JSON response object diff --git a/pkg/kor/crds.go b/pkg/kor/crds.go new file mode 100644 index 00000000..7c13d307 --- /dev/null +++ b/pkg/kor/crds.go @@ -0,0 +1,99 @@ +package kor + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" + "sigs.k8s.io/yaml" +) + +func processCrds(apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface) ([]string, error) { + + var unusedCRDs []string + + crds, err := apiExtClient.ApiextensionsV1().CustomResourceDefinitions().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + for _, crd := range crds.Items { + if crd.Labels["kor/used"] == "true" { + continue + } + + gvr := schema.GroupVersionResource{ + Group: crd.Spec.Group, + Version: crd.Spec.Versions[0].Name, // We're checking the first version. + Resource: crd.Spec.Names.Plural, + } + instances, err := dynamicClient.Resource(gvr).Namespace("").List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + if len(instances.Items) == 0 { + unusedCRDs = append(unusedCRDs, crd.Name) + } + } + return unusedCRDs, nil +} + +func GetUnusedCrds(apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface, slackOpts SlackOpts) { + + var outputBuffer bytes.Buffer + diff, err := processCrds(apiExtClient, dynamicClient) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to process crds %v\n", err) + } + output := FormatOutput("", diff, "Crds") + + outputBuffer.WriteString(output) + outputBuffer.WriteString("\n") + + if slackOpts != (SlackOpts{}) { + if err := SendToSlack(SlackMessage{}, slackOpts, outputBuffer.String()); err != nil { + fmt.Fprintf(os.Stderr, "Failed to send message to slack: %v\n", err) + os.Exit(1) + } + } else { + fmt.Println(outputBuffer.String()) + } +} + +func GetUnusedCrdsStructured(apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface, outputFormat string) (string, error) { + response := make(map[string]map[string][]string) + + diff, err := processCrds(apiExtClient, dynamicClient) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to process crds: %v\n", err) + } + if len(diff) > 0 { + // We consider cluster scope resources in "" (empty string) namesapce, as it is common in k8s + if response[""] == nil { + response[""] = make(map[string][]string) + } + response[""]["Crd"] = diff + } + + jsonResponse, err := json.MarshalIndent(response, "", " ") + if err != nil { + return "", err + } + + if outputFormat == "yaml" { + yamlResponse, err := yaml.JSONToYAML(jsonResponse) + if err != nil { + fmt.Printf("err: %v\n", err) + } + return string(yamlResponse), nil + } else { + return string(jsonResponse), nil + } +} diff --git a/pkg/kor/exporter.go b/pkg/kor/exporter.go index e2180682..e77a7d0f 100644 --- a/pkg/kor/exporter.go +++ b/pkg/kor/exporter.go @@ -10,6 +10,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" ) @@ -28,16 +30,16 @@ func init() { } // TODO: add option to change port / url !? -func Exporter(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, outputFormat string) { +func Exporter(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface, outputFormat string) { http.Handle("/metrics", promhttp.Handler()) fmt.Println("Server listening on :8080") - go exportMetrics(includeExcludeLists, clientset, outputFormat) // Start exporting metrics in the background + go exportMetrics(includeExcludeLists, clientset, apiExtClient, dynamicClient, outputFormat) // Start exporting metrics in the background if err := http.ListenAndServe(":8080", nil); err != nil { fmt.Println(err) } } -func exportMetrics(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, outputFormat string) { +func exportMetrics(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface, outputFormat string) { exporterInterval := os.Getenv("EXPORTER_INTERVAL") if exporterInterval == "" { exporterInterval = "10" @@ -49,7 +51,7 @@ func exportMetrics(includeExcludeLists IncludeExcludeLists, clientset kubernetes } for { - if korOutput, err := GetUnusedAllStructured(includeExcludeLists, clientset, outputFormat); err != nil { + if korOutput, err := GetUnusedAllStructured(includeExcludeLists, clientset, apiExtClient, dynamicClient, outputFormat); err != nil { fmt.Println(err) os.Exit(1) } else { diff --git a/pkg/kor/kor.go b/pkg/kor/kor.go index d2ebe91f..ae6eaeaf 100644 --- a/pkg/kor/kor.go +++ b/pkg/kor/kor.go @@ -10,7 +10,9 @@ import ( "strings" "github.com/olekukonko/tablewriter" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -80,6 +82,78 @@ func GetKubeClient(kubeconfig string) *kubernetes.Clientset { return clientset } +func GetAPIExtensionsClient(kubeconfig string) *apiextensionsclientset.Clientset { + if _, err := os.Stat("/var/run/secrets/kubernetes.io/serviceaccount/token"); err == nil { + config, err := rest.InClusterConfig() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to load kubeconfig: %v\n", err) + os.Exit(1) + } + + clientset, err := apiextensionsclientset.NewForConfig(config) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to create Kubernetes client: %v\n", err) + os.Exit(1) + } + return clientset + } + if kubeconfig == "" { + if configEnv := os.Getenv("KUBECONFIG"); configEnv != "" { + kubeconfig = configEnv + } else { + kubeconfig = GetKubeConfigPath() + } + } + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to load kubeconfig: %v\n", err) + os.Exit(1) + } + + clientset, err := apiextensionsclientset.NewForConfig(config) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to create Kubernetes client: %v\n", err) + os.Exit(1) + } + return clientset +} + +func GetDynamicClient(kubeconfig string) *dynamic.DynamicClient { + if _, err := os.Stat("/var/run/secrets/kubernetes.io/serviceaccount/token"); err == nil { + config, err := rest.InClusterConfig() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to load kubeconfig: %v\n", err) + os.Exit(1) + } + + clientset, err := dynamic.NewForConfig(config) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to create Kubernetes client: %v\n", err) + os.Exit(1) + } + return clientset + } + if kubeconfig == "" { + if configEnv := os.Getenv("KUBECONFIG"); configEnv != "" { + kubeconfig = configEnv + } else { + kubeconfig = GetKubeConfigPath() + } + } + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to load kubeconfig: %v\n", err) + os.Exit(1) + } + + clientset, err := dynamic.NewForConfig(config) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to create Kubernetes client: %v\n", err) + os.Exit(1) + } + return clientset +} + func SetNamespaceList(namespaceLists IncludeExcludeLists, clientset kubernetes.Interface) []string { namespaces := make([]string, 0) namespacesMap := make(map[string]bool)