feat: add support to detect unused CRDs

.gitignore

@@ -1,2 +1,3 @@
 *.iml
 .idea/
+kor

README.md

@@ -8,7 +8,8 @@
 ![Kor Logo](/images/kor_logo.png)
 
 Kor is a tool to discover unused Kubernetes resources. Currently, Kor can identify and list unused:
-- ConfigMaps  
+
+- ConfigMaps
 - Secrets
 - Services
 - ServiceAccounts
@@ -19,6 +20,7 @@ Kor is a tool to discover unused Kubernetes resources. Currently, Kor can identi
 - PVCs
 - Ingresses
 - PDBs
+- CRDs
 
 ![Kor Screenshot](/images/screenshot.png)
 
@@ -64,7 +66,7 @@ helm upgrade -i kor \
 ```
 
 
-For more information see [in cluster usage](#in-cluster-usage) 
+For more information see [in cluster usage](#in-cluster-usage)
 
 ## Usage
 
@@ -82,6 +84,7 @@ Kor provides various subcommands to identify and list unused resources. The avai
 - `pvc` - Gets unused PVCs for the specified namespace or all namespaces.
 - `ingress` - Gets unused Ingresses for the specified namespace or all namespaces.
 - `pdb` - Gets unused PDBs for the specified namespace or all namespaces.
+- `crd` - Gets unused CRDs in the cluster
 - `exporter` - Export Prometheus metrics.
 
 ### Supported Flags
@@ -198,4 +201,3 @@ Contributions are welcome! If you encounter any bugs or have suggestions for imp
 ## License
 
 This open-source project is available under the [MIT License](LICENSE). Feel free to use, modify, and distribute it as per the terms of the license.
-

cmd/kor/all.go

@@ -13,14 +13,16 @@ var allCmd = &cobra.Command{
 	Args:  cobra.ExactArgs(0),
 	Run: func(cmd *cobra.Command, args []string) {
 		clientset := kor.GetKubeClient(kubeconfig)
+		apiExtClient := kor.GetAPIExtensionsClient(kubeconfig)
+		dynamicClient := kor.GetDynamicClient(kubeconfig)
 		if outputFormat == "json" || outputFormat == "yaml" {
-			if response, err := kor.GetUnusedAllStructured(includeExcludeLists, clientset, outputFormat); err != nil {
+			if response, err := kor.GetUnusedAllStructured(includeExcludeLists, clientset, apiExtClient, dynamicClient, outputFormat); err != nil {
 				fmt.Println(err)
 			} else {
 				fmt.Println(response)
 			}
 		} else {
-			kor.GetUnusedAll(includeExcludeLists, clientset, slackOpts)
+			kor.GetUnusedAll(includeExcludeLists, clientset, apiExtClient, dynamicClient, slackOpts)
 		}
 
 	},

cmd/kor/crds.go

@@ -0,0 +1,32 @@
+package kor
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"github.com/yonahd/kor/pkg/kor"
+)
+
+var crdCmd = &cobra.Command{
+	Use:     "customresourcedefinition",
+	Aliases: []string{"crd", "customresourcedefinitions"},
+	Short:   "Gets unused crds",
+	Args:    cobra.NoArgs,
+	Run: func(cmd *cobra.Command, args []string) {
+		apiExtClient := kor.GetAPIExtensionsClient(kubeconfig)
+		dynamicClient := kor.GetDynamicClient(kubeconfig)
+		if outputFormat == "json" || outputFormat == "yaml" {
+			if response, err := kor.GetUnusedCrdsStructured(apiExtClient, dynamicClient, outputFormat); err != nil {
+				fmt.Println(err)
+			} else {
+				fmt.Println(response)
+			}
+		} else {
+			kor.GetUnusedCrds(apiExtClient, dynamicClient, slackOpts)
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(crdCmd)
+}

cmd/kor/exporter.go

@@ -11,7 +11,9 @@ var exporterCmd = &cobra.Command{
 	Args:  cobra.ExactArgs(0),
 	Run: func(cmd *cobra.Command, args []string) {
 		clientset := kor.GetKubeClient(kubeconfig)
-		kor.Exporter(includeExcludeLists, clientset, "json")
+		apiExtClient := kor.GetAPIExtensionsClient(kubeconfig)
+		dynamicClient := kor.GetDynamicClient(kubeconfig)
+		kor.Exporter(includeExcludeLists, clientset, apiExtClient, dynamicClient, "json")
 
 	},
 }

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/prometheus/client_golang v1.17.0
 	github.com/spf13/cobra v1.7.0
 	k8s.io/api v0.28.2
+	k8s.io/apiextensions-apiserver v0.28.2
 	k8s.io/apimachinery v0.28.2
 	k8s.io/client-go v0.28.2
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2

go.sum

@@ -177,6 +177,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw=
 k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg=
+k8s.io/apiextensions-apiserver v0.28.2 h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU=
+k8s.io/apiextensions-apiserver v0.28.2/go.mod h1:5tnkxLGa9nefefYzWuAlWZ7RZYuN/765Au8cWLA6SRg=
 k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ=
 k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU=
 k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY=

pkg/kor/all.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"os"
 
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/yaml"
 )
@@ -119,7 +121,16 @@ func getUnusedPdbs(clientset kubernetes.Interface, namespace string) ResourceDif
 	return namespacePdbDiff
 }
 
-func GetUnusedAll(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, slackOpts SlackOpts) {
+func getUnusedCrds(apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface) ResourceDiff {
+	crdDiff, err := processCrds(apiExtClient, dynamicClient)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to get crds: %v\n", err)
+	}
+	namespaceCrdDiff := ResourceDiff{"Crd", crdDiff} // This is only named namespaceCrd for inconsistency
+	return namespaceCrdDiff
+}
+
+func GetUnusedAll(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface, slackOpts SlackOpts) {
 	namespaces := SetNamespaceList(includeExcludeLists, clientset)
 
 	var outputBuffer bytes.Buffer
@@ -155,6 +166,16 @@ func GetUnusedAll(includeExcludeLists IncludeExcludeLists, clientset kubernetes.
 		outputBuffer.WriteString("\n")
 	}
 
+	// cluster scope diffs
+	var allDiffs []ResourceDiff
+	crdDiff := getUnusedCrds(apiExtClient, dynamicClient)
+	allDiffs = append(allDiffs, crdDiff)
+
+	output := FormatOutputAll("", allDiffs)
+
+	outputBuffer.WriteString(output)
+	outputBuffer.WriteString("\n")
+
 	if slackOpts != (SlackOpts{}) {
 		if err := SendToSlack(SlackMessage{}, slackOpts, outputBuffer.String()); err != nil {
 			fmt.Fprintf(os.Stderr, "Failed to send message to slack: %v\n", err)
@@ -165,7 +186,7 @@ func GetUnusedAll(includeExcludeLists IncludeExcludeLists, clientset kubernetes.
 	}
 }
 
-func GetUnusedAllStructured(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, outputFormat string) (string, error) {
+func GetUnusedAllStructured(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface, outputFormat string) (string, error) {
 	namespaces := SetNamespaceList(includeExcludeLists, clientset)
 
 	// Create the JSON response object

pkg/kor/crds.go

@@ -0,0 +1,99 @@
+package kor
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
+	"sigs.k8s.io/yaml"
+)
+
+func processCrds(apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface) ([]string, error) {
+
+	var unusedCRDs []string
+
+	crds, err := apiExtClient.ApiextensionsV1().CustomResourceDefinitions().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, crd := range crds.Items {
+		if crd.Labels["kor/used"] == "true" {
+			continue
+		}
+
+		gvr := schema.GroupVersionResource{
+			Group:    crd.Spec.Group,
+			Version:  crd.Spec.Versions[0].Name, // We're checking the first version.
+			Resource: crd.Spec.Names.Plural,
+		}
+		instances, err := dynamicClient.Resource(gvr).Namespace("").List(context.TODO(), metav1.ListOptions{})
+		if err != nil {
+			return nil, err
+		}
+		if len(instances.Items) == 0 {
+			unusedCRDs = append(unusedCRDs, crd.Name)
+		}
+	}
+	return unusedCRDs, nil
+}
+
+func GetUnusedCrds(apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface, slackOpts SlackOpts) {
+
+	var outputBuffer bytes.Buffer
+	diff, err := processCrds(apiExtClient, dynamicClient)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to process crds %v\n", err)
+	}
+	output := FormatOutput("", diff, "Crds")
+
+	outputBuffer.WriteString(output)
+	outputBuffer.WriteString("\n")
+
+	if slackOpts != (SlackOpts{}) {
+		if err := SendToSlack(SlackMessage{}, slackOpts, outputBuffer.String()); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to send message to slack: %v\n", err)
+			os.Exit(1)
+		}
+	} else {
+		fmt.Println(outputBuffer.String())
+	}
+}
+
+func GetUnusedCrdsStructured(apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface, outputFormat string) (string, error) {
+	response := make(map[string]map[string][]string)
+
+	diff, err := processCrds(apiExtClient, dynamicClient)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to process crds: %v\n", err)
+	}
+	if len(diff) > 0 {
+		// We consider cluster scope resources in "" (empty string) namesapce, as it is common in k8s
+		if response[""] == nil {
+			response[""] = make(map[string][]string)
+		}
+		response[""]["Crd"] = diff
+	}
+
+	jsonResponse, err := json.MarshalIndent(response, "", "  ")
+	if err != nil {
+		return "", err
+	}
+
+	if outputFormat == "yaml" {
+		yamlResponse, err := yaml.JSONToYAML(jsonResponse)
+		if err != nil {
+			fmt.Printf("err: %v\n", err)
+		}
+		return string(yamlResponse), nil
+	} else {
+		return string(jsonResponse), nil
+	}
+}

pkg/kor/exporter.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -28,16 +30,16 @@ func init() {
 }
 
 // TODO: add option to change port / url !?
-func Exporter(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, outputFormat string) {
+func Exporter(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface, outputFormat string) {
 	http.Handle("/metrics", promhttp.Handler())
 	fmt.Println("Server listening on :8080")
-	go exportMetrics(includeExcludeLists, clientset, outputFormat) // Start exporting metrics in the background
+	go exportMetrics(includeExcludeLists, clientset, apiExtClient, dynamicClient, outputFormat) // Start exporting metrics in the background
 	if err := http.ListenAndServe(":8080", nil); err != nil {
 		fmt.Println(err)
 	}
 }
 
-func exportMetrics(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, outputFormat string) {
+func exportMetrics(includeExcludeLists IncludeExcludeLists, clientset kubernetes.Interface, apiExtClient apiextensionsclientset.Interface, dynamicClient dynamic.Interface, outputFormat string) {
 	exporterInterval := os.Getenv("EXPORTER_INTERVAL")
 	if exporterInterval == "" {
 		exporterInterval = "10"
@@ -49,7 +51,7 @@ func exportMetrics(includeExcludeLists IncludeExcludeLists, clientset kubernetes
 	}
 
 	for {
-		if korOutput, err := GetUnusedAllStructured(includeExcludeLists, clientset, outputFormat); err != nil {
+		if korOutput, err := GetUnusedAllStructured(includeExcludeLists, clientset, apiExtClient, dynamicClient, outputFormat); err != nil {
 			fmt.Println(err)
 			os.Exit(1)
 		} else {