From e54279defb5b079e3c1c3dfb6362186d7d45179c Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Wed, 10 May 2023 16:08:25 +0000 Subject: [PATCH 01/24] Initial add of new source --- .devcontainer/setup.sh | 2 +- docker-compose.yml | 26 --- go.mod | 2 +- internal/sources/generic_source.go | 197 ++++++++++++++++++++ internal/sources/generic_source_test.go | 211 ++++++++++++++++++++++ internal/sources/shared_resourcesource.go | 7 +- internal/sources/shared_util.go | 14 +- internal/sources/shared_util_test.go | 8 +- 8 files changed, 424 insertions(+), 43 deletions(-) delete mode 100644 docker-compose.yml create mode 100644 internal/sources/generic_source.go create mode 100644 internal/sources/generic_source_test.go diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 330b4cb..4990152 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -1,3 +1,3 @@ #!/bin/bash -curl -Lo ./kind \"https://kind.sigs.k8s.io/dl/v0.14.0/kind-$(uname)-amd64\" && chmod +x ./kind && sudo mv ./kind /usr/local/bin/kind +go install sigs.k8s.io/kind@latest \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 5b1700b..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: "3" -services: - nats: - image: nats - command: "-c /etc/nats/nats.conf -DV" #-c /etc/nats/nats.conf --cluster nats://0.0.0.0:6222 --routes=nats://ruser:T0pS3cr3t@nats:6222 - ports: - - "4222:4222" - - "8222:8222" - - "6222:6222" - - "4433:4433" - volumes: - - ./acceptance/nats-server.conf:/etc/nats/nats.conf - # nats-1: - # image: nats - # command: "-c nats-server.conf --routes=nats-route://ruser:T0pS3cr3t@nats:6222 -DV" - #link: - # # Will build from a local copy - # build: ../redacted_link - # environment: - # - REDACTED_NATS_URLS=nats - # - REDACTED_VERBOSITY=debug - -networks: - default: - external: - name: nats diff --git a/go.mod b/go.mod index abb821e..90cf843 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 + github.com/google/uuid v1.3.0 github.com/spf13/viper v1.15.0 k8s.io/api v0.27.1 k8s.io/apimachinery v0.27.1 @@ -42,7 +43,6 @@ require ( github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.15 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/internal/sources/generic_source.go b/internal/sources/generic_source.go new file mode 100644 index 0000000..4ed0a60 --- /dev/null +++ b/internal/sources/generic_source.go @@ -0,0 +1,197 @@ +package sources + +import ( + "context" + "fmt" + + "github.com/overmindtech/sdp-go" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NamespacedInterfaceBuilder The function that create a client to query a +// namespaced resource. e.g. `CoreV1().Pods` +type NamespacedInterfaceBuilder[Resource metav1.Object, ResourceList any] func(namespace string) ItemInterface[Resource, ResourceList] + +// ClusterInterfaceBuilder The function that create a client to query a +// cluster-wide resource. e.g. `CoreV1().Nodes` +type ClusterInterfaceBuilder[Resource metav1.Object, ResourceList any] func() ItemInterface[Resource, ResourceList] + +// ItemInterface An interface that matches the `Get` and `List` methods for K8s +// resources since these are the ones that we use for getting Overmind data. +// Kube's clients are usually namespaced when they are created, so this +// interface is expected to only returns items from a single namespace +type ItemInterface[Resource metav1.Object, ResourceList any] interface { + Get(ctx context.Context, name string, opts metav1.GetOptions) (Resource, error) + List(ctx context.Context, opts metav1.ListOptions) (ResourceList, error) +} + +type KubeTypeSource[Resource metav1.Object, ResourceList any] struct { + // The function that creates a client to query a namespaced resource. e.g. + // `CoreV1().Pods`. Either this or `NamespacedInterfaceBuilder` must be + // specified + ClusterInterfaceBuilder ClusterInterfaceBuilder[Resource, ResourceList] + + // The function that creates a client to query a cluster-wide resource. e.g. + // `CoreV1().Nodes`. Either this or `ClusterInterfaceBuilder` must be + // specified + NamespacedInterfaceBuilder NamespacedInterfaceBuilder[Resource, ResourceList] + + // A function that extracts a slice of Resources from a ResourceList + ListExtractor func(ResourceList) ([]Resource, error) + + // A function that returns a list of linked item queries for a given + // resource + LinkedItemQueryExtractor func(Resource) ([]*sdp.Query, error) + + // The type of items that this source should return + TypeName string + // List of namespaces that this source should query + Namespaces []string + // The name of the cluster that this source is for. This is used to generate + // scopes + ClusterName string +} + +// validate Validates that the source is correctly set up +func (k *KubeTypeSource[Resource, ResourceList]) Validate() error { + if k.NamespacedInterfaceBuilder == nil && k.ClusterInterfaceBuilder == nil { + return fmt.Errorf("either NamespacedInterfaceBuilder or ClusterInterfaceBuilder must be specified") + } + + if k.ListExtractor == nil { + return fmt.Errorf("ListExtractor must be specified") + } + + if k.TypeName == "" { + return fmt.Errorf("TypeName must be specified") + } + + if k.namespaced() && len(k.Namespaces) == 0 { + return fmt.Errorf("Namespaces must be specified when NamespacedInterfaceBuilder is specified") + } + + if k.ClusterName == "" { + return fmt.Errorf("ClusterName must be specified") + } + + return nil +} + +// namespaced Returns whether the source is namespaced or not +func (k *KubeTypeSource[Resource, ResourceList]) namespaced() bool { + return k.NamespacedInterfaceBuilder != nil +} + +func (k *KubeTypeSource[Resource, ResourceList]) Type() string { + return k.TypeName +} + +func (k *KubeTypeSource[Resource, ResourceList]) Name() string { + return "TODO" +} + +func (k *KubeTypeSource[Resource, ResourceList]) Scopes() []string { + namespaces := make([]string, 0) + + if k.namespaced() { + for _, ns := range k.Namespaces { + sd := ScopeDetails{ + ClusterName: k.ClusterName, + Namespace: ns, + } + + namespaces = append(namespaces, sd.String()) + } + } else { + sd := ScopeDetails{ + ClusterName: k.ClusterName, + } + + namespaces = append(namespaces, sd.String()) + } + + return namespaces +} + +func (k *KubeTypeSource[Resource, ResourceList]) Get(ctx context.Context, scope string, query string) (*sdp.Item, error) { + i := k.itemInterface(scope) + resource, err := i.Get(ctx, query, metav1.GetOptions{}) + + if err != nil { + // TODO: Handle not found + return nil, err + } + + item, err := mapK8sObject(k.TypeName, resource) + + if err != nil { + return nil, err + } + + if k.LinkedItemQueryExtractor != nil { + // Add linked items + item.LinkedItemQueries, err = k.LinkedItemQueryExtractor(resource) + + if err != nil { + return nil, err + } + } + + return item, nil +} + +func (k *KubeTypeSource[Resource, ResourceList]) List(ctx context.Context, scope string) ([]*sdp.Item, error) { + i := k.itemInterface(scope) + list, err := i.List(ctx, metav1.ListOptions{}) + + if err != nil { + return nil, err + } + + resourceList, err := k.ListExtractor(list) + + if err != nil { + return nil, err + } + + items := make([]*sdp.Item, 0) + + for _, resource := range resourceList { + item, err := mapK8sObject(k.TypeName, resource) + + if err != nil { + return nil, err + } + + if k.LinkedItemQueryExtractor != nil { + // Add linked items + item.LinkedItemQueries, err = k.LinkedItemQueryExtractor(resource) + + if err != nil { + return nil, err + } + } + + items = append(items, item) + } + + return items, nil +} + +func (k *KubeTypeSource[Resource, ResourceList]) Search(ctx context.Context, scope string, query string) ([]*sdp.Item, error) { + return nil, nil +} + +// itemInterface Returns the correct interface depending on whether the source +// is namespaced or not +func (k *KubeTypeSource[Resource, ResourceList]) itemInterface(scope string) ItemInterface[Resource, ResourceList] { + // If this is a namespaced resource, then parse the scope to get the + // namespace + if k.namespaced() { + details := ParseScope(scope) + + return k.NamespacedInterfaceBuilder(details.Namespace) + } else { + return k.ClusterInterfaceBuilder() + } +} diff --git a/internal/sources/generic_source_test.go b/internal/sources/generic_source_test.go new file mode 100644 index 0000000..a0d46fe --- /dev/null +++ b/internal/sources/generic_source_test.go @@ -0,0 +1,211 @@ +package sources + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/overmindtech/sdp-go" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +type PodClient struct { + GetError error + ListError error +} + +func (p PodClient) Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Pod, error) { + if p.GetError != nil { + return nil, p.GetError + } + + uid := uuid.NewString() + + return &v1.Pod{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + UID: types.UID(uid), + ResourceVersion: "9164", + CreationTimestamp: metav1.NewTime(time.Now()), + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "kube-api-access-hgq4d", + }, + }, + RestartPolicy: "Always", + DNSPolicy: "ClusterFirst", + ServiceAccountName: "default", + NodeName: "minikube", + }, + Status: v1.PodStatus{ + Phase: "Running", + HostIP: "10.0.0.1", + PodIP: "10.244.0.6", + }, + }, nil +} + +func (p PodClient) List(ctx context.Context, opts metav1.ListOptions) (*v1.PodList, error) { + if p.ListError != nil { + return nil, p.ListError + } + + uid := uuid.NewString() + + return &v1.PodList{ + Items: []v1.Pod{ + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + UID: types.UID(uid), + ResourceVersion: "9164", + CreationTimestamp: metav1.NewTime(time.Now()), + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "kube-api-access-hgq4d", + }, + }, + RestartPolicy: "Always", + DNSPolicy: "ClusterFirst", + ServiceAccountName: "default", + NodeName: "minikube", + }, + Status: v1.PodStatus{ + Phase: "Running", + HostIP: "10.0.0.1", + PodIP: "10.244.0.6", + }, + }, + }, + }, nil +} + +func createSource() KubeTypeSource[*v1.Pod, *v1.PodList] { + return KubeTypeSource[*v1.Pod, *v1.PodList]{ + ClusterInterfaceBuilder: func() ItemInterface[*v1.Pod, *v1.PodList] { + return PodClient{} + }, + ListExtractor: func(p *v1.PodList) ([]*v1.Pod, error) { + pods := make([]*v1.Pod, len(p.Items)) + + for i := range p.Items { + pods[i] = &p.Items[i] + } + + return pods, nil + }, + LinkedItemQueryExtractor: func(p *v1.Pod) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) + + if p.Spec.NodeName == "" { + queries = append(queries, &sdp.Query{ + Type: "node", + Method: sdp.QueryMethod_GET, + Query: p.Spec.NodeName, + Scope: "foo", + }) + } + + return queries, nil + }, + TypeName: "pod", + ClusterName: "minikube", + } +} + +func TestSourceValidate(t *testing.T) { + t.Run("fully populated source", func(t *testing.T) { + t.Parallel() + source := createSource() + err := source.Validate() + + if err != nil { + t.Errorf("expected no error, got %s", err) + } + }) + + t.Run("missing ClusterInterfaceBuilder", func(t *testing.T) { + t.Parallel() + source := createSource() + source.ClusterInterfaceBuilder = nil + + err := source.Validate() + + if err == nil { + t.Errorf("expected error, got none") + } + }) + + t.Run("missing ListExtractor", func(t *testing.T) { + t.Parallel() + source := createSource() + source.ListExtractor = nil + + err := source.Validate() + + if err == nil { + t.Errorf("expected error, got none") + } + }) + + t.Run("missing TypeName", func(t *testing.T) { + t.Parallel() + source := createSource() + source.TypeName = "" + + err := source.Validate() + + if err == nil { + t.Errorf("expected error, got none") + } + }) +} + +func TestSourceGet(t *testing.T) { + t.Run("get existing item", func(t *testing.T) { + source := createSource() + + item, err := source.Get(context.Background(), "foo", "example") + + if err != nil { + t.Errorf("expected no error, got %s", err) + } + + if item == nil { + t.Errorf("expected item, got none") + } + + if item.UniqueAttributeValue() != "example" { + t.Errorf("expected item with unique attribute value 'example', got %s", item.UniqueAttributeValue()) + } + }) + + t.Run("get non-existent item", func(t *testing.T) { + source := createSource() + source.ClusterInterfaceBuilder = func() ItemInterface[*v1.Pod, *v1.PodList] { + return PodClient{ + GetError: &sdp.QueryError{ + ErrorType: sdp.QueryError_NOTFOUND, + ErrorString: "not found", + }, + } + } + + _, err := source.Get(context.Background(), "foo", "example") + + if err == nil { + t.Errorf("expected error, got none") + } + }) +} diff --git a/internal/sources/shared_resourcesource.go b/internal/sources/shared_resourcesource.go index 46fcbca..c673e3f 100644 --- a/internal/sources/shared_resourcesource.go +++ b/internal/sources/shared_resourcesource.go @@ -391,12 +391,7 @@ func (rs *ResourceSource) Weight() int { // will allow us to call Get and List functions which will in turn actually // execute API queries against K8s func (rs *ResourceSource) interactionInterface(itemScope string) (reflect.Value, error) { - contextDetails, err := ParseScope(itemScope) - - if err != nil { - return reflect.Value{}, err - } - + contextDetails := ParseScope(itemScope) interfaceFunctionArgs := make([]reflect.Value, 0) if rs.Namespaced { diff --git a/internal/sources/shared_util.go b/internal/sources/shared_util.go index 9828cef..bc8b33a 100644 --- a/internal/sources/shared_util.go +++ b/internal/sources/shared_util.go @@ -17,6 +17,14 @@ type ScopeDetails struct { Namespace string } +func (sd ScopeDetails) String() string { + if sd.Namespace == "" { + return sd.ClusterName + } + + return fmt.Sprintf("%v.%v", sd.ClusterName, sd.Namespace) +} + // ClusterNamespaceRegex matches the cluster name and namespace from a string // that is in the format {clusterName}.{namespace} // @@ -25,20 +33,20 @@ var ClusterNamespaceRegex = regexp.MustCompile(`(?P.+:.+?)(\.(?P Date: Thu, 11 May 2023 10:14:29 +0000 Subject: [PATCH 02/24] Rationalise list and search --- internal/sources/generic_source.go | 37 ++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/internal/sources/generic_source.go b/internal/sources/generic_source.go index 4ed0a60..a2a1bd3 100644 --- a/internal/sources/generic_source.go +++ b/internal/sources/generic_source.go @@ -87,7 +87,7 @@ func (k *KubeTypeSource[Resource, ResourceList]) Type() string { } func (k *KubeTypeSource[Resource, ResourceList]) Name() string { - return "TODO" + return fmt.Sprintf("k8s-%v", k.TypeName) } func (k *KubeTypeSource[Resource, ResourceList]) Scopes() []string { @@ -141,8 +141,14 @@ func (k *KubeTypeSource[Resource, ResourceList]) Get(ctx context.Context, scope } func (k *KubeTypeSource[Resource, ResourceList]) List(ctx context.Context, scope string) ([]*sdp.Item, error) { + return k.listWithOptions(ctx, scope, metav1.ListOptions{}) +} + +// listWithOptions Runs the inbuilt list method with the given options +func (k *KubeTypeSource[Resource, ResourceList]) listWithOptions(ctx context.Context, scope string, opts metav1.ListOptions) ([]*sdp.Item, error) { i := k.itemInterface(scope) - list, err := i.List(ctx, metav1.ListOptions{}) + + list, err := i.List(ctx, opts) if err != nil { return nil, err @@ -154,10 +160,21 @@ func (k *KubeTypeSource[Resource, ResourceList]) List(ctx context.Context, scope return nil, err } - items := make([]*sdp.Item, 0) + items, err := k.resourcesToItems(resourceList) + + if err != nil { + return nil, err + } + + return items, nil +} - for _, resource := range resourceList { - item, err := mapK8sObject(k.TypeName, resource) +// resourcesToItems Converts a slice of resources to a slice of items +func (k *KubeTypeSource[Resource, ResourceList]) resourcesToItems(resourceList []Resource) ([]*sdp.Item, error) { + items := make([]*sdp.Item, len(resourceList)) + + for i := range resourceList { + item, err := mapK8sObject(k.TypeName, resourceList[i]) if err != nil { return nil, err @@ -165,7 +182,7 @@ func (k *KubeTypeSource[Resource, ResourceList]) List(ctx context.Context, scope if k.LinkedItemQueryExtractor != nil { // Add linked items - item.LinkedItemQueries, err = k.LinkedItemQueryExtractor(resource) + item.LinkedItemQueries, err = k.LinkedItemQueryExtractor(resourceList[i]) if err != nil { return nil, err @@ -179,7 +196,13 @@ func (k *KubeTypeSource[Resource, ResourceList]) List(ctx context.Context, scope } func (k *KubeTypeSource[Resource, ResourceList]) Search(ctx context.Context, scope string, query string) ([]*sdp.Item, error) { - return nil, nil + opts, err := QueryToListOptions(query) + + if err != nil { + return nil, err + } + + return k.listWithOptions(ctx, scope, opts) } // itemInterface Returns the correct interface depending on whether the source From d40c0e12d96e7dd16583c1f3230088e0beff2616 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Thu, 11 May 2023 10:46:41 +0000 Subject: [PATCH 03/24] Added handling for not found errors --- internal/sources/generic_source.go | 12 ++++++- internal/sources/generic_source_test.go | 46 +++++++++++++++++++++++++ internal/sources/shared_util.go | 4 ++- 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/internal/sources/generic_source.go b/internal/sources/generic_source.go index a2a1bd3..f245c2f 100644 --- a/internal/sources/generic_source.go +++ b/internal/sources/generic_source.go @@ -2,9 +2,11 @@ package sources import ( "context" + "errors" "fmt" "github.com/overmindtech/sdp-go" + k8serr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -118,7 +120,15 @@ func (k *KubeTypeSource[Resource, ResourceList]) Get(ctx context.Context, scope resource, err := i.Get(ctx, query, metav1.GetOptions{}) if err != nil { - // TODO: Handle not found + statusErr := new(k8serr.StatusError) + + if errors.As(err, &statusErr) && statusErr.ErrStatus.Code == 404 { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_NOTFOUND, + ErrorString: statusErr.ErrStatus.Message, + } + } + return nil, err } diff --git a/internal/sources/generic_source_test.go b/internal/sources/generic_source_test.go index a0d46fe..77aeecd 100644 --- a/internal/sources/generic_source_test.go +++ b/internal/sources/generic_source_test.go @@ -2,6 +2,7 @@ package sources import ( "context" + "errors" "testing" "time" @@ -209,3 +210,48 @@ func TestSourceGet(t *testing.T) { } }) } + +func TestRealGet(t *testing.T) { + source := KubeTypeSource[*v1.Pod, *v1.PodList]{ + TypeName: "pod", + Namespaces: []string{"default"}, + ClusterName: "minikube", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.Pod, *v1.PodList] { + return CurrentCluster.ClientSet.CoreV1().Pods(namespace) + }, + ListExtractor: func(p *v1.PodList) ([]*v1.Pod, error) { + pods := make([]*v1.Pod, len(p.Items)) + + for i := range p.Items { + pods[i] = &p.Items[i] + } + + return pods, nil + }, + LinkedItemQueryExtractor: func(p *v1.Pod) ([]*sdp.Query, error) { + return []*sdp.Query{}, nil + }, + } + + err := source.Validate() + + if err != nil { + t.Fatalf("source validation failed: %s", err) + } + + _, err = source.Get(context.Background(), "minikube:8080.default", "not-real-pod") + + if err == nil { + t.Error("expected error, got none") + } + + sdpErr := new(sdp.QueryError) + + if errors.As(err, &sdpErr) { + if sdpErr.ErrorType != sdp.QueryError_NOTFOUND { + t.Errorf("expected not found error, got %s", sdpErr.ErrorType) + } + } else { + t.Errorf("expected sdp.QueryError, got %s", err) + } +} diff --git a/internal/sources/shared_util.go b/internal/sources/shared_util.go index bc8b33a..39d80b5 100644 --- a/internal/sources/shared_util.go +++ b/internal/sources/shared_util.go @@ -28,7 +28,9 @@ func (sd ScopeDetails) String() string { // ClusterNamespaceRegex matches the cluster name and namespace from a string // that is in the format {clusterName}.{namespace} // -// This is possible due to the fact namespaces have a limited set of characters so we can use a regex to find the last instance of a namespace-compliant string after a trailing +// This is possible due to the fact namespaces have a limited set of characters +// so we can use a regex to find the last instance of a namespace-compliant +// string after a trailing var ClusterNamespaceRegex = regexp.MustCompile(`(?P.+:.+?)(\.(?P[a-z0-9]([-a-z0-9]*[a-z0-9])?))?$`) // ParseScope Parses the custer and scope name out of a given SDP scope From 1a98e7785ab7196119a9c858b3535a65c7969cfa Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Thu, 11 May 2023 12:13:47 +0000 Subject: [PATCH 04/24] Added more tests --- internal/sources/generic_source.go | 21 +- internal/sources/generic_source_test.go | 270 ++++++++++++++++++++---- 2 files changed, 235 insertions(+), 56 deletions(-) diff --git a/internal/sources/generic_source.go b/internal/sources/generic_source.go index f245c2f..3b6aadb 100644 --- a/internal/sources/generic_source.go +++ b/internal/sources/generic_source.go @@ -42,10 +42,11 @@ type KubeTypeSource[Resource metav1.Object, ResourceList any] struct { ListExtractor func(ResourceList) ([]Resource, error) // A function that returns a list of linked item queries for a given - // resource - LinkedItemQueryExtractor func(Resource) ([]*sdp.Query, error) + // resource and scope + LinkedItemQueryExtractor func(resource Resource, scope string) ([]*sdp.Query, error) - // The type of items that this source should return + // The type of items that this source should return. This should be the + // "Kind" of the kubernetes resources, e.g. "Pod", "Node", "ServiceAccount" TypeName string // List of namespaces that this source should query Namespaces []string @@ -140,7 +141,7 @@ func (k *KubeTypeSource[Resource, ResourceList]) Get(ctx context.Context, scope if k.LinkedItemQueryExtractor != nil { // Add linked items - item.LinkedItemQueries, err = k.LinkedItemQueryExtractor(resource) + item.LinkedItemQueries, err = k.LinkedItemQueryExtractor(resource, scope) if err != nil { return nil, err @@ -170,7 +171,7 @@ func (k *KubeTypeSource[Resource, ResourceList]) listWithOptions(ctx context.Con return nil, err } - items, err := k.resourcesToItems(resourceList) + items, err := k.resourcesToItems(resourceList, scope) if err != nil { return nil, err @@ -180,11 +181,13 @@ func (k *KubeTypeSource[Resource, ResourceList]) listWithOptions(ctx context.Con } // resourcesToItems Converts a slice of resources to a slice of items -func (k *KubeTypeSource[Resource, ResourceList]) resourcesToItems(resourceList []Resource) ([]*sdp.Item, error) { +func (k *KubeTypeSource[Resource, ResourceList]) resourcesToItems(resourceList []Resource, scope string) ([]*sdp.Item, error) { items := make([]*sdp.Item, len(resourceList)) + var err error + for i := range resourceList { - item, err := mapK8sObject(k.TypeName, resourceList[i]) + items[i], err = mapK8sObject(k.TypeName, resourceList[i]) if err != nil { return nil, err @@ -192,14 +195,12 @@ func (k *KubeTypeSource[Resource, ResourceList]) resourcesToItems(resourceList [ if k.LinkedItemQueryExtractor != nil { // Add linked items - item.LinkedItemQueries, err = k.LinkedItemQueryExtractor(resourceList[i]) + items[i].LinkedItemQueries, err = k.LinkedItemQueryExtractor(resourceList[i], scope) if err != nil { return nil, err } } - - items = append(items, item) } return items, nil diff --git a/internal/sources/generic_source_test.go b/internal/sources/generic_source_test.go index 77aeecd..f234d75 100644 --- a/internal/sources/generic_source_test.go +++ b/internal/sources/generic_source_test.go @@ -88,15 +88,53 @@ func (p PodClient) List(ctx context.Context, opts metav1.ListOptions) (*v1.PodLi PodIP: "10.244.0.6", }, }, + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Namespace: "default", + UID: types.UID(uid), + ResourceVersion: "9164", + CreationTimestamp: metav1.NewTime(time.Now()), + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "kube-api-access-c43w1", + }, + }, + RestartPolicy: "Always", + DNSPolicy: "ClusterFirst", + ServiceAccountName: "default", + NodeName: "minikube", + }, + Status: v1.PodStatus{ + Phase: "Running", + HostIP: "10.0.0.1", + PodIP: "10.244.0.7", + }, + }, }, }, nil } -func createSource() KubeTypeSource[*v1.Pod, *v1.PodList] { - return KubeTypeSource[*v1.Pod, *v1.PodList]{ - ClusterInterfaceBuilder: func() ItemInterface[*v1.Pod, *v1.PodList] { +func createSource(namespaced bool) KubeTypeSource[*v1.Pod, *v1.PodList] { + var clusterInterfaceBuilder ClusterInterfaceBuilder[*v1.Pod, *v1.PodList] + var namespacedInterfaceBuilder NamespacedInterfaceBuilder[*v1.Pod, *v1.PodList] + + if namespaced { + namespacedInterfaceBuilder = func(namespace string) ItemInterface[*v1.Pod, *v1.PodList] { return PodClient{} - }, + } + } else { + clusterInterfaceBuilder = func() ItemInterface[*v1.Pod, *v1.PodList] { + return PodClient{} + } + } + + return KubeTypeSource[*v1.Pod, *v1.PodList]{ + ClusterInterfaceBuilder: clusterInterfaceBuilder, + NamespacedInterfaceBuilder: namespacedInterfaceBuilder, ListExtractor: func(p *v1.PodList) ([]*v1.Pod, error) { pods := make([]*v1.Pod, len(p.Items)) @@ -106,7 +144,7 @@ func createSource() KubeTypeSource[*v1.Pod, *v1.PodList] { return pods, nil }, - LinkedItemQueryExtractor: func(p *v1.Pod) ([]*sdp.Query, error) { + LinkedItemQueryExtractor: func(p *v1.Pod, scope string) ([]*sdp.Query, error) { queries := make([]*sdp.Query, 0) if p.Spec.NodeName == "" { @@ -114,21 +152,22 @@ func createSource() KubeTypeSource[*v1.Pod, *v1.PodList] { Type: "node", Method: sdp.QueryMethod_GET, Query: p.Spec.NodeName, - Scope: "foo", + Scope: scope, }) } return queries, nil }, - TypeName: "pod", + TypeName: "Pod", ClusterName: "minikube", + Namespaces: []string{"default", "app1"}, } } func TestSourceValidate(t *testing.T) { t.Run("fully populated source", func(t *testing.T) { t.Parallel() - source := createSource() + source := createSource(false) err := source.Validate() if err != nil { @@ -138,7 +177,7 @@ func TestSourceValidate(t *testing.T) { t.Run("missing ClusterInterfaceBuilder", func(t *testing.T) { t.Parallel() - source := createSource() + source := createSource(false) source.ClusterInterfaceBuilder = nil err := source.Validate() @@ -150,7 +189,7 @@ func TestSourceValidate(t *testing.T) { t.Run("missing ListExtractor", func(t *testing.T) { t.Parallel() - source := createSource() + source := createSource(false) source.ListExtractor = nil err := source.Validate() @@ -162,7 +201,7 @@ func TestSourceValidate(t *testing.T) { t.Run("missing TypeName", func(t *testing.T) { t.Parallel() - source := createSource() + source := createSource(false) source.TypeName = "" err := source.Validate() @@ -171,11 +210,100 @@ func TestSourceValidate(t *testing.T) { t.Errorf("expected error, got none") } }) + + t.Run("missing ClusterName", func(t *testing.T) { + t.Parallel() + source := createSource(false) + source.ClusterName = "" + + err := source.Validate() + + if err == nil { + t.Errorf("expected error, got none") + } + }) + + t.Run("missing namespaces", func(t *testing.T) { + t.Run("when namespaced", func(t *testing.T) { + t.Parallel() + source := createSource(true) + source.Namespaces = nil + + err := source.Validate() + + if err == nil { + t.Errorf("expected error, got none") + } + + source.Namespaces = []string{} + + err = source.Validate() + + if err == nil { + t.Errorf("expected error, got none") + } + }) + + t.Run("when not namespaced", func(t *testing.T) { + t.Parallel() + source := createSource(false) + source.Namespaces = nil + + err := source.Validate() + + if err != nil { + t.Errorf("expected no error, got %s", err) + } + + source.Namespaces = []string{} + + err = source.Validate() + + if err != nil { + t.Errorf("expected no error, got %s", err) + } + }) + + }) +} + +func TestType(t *testing.T) { + source := createSource(false) + + if source.Type() != "Pod" { + t.Errorf("expected type 'Pod', got %s", source.Type()) + } +} + +func TestName(t *testing.T) { + source := createSource(false) + + if source.Name() == "" { + t.Errorf("expected non-empty name, got none") + } +} + +func TestScopes(t *testing.T) { + t.Run("when namespaced", func(t *testing.T) { + source := createSource(true) + + if len(source.Scopes()) != len(source.Namespaces) { + t.Errorf("expected %d scopes, got %d", len(source.Namespaces), len(source.Scopes())) + } + }) + + t.Run("when not namespaced", func(t *testing.T) { + source := createSource(false) + + if len(source.Scopes()) != 1 { + t.Errorf("expected 1 scope, got %d", len(source.Scopes())) + } + }) } func TestSourceGet(t *testing.T) { t.Run("get existing item", func(t *testing.T) { - source := createSource() + source := createSource(false) item, err := source.Get(context.Background(), "foo", "example") @@ -193,7 +321,7 @@ func TestSourceGet(t *testing.T) { }) t.Run("get non-existent item", func(t *testing.T) { - source := createSource() + source := createSource(false) source.ClusterInterfaceBuilder = func() ItemInterface[*v1.Pod, *v1.PodList] { return PodClient{ GetError: &sdp.QueryError{ @@ -211,47 +339,97 @@ func TestSourceGet(t *testing.T) { }) } -func TestRealGet(t *testing.T) { - source := KubeTypeSource[*v1.Pod, *v1.PodList]{ - TypeName: "pod", - Namespaces: []string{"default"}, - ClusterName: "minikube", - NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.Pod, *v1.PodList] { - return CurrentCluster.ClientSet.CoreV1().Pods(namespace) - }, - ListExtractor: func(p *v1.PodList) ([]*v1.Pod, error) { - pods := make([]*v1.Pod, len(p.Items)) +func TestFailingQueryExtractor(t *testing.T) { + source := createSource(false) + source.LinkedItemQueryExtractor = func(_ *v1.Pod, _ string) ([]*sdp.Query, error) { + return nil, errors.New("failed to extract queries") + } - for i := range p.Items { - pods[i] = &p.Items[i] - } + _, err := source.Get(context.Background(), "foo", "example") - return pods, nil - }, - LinkedItemQueryExtractor: func(p *v1.Pod) ([]*sdp.Query, error) { - return []*sdp.Query{}, nil - }, + if err == nil { + t.Errorf("expected error, got none") } +} - err := source.Validate() +func TestList(t *testing.T) { + t.Run("when namespaced", func(t *testing.T) { + source := createSource(true) - if err != nil { - t.Fatalf("source validation failed: %s", err) - } + items, err := source.List(context.Background(), "foo") - _, err = source.Get(context.Background(), "minikube:8080.default", "not-real-pod") + if err != nil { + t.Errorf("expected no error, got %s", err) + } - if err == nil { - t.Error("expected error, got none") - } + if len(items) != 2 { + t.Errorf("expected 2 items, got %d", len(items)) + } + }) + + t.Run("when not namespaced", func(t *testing.T) { + source := createSource(false) - sdpErr := new(sdp.QueryError) + items, err := source.List(context.Background(), "foo") + + if err != nil { + t.Errorf("expected no error, got %s", err) + } - if errors.As(err, &sdpErr) { - if sdpErr.ErrorType != sdp.QueryError_NOTFOUND { - t.Errorf("expected not found error, got %s", sdpErr.ErrorType) + if len(items) != 2 { + t.Errorf("expected 2 items, got %d", len(items)) } - } else { - t.Errorf("expected sdp.QueryError, got %s", err) - } + }) + + t.Run("with failing list extractor", func(t *testing.T) { + source := createSource(false) + source.ListExtractor = func(_ *v1.PodList) ([]*v1.Pod, error) { + return nil, errors.New("failed to extract list") + } + + _, err := source.List(context.Background(), "foo") + + if err == nil { + t.Errorf("expected error, got none") + } + }) + + t.Run("with failing query extractor", func(t *testing.T) { + source := createSource(false) + source.LinkedItemQueryExtractor = func(_ *v1.Pod, _ string) ([]*sdp.Query, error) { + return nil, errors.New("failed to extract queries") + } + + _, err := source.List(context.Background(), "foo") + + if err == nil { + t.Errorf("expected error, got none") + } + }) +} + +func TestSearch(t *testing.T) { + t.Run("with a valid query", func(t *testing.T) { + source := createSource(false) + + items, err := source.Search(context.Background(), "foo", "{\"labelSelector\":\"app=foo\"}") + + if err != nil { + t.Errorf("expected no error, got %s", err) + } + + if len(items) != 2 { + t.Errorf("expected 2 item, got %d", len(items)) + } + }) + + t.Run("with an invalid query", func(t *testing.T) { + source := createSource(false) + + _, err := source.Search(context.Background(), "foo", "{{{{}") + + if err == nil { + t.Errorf("expected error, got none") + } + }) } From f541ebe7c2b8439b4e465e899fb3013f2e7712cf Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Thu, 11 May 2023 15:05:20 +0000 Subject: [PATCH 05/24] Added cluster role binding source --- internal/sources/clusterrolebinding.go | 115 +-- internal/sources/clusterrolebinding_test.go | 59 ++ internal/sources/generic_source.go | 64 +- internal/sources/generic_source_test.go | 104 +++ internal/sources/shared_resourcesource.go | 1 - internal/sources/shared_test.go | 822 ++++++++++---------- 6 files changed, 675 insertions(+), 490 deletions(-) create mode 100644 internal/sources/clusterrolebinding_test.go diff --git a/internal/sources/clusterrolebinding.go b/internal/sources/clusterrolebinding.go index 6dcea33..245866b 100644 --- a/internal/sources/clusterrolebinding.go +++ b/internal/sources/clusterrolebinding.go @@ -1,96 +1,59 @@ package sources import ( - "fmt" - "strings" - - rbacV1beta1 "k8s.io/api/rbac/v1beta1" + v1 "k8s.io/api/rbac/v1" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// ClusterRoleBindingSource returns a ResourceSource for ClusterRoleBindingClaims for a given -// client -func ClusterRoleBindingSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "clusterrolebinding", - MapGet: MapClusterRoleBindingGet, - MapList: MapClusterRoleBindingList, - Namespaced: false, - } - - err := source.LoadFunction( - cs.RbacV1beta1().ClusterRoleBindings, - ) - - return source, err -} - -// MapClusterRoleBindingList maps an interface that is underneath a -// *rbacV1Beta1.ClusterRoleBindingList to a list of Items -func MapClusterRoleBindingList(i interface{}) ([]*sdp.Item, error) { - var objectList *rbacV1beta1.ClusterRoleBindingList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*rbacV1beta1.ClusterRoleBindingList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *rbacV1Beta1.ClusterRoleBindingList", i) - } - - for _, object := range objectList.Items { - if item, err = MapClusterRoleBindingGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} - -// MapClusterRoleBindingGet maps an interface that is underneath a *rbacV1Beta1.ClusterRoleBinding to an item. If -// the interface isn't actually a *rbacV1Beta1.ClusterRoleBinding this will fail -func MapClusterRoleBindingGet(i interface{}) (*sdp.Item, error) { - var object *rbacV1beta1.ClusterRoleBinding - var ok bool - - // Expect this to be a *rbacV1Beta1.ClusterRoleBinding - if object, ok = i.(*rbacV1beta1.ClusterRoleBinding); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *rbacV1Beta1.ClusterRolebinding", i) - } - - item, err := mapK8sObject("clusterrolebinding", object) - - if err != nil { - return &sdp.Item{}, err - } +func clusterRoleBindingExtractor(resource *v1.ClusterRoleBinding, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: ClusterName, + queries = append(queries, &sdp.Query{ + Scope: scope, Method: sdp.QueryMethod_GET, - Query: object.RoleRef.Name, - Type: strings.ToLower(object.RoleRef.Kind), + Query: resource.RoleRef.Name, + Type: resource.RoleRef.Kind, }) - for _, subject := range object.Subjects { - var scope string - if subject.Namespace == "" { - scope = ClusterName - } else { - scope = ClusterName + "." + subject.Namespace + for _, subject := range resource.Subjects { + sd := ScopeDetails{ + ClusterName: scope, // Since this is a cluster role binding, the scope is the cluster name + } + + if subject.Namespace != "" { + sd.Namespace = subject.Namespace } - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: scope, + queries = append(queries, &sdp.Query{ + Scope: sd.String(), Method: sdp.QueryMethod_GET, Query: subject.Name, - Type: strings.ToLower(subject.Kind), + Type: subject.Kind, }) } - return item, nil + return queries, nil +} + +func NewClusterRoleBindingSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ClusterRoleBinding, *v1.ClusterRoleBindingList] { + return KubeTypeSource[*v1.ClusterRoleBinding, *v1.ClusterRoleBindingList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "ClusterRoleBinding", + ClusterInterfaceBuilder: func() ItemInterface[*v1.ClusterRoleBinding, *v1.ClusterRoleBindingList] { + return cs.RbacV1().ClusterRoleBindings() + }, + ListExtractor: func(list *v1.ClusterRoleBindingList) ([]*v1.ClusterRoleBinding, error) { + bindings := make([]*v1.ClusterRoleBinding, len(list.Items)) + + for i, crb := range list.Items { + bindings[i] = &crb + } + + return bindings, nil + }, + LinkedItemQueryExtractor: clusterRoleBindingExtractor, + } } diff --git a/internal/sources/clusterrolebinding_test.go b/internal/sources/clusterrolebinding_test.go new file mode 100644 index 0000000..2942958 --- /dev/null +++ b/internal/sources/clusterrolebinding_test.go @@ -0,0 +1,59 @@ +package sources + +import ( + "testing" + + "github.com/overmindtech/sdp-go" +) + +var clusterRoleBindingYAML = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: admin-binding +subjects: +- kind: Group + name: system:serviceaccounts:default + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: admin + apiGroup: rbac.authorization.k8s.io +` + +func TestClusterRoleBindingSource(t *testing.T) { + err := CurrentCluster.Apply(clusterRoleBindingYAML) + + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + CurrentCluster.Delete(clusterRoleBindingYAML) + }) + + source := NewClusterRoleBindingSource(CurrentCluster.ClientSet, CurrentCluster.Name, []string{}) + + st := SourceTests{ + Source: &source, + GetQuery: "admin-binding", + GetScope: CurrentCluster.Name, + SetupYAML: clusterRoleBindingYAML, + GetQueryTests: QueryTests{ + { + ExpectedType: "ClusterRole", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "admin", + ExpectedScope: CurrentCluster.Name, + }, + { + ExpectedType: "Group", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "system:serviceaccounts:default", + ExpectedScope: CurrentCluster.Name, + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/generic_source.go b/internal/sources/generic_source.go index 3b6aadb..9e3b5c5 100644 --- a/internal/sources/generic_source.go +++ b/internal/sources/generic_source.go @@ -93,6 +93,10 @@ func (k *KubeTypeSource[Resource, ResourceList]) Name() string { return fmt.Sprintf("k8s-%v", k.TypeName) } +func (k *KubeTypeSource[Resource, ResourceList]) Weight() int { + return 10 +} + func (k *KubeTypeSource[Resource, ResourceList]) Scopes() []string { namespaces := make([]string, 0) @@ -133,7 +137,7 @@ func (k *KubeTypeSource[Resource, ResourceList]) Get(ctx context.Context, scope return nil, err } - item, err := mapK8sObject(k.TypeName, resource) + item, err := resourceToObject(resource, k.ClusterName) if err != nil { return nil, err @@ -187,7 +191,7 @@ func (k *KubeTypeSource[Resource, ResourceList]) resourcesToItems(resourceList [ var err error for i := range resourceList { - items[i], err = mapK8sObject(k.TypeName, resourceList[i]) + items[i], err = resourceToObject(resourceList[i], k.ClusterName) if err != nil { return nil, err @@ -229,3 +233,59 @@ func (k *KubeTypeSource[Resource, ResourceList]) itemInterface(scope string) Ite return k.ClusterInterfaceBuilder() } } + +var ignoredMetadataFields = []string{ + "managedFields", + "binaryData", + "immutable", + "stringData", +} + +func ignored(key string) bool { + for _, ignoredKey := range ignoredMetadataFields { + if key == ignoredKey { + return true + } + } + + return false +} + +// resourceToObject Converts a resource to an item +func resourceToObject(resource metav1.Object, cluster string) (*sdp.Item, error) { + sd := ScopeDetails{ + ClusterName: cluster, + Namespace: resource.GetNamespace(), + } + + attributes, err := sdp.ToAttributesViaJson(resource) + + if err != nil { + return nil, err + } + + // Promote the metadata to the top level + if metadata, err := attributes.Get("metadata"); err == nil { + // Cast to a type we can iterate over + if metadataMap, ok := metadata.(map[string]interface{}); ok { + for key, value := range metadataMap { + // Check that the key isn't in the ignored list + if !ignored(key) { + attributes.Set(key, value) + } + } + } + + // Remove the metadata attribute + attributes.AttrStruct.Fields["metadata"] = nil + } + + item := &sdp.Item{ + Type: resource.GetName(), + UniqueAttribute: "name", + Scope: sd.String(), + Attributes: attributes, + } + + return item, nil +} diff --git a/internal/sources/generic_source_test.go b/internal/sources/generic_source_test.go index f234d75..8250e45 100644 --- a/internal/sources/generic_source_test.go +++ b/internal/sources/generic_source_test.go @@ -3,10 +3,12 @@ package sources import ( "context" "errors" + "fmt" "testing" "time" "github.com/google/uuid" + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -433,3 +435,105 @@ func TestSearch(t *testing.T) { } }) } + +type QueryTest struct { + ExpectedType string + ExpectedMethod sdp.QueryMethod + ExpectedQuery string + ExpectedScope string +} + +type QueryTests []QueryTest + +func (i QueryTests) Execute(t *testing.T, item *sdp.Item) { + for _, test := range i { + var found bool + + for _, lir := range item.LinkedItemQueries { + if lirMatches(test, lir) { + found = true + break + } + } + + if !found { + t.Errorf("could not find linked item request in %v requests.\nType: %v\nQuery: %v\nScope: %v", len(item.LinkedItemQueries), test.ExpectedType, test.ExpectedQuery, test.ExpectedScope) + } + } +} + +func lirMatches(test QueryTest, req *sdp.Query) bool { + return (test.ExpectedMethod == req.Method && + test.ExpectedQuery == req.Query && + test.ExpectedScope == req.Scope && + test.ExpectedType == req.Type) +} + +type SourceTests struct { + // The source under test + Source discovery.Source + + // The get query to test + GetQuery string + GetScope string + GetQueryTests QueryTests + + // YAML to apply before testing, it will be removed after + SetupYAML string +} + +func (s SourceTests) Execute(t *testing.T) { + t.Parallel() + + if s.SetupYAML != "" { + err := CurrentCluster.Apply(s.SetupYAML) + + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + CurrentCluster.Delete(s.SetupYAML) + }) + } + + t.Run(s.Source.Name(), func(t *testing.T) { + if s.GetQuery != "" { + t.Run(fmt.Sprintf("GET: %v", s.GetQuery), func(t *testing.T) { + item, err := s.Source.Get(context.Background(), s.GetScope, s.GetQuery) + + if err != nil { + t.Fatal(err) + } + + if item == nil { + t.Errorf("expected item, got none") + } + + if err = item.Validate(); err != nil { + t.Error(err) + } + + s.GetQueryTests.Execute(t, item) + }) + } + + t.Run("LIST", func(t *testing.T) { + items, err := s.Source.List(context.Background(), s.GetScope) + + if err != nil { + t.Fatal(err) + } + + if len(items) == 0 { + t.Errorf("expected items, got none") + } + + for _, item := range items { + if err = item.Validate(); err != nil { + t.Error(err) + } + } + }) + }) +} diff --git a/internal/sources/shared_resourcesource.go b/internal/sources/shared_resourcesource.go index c673e3f..fe4b9cd 100644 --- a/internal/sources/shared_resourcesource.go +++ b/internal/sources/shared_resourcesource.go @@ -70,7 +70,6 @@ var SourceFunctions = []SourceFunction{ NodeSource, PersistentVolumeSource, ClusterRoleSource, - ClusterRoleBindingSource, StorageClassSource, PriorityClassSource, } diff --git a/internal/sources/shared_test.go b/internal/sources/shared_test.go index 915f737..35df6b3 100644 --- a/internal/sources/shared_test.go +++ b/internal/sources/shared_test.go @@ -114,9 +114,9 @@ func (t *TestCluster) Start() error { return nil } -func (t *TestCluster) ApplyBaselineConfig() error { - return t.Apply(ClusterBaseline) -} +// func (t *TestCluster) ApplyBaselineConfig() error { +// return t.Apply(ClusterBaseline) +// } // Apply Runs of `kubectl apply -f` for a given string of YAML func (t *TestCluster) Apply(yaml string) error { @@ -206,411 +206,411 @@ func TestMain(m *testing.M) { os.Exit(code) } -const ClusterBaseline string = ` -apiVersion: v1 -kind: PersistentVolume -metadata: - name: d1 - labels: - type: local -spec: - storageClassName: manual - capacity: - storage: 20Gi - accessModes: - - ReadWriteOnce - hostPath: - path: "/mnt/d1" ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: d2 - labels: - type: local -spec: - storageClassName: manual - capacity: - storage: 20Gi - accessModes: - - ReadWriteOnce - hostPath: - path: "/mnt/d2" ---- -apiVersion: v1 -kind: Service -metadata: - name: wordpress-mysql - labels: - app: wordpress -spec: - ports: - - port: 3306 - selector: - app: wordpress - tier: mysql - clusterIP: None ---- -apiVersion: v1 -kind: LimitRange -metadata: - name: test-lr2 -spec: - limits: - - max: - cpu: "200m" - min: - cpu: "50m" - type: Container ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: mysql-pv-claim - labels: - app: wordpress -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi ---- -apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 -kind: Deployment -metadata: - name: wordpress-mysql - labels: - app: wordpress -spec: - selector: - matchLabels: - app: wordpress - tier: mysql - strategy: - type: Recreate - template: - metadata: - labels: - app: wordpress - tier: mysql - spec: - containers: - - image: mysql:5.6 - name: mysql - env: - - name: MYSQL_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: mysql-pass - key: password - ports: - - containerPort: 3306 - name: mysql - volumeMounts: - - name: mysql-persistent-storage - mountPath: /var/lib/mysql - volumes: - - name: mysql-persistent-storage - persistentVolumeClaim: - claimName: mysql-pv-claim ---- -apiVersion: autoscaling/v1 -kind: HorizontalPodAutoscaler -metadata: - name: wordpress-mysql-as -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: wordpress-mysql - minReplicas: 1 - maxReplicas: 3 - targetCPUUtilizationPercentage: 50 ---- -apiVersion: v1 -kind: Service -metadata: - name: wordpress - labels: - app: wordpress -spec: - ports: - - port: 8088 - selector: - app: wordpress - tier: frontend - type: LoadBalancer ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: wordpress-ingress - annotations: - nginx.ingress.kubernetes.io/rewrite-target: / -spec: - rules: - - http: - paths: - - path: /foo - pathType: Prefix - backend: - service: - name: wordpress - port: - number: 8088 ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: wp-pv-claim - labels: - app: wordpress -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi ---- -apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 -kind: Deployment -metadata: - name: wordpress - labels: - app: wordpress -spec: - selector: - matchLabels: - app: wordpress - tier: frontend - strategy: - type: Recreate - template: - metadata: - labels: - app: wordpress - tier: frontend - spec: - containers: - - image: wordpress:4.8-apache - name: wordpress - env: - - name: WORDPRESS_DB_HOST - value: wordpress-mysql - - name: WORDPRESS_DB_PASSWORD - valueFrom: - secretKeyRef: - name: mysql-pass - key: password - resources: - limits: - cpu: 200m - requests: - cpu: 200m - ports: - - containerPort: 80 - name: wordpress - volumeMounts: - - name: wordpress-persistent-storage - mountPath: /var/www/html - volumes: - - name: wordpress-persistent-storage - persistentVolumeClaim: - claimName: wp-pv-claim ---- -# Example replication controller. These are old school and basically replaced -# by deployments -apiVersion: v1 -kind: ReplicationController -metadata: - name: nginx -spec: - replicas: 1 - selector: - app: nginx - template: - metadata: - name: nginx - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 ---- -apiVersion: v1 -kind: ResourceQuota -metadata: - name: pods-high -spec: - hard: - cpu: "1000" - memory: 200Gi - pods: "10" - scopeSelector: - matchExpressions: - - operator : In - scopeName: PriorityClass - values: ["high"] ---- -apiVersion: v1 -kind: ResourceQuota -metadata: - name: pods-medium -spec: - hard: - cpu: "10" - memory: 20Gi - pods: "10" - scopeSelector: - matchExpressions: - - operator : In - scopeName: PriorityClass - values: ["medium"] ---- -apiVersion: v1 -kind: ResourceQuota -metadata: - name: pods-low -spec: - hard: - cpu: "5" - memory: 10Gi - pods: "10" - scopeSelector: - matchExpressions: - - operator : In - scopeName: PriorityClass - values: ["low"] ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: fluentd-elasticsearch - labels: - k8s-app: fluentd-logging -spec: - selector: - matchLabels: - name: fluentd-elasticsearch - template: - metadata: - labels: - name: fluentd-elasticsearch - spec: - containers: - - name: fluentd-elasticsearch - image: quay.io/fluentd_elasticsearch/fluentd:v2.5.2 - resources: - limits: - memory: 200Mi - requests: - cpu: 50m - memory: 200Mi - volumeMounts: - - name: varlog - mountPath: /var/log - - name: varlibdockercontainers - mountPath: /var/lib/docker/containers - readOnly: true - terminationGracePeriodSeconds: 30 - volumes: - - name: varlog - hostPath: - path: /var/log - - name: varlibdockercontainers - hostPath: - path: /var/lib/docker/containers ---- -# Example stateful set -apiVersion: v1 -kind: Service -metadata: - name: nginx - labels: - app: nginx -spec: - ports: - - port: 8089 - name: web - clusterIP: None - selector: - app: nginx ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: web -spec: - serviceName: "nginx" - replicas: 1 - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: k8s.gcr.io/nginx-slim:0.8 - ports: - - containerPort: 80 - name: web - volumeMounts: - - name: www - mountPath: /usr/share/nginx/html - resources: - limits: - cpu: 50m - requests: - cpu: 50m - volumeClaimTemplates: - - metadata: - name: www - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi ---- -# job example -apiVersion: batch/v1 -kind: Job -metadata: - name: pi -spec: - template: - spec: - containers: - - name: pi - image: perl - command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"] - restartPolicy: Never - backoffLimit: 4 - parallelism: 3 ---- -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: hello -spec: - schedule: "*/1 * * * *" - jobTemplate: - spec: - template: - spec: - containers: - - name: hello - image: busybox - args: - - /bin/sh - - -c - - date; echo Hello from the Kubernetes cluster - restartPolicy: OnFailure - -` +// const ClusterBaseline string = ` +// apiVersion: v1 +// kind: PersistentVolume +// metadata: +// name: d1 +// labels: +// type: local +// spec: +// storageClassName: manual +// capacity: +// storage: 20Gi +// accessModes: +// - ReadWriteOnce +// hostPath: +// path: "/mnt/d1" +// --- +// apiVersion: v1 +// kind: PersistentVolume +// metadata: +// name: d2 +// labels: +// type: local +// spec: +// storageClassName: manual +// capacity: +// storage: 20Gi +// accessModes: +// - ReadWriteOnce +// hostPath: +// path: "/mnt/d2" +// --- +// apiVersion: v1 +// kind: Service +// metadata: +// name: wordpress-mysql +// labels: +// app: wordpress +// spec: +// ports: +// - port: 3306 +// selector: +// app: wordpress +// tier: mysql +// clusterIP: None +// --- +// apiVersion: v1 +// kind: LimitRange +// metadata: +// name: test-lr2 +// spec: +// limits: +// - max: +// cpu: "200m" +// min: +// cpu: "50m" +// type: Container +// --- +// apiVersion: v1 +// kind: PersistentVolumeClaim +// metadata: +// name: mysql-pv-claim +// labels: +// app: wordpress +// spec: +// accessModes: +// - ReadWriteOnce +// resources: +// requests: +// storage: 1Gi +// --- +// apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +// kind: Deployment +// metadata: +// name: wordpress-mysql +// labels: +// app: wordpress +// spec: +// selector: +// matchLabels: +// app: wordpress +// tier: mysql +// strategy: +// type: Recreate +// template: +// metadata: +// labels: +// app: wordpress +// tier: mysql +// spec: +// containers: +// - image: mysql:5.6 +// name: mysql +// env: +// - name: MYSQL_ROOT_PASSWORD +// valueFrom: +// secretKeyRef: +// name: mysql-pass +// key: password +// ports: +// - containerPort: 3306 +// name: mysql +// volumeMounts: +// - name: mysql-persistent-storage +// mountPath: /var/lib/mysql +// volumes: +// - name: mysql-persistent-storage +// persistentVolumeClaim: +// claimName: mysql-pv-claim +// --- +// apiVersion: autoscaling/v1 +// kind: HorizontalPodAutoscaler +// metadata: +// name: wordpress-mysql-as +// spec: +// scaleTargetRef: +// apiVersion: apps/v1 +// kind: Deployment +// name: wordpress-mysql +// minReplicas: 1 +// maxReplicas: 3 +// targetCPUUtilizationPercentage: 50 +// --- +// apiVersion: v1 +// kind: Service +// metadata: +// name: wordpress +// labels: +// app: wordpress +// spec: +// ports: +// - port: 8088 +// selector: +// app: wordpress +// tier: frontend +// type: LoadBalancer +// --- +// apiVersion: networking.k8s.io/v1 +// kind: Ingress +// metadata: +// name: wordpress-ingress +// annotations: +// nginx.ingress.kubernetes.io/rewrite-target: / +// spec: +// rules: +// - http: +// paths: +// - path: /foo +// pathType: Prefix +// backend: +// service: +// name: wordpress +// port: +// number: 8088 +// --- +// apiVersion: v1 +// kind: PersistentVolumeClaim +// metadata: +// name: wp-pv-claim +// labels: +// app: wordpress +// spec: +// accessModes: +// - ReadWriteOnce +// resources: +// requests: +// storage: 1Gi +// --- +// apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +// kind: Deployment +// metadata: +// name: wordpress +// labels: +// app: wordpress +// spec: +// selector: +// matchLabels: +// app: wordpress +// tier: frontend +// strategy: +// type: Recreate +// template: +// metadata: +// labels: +// app: wordpress +// tier: frontend +// spec: +// containers: +// - image: wordpress:4.8-apache +// name: wordpress +// env: +// - name: WORDPRESS_DB_HOST +// value: wordpress-mysql +// - name: WORDPRESS_DB_PASSWORD +// valueFrom: +// secretKeyRef: +// name: mysql-pass +// key: password +// resources: +// limits: +// cpu: 200m +// requests: +// cpu: 200m +// ports: +// - containerPort: 80 +// name: wordpress +// volumeMounts: +// - name: wordpress-persistent-storage +// mountPath: /var/www/html +// volumes: +// - name: wordpress-persistent-storage +// persistentVolumeClaim: +// claimName: wp-pv-claim +// --- +// # Example replication controller. These are old school and basically replaced +// # by deployments +// apiVersion: v1 +// kind: ReplicationController +// metadata: +// name: nginx +// spec: +// replicas: 1 +// selector: +// app: nginx +// template: +// metadata: +// name: nginx +// labels: +// app: nginx +// spec: +// containers: +// - name: nginx +// image: nginx +// ports: +// - containerPort: 80 +// --- +// apiVersion: v1 +// kind: ResourceQuota +// metadata: +// name: pods-high +// spec: +// hard: +// cpu: "1000" +// memory: 200Gi +// pods: "10" +// scopeSelector: +// matchExpressions: +// - operator : In +// scopeName: PriorityClass +// values: ["high"] +// --- +// apiVersion: v1 +// kind: ResourceQuota +// metadata: +// name: pods-medium +// spec: +// hard: +// cpu: "10" +// memory: 20Gi +// pods: "10" +// scopeSelector: +// matchExpressions: +// - operator : In +// scopeName: PriorityClass +// values: ["medium"] +// --- +// apiVersion: v1 +// kind: ResourceQuota +// metadata: +// name: pods-low +// spec: +// hard: +// cpu: "5" +// memory: 10Gi +// pods: "10" +// scopeSelector: +// matchExpressions: +// - operator : In +// scopeName: PriorityClass +// values: ["low"] +// --- +// apiVersion: apps/v1 +// kind: DaemonSet +// metadata: +// name: fluentd-elasticsearch +// labels: +// k8s-app: fluentd-logging +// spec: +// selector: +// matchLabels: +// name: fluentd-elasticsearch +// template: +// metadata: +// labels: +// name: fluentd-elasticsearch +// spec: +// containers: +// - name: fluentd-elasticsearch +// image: quay.io/fluentd_elasticsearch/fluentd:v2.5.2 +// resources: +// limits: +// memory: 200Mi +// requests: +// cpu: 50m +// memory: 200Mi +// volumeMounts: +// - name: varlog +// mountPath: /var/log +// - name: varlibdockercontainers +// mountPath: /var/lib/docker/containers +// readOnly: true +// terminationGracePeriodSeconds: 30 +// volumes: +// - name: varlog +// hostPath: +// path: /var/log +// - name: varlibdockercontainers +// hostPath: +// path: /var/lib/docker/containers +// --- +// # Example stateful set +// apiVersion: v1 +// kind: Service +// metadata: +// name: nginx +// labels: +// app: nginx +// spec: +// ports: +// - port: 8089 +// name: web +// clusterIP: None +// selector: +// app: nginx +// --- +// apiVersion: apps/v1 +// kind: StatefulSet +// metadata: +// name: web +// spec: +// serviceName: "nginx" +// replicas: 1 +// selector: +// matchLabels: +// app: nginx +// template: +// metadata: +// labels: +// app: nginx +// spec: +// containers: +// - name: nginx +// image: k8s.gcr.io/nginx-slim:0.8 +// ports: +// - containerPort: 80 +// name: web +// volumeMounts: +// - name: www +// mountPath: /usr/share/nginx/html +// resources: +// limits: +// cpu: 50m +// requests: +// cpu: 50m +// volumeClaimTemplates: +// - metadata: +// name: www +// spec: +// accessModes: [ "ReadWriteOnce" ] +// resources: +// requests: +// storage: 1Gi +// --- +// # job example +// apiVersion: batch/v1 +// kind: Job +// metadata: +// name: pi +// spec: +// template: +// spec: +// containers: +// - name: pi +// image: perl +// command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"] +// restartPolicy: Never +// backoffLimit: 4 +// parallelism: 3 +// --- +// apiVersion: batch/v1beta1 +// kind: CronJob +// metadata: +// name: hello +// spec: +// schedule: "*/1 * * * *" +// jobTemplate: +// spec: +// template: +// spec: +// containers: +// - name: hello +// image: busybox +// args: +// - /bin/sh +// - -c +// - date; echo Hello from the Kubernetes cluster +// restartPolicy: OnFailure + +// ` From 528076cf1648f49f69e4a4fed247a662145f7b38 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Thu, 11 May 2023 15:49:01 +0000 Subject: [PATCH 06/24] Added cluster role and config map --- internal/sources/clusterrole.go | 84 ++++--------- internal/sources/clusterrole_test.go | 49 ++++++++ internal/sources/clusterrolebinding_test.go | 58 +++------ internal/sources/configmap.go | 83 ++++--------- internal/sources/configmap_test.go | 32 +++++ internal/sources/generic_source.go | 31 ++++- internal/sources/generic_source_test.go | 2 +- internal/sources/shared_resourcesource.go | 4 +- internal/sources/shared_util.go | 50 ++++---- internal/sources/shared_util_test.go | 126 +++++++++++++++----- 10 files changed, 288 insertions(+), 231 deletions(-) create mode 100644 internal/sources/clusterrole_test.go create mode 100644 internal/sources/configmap_test.go diff --git a/internal/sources/clusterrole.go b/internal/sources/clusterrole.go index 8649e32..273cdb6 100644 --- a/internal/sources/clusterrole.go +++ b/internal/sources/clusterrole.go @@ -1,72 +1,32 @@ package sources import ( - "fmt" - - rbacV1Beta1 "k8s.io/api/rbac/v1beta1" + v1 "k8s.io/api/rbac/v1" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// ClusterRoleSource returns a ResourceSource for ClusterRoleClaims for a given -// client -func ClusterRoleSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "clusterrole", - MapGet: MapClusterRoleGet, - MapList: MapClusterRoleList, - Namespaced: false, - } - - err := source.LoadFunction( - cs.RbacV1beta1().ClusterRoles, - ) - - return source, err -} - -// MapClusterRoleList maps an interface that is underneath a -// *rbacV1Beta1.ClusterRoleList to a list of Items -func MapClusterRoleList(i interface{}) ([]*sdp.Item, error) { - var objectList *rbacV1Beta1.ClusterRoleList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*rbacV1Beta1.ClusterRoleList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *rbacV1Beta1.ClusterRoleList", i) - } - - for _, object := range objectList.Items { - if item, err = MapClusterRoleGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } +func NewClusterRoleSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ClusterRole, *v1.ClusterRoleList] { + return KubeTypeSource[*v1.ClusterRole, *v1.ClusterRoleList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "ClusterRole", + ClusterInterfaceBuilder: func() ItemInterface[*v1.ClusterRole, *v1.ClusterRoleList] { + return cs.RbacV1().ClusterRoles() + }, + ListExtractor: func(list *v1.ClusterRoleList) ([]*v1.ClusterRole, error) { + bindings := make([]*v1.ClusterRole, len(list.Items)) + + for i, cr := range list.Items { + bindings[i] = &cr + } + + return bindings, nil + }, + LinkedItemQueryExtractor: func(resource *v1.ClusterRole, scope string) ([]*sdp.Query, error) { + // No linked items + return []*sdp.Query{}, nil + }, } - - return items, nil -} - -// MapClusterRoleGet maps an interface that is underneath a *rbacV1Beta1.ClusterRole to an item. If -// the interface isn't actually a *rbacV1Beta1.ClusterRole this will fail -func MapClusterRoleGet(i interface{}) (*sdp.Item, error) { - var object *rbacV1Beta1.ClusterRole - var ok bool - - // Expect this to be a *rbacV1Beta1.ClusterRole - if object, ok = i.(*rbacV1Beta1.ClusterRole); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *rbacV1Beta1.ClusterRole", i) - } - - item, err := mapK8sObject("clusterrole", object) - - if err != nil { - return &sdp.Item{}, err - } - - return item, nil } diff --git a/internal/sources/clusterrole_test.go b/internal/sources/clusterrole_test.go new file mode 100644 index 0000000..33dbf5d --- /dev/null +++ b/internal/sources/clusterrole_test.go @@ -0,0 +1,49 @@ +package sources + +import ( + "testing" + + "github.com/overmindtech/sdp-go" +) + +var clusterRoleBindingYAML = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: admin-binding +subjects: +- kind: Group + name: system:serviceaccounts:default + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: admin + apiGroup: rbac.authorization.k8s.io +` + +func TestClusterRoleBindingSource(t *testing.T) { + source := NewClusterRoleBindingSource(CurrentCluster.ClientSet, CurrentCluster.Name, []string{}) + + st := SourceTests{ + Source: &source, + GetQuery: "admin-binding", + GetScope: CurrentCluster.Name, + SetupYAML: clusterRoleBindingYAML, + GetQueryTests: QueryTests{ + { + ExpectedType: "ClusterRole", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "admin", + ExpectedScope: CurrentCluster.Name, + }, + { + ExpectedType: "Group", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "system:serviceaccounts:default", + ExpectedScope: CurrentCluster.Name, + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/clusterrolebinding_test.go b/internal/sources/clusterrolebinding_test.go index 2942958..26feebf 100644 --- a/internal/sources/clusterrolebinding_test.go +++ b/internal/sources/clusterrolebinding_test.go @@ -2,57 +2,29 @@ package sources import ( "testing" - - "github.com/overmindtech/sdp-go" ) -var clusterRoleBindingYAML = ` +var clusterRoleYAML = ` apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: ClusterRole metadata: - name: admin-binding -subjects: -- kind: Group - name: system:serviceaccounts:default - apiGroup: rbac.authorization.k8s.io -roleRef: - kind: ClusterRole - name: admin - apiGroup: rbac.authorization.k8s.io -` - -func TestClusterRoleBindingSource(t *testing.T) { - err := CurrentCluster.Apply(clusterRoleBindingYAML) + name: read-only +rules: +- apiGroups: [""] + resources: ["*"] + verbs: ["get", "list", "watch"] - if err != nil { - t.Fatal(err) - } - - t.Cleanup(func() { - CurrentCluster.Delete(clusterRoleBindingYAML) - }) +` - source := NewClusterRoleBindingSource(CurrentCluster.ClientSet, CurrentCluster.Name, []string{}) +func TestClusterRoleSource(t *testing.T) { + source := NewClusterRoleSource(CurrentCluster.ClientSet, CurrentCluster.Name, []string{}) st := SourceTests{ - Source: &source, - GetQuery: "admin-binding", - GetScope: CurrentCluster.Name, - SetupYAML: clusterRoleBindingYAML, - GetQueryTests: QueryTests{ - { - ExpectedType: "ClusterRole", - ExpectedMethod: sdp.QueryMethod_GET, - ExpectedQuery: "admin", - ExpectedScope: CurrentCluster.Name, - }, - { - ExpectedType: "Group", - ExpectedMethod: sdp.QueryMethod_GET, - ExpectedQuery: "system:serviceaccounts:default", - ExpectedScope: CurrentCluster.Name, - }, - }, + Source: &source, + GetQuery: "read-only", + GetScope: CurrentCluster.Name, + SetupYAML: clusterRoleYAML, + GetQueryTests: QueryTests{}, } st.Execute(t) diff --git a/internal/sources/configmap.go b/internal/sources/configmap.go index cf0b8da..69d270a 100644 --- a/internal/sources/configmap.go +++ b/internal/sources/configmap.go @@ -1,71 +1,30 @@ package sources import ( - "fmt" - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -// ConfigMapSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func ConfigMapSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "configMap", - MapGet: MapConfigMapGet, - MapList: MapConfigMapList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.CoreV1().ConfigMaps, - ) - - return source, err -} - -// MapConfigMapList maps an interface that is underneath a -// *coreV1.ConfigMapList to a list of Items -func MapConfigMapList(i interface{}) ([]*sdp.Item, error) { - var objectList *coreV1.ConfigMapList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*coreV1.ConfigMapList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.ConfigMapList", i) - } - - for _, object := range objectList.Items { - if item, err = MapConfigMapGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } +func NewConfigMapSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ConfigMap, *v1.ConfigMapList] { + return KubeTypeSource[*v1.ConfigMap, *v1.ConfigMapList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "ConfigMap", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.ConfigMap, *v1.ConfigMapList] { + return cs.CoreV1().ConfigMaps(namespace) + }, + ListExtractor: func(list *v1.ConfigMapList) ([]*v1.ConfigMap, error) { + bindings := make([]*v1.ConfigMap, len(list.Items)) + + for i, crb := range list.Items { + bindings[i] = &crb + } + + return bindings, nil + }, + LinkedItemQueryExtractor: func(resource *v1.ConfigMap, scope string) ([]*sdp.Query, error) { + return []*sdp.Query{}, nil + }, } - - return items, nil -} - -// MapConfigMapGet maps an interface that is underneath a *coreV1.ConfigMap to an item. If -// the interface isn't actually a *coreV1.ConfigMap this will fail -func MapConfigMapGet(i interface{}) (*sdp.Item, error) { - var object *coreV1.ConfigMap - var ok bool - - // Expect this to be a *coreV1.ConfigMap - if object, ok = i.(*coreV1.ConfigMap); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.ConfigMap", i) - } - - item, err := mapK8sObject("configMap", object) - - if err != nil { - return &sdp.Item{}, err - } - - return item, nil } diff --git a/internal/sources/configmap_test.go b/internal/sources/configmap_test.go new file mode 100644 index 0000000..5934b97 --- /dev/null +++ b/internal/sources/configmap_test.go @@ -0,0 +1,32 @@ +package sources + +import "testing" + +var configMapYAML = ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-configmap +data: + DATABASE_URL: "postgres://myuser:mypassword@mydbhost:5432/mydatabase" + APP_SECRET_KEY: "mysecretkey123" +` + +func TestConfigMapSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewConfigMapSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "my-configmap", + GetScope: sd.String(), + SetupYAML: configMapYAML, + GetQueryTests: QueryTests{}, + } + + st.Execute(t) +} diff --git a/internal/sources/generic_source.go b/internal/sources/generic_source.go index 9e3b5c5..68c1401 100644 --- a/internal/sources/generic_source.go +++ b/internal/sources/generic_source.go @@ -121,7 +121,15 @@ func (k *KubeTypeSource[Resource, ResourceList]) Scopes() []string { } func (k *KubeTypeSource[Resource, ResourceList]) Get(ctx context.Context, scope string, query string) (*sdp.Item, error) { - i := k.itemInterface(scope) + i, err := k.itemInterface(scope) + + if err != nil { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_NOSCOPE, + ErrorString: err.Error(), + } + } + resource, err := i.Get(ctx, query, metav1.GetOptions{}) if err != nil { @@ -161,7 +169,14 @@ func (k *KubeTypeSource[Resource, ResourceList]) List(ctx context.Context, scope // listWithOptions Runs the inbuilt list method with the given options func (k *KubeTypeSource[Resource, ResourceList]) listWithOptions(ctx context.Context, scope string, opts metav1.ListOptions) ([]*sdp.Item, error) { - i := k.itemInterface(scope) + i, err := k.itemInterface(scope) + + if err != nil { + return nil, &sdp.QueryError{ + ErrorType: sdp.QueryError_NOSCOPE, + ErrorString: err.Error(), + } + } list, err := i.List(ctx, opts) @@ -222,15 +237,19 @@ func (k *KubeTypeSource[Resource, ResourceList]) Search(ctx context.Context, sco // itemInterface Returns the correct interface depending on whether the source // is namespaced or not -func (k *KubeTypeSource[Resource, ResourceList]) itemInterface(scope string) ItemInterface[Resource, ResourceList] { +func (k *KubeTypeSource[Resource, ResourceList]) itemInterface(scope string) (ItemInterface[Resource, ResourceList], error) { // If this is a namespaced resource, then parse the scope to get the // namespace if k.namespaced() { - details := ParseScope(scope) + details, err := ParseScope(scope, k.namespaced()) + + if err != nil { + return nil, err + } - return k.NamespacedInterfaceBuilder(details.Namespace) + return k.NamespacedInterfaceBuilder(details.Namespace), nil } else { - return k.ClusterInterfaceBuilder() + return k.ClusterInterfaceBuilder(), nil } } diff --git a/internal/sources/generic_source_test.go b/internal/sources/generic_source_test.go index 8250e45..d09dd91 100644 --- a/internal/sources/generic_source_test.go +++ b/internal/sources/generic_source_test.go @@ -499,7 +499,7 @@ func (s SourceTests) Execute(t *testing.T) { t.Run(s.Source.Name(), func(t *testing.T) { if s.GetQuery != "" { - t.Run(fmt.Sprintf("GET: %v", s.GetQuery), func(t *testing.T) { + t.Run(fmt.Sprintf("GET:%v", s.GetQuery), func(t *testing.T) { item, err := s.Source.Get(context.Background(), s.GetScope, s.GetQuery) if err != nil { diff --git a/internal/sources/shared_resourcesource.go b/internal/sources/shared_resourcesource.go index fe4b9cd..84161ea 100644 --- a/internal/sources/shared_resourcesource.go +++ b/internal/sources/shared_resourcesource.go @@ -47,7 +47,6 @@ var SourceFunctions = []SourceFunction{ ServiceSource, PVCSource, SecretSource, - ConfigMapSource, EndpointSource, ServiceAccountSource, LimitRangeSource, @@ -69,7 +68,6 @@ var SourceFunctions = []SourceFunction{ NamespaceSource, NodeSource, PersistentVolumeSource, - ClusterRoleSource, StorageClassSource, PriorityClassSource, } @@ -390,7 +388,7 @@ func (rs *ResourceSource) Weight() int { // will allow us to call Get and List functions which will in turn actually // execute API queries against K8s func (rs *ResourceSource) interactionInterface(itemScope string) (reflect.Value, error) { - contextDetails := ParseScope(itemScope) + contextDetails, _ := ParseScope(itemScope, rs.Namespaced) interfaceFunctionArgs := make([]reflect.Value, 0) if rs.Namespaced { diff --git a/internal/sources/shared_util.go b/internal/sources/shared_util.go index 39d80b5..cb9a90f 100644 --- a/internal/sources/shared_util.go +++ b/internal/sources/shared_util.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "reflect" - "regexp" "strings" "github.com/overmindtech/sdp-go" @@ -25,30 +24,39 @@ func (sd ScopeDetails) String() string { return fmt.Sprintf("%v.%v", sd.ClusterName, sd.Namespace) } -// ClusterNamespaceRegex matches the cluster name and namespace from a string -// that is in the format {clusterName}.{namespace} -// -// This is possible due to the fact namespaces have a limited set of characters -// so we can use a regex to find the last instance of a namespace-compliant -// string after a trailing -var ClusterNamespaceRegex = regexp.MustCompile(`(?P.+:.+?)(\.(?P[a-z0-9]([-a-z0-9]*[a-z0-9])?))?$`) - -// ParseScope Parses the custer and scope name out of a given SDP scope -// given that the naming convention is {clusterName}.{namespace} -func ParseScope(itemScope string) ScopeDetails { - matches := ClusterNamespaceRegex.FindStringSubmatch(itemScope) - - if len(matches) != 5 { - return ScopeDetails{ - ClusterName: itemScope, - Namespace: "", +// ParseScope Parses the custer and scope name out of a given SDP scope given +// that the naming convention is {clusterName}.{namespace}. Since all sources +// know whether they are namespaced or not, we can just pass that in to make +// parsing easier +func ParseScope(itemScope string, namespaced bool) (ScopeDetails, error) { + sections := strings.Split(itemScope, ".") + + var namespace string + var clusterEnd int + var clusterName string + + if namespaced { + if len(sections) < 2 { + return ScopeDetails{}, fmt.Errorf("scope %v does not contain a namespace in the format: {clusterName}.{namespace}", itemScope) } + + namespace = sections[len(sections)-1] + clusterEnd = len(sections) - 1 + } else { + namespace = "" + clusterEnd = len(sections) } - return ScopeDetails{ - ClusterName: matches[ClusterNamespaceRegex.SubexpIndex("clusterName")], - Namespace: matches[ClusterNamespaceRegex.SubexpIndex("namespace")], + clusterName = strings.Join(sections[:clusterEnd], ".") + + if clusterName == "" { + return ScopeDetails{}, fmt.Errorf("cluster name was blank for scope %v", itemScope) } + + return ScopeDetails{ + ClusterName: clusterName, + Namespace: namespace, + }, nil } // Selector represents a set of key value pairs that we are going to use as a diff --git a/internal/sources/shared_util_test.go b/internal/sources/shared_util_test.go index 7a594f3..6eafc41 100644 --- a/internal/sources/shared_util_test.go +++ b/internal/sources/shared_util_test.go @@ -4,64 +4,124 @@ import "testing" func TestParseScope(t *testing.T) { type ParseTest struct { - Input string - ClusterName string - Namespace string + Input string + ClusterName string + Namespace string + IsNamespaced bool + ExpectError bool } tests := []ParseTest{ { - Input: "127.0.0.1:61081.default", - ClusterName: "127.0.0.1:61081", - Namespace: "default", + Input: "127.0.0.1:61081.default", + ClusterName: "127.0.0.1:61081", + Namespace: "default", + IsNamespaced: true, }, { - Input: "127.0.0.1:61081.kube-node-lease", - ClusterName: "127.0.0.1:61081", - Namespace: "kube-node-lease", + Input: "127.0.0.1:61081.kube-node-lease", + ClusterName: "127.0.0.1:61081", + Namespace: "kube-node-lease", + IsNamespaced: true, }, { - Input: "127.0.0.1:61081.kube-public", - ClusterName: "127.0.0.1:61081", - Namespace: "kube-public", + Input: "127.0.0.1:61081.kube-public", + ClusterName: "127.0.0.1:61081", + Namespace: "kube-public", + IsNamespaced: true, }, { - Input: "127.0.0.1:61081.kube-system", - ClusterName: "127.0.0.1:61081", - Namespace: "kube-system", + Input: "127.0.0.1:61081.kube-system", + ClusterName: "127.0.0.1:61081", + Namespace: "kube-system", + IsNamespaced: true, }, { - Input: "127.0.0.1:61081", - ClusterName: "127.0.0.1:61081", - Namespace: "", + Input: "127.0.0.1:61081", + ClusterName: "127.0.0.1:61081", + Namespace: "", + IsNamespaced: false, }, { - Input: "cluster1.k8s.company.com:443", - ClusterName: "cluster1.k8s.company.com:443", - Namespace: "", + Input: "cluster1.k8s.company.com:443", + ClusterName: "cluster1.k8s.company.com:443", + Namespace: "", + IsNamespaced: false, }, { - Input: "cluster1.k8s.company.com", - ClusterName: "cluster1.k8s.company.com", - Namespace: "", + Input: "cluster1.k8s.company.com", + ClusterName: "cluster1.k8s.company.com", + Namespace: "", + IsNamespaced: false, }, { - Input: "test", - ClusterName: "test", - Namespace: "", + Input: "test", + ClusterName: "test", + Namespace: "", + IsNamespaced: false, + }, + { + Input: "prod.default", + ClusterName: "prod", + Namespace: "default", + IsNamespaced: true, + }, + { + Input: "prod", + ClusterName: "", + Namespace: "prod", + IsNamespaced: true, + ExpectError: true, + }, + { + Input: "prod.default.test", + ClusterName: "prod.default.test", + Namespace: "", + IsNamespaced: false, + }, + { + Input: "prod.default.test", + ClusterName: "prod.default", + Namespace: "test", + IsNamespaced: true, + }, + { + Input: "", + ClusterName: "", + Namespace: "", + IsNamespaced: false, + ExpectError: true, + }, + { + Input: "", + ClusterName: "", + Namespace: "", + IsNamespaced: true, + ExpectError: true, }, } for _, test := range tests { - result := ParseScope(test.Input) + result, err := ParseScope(test.Input, test.IsNamespaced) - if test.ClusterName != result.ClusterName { - t.Errorf("ClusterName did not match, expected %v, got %v", test.ClusterName, result.ClusterName) - } + if test.ExpectError { + if err == nil { + t.Errorf("Expected error, but got none. Test %v", test) + } + } else { + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + if test.ClusterName != result.ClusterName { + t.Errorf("ClusterName did not match, expected %v, got %v", test.ClusterName, result.ClusterName) + } - if test.Namespace != result.Namespace { - t.Errorf("Namespace did not match, expected %v, got %v", test.Namespace, result.Namespace) + if test.Namespace != result.Namespace { + t.Errorf("Namespace did not match, expected %v, got %v", test.Namespace, result.Namespace) + } } + } } From 25d1449ce7b4bc4bb0f8b6c107e6a9fcc2425f6f Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Fri, 12 May 2023 10:12:18 +0000 Subject: [PATCH 07/24] Added cronjobs and jobs --- .devcontainer/setup.sh | 10 +- internal/sources/clusterrole.go | 4 +- internal/sources/clusterrolebinding.go | 4 +- internal/sources/configmap.go | 4 +- internal/sources/cronjob.go | 83 ++++------------ internal/sources/cronjob_test.go | 76 +++++++++++++++ internal/sources/generic_source.go | 22 ++++- internal/sources/generic_source_test.go | 46 ++++++++- internal/sources/job.go | 109 ++++++---------------- internal/sources/job_test.go | 68 ++++++++++++++ internal/sources/shared_resourcesource.go | 2 - internal/sources/shared_test.go | 2 +- 12 files changed, 267 insertions(+), 163 deletions(-) create mode 100644 internal/sources/cronjob_test.go create mode 100644 internal/sources/job_test.go diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 4990152..8631758 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -1,3 +1,11 @@ #!/bin/bash -go install sigs.k8s.io/kind@latest \ No newline at end of file +go install sigs.k8s.io/kind@latest + +# Create the test cluster (the tests also do this) but also set local kube +# config +kind create cluster --name k8s-source-tests +kind export kubeconfig --name k8s-source-tests + +# Install k9s +curl -sS https://webinstall.dev/k9s | bash diff --git a/internal/sources/clusterrole.go b/internal/sources/clusterrole.go index 273cdb6..eb9ffb4 100644 --- a/internal/sources/clusterrole.go +++ b/internal/sources/clusterrole.go @@ -18,8 +18,8 @@ func NewClusterRoleSource(cs *kubernetes.Clientset, cluster string, namespaces [ ListExtractor: func(list *v1.ClusterRoleList) ([]*v1.ClusterRole, error) { bindings := make([]*v1.ClusterRole, len(list.Items)) - for i, cr := range list.Items { - bindings[i] = &cr + for i := range list.Items { + bindings[i] = &list.Items[i] } return bindings, nil diff --git a/internal/sources/clusterrolebinding.go b/internal/sources/clusterrolebinding.go index 245866b..9741bd9 100644 --- a/internal/sources/clusterrolebinding.go +++ b/internal/sources/clusterrolebinding.go @@ -48,8 +48,8 @@ func NewClusterRoleBindingSource(cs *kubernetes.Clientset, cluster string, names ListExtractor: func(list *v1.ClusterRoleBindingList) ([]*v1.ClusterRoleBinding, error) { bindings := make([]*v1.ClusterRoleBinding, len(list.Items)) - for i, crb := range list.Items { - bindings[i] = &crb + for i := range list.Items { + bindings[i] = &list.Items[i] } return bindings, nil diff --git a/internal/sources/configmap.go b/internal/sources/configmap.go index 69d270a..dc2145a 100644 --- a/internal/sources/configmap.go +++ b/internal/sources/configmap.go @@ -17,8 +17,8 @@ func NewConfigMapSource(cs *kubernetes.Clientset, cluster string, namespaces []s ListExtractor: func(list *v1.ConfigMapList) ([]*v1.ConfigMap, error) { bindings := make([]*v1.ConfigMap, len(list.Items)) - for i, crb := range list.Items { - bindings[i] = &crb + for i := range list.Items { + bindings[i] = &list.Items[i] } return bindings, nil diff --git a/internal/sources/cronjob.go b/internal/sources/cronjob.go index 7da3088..1da57bb 100644 --- a/internal/sources/cronjob.go +++ b/internal/sources/cronjob.go @@ -1,72 +1,29 @@ package sources import ( - "fmt" + v1 "k8s.io/api/batch/v1" - batchV1beta1 "k8s.io/api/batch/v1beta1" - - "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// CronJobSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func CronJobSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "cronjob", - MapGet: MapCronJobGet, - MapList: MapCronJobList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.BatchV1beta1().CronJobs, - ) - - return source, err -} - -// MapCronJobList maps an interface that is underneath a -// *batchV1beta1.CronJobList to a list of Items -func MapCronJobList(i interface{}) ([]*sdp.Item, error) { - var objectList *batchV1beta1.CronJobList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*batchV1beta1.CronJobList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *batchV1beta1.CronJobList", i) - } - - for _, object := range objectList.Items { - if item, err = MapCronJobGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } +func NewCronJobSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.CronJob, *v1.CronJobList] { + return KubeTypeSource[*v1.CronJob, *v1.CronJobList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "CronJob", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.CronJob, *v1.CronJobList] { + return cs.BatchV1().CronJobs(namespace) + }, + ListExtractor: func(list *v1.CronJobList) ([]*v1.CronJob, error) { + bindings := make([]*v1.CronJob, len(list.Items)) + + for i := range list.Items { + bindings[i] = &list.Items[i] + } + + return bindings, nil + }, + // Cronjobs don't need linked items as the jobs they produce are linked + // automatically } - - return items, nil -} - -// MapCronJobGet maps an interface that is underneath a *batchV1beta1.CronJob to an item. If -// the interface isn't actually a *batchV1beta1.CronJob this will fail -func MapCronJobGet(i interface{}) (*sdp.Item, error) { - var object *batchV1beta1.CronJob - var ok bool - - // Expect this to be a *batchV1beta1.CronJob - if object, ok = i.(*batchV1beta1.CronJob); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *batchV1beta1.CronJob", i) - } - - item, err := mapK8sObject("cronjob", object) - - if err != nil { - return &sdp.Item{}, err - } - - return item, nil } diff --git a/internal/sources/cronjob_test.go b/internal/sources/cronjob_test.go new file mode 100644 index 0000000..ef21f96 --- /dev/null +++ b/internal/sources/cronjob_test.go @@ -0,0 +1,76 @@ +package sources + +import ( + "context" + "testing" + "time" +) + +var cronJobYAML = ` +apiVersion: batch/v1 +kind: CronJob +metadata: + name: my-cronjob +spec: + schedule: "* * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: my-container + image: alpine + command: ["/bin/sh", "-c"] + args: + - sleep 10; echo "Hello, world!" + restartPolicy: OnFailure +` + +func TestCronJobSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewCronJobSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "my-cronjob", + GetScope: sd.String(), + SetupYAML: cronJobYAML, + GetQueryTests: QueryTests{}, + } + + st.Execute(t) + + // Additionally, make sure that the job has a link back to the cronjob that + // created it + jobSource := NewJobSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + // Wait for the job to be created + err := WaitFor(10*time.Second, func() bool { + jobs, err := jobSource.List(context.Background(), sd.String()) + + if err != nil { + t.Fatal(err) + return false + } + + // Ensure that the job has a link back to the cronjob + for _, job := range jobs { + for _, q := range job.LinkedItemQueries { + if q.Query == "my-cronjob" { + return true + } + } + + } + + return false + }) + + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/sources/generic_source.go b/internal/sources/generic_source.go index 68c1401..1b79784 100644 --- a/internal/sources/generic_source.go +++ b/internal/sources/generic_source.go @@ -145,7 +145,7 @@ func (k *KubeTypeSource[Resource, ResourceList]) Get(ctx context.Context, scope return nil, err } - item, err := resourceToObject(resource, k.ClusterName) + item, err := resourceToItem(resource, k.ClusterName) if err != nil { return nil, err @@ -206,7 +206,7 @@ func (k *KubeTypeSource[Resource, ResourceList]) resourcesToItems(resourceList [ var err error for i := range resourceList { - items[i], err = resourceToObject(resourceList[i], k.ClusterName) + items[i], err = resourceToItem(resourceList[i], k.ClusterName) if err != nil { return nil, err @@ -214,11 +214,13 @@ func (k *KubeTypeSource[Resource, ResourceList]) resourcesToItems(resourceList [ if k.LinkedItemQueryExtractor != nil { // Add linked items - items[i].LinkedItemQueries, err = k.LinkedItemQueryExtractor(resourceList[i], scope) + newQueries, err := k.LinkedItemQueryExtractor(resourceList[i], scope) if err != nil { return nil, err } + + items[i].LinkedItemQueries = append(items[i].LinkedItemQueries, newQueries...) } } @@ -270,8 +272,8 @@ func ignored(key string) bool { return false } -// resourceToObject Converts a resource to an item -func resourceToObject(resource metav1.Object, cluster string) (*sdp.Item, error) { +// resourceToItem Converts a resource to an item +func resourceToItem(resource metav1.Object, cluster string) (*sdp.Item, error) { sd := ScopeDetails{ ClusterName: cluster, Namespace: resource.GetNamespace(), @@ -306,5 +308,15 @@ func resourceToObject(resource metav1.Object, cluster string) (*sdp.Item, error) Attributes: attributes, } + // Automatically create links to owner references + for _, ref := range resource.GetOwnerReferences() { + item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ + Type: ref.Kind, + Method: sdp.QueryMethod_GET, + Query: ref.Name, + Scope: sd.String(), + }) + } + return item, nil } diff --git a/internal/sources/generic_source_test.go b/internal/sources/generic_source_test.go index d09dd91..9cd591d 100644 --- a/internal/sources/generic_source_test.go +++ b/internal/sources/generic_source_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "regexp" "testing" "time" @@ -441,6 +442,10 @@ type QueryTest struct { ExpectedMethod sdp.QueryMethod ExpectedQuery string ExpectedScope string + + // Expect the query to match a regex, this takes precedence over + // ExpectedQuery + ExpectedQueryMatches *regexp.Regexp } type QueryTests []QueryTest @@ -463,10 +468,18 @@ func (i QueryTests) Execute(t *testing.T, item *sdp.Item) { } func lirMatches(test QueryTest, req *sdp.Query) bool { - return (test.ExpectedMethod == req.Method && - test.ExpectedQuery == req.Query && - test.ExpectedScope == req.Scope && - test.ExpectedType == req.Type) + methodOK := test.ExpectedMethod == req.Method + scopeOK := test.ExpectedScope == req.Scope + typeOK := test.ExpectedType == req.Type + var queryOK bool + + if test.ExpectedQueryMatches != nil { + queryOK = test.ExpectedQueryMatches.MatchString(req.Query) + } else { + queryOK = test.ExpectedQuery == req.Query + } + + return methodOK && scopeOK && typeOK && queryOK } type SourceTests struct { @@ -529,11 +542,36 @@ func (s SourceTests) Execute(t *testing.T) { t.Errorf("expected items, got none") } + itemMap := make(map[string]*sdp.Item) + for _, item := range items { + itemMap[item.UniqueAttributeValue()] = item + if err = item.Validate(); err != nil { t.Error(err) } } + + if len(itemMap) != len(items) { + t.Errorf("expected %v unique items, got %v", len(items), len(itemMap)) + } }) }) } + +// WaitFor waits for a condition to be true, or returns an error if the timeout +func WaitFor(timeout time.Duration, run func() bool) error { + start := time.Now() + + for { + if run() { + return nil + } + + if time.Since(start) > timeout { + return fmt.Errorf("timeout exceeded") + } + + time.Sleep(250 * time.Millisecond) + } +} diff --git a/internal/sources/job.go b/internal/sources/job.go index b3af65d..7208207 100644 --- a/internal/sources/job.go +++ b/internal/sources/job.go @@ -1,97 +1,44 @@ package sources import ( - "fmt" - - batchV1 "k8s.io/api/batch/v1" + v1 "k8s.io/api/batch/v1" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// JobSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func JobSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "job", - MapGet: MapJobGet, - MapList: MapJobList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.BatchV1().Jobs, - ) - - return source, err -} +func jobExtractor(resource *v1.Job, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) -// MapJobList maps an interface that is underneath a -// *batchV1.JobList to a list of Items -func MapJobList(i interface{}) ([]*sdp.Item, error) { - var objectList *batchV1.JobList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*batchV1.JobList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *batchV1.JobList", i) - } - - for _, object := range objectList.Items { - if item, err = MapJobGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } + if resource.Spec.Selector != nil { + queries = append(queries, &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(resource.Spec.Selector), + Type: "Pod", + }) } - return items, nil + return queries, nil } -// MapJobGet maps an interface that is underneath a *batchV1.Job to an item. If -// the interface isn't actually a *batchV1.Job this will fail -func MapJobGet(i interface{}) (*sdp.Item, error) { - var object *batchV1.Job - var ok bool - - // Expect this to be a *batchV1.Job - if object, ok = i.(*batchV1.Job); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *batchV1.Job", i) - } - - item, err := mapK8sObject("job", object) - - if err != nil { - return &sdp.Item{}, err - } - - if object.Spec.Selector != nil { - item.LinkedItemQueries = []*sdp.Query{ - { - Scope: item.Scope, - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(object.Spec.Selector), - Type: "pod", - }, - } - } - - // Check owner references to see if it was created by a cronjob - for _, o := range object.ObjectMeta.OwnerReferences { - if o.Kind == "CronJob" { - item.LinkedItemQueries = []*sdp.Query{ - { - Scope: item.Scope, - Method: sdp.QueryMethod_GET, - Query: o.Name, - Type: "cronjob", - }, +func NewJobSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Job, *v1.JobList] { + return KubeTypeSource[*v1.Job, *v1.JobList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "Job", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.Job, *v1.JobList] { + return cs.BatchV1().Jobs(namespace) + }, + ListExtractor: func(list *v1.JobList) ([]*v1.Job, error) { + bindings := make([]*v1.Job, len(list.Items)) + + for i := range list.Items { + bindings[i] = &list.Items[i] } - } - } - return item, nil + return bindings, nil + }, + LinkedItemQueryExtractor: jobExtractor, + } } diff --git a/internal/sources/job_test.go b/internal/sources/job_test.go new file mode 100644 index 0000000..8d3f3fa --- /dev/null +++ b/internal/sources/job_test.go @@ -0,0 +1,68 @@ +package sources + +import ( + "regexp" + "testing" + + "github.com/overmindtech/sdp-go" +) + +var jobYAML = ` +apiVersion: batch/v1 +kind: Job +metadata: + name: my-job +spec: + template: + spec: + containers: + - name: my-container + image: nginx + command: ["/bin/sh", "-c"] + args: + - echo "Hello, world!"; sleep 5 + restartPolicy: OnFailure + backoffLimit: 4 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: my-job2 +spec: + template: + spec: + containers: + - name: my-container + image: nginx + command: ["/bin/sh", "-c"] + args: + - echo "Hello, world!"; sleep 5 + restartPolicy: OnFailure + backoffLimit: 4 +` + +func TestJobSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewJobSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "my-job", + GetScope: sd.String(), + SetupYAML: jobYAML, + GetQueryTests: QueryTests{ + { + ExpectedQueryMatches: regexp.MustCompile("controller-uid="), + ExpectedType: "Pod", + ExpectedMethod: sdp.QueryMethod_SEARCH, + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/shared_resourcesource.go b/internal/sources/shared_resourcesource.go index 84161ea..314bced 100644 --- a/internal/sources/shared_resourcesource.go +++ b/internal/sources/shared_resourcesource.go @@ -57,8 +57,6 @@ var SourceFunctions = []SourceFunction{ DeploymentSource, HorizontalPodAutoscalerSource, StatefulSetSource, - JobSource, - CronJobSource, IngressSource, NetworkPolicySource, PodDisruptionBudgetSource, diff --git a/internal/sources/shared_test.go b/internal/sources/shared_test.go index 35df6b3..8333f0d 100644 --- a/internal/sources/shared_test.go +++ b/internal/sources/shared_test.go @@ -153,7 +153,7 @@ func (t *TestCluster) kubectl(method string, yaml string) error { err = cmd.Run() if err != nil { - return err + return fmt.Errorf("%w\nstdout: %v\nstderr: %v", err, stdout.String(), stderr.String()) } if e := stderr.String(); e != "" { From e2b2811f72b9e2623a961b880b2f15ba00293e20 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Fri, 12 May 2023 11:52:15 +0000 Subject: [PATCH 08/24] Added more sources --- internal/sources/daemonset.go | 94 ++++------------ internal/sources/daemonset_test.go | 45 ++++++++ internal/sources/deployment.go | 94 ++++------------ internal/sources/deployment_test.go | 45 ++++++++ internal/sources/endpoint.go | 120 ++++++++++---------- internal/sources/endpoints_test.go | 83 ++++++++++++++ internal/sources/endpointslice.go | 127 +++++++++++----------- internal/sources/endpointslice_test.go | 83 ++++++++++++++ internal/sources/generic_source.go | 21 ++++ internal/sources/generic_source_test.go | 64 ++++++++++- internal/sources/job.go | 6 +- internal/sources/shared_resourcesource.go | 4 - 12 files changed, 498 insertions(+), 288 deletions(-) create mode 100644 internal/sources/daemonset_test.go create mode 100644 internal/sources/deployment_test.go create mode 100644 internal/sources/endpoints_test.go create mode 100644 internal/sources/endpointslice_test.go diff --git a/internal/sources/daemonset.go b/internal/sources/daemonset.go index 1c5059a..68a0f7a 100644 --- a/internal/sources/daemonset.go +++ b/internal/sources/daemonset.go @@ -1,84 +1,28 @@ package sources import ( - "fmt" + v1 "k8s.io/api/apps/v1" - appsV1 "k8s.io/api/apps/v1" - - "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// DaemonSetSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func DaemonSetSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "daemonset", - MapGet: MapDaemonSetGet, - MapList: MapDaemonSetList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.AppsV1().DaemonSets, - ) - - return source, err -} - -// MapDaemonSetList maps an interface that is underneath a -// *appsV1.DaemonSetList to a list of Items -func MapDaemonSetList(i interface{}) ([]*sdp.Item, error) { - var objectList *appsV1.DaemonSetList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*appsV1.DaemonSetList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *appsV1.DaemonSetList", i) - } - - for _, object := range objectList.Items { - if item, err = MapDaemonSetGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} - -// MapDaemonSetGet maps an interface that is underneath a *appsV1.DaemonSet to an item. If -// the interface isn't actually a *appsV1.DaemonSet this will fail -func MapDaemonSetGet(i interface{}) (*sdp.Item, error) { - var object *appsV1.DaemonSet - var ok bool - - // Expect this to be a *appsV1.DaemonSet - if object, ok = i.(*appsV1.DaemonSet); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *appsV1.DaemonSet", i) - } - - item, err := mapK8sObject("daemonset", object) - - if err != nil { - return &sdp.Item{}, err +func NewDaemonSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.DaemonSet, *v1.DaemonSetList] { + return KubeTypeSource[*v1.DaemonSet, *v1.DaemonSetList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "DaemonSet", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.DaemonSet, *v1.DaemonSetList] { + return cs.AppsV1().DaemonSets(namespace) + }, + ListExtractor: func(list *v1.DaemonSetList) ([]*v1.DaemonSet, error) { + extracted := make([]*v1.DaemonSet, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + // Pods are linked automatically } - - if object.Spec.Selector != nil { - item.LinkedItemQueries = []*sdp.Query{ - // Services are linked to pods via their selector - { - Scope: item.Scope, - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(object.Spec.Selector), - Type: "pod", - }, - } - } - - return item, nil } diff --git a/internal/sources/daemonset_test.go b/internal/sources/daemonset_test.go new file mode 100644 index 0000000..ed790b4 --- /dev/null +++ b/internal/sources/daemonset_test.go @@ -0,0 +1,45 @@ +package sources + +import ( + "testing" +) + +var daemonSetYAML = ` +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: my-daemonset +spec: + selector: + matchLabels: + app: my-app + template: + metadata: + labels: + app: my-app + spec: + containers: + - name: my-container + image: nginx:latest + ports: + - containerPort: 80 + +` + +func TestDaemonSetSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewDaemonSetSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "my-daemonset", + GetScope: sd.String(), + SetupYAML: daemonSetYAML, + } + + st.Execute(t) +} diff --git a/internal/sources/deployment.go b/internal/sources/deployment.go index 660c236..2f4c308 100644 --- a/internal/sources/deployment.go +++ b/internal/sources/deployment.go @@ -1,84 +1,28 @@ package sources import ( - "fmt" + v1 "k8s.io/api/apps/v1" - appsV1 "k8s.io/api/apps/v1" - - "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// DeploymentSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func DeploymentSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "deployment", - MapGet: MapDeploymentGet, - MapList: MapDeploymentList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.AppsV1().Deployments, - ) - - return source, err -} - -// MapDeploymentList maps an interface that is underneath a -// *appsV1.DeploymentList to a list of Items -func MapDeploymentList(i interface{}) ([]*sdp.Item, error) { - var objectList *appsV1.DeploymentList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*appsV1.DeploymentList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *appsV1.DeploymentList", i) - } - - for _, object := range objectList.Items { - if item, err = MapDeploymentGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} - -// MapDeploymentGet maps an interface that is underneath a *appsV1.Deployment to an item. If -// the interface isn't actually a *appsV1.Deployment this will fail -func MapDeploymentGet(i interface{}) (*sdp.Item, error) { - var object *appsV1.Deployment - var ok bool - - // Expect this to be a *appsV1.Deployment - if object, ok = i.(*appsV1.Deployment); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *appsV1.Deployment", i) - } - - item, err := mapK8sObject("deployment", object) - - if err != nil { - return &sdp.Item{}, err +func NewDeploymentSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Deployment, *v1.DeploymentList] { + return KubeTypeSource[*v1.Deployment, *v1.DeploymentList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "Deployment", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.Deployment, *v1.DeploymentList] { + return cs.AppsV1().Deployments(namespace) + }, + ListExtractor: func(list *v1.DeploymentList) ([]*v1.Deployment, error) { + extracted := make([]*v1.Deployment, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + // Replicasets are linked automatically } - - if object.Spec.Selector != nil { - item.LinkedItemQueries = []*sdp.Query{ - // Services are linked to pods via their selector - { - Scope: item.Scope, - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(object.Spec.Selector), - Type: "replicaset", - }, - } - } - - return item, nil } diff --git a/internal/sources/deployment_test.go b/internal/sources/deployment_test.go new file mode 100644 index 0000000..331a833 --- /dev/null +++ b/internal/sources/deployment_test.go @@ -0,0 +1,45 @@ +package sources + +import ( + "testing" +) + +var deploymentYAML = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: my-deployment + template: + metadata: + labels: + app: my-deployment + spec: + containers: + - name: my-container + image: nginx:latest + ports: + - containerPort: 80 +` + +func TestDeploymentSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewDeploymentSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "my-deployment", + GetScope: sd.String(), + SetupYAML: deploymentYAML, + } + + st.Execute(t) +} diff --git a/internal/sources/endpoint.go b/internal/sources/endpoint.go index 37d4840..3f3e1fa 100644 --- a/internal/sources/endpoint.go +++ b/internal/sources/endpoint.go @@ -1,86 +1,76 @@ package sources import ( - "fmt" - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -// EndpointSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func EndpointSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "endpoint", - MapGet: MapEndpointGet, - MapList: MapEndpointList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.CoreV1().Endpoints, - ) +func EndpointsExtractor(resource *v1.Endpoints, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) - return source, err -} - -// MapEndpointList maps an interface that is underneath a -// *coreV1.EndpointsList to a list of Items -func MapEndpointList(i interface{}) ([]*sdp.Item, error) { - var objectList *coreV1.EndpointsList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*coreV1.EndpointsList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.EndpointsList", i) - } + sd, err := ParseScope(scope, true) - for _, object := range objectList.Items { - if item, err = MapEndpointGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } + if err != nil { + return nil, err } - return items, nil -} - -// MapEndpointGet maps an interface that is underneath a *coreV1.Endpoints to an item. If -// the interface isn't actually a *coreV1.Endpoints this will fail -func MapEndpointGet(i interface{}) (*sdp.Item, error) { - var object *coreV1.Endpoints - var ok bool - - // Expect this to be a *coreV1.Endpoints - if object, ok = i.(*coreV1.Endpoints); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.Endpoints", i) - } + for _, subset := range resource.Subsets { + for _, address := range subset.Addresses { + if address.Hostname != "" { + queries = append(queries, &sdp.Query{ + Scope: "global", + Method: sdp.QueryMethod_GET, + Query: address.Hostname, + Type: "dns", + }) + } - item, err := mapK8sObject("endpoint", object) + if address.NodeName != nil { + queries = append(queries, &sdp.Query{ + Type: "Node", + Scope: sd.ClusterName, + Method: sdp.QueryMethod_GET, + Query: *address.NodeName, + }) + } - if err != nil { - return &sdp.Item{}, err - } + if address.IP != "" { + queries = append(queries, &sdp.Query{ + Type: "ip", + Method: sdp.QueryMethod_GET, + Query: address.IP, + Scope: "global", + }) + } - // Create linked item requests for all ObjectReferences - for _, subset := range object.Subsets { - for _, address := range subset.Addresses { if address.TargetRef != nil { - item.LinkedItemQueries = append(item.LinkedItemQueries, ObjectReferenceToLIR(address.TargetRef, ClusterName)) + targetQuery := ObjectReferenceToQuery(address.TargetRef, sd) + queries = append(queries, targetQuery) } } + } + + return queries, nil +} - for _, notReadAddress := range subset.NotReadyAddresses { - if notReadAddress.TargetRef != nil { - item.LinkedItemQueries = append(item.LinkedItemQueries, ObjectReferenceToLIR(notReadAddress.TargetRef, ClusterName)) +func NewEndpointsSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Endpoints, *v1.EndpointsList] { + return KubeTypeSource[*v1.Endpoints, *v1.EndpointsList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "Endpoints", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.Endpoints, *v1.EndpointsList] { + return cs.CoreV1().Endpoints(namespace) + }, + ListExtractor: func(list *v1.EndpointsList) ([]*v1.Endpoints, error) { + extracted := make([]*v1.Endpoints, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] } - } - } - return item, nil + return extracted, nil + }, + LinkedItemQueryExtractor: EndpointsExtractor, + } } diff --git a/internal/sources/endpoints_test.go b/internal/sources/endpoints_test.go new file mode 100644 index 0000000..43aff53 --- /dev/null +++ b/internal/sources/endpoints_test.go @@ -0,0 +1,83 @@ +package sources + +import ( + "regexp" + "testing" + + "github.com/overmindtech/sdp-go" +) + +var endpointsYAML = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: endpoint-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: endpoint-test + template: + metadata: + labels: + app: endpoint-test + spec: + containers: + - name: endpoint-test + image: nginx:latest + ports: + - containerPort: 80 + +--- +apiVersion: v1 +kind: Service +metadata: + name: endpoint-service +spec: + selector: + app: endpoint-test + ports: + - name: http + port: 80 + targetPort: 80 + type: ClusterIP + +` + +func TestEndpointsSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewEndpointsSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "endpoint-service", + GetScope: sd.String(), + SetupYAML: endpointsYAML, + GetQueryTests: QueryTests{ + { + ExpectedQueryMatches: regexp.MustCompile(`^10\.`), + ExpectedType: "ip", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedScope: "global", + }, + { + ExpectedType: "Node", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "k8s-source-tests-control-plane", + ExpectedScope: CurrentCluster.Name, + }, + { + ExpectedType: "Pod", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQueryMatches: regexp.MustCompile("endpoint-deployment"), + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/endpointslice.go b/internal/sources/endpointslice.go index bcd6f4b..655857e 100644 --- a/internal/sources/endpointslice.go +++ b/internal/sources/endpointslice.go @@ -1,84 +1,85 @@ package sources import ( - "fmt" - "strings" - - discoveryV1beta1 "k8s.io/api/discovery/v1beta1" + v1 "k8s.io/api/discovery/v1" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// EndpointSliceSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func EndpointSliceSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "endpointslice", - MapGet: MapEndpointSliceGet, - MapList: MapEndpointSliceList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.DiscoveryV1beta1().EndpointSlices, - ) +func endpointSliceExtractor(resource *v1.EndpointSlice, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) - return source, err -} - -// MapEndpointSliceList maps an interface that is underneath a -// *discoveryV1beta1.EndpointSliceList to a list of Items -func MapEndpointSliceList(i interface{}) ([]*sdp.Item, error) { - var objectList *discoveryV1beta1.EndpointSliceList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error + sd, err := ParseScope(scope, true) - // Expect this to be a objectList - if objectList, ok = i.(*discoveryV1beta1.EndpointSliceList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *discoveryV1beta1.EndpointSliceList", i) + if err != nil { + return nil, err } - for _, object := range objectList.Items { - if item, err = MapEndpointSliceGet(&object); err == nil { - items = append(items, item) - } else { - return items, err + for _, endpoint := range resource.Endpoints { + if endpoint.Hostname != nil { + queries = append(queries, &sdp.Query{ + Type: "dns", + Method: sdp.QueryMethod_GET, + Query: *endpoint.Hostname, + Scope: "global", + }) } - } - return items, nil -} + if endpoint.NodeName != nil { + queries = append(queries, &sdp.Query{ + Type: "Node", + Method: sdp.QueryMethod_GET, + Query: *endpoint.NodeName, + Scope: sd.ClusterName, + }) + } -// MapEndpointSliceGet maps an interface that is underneath a *discoveryV1beta1.EndpointSlice to an item. If -// the interface isn't actually a *discoveryV1beta1.EndpointSlice this will fail -func MapEndpointSliceGet(i interface{}) (*sdp.Item, error) { - var object *discoveryV1beta1.EndpointSlice - var ok bool + if endpoint.TargetRef != nil { + newQuery := ObjectReferenceToQuery(endpoint.TargetRef, sd) + queries = append(queries, newQuery) + } - // Expect this to be a *discoveryV1beta1.EndpointSlice - if object, ok = i.(*discoveryV1beta1.EndpointSlice); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *discoveryV1beta1.EndpointSlice", i) + for _, address := range endpoint.Addresses { + switch resource.AddressType { + case v1.AddressTypeIPv4, v1.AddressTypeIPv6: + queries = append(queries, &sdp.Query{ + Type: "ip", + Method: sdp.QueryMethod_GET, + Query: address, + Scope: "global", + }) + case v1.AddressTypeFQDN: + queries = append(queries, &sdp.Query{ + Type: "dns", + Method: sdp.QueryMethod_GET, + Query: address, + Scope: "global", + }) + } + } } - item, err := mapK8sObject("endpointslice", object) - - if err != nil { - return &sdp.Item{}, err - } + return queries, nil +} - for _, endpoint := range object.Endpoints { - if tr := endpoint.TargetRef; tr != nil { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, - Method: sdp.QueryMethod_GET, - Query: tr.Name, - Type: strings.ToLower(tr.Kind), - }) - } +func NewEndpointSliceSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.EndpointSlice, *v1.EndpointSliceList] { + return KubeTypeSource[*v1.EndpointSlice, *v1.EndpointSliceList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "EndpointSlice", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.EndpointSlice, *v1.EndpointSliceList] { + return cs.DiscoveryV1().EndpointSlices(namespace) + }, + ListExtractor: func(list *v1.EndpointSliceList) ([]*v1.EndpointSlice, error) { + extracted := make([]*v1.EndpointSlice, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + LinkedItemQueryExtractor: endpointSliceExtractor, } - - return item, nil } diff --git a/internal/sources/endpointslice_test.go b/internal/sources/endpointslice_test.go new file mode 100644 index 0000000..6155d32 --- /dev/null +++ b/internal/sources/endpointslice_test.go @@ -0,0 +1,83 @@ +package sources + +import ( + "regexp" + "testing" + + "github.com/overmindtech/sdp-go" +) + +var endpointSliceYAML = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: endpointslice-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: endpointslice-test + template: + metadata: + labels: + app: endpointslice-test + spec: + containers: + - name: endpointslice-test + image: nginx:latest + ports: + - containerPort: 80 + +--- +apiVersion: v1 +kind: Service +metadata: + name: endpointslice-service +spec: + selector: + app: endpointslice-test + ports: + - name: http + port: 80 + targetPort: 80 + type: ClusterIP + +` + +func TestEndpointSliceSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewEndpointSliceSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQueryRegexp: regexp.MustCompile("endpoint-service"), + GetScope: sd.String(), + SetupYAML: endpointSliceYAML, + GetQueryTests: QueryTests{ + { + ExpectedQueryMatches: regexp.MustCompile(`^10\.`), + ExpectedType: "ip", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedScope: "global", + }, + { + ExpectedType: "Node", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "k8s-source-tests-control-plane", + ExpectedScope: CurrentCluster.Name, + }, + { + ExpectedType: "Pod", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQueryMatches: regexp.MustCompile("endpoint-deployment"), + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/generic_source.go b/internal/sources/generic_source.go index 1b79784..b33fe12 100644 --- a/internal/sources/generic_source.go +++ b/internal/sources/generic_source.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/overmindtech/sdp-go" + corev1 "k8s.io/api/core/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -320,3 +321,23 @@ func resourceToItem(resource metav1.Object, cluster string) (*sdp.Item, error) { return item, nil } + +// ObjectReferenceToQuery Converts a K8s ObjectReference to a linked item +// request. Note that you must provide the parent scope since the reference +// could be an object in a different namespace, if it is we need to re-use the +// cluster name from the parent scope +func ObjectReferenceToQuery(ref *corev1.ObjectReference, parentScope ScopeDetails) *sdp.Query { + if ref == nil { + return nil + } + + // Update the namespace, but keep the cluster the same + parentScope.Namespace = ref.Namespace + + return &sdp.Query{ + Type: ref.Kind, + Method: sdp.QueryMethod_GET, // Object references are to a specific object + Query: ref.Name, + Scope: parentScope.String(), + } +} diff --git a/internal/sources/generic_source_test.go b/internal/sources/generic_source_test.go index 9cd591d..7892da0 100644 --- a/internal/sources/generic_source_test.go +++ b/internal/sources/generic_source_test.go @@ -491,6 +491,10 @@ type SourceTests struct { GetScope string GetQueryTests QueryTests + // If this is set,. the get query is determined by running a list, then + // finding the first item that matches this regexp + GetQueryRegexp *regexp.Regexp + // YAML to apply before testing, it will be removed after SetupYAML string } @@ -511,9 +515,28 @@ func (s SourceTests) Execute(t *testing.T) { } t.Run(s.Source.Name(), func(t *testing.T) { - if s.GetQuery != "" { - t.Run(fmt.Sprintf("GET:%v", s.GetQuery), func(t *testing.T) { - item, err := s.Source.Get(context.Background(), s.GetScope, s.GetQuery) + var getQuery string + + if s.GetQueryRegexp != nil { + items, err := s.Source.List(context.Background(), s.GetScope) + + if err != nil { + t.Fatal(err) + } + + for _, item := range items { + if s.GetQueryRegexp.MatchString(item.UniqueAttributeValue()) { + getQuery = item.UniqueAttributeValue() + break + } + } + } else { + getQuery = s.GetQuery + } + + if getQuery != "" { + t.Run(fmt.Sprintf("GET:%v", getQuery), func(t *testing.T) { + item, err := s.Source.Get(context.Background(), s.GetScope, getQuery) if err != nil { t.Fatal(err) @@ -575,3 +598,38 @@ func WaitFor(timeout time.Duration, run func() bool) error { time.Sleep(250 * time.Millisecond) } } + +func TestObjectReferenceToQuery(t *testing.T) { + t.Run("with a valid object reference", func(t *testing.T) { + ref := &v1.ObjectReference{ + Kind: "Pod", + Namespace: "default", + Name: "foo", + } + + query := ObjectReferenceToQuery(ref, ScopeDetails{ + ClusterName: "test-cluster", + Namespace: "default", + }) + + if query.Type != "Pod" { + t.Errorf("expected type Pod, got %s", query.Type) + } + + if query.Query != "foo" { + t.Errorf("expected query to be foo, got %s", query.Query) + } + + if query.Scope != "test-cluster.default" { + t.Errorf("expected scope to be test-cluster.default, got %s", query.Scope) + } + }) + + t.Run("with a nil object reference", func(t *testing.T) { + query := ObjectReferenceToQuery(nil, ScopeDetails{}) + + if query != nil { + t.Errorf("expected nil query, got %v", query) + } + }) +} diff --git a/internal/sources/job.go b/internal/sources/job.go index 7208207..7b5f48c 100644 --- a/internal/sources/job.go +++ b/internal/sources/job.go @@ -31,13 +31,13 @@ func NewJobSource(cs *kubernetes.Clientset, cluster string, namespaces []string) return cs.BatchV1().Jobs(namespace) }, ListExtractor: func(list *v1.JobList) ([]*v1.Job, error) { - bindings := make([]*v1.Job, len(list.Items)) + extracted := make([]*v1.Job, len(list.Items)) for i := range list.Items { - bindings[i] = &list.Items[i] + extracted[i] = &list.Items[i] } - return bindings, nil + return extracted, nil }, LinkedItemQueryExtractor: jobExtractor, } diff --git a/internal/sources/shared_resourcesource.go b/internal/sources/shared_resourcesource.go index 314bced..988ac37 100644 --- a/internal/sources/shared_resourcesource.go +++ b/internal/sources/shared_resourcesource.go @@ -47,14 +47,11 @@ var SourceFunctions = []SourceFunction{ ServiceSource, PVCSource, SecretSource, - EndpointSource, ServiceAccountSource, LimitRangeSource, ReplicationControllerSource, ResourceQuotaSource, - DaemonSetSource, ReplicaSetSource, - DeploymentSource, HorizontalPodAutoscalerSource, StatefulSetSource, IngressSource, @@ -62,7 +59,6 @@ var SourceFunctions = []SourceFunction{ PodDisruptionBudgetSource, RoleBindingSource, RoleSource, - EndpointSliceSource, NamespaceSource, NodeSource, PersistentVolumeSource, From 4e62ae9c52f7467439f51247ce3b0f09c3c569fd Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Fri, 12 May 2023 12:54:52 +0000 Subject: [PATCH 09/24] Added HPA and ingress --- internal/sources/horizontalpodautoscaler.go | 91 +++-------- .../sources/horizontalpodautoscaler_test.go | 74 +++++++++ internal/sources/ingress.go | 145 +++++++++--------- internal/sources/ingress_test.go | 91 +++++++++++ internal/sources/shared_resourcesource.go | 2 - 5 files changed, 261 insertions(+), 142 deletions(-) create mode 100644 internal/sources/horizontalpodautoscaler_test.go create mode 100644 internal/sources/ingress_test.go diff --git a/internal/sources/horizontalpodautoscaler.go b/internal/sources/horizontalpodautoscaler.go index 92d7452..dc4fb0a 100644 --- a/internal/sources/horizontalpodautoscaler.go +++ b/internal/sources/horizontalpodautoscaler.go @@ -1,83 +1,42 @@ package sources import ( - "fmt" - "strings" - - autoscalingV1 "k8s.io/api/autoscaling/v1" + v2 "k8s.io/api/autoscaling/v2" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// HorizontalPodAutoscalerSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func HorizontalPodAutoscalerSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "horizontalpodautoscaler", - MapGet: MapHorizontalPodAutoscalerGet, - MapList: MapHorizontalPodAutoscalerList, - Namespaced: true, - } +func horizontalPodAutoscalerExtractor(resource *v2.HorizontalPodAutoscaler, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) - err := source.LoadFunction( - cs.AutoscalingV1().HorizontalPodAutoscalers, - ) + queries = append(queries, &sdp.Query{ + Type: resource.Spec.ScaleTargetRef.Kind, + Method: sdp.QueryMethod_GET, + Query: resource.Spec.ScaleTargetRef.Name, + Scope: scope, + }) - return source, err + return queries, nil } -// MapHorizontalPodAutoscalerList maps an interface that is underneath a -// *autoscalingV1.HorizontalPodAutoscalerList to a list of Items -func MapHorizontalPodAutoscalerList(i interface{}) ([]*sdp.Item, error) { - var objectList *autoscalingV1.HorizontalPodAutoscalerList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*autoscalingV1.HorizontalPodAutoscalerList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *autoscalingV1.HorizontalPodAutoscalerList", i) - } - - for _, object := range objectList.Items { - if item, err = MapHorizontalPodAutoscalerGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} - -// MapHorizontalPodAutoscalerGet maps an interface that is underneath a *autoscalingV1.HorizontalPodAutoscaler to an item. If -// the interface isn't actually a *autoscalingV1.HorizontalPodAutoscaler this will fail -func MapHorizontalPodAutoscalerGet(i interface{}) (*sdp.Item, error) { - var object *autoscalingV1.HorizontalPodAutoscaler - var ok bool - - // Expect this to be a *autoscalingV1.HorizontalPodAutoscaler - if object, ok = i.(*autoscalingV1.HorizontalPodAutoscaler); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *autoscalingV1.HorizontalPodAutoscaler", i) - } - - item, err := mapK8sObject("horizontalpodautoscaler", object) +func NewHorizontalPodAutoscalerSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v2.HorizontalPodAutoscaler, *v2.HorizontalPodAutoscalerList] { + return KubeTypeSource[*v2.HorizontalPodAutoscaler, *v2.HorizontalPodAutoscalerList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "HorizontalPodAutoscaler", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v2.HorizontalPodAutoscaler, *v2.HorizontalPodAutoscalerList] { + return cs.AutoscalingV2().HorizontalPodAutoscalers(namespace) + }, + ListExtractor: func(list *v2.HorizontalPodAutoscalerList) ([]*v2.HorizontalPodAutoscaler, error) { + extracted := make([]*v2.HorizontalPodAutoscaler, len(list.Items)) - if err != nil { - return &sdp.Item{}, err - } + for i := range list.Items { + extracted[i] = &list.Items[i] + } - item.LinkedItemQueries = []*sdp.Query{ - // Services are linked to pods via their selector - { - Scope: item.Scope, - Method: sdp.QueryMethod_GET, - Query: object.Spec.ScaleTargetRef.Name, - Type: strings.ToLower(object.Spec.ScaleTargetRef.Kind), + return extracted, nil }, + LinkedItemQueryExtractor: horizontalPodAutoscalerExtractor, } - - return item, nil } diff --git a/internal/sources/horizontalpodautoscaler_test.go b/internal/sources/horizontalpodautoscaler_test.go new file mode 100644 index 0000000..6facd93 --- /dev/null +++ b/internal/sources/horizontalpodautoscaler_test.go @@ -0,0 +1,74 @@ +package sources + +import ( + "testing" + + "github.com/overmindtech/sdp-go" +) + +var horizontalPodAutoscalerYAML = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hpa-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: hpa-app + template: + metadata: + labels: + app: hpa-app + spec: + containers: + - name: hpa-container + image: nginx:latest + ports: + - containerPort: 80 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: my-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: hpa-deployment + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 +` + +func TestHorizontalPodAutoscalerSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewHorizontalPodAutoscalerSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "my-hpa", + GetScope: sd.String(), + SetupYAML: horizontalPodAutoscalerYAML, + GetQueryTests: QueryTests{ + { + ExpectedType: "Deployment", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedScope: sd.String(), + ExpectedQuery: "hpa-deployment", + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/ingress.go b/internal/sources/ingress.go index a1bf76c..1ef96ac 100644 --- a/internal/sources/ingress.go +++ b/internal/sources/ingress.go @@ -1,100 +1,97 @@ package sources import ( - "fmt" - - networkingV1 "k8s.io/api/networking/v1" + v1 "k8s.io/api/networking/v1" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// IngressSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func IngressSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "ingress", - MapGet: MapIngressGet, - MapList: MapIngressList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.NetworkingV1().Ingresses, - ) - - return source, err -} - -// MapIngressList maps an interface that is underneath a -// *networkingV1.IngressList to a list of Items -func MapIngressList(i interface{}) ([]*sdp.Item, error) { - var objectList *networkingV1.IngressList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error +func ingressExtractor(resource *v1.Ingress, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) - // Expect this to be a objectList - if objectList, ok = i.(*networkingV1.IngressList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *networkingV1.IngressList", i) + if resource.Spec.IngressClassName != nil { + queries = append(queries, &sdp.Query{ + Type: "IngressClass", + Method: sdp.QueryMethod_GET, + Query: *resource.Spec.IngressClassName, + Scope: scope, + }) } - for _, object := range objectList.Items { - if item, err = MapIngressGet(&object); err == nil { - items = append(items, item) - } else { - return items, err + if resource.Spec.DefaultBackend != nil { + if resource.Spec.DefaultBackend.Service != nil { + queries = append(queries, &sdp.Query{ + Type: "Service", + Method: sdp.QueryMethod_GET, + Query: resource.Spec.DefaultBackend.Service.Name, + Scope: scope, + }) } - } - - return items, nil -} - -// MapIngressGet maps an interface that is underneath a *networkingV1.Ingress to an item. If -// the interface isn't actually a *networkingV1.Ingress this will fail -func MapIngressGet(i interface{}) (*sdp.Item, error) { - var object *networkingV1.Ingress - var ok bool - // Expect this to be a *networkingV1.Ingress - if object, ok = i.(*networkingV1.Ingress); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *networkingV1.Ingress", i) + if linkRes := resource.Spec.DefaultBackend.Resource; linkRes != nil { + queries = append(queries, &sdp.Query{ + Type: linkRes.Kind, + Method: sdp.QueryMethod_GET, + Query: linkRes.Name, + Scope: scope, + }) + } } - item, err := mapK8sObject("ingress", object) + for _, rule := range resource.Spec.Rules { + if rule.Host != "" { + queries = append(queries, &sdp.Query{ + Type: "dns", + Method: sdp.QueryMethod_GET, + Query: rule.Host, + Scope: "global", + }) + } - if err != nil { - return &sdp.Item{}, err - } + if rule.HTTP != nil { + for _, path := range rule.HTTP.Paths { + if path.Backend.Service != nil { + queries = append(queries, &sdp.Query{ + Type: "Service", + Method: sdp.QueryMethod_GET, + Query: path.Backend.Service.Name, + Scope: scope, + }) + } - // Link services from each path - for _, rule := range object.Spec.Rules { - if http := rule.HTTP; http != nil { - for _, path := range http.Paths { - if service := path.Backend.Service; service != nil { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + if path.Backend.Resource != nil { + queries = append(queries, &sdp.Query{ + Type: path.Backend.Resource.Kind, Method: sdp.QueryMethod_GET, - Query: service.Name, - Type: "service", + Query: path.Backend.Resource.Name, + Scope: scope, }) } } } } - // Link default if it exists - if db := object.Spec.DefaultBackend; db != nil { - if service := db.Service; service != nil { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, - Method: sdp.QueryMethod_GET, - Query: service.Name, - Type: "service", - }) - } - } + return queries, nil +} - return item, nil +func NewIngressSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Ingress, *v1.IngressList] { + return KubeTypeSource[*v1.Ingress, *v1.IngressList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "Ingress", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.Ingress, *v1.IngressList] { + return cs.NetworkingV1().Ingresses(namespace) + }, + ListExtractor: func(list *v1.IngressList) ([]*v1.Ingress, error) { + extracted := make([]*v1.Ingress, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + LinkedItemQueryExtractor: ingressExtractor, + } } diff --git a/internal/sources/ingress_test.go b/internal/sources/ingress_test.go new file mode 100644 index 0000000..4273334 --- /dev/null +++ b/internal/sources/ingress_test.go @@ -0,0 +1,91 @@ +package sources + +import ( + "testing" + + "github.com/overmindtech/sdp-go" +) + +var ingressYAML = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ingress-app +spec: + replicas: 3 + selector: + matchLabels: + app: ingress-app + template: + metadata: + labels: + app: ingress-app + spec: + containers: + - name: ingress-app + image: nginx + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: ingress-app +spec: + selector: + app: ingress-app + ports: + - name: http + port: 80 + targetPort: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-app +spec: + rules: + - host: example.com + http: + paths: + - path: /ingress-app + pathType: Prefix + backend: + service: + name: ingress-app + port: + name: http + +` + +func TestIngressSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewIngressSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "ingress-app", + GetScope: sd.String(), + SetupYAML: ingressYAML, + GetQueryTests: QueryTests{ + { + ExpectedType: "dns", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "example.com", + ExpectedScope: "global", + }, + { + ExpectedType: "Service", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "ingress-app", + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/shared_resourcesource.go b/internal/sources/shared_resourcesource.go index 988ac37..ceeec5e 100644 --- a/internal/sources/shared_resourcesource.go +++ b/internal/sources/shared_resourcesource.go @@ -52,9 +52,7 @@ var SourceFunctions = []SourceFunction{ ReplicationControllerSource, ResourceQuotaSource, ReplicaSetSource, - HorizontalPodAutoscalerSource, StatefulSetSource, - IngressSource, NetworkPolicySource, PodDisruptionBudgetSource, RoleBindingSource, From 445c2d2c0506ff071ca0d78d8c6bafaeacfb978d Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Fri, 12 May 2023 14:22:23 +0000 Subject: [PATCH 10/24] Added more sources --- internal/sources/limitrange.go | 84 +++---------- internal/sources/limitrange_test.go | 51 ++++++++ internal/sources/namespace.go | 84 ------------- internal/sources/networkpolicy.go | 109 ++++++---------- internal/sources/networkpolicy_test.go | 55 ++++++++ internal/sources/node.go | 102 +++------------ internal/sources/node_test.go | 23 ++++ internal/sources/persistentvolume.go | 118 +++++++----------- internal/sources/persistentvolume_test.go | 39 ++++++ internal/sources/persistentvolumeclaim.go | 26 ++++ .../sources/persistentvolumeclaim_test.go | 65 ++++++++++ internal/sources/pods.go | 2 +- internal/sources/pvc.go | 76 ----------- internal/sources/shared_resourcesource.go | 6 - 14 files changed, 375 insertions(+), 465 deletions(-) create mode 100644 internal/sources/limitrange_test.go delete mode 100644 internal/sources/namespace.go create mode 100644 internal/sources/networkpolicy_test.go create mode 100644 internal/sources/node_test.go create mode 100644 internal/sources/persistentvolume_test.go create mode 100644 internal/sources/persistentvolumeclaim.go create mode 100644 internal/sources/persistentvolumeclaim_test.go delete mode 100644 internal/sources/pvc.go diff --git a/internal/sources/limitrange.go b/internal/sources/limitrange.go index 12875ed..c3f9640 100644 --- a/internal/sources/limitrange.go +++ b/internal/sources/limitrange.go @@ -1,74 +1,26 @@ package sources import ( - "fmt" - - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -// LimitRangeSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func LimitRangeSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "limitrange", - MapGet: MapLimitRangeGet, - MapList: MapLimitRangeList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.CoreV1().LimitRanges, - ) - - return source, err -} - -// MapLimitRangeList maps an interface that is underneath a -// *coreV1.LimitRangeList to a list of Items -func MapLimitRangeList(i interface{}) ([]*sdp.Item, error) { - var objectList *coreV1.LimitRangeList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*coreV1.LimitRangeList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.LimitRangeList", i) +func NewLimitRangeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.LimitRange, *v1.LimitRangeList] { + return KubeTypeSource[*v1.LimitRange, *v1.LimitRangeList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "LimitRange", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.LimitRange, *v1.LimitRangeList] { + return cs.CoreV1().LimitRanges(namespace) + }, + ListExtractor: func(list *v1.LimitRangeList) ([]*v1.LimitRange, error) { + extracted := make([]*v1.LimitRange, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, } - - for _, object := range objectList.Items { - if item, err = MapLimitRangeGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} - -// MapLimitRangeGet maps an interface that is underneath a *coreV1.LimitRange to an item. If -// the interface isn't actually a *coreV1.LimitRange this will fail -func MapLimitRangeGet(i interface{}) (*sdp.Item, error) { - var object *coreV1.LimitRange - var ok bool - - // Expect this to be a *coreV1.LimitRange - if object, ok = i.(*coreV1.LimitRange); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.LimitRange", i) - } - - item, err := mapK8sObject("limitrange", object) - - if err != nil { - return &sdp.Item{}, err - } - - // TODO: Should these be linked to a namespace? The only thing that a limit - // range is actually related to is the namespace - - return item, nil } diff --git a/internal/sources/limitrange_test.go b/internal/sources/limitrange_test.go new file mode 100644 index 0000000..5270efa --- /dev/null +++ b/internal/sources/limitrange_test.go @@ -0,0 +1,51 @@ +package sources + +import ( + "testing" +) + +var limitRangeYAML = ` +apiVersion: v1 +kind: LimitRange +metadata: + name: example-limit-range +spec: + limits: + - type: Pod + max: + memory: 200Mi + min: + cpu: 50m + - type: Container + max: + memory: 150Mi + cpu: 100m + min: + memory: 50Mi + cpu: 50m + default: + memory: 100Mi + cpu: 50m + defaultRequest: + memory: 80Mi + cpu: 50m +` + +func TestLimitRangeSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewLimitRangeSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "example-limit-range", + GetScope: sd.String(), + SetupYAML: limitRangeYAML, + GetQueryTests: QueryTests{}, + } + + st.Execute(t) +} diff --git a/internal/sources/namespace.go b/internal/sources/namespace.go deleted file mode 100644 index b218001..0000000 --- a/internal/sources/namespace.go +++ /dev/null @@ -1,84 +0,0 @@ -package sources - -import ( - "fmt" - "strings" - - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" - "k8s.io/client-go/kubernetes" -) - -// NamespaceSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func NamespaceSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "namespace", - MapGet: MapNamespaceGet, - MapList: MapNamespaceList, - Namespaced: false, - } - - err := source.LoadFunction( - cs.CoreV1().Namespaces, - ) - - return source, err -} - -// MapNamespaceList maps an interface that is underneath a -// *coreV1.NamespaceList to a list of Items -func MapNamespaceList(i interface{}) ([]*sdp.Item, error) { - var objectList *coreV1.NamespaceList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*coreV1.NamespaceList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.NamespaceList", i) - } - - for _, object := range objectList.Items { - if item, err = MapNamespaceGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} - -// MapNamespaceGet maps an interface that is underneath a *coreV1.Namespace to an item. If -// the interface isn't actually a *coreV1.Namespace this will fail -func MapNamespaceGet(i interface{}) (*sdp.Item, error) { - var object *coreV1.Namespace - var ok bool - - // Expect this to be a *coreV1.Namespace - if object, ok = i.(*coreV1.Namespace); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.Namespace", i) - } - - item, err := mapK8sObject("namespace", object) - - if err != nil { - return &sdp.Item{}, err - } - - scope := strings.Join([]string{ClusterName, object.Name}, ".") - - // Link to all items in the namespace - item.LinkedItemQueries = []*sdp.Query{ - // Search all types within the namespace's scope - { - Scope: scope, - Method: sdp.QueryMethod_LIST, - Type: "*", - }, - } - - return item, nil -} diff --git a/internal/sources/networkpolicy.go b/internal/sources/networkpolicy.go index 737208f..db168ef 100644 --- a/internal/sources/networkpolicy.go +++ b/internal/sources/networkpolicy.go @@ -1,87 +1,29 @@ package sources import ( - "fmt" - - networkingV1 "k8s.io/api/networking/v1" + v1 "k8s.io/api/networking/v1" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// NetworkPolicySource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func NetworkPolicySource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "networkpolicy", - MapGet: MapNetworkPolicyGet, - MapList: MapNetworkPolicyList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.NetworkingV1().NetworkPolicies, - ) - - return source, err -} - -// MapNetworkPolicyList maps an interface that is underneath a -// *networkingV1.NetworkPolicyList to a list of Items -func MapNetworkPolicyList(i interface{}) ([]*sdp.Item, error) { - var objectList *networkingV1.NetworkPolicyList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*networkingV1.NetworkPolicyList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *networkingV1.NetworkPolicyList", i) - } - - for _, object := range objectList.Items { - if item, err = MapNetworkPolicyGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} - -// MapNetworkPolicyGet maps an interface that is underneath a *networkingV1.NetworkPolicy to an item. If -// the interface isn't actually a *networkingV1.NetworkPolicy this will fail -func MapNetworkPolicyGet(i interface{}) (*sdp.Item, error) { - var object *networkingV1.NetworkPolicy - var ok bool - - // Expect this to be a *networkingV1.NetworkPolicy - if object, ok = i.(*networkingV1.NetworkPolicy); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *networkingV1.NetworkPolicy", i) - } - - item, err := mapK8sObject("networkpolicy", object) +func NetworkPolicyExtractor(resource *v1.NetworkPolicy, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) - if err != nil { - return &sdp.Item{}, err - } - - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, - Method: sdp.QueryMethod_GET, - Query: LabelSelectorToQuery(&object.Spec.PodSelector), - Type: "pod", + queries = append(queries, &sdp.Query{ + Type: "Pod", + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(&resource.Spec.PodSelector), + Scope: scope, }) - var peers []networkingV1.NetworkPolicyPeer + var peers []v1.NetworkPolicyPeer - for _, ig := range object.Spec.Ingress { + for _, ig := range resource.Spec.Ingress { peers = append(peers, ig.From...) } - for _, eg := range object.Spec.Egress { + for _, eg := range resource.Spec.Egress { peers = append(peers, eg.To...) } @@ -93,14 +35,35 @@ func MapNetworkPolicyGet(i interface{}) (*sdp.Item, error) { // matchLabels: // project: something - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + queries = append(queries, &sdp.Query{ + Scope: scope, Method: sdp.QueryMethod_GET, Query: LabelSelectorToQuery(ps), - Type: "pod", + Type: "Pod", }) } } - return item, nil + return queries, nil +} + +func NewNetworkPolicySource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.NetworkPolicy, *v1.NetworkPolicyList] { + return KubeTypeSource[*v1.NetworkPolicy, *v1.NetworkPolicyList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "NetworkPolicy", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.NetworkPolicy, *v1.NetworkPolicyList] { + return cs.NetworkingV1().NetworkPolicies(namespace) + }, + ListExtractor: func(list *v1.NetworkPolicyList) ([]*v1.NetworkPolicy, error) { + extracted := make([]*v1.NetworkPolicy, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + LinkedItemQueryExtractor: NetworkPolicyExtractor, + } } diff --git a/internal/sources/networkpolicy_test.go b/internal/sources/networkpolicy_test.go new file mode 100644 index 0000000..463b914 --- /dev/null +++ b/internal/sources/networkpolicy_test.go @@ -0,0 +1,55 @@ +package sources + +import ( + "regexp" + "testing" + + "github.com/overmindtech/sdp-go" +) + +var NetworkPolicyYAML = ` +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-nginx +spec: + podSelector: + matchLabels: + app: nginx + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: frontend + ports: + - protocol: TCP + port: 80 +` + +func TestNetworkPolicySource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewNetworkPolicySource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "allow-nginx", + GetScope: sd.String(), + SetupYAML: NetworkPolicyYAML, + GetQueryTests: QueryTests{ + { + ExpectedQueryMatches: regexp.MustCompile("nginx"), + ExpectedType: "Pod", + ExpectedMethod: sdp.QueryMethod_SEARCH, + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/node.go b/internal/sources/node.go index d3afc53..3b684bd 100644 --- a/internal/sources/node.go +++ b/internal/sources/node.go @@ -1,93 +1,27 @@ package sources import ( - "fmt" - "strings" - - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" - metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -// NodeSource returns a ResourceSource for PersistentVolumeClaims for a given -// client -func NodeSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "node", - Namespaced: false, +func NewNodeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Node, *v1.NodeList] { + return KubeTypeSource[*v1.Node, *v1.NodeList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "Node", + ClusterInterfaceBuilder: func() ItemInterface[*v1.Node, *v1.NodeList] { + return cs.CoreV1().Nodes() + }, + ListExtractor: func(list *v1.NodeList) ([]*v1.Node, error) { + extracted := make([]*v1.Node, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, } - - source.MapGet = source.MapNodeGet - source.MapList = source.MapNodeList - - err := source.LoadFunction( - cs.CoreV1().Nodes, - ) - - return source, err -} - -// MapNodeList maps an interface that is underneath a -// *coreV1.NodeList to a list of Items -func (rs *ResourceSource) MapNodeList(i interface{}) ([]*sdp.Item, error) { - var objectList *coreV1.NodeList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*coreV1.NodeList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.NodeList", i) - } - - for _, object := range objectList.Items { - if item, err = rs.MapNodeGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} - -// MapNodeGet maps an interface that is underneath a *coreV1.Node to an item. If -// the interface isn't actually a *coreV1.Node this will fail -func (rs *ResourceSource) MapNodeGet(i interface{}) (*sdp.Item, error) { - var object *coreV1.Node - var ok bool - - // Expect this to be a *coreV1.Node - if object, ok = i.(*coreV1.Node); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.Node", i) - } - - item, err := mapK8sObject("node", object) - - if err != nil { - return &sdp.Item{}, err - } - - // Query based onf fields not labels - hostQuery := metaV1.ListOptions{ - FieldSelector: fmt.Sprintf("spec.nodeName=%v", object.Name), - } - - namespaces, _ := rs.NSS.Namespaces() - - for _, namespace := range namespaces { - scope := strings.Join([]string{ClusterName, namespace}, ".") - - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_SEARCH, - Type: "pod", - Query: ListOptionsToQuery(&hostQuery), - }) - } - - return item, nil } diff --git a/internal/sources/node_test.go b/internal/sources/node_test.go new file mode 100644 index 0000000..ab10596 --- /dev/null +++ b/internal/sources/node_test.go @@ -0,0 +1,23 @@ +package sources + +import ( + "testing" +) + +func TestNodeSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewNodeSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "k8s-source-tests-control-plane", + GetScope: sd.String(), + GetQueryTests: QueryTests{}, + } + + st.Execute(t) +} diff --git a/internal/sources/persistentvolume.go b/internal/sources/persistentvolume.go index 6979e80..b4264d9 100644 --- a/internal/sources/persistentvolume.go +++ b/internal/sources/persistentvolume.go @@ -1,95 +1,63 @@ package sources import ( - "fmt" - "strings" - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -// PersistentVolumeSource returns a ResourceSource for PersistentVolumeClaims for a given -// client -func PersistentVolumeSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "persistentvolume", - MapGet: MapPersistentVolumeGet, - MapList: MapPersistentVolumeList, - Namespaced: false, - } - - err := source.LoadFunction( - cs.CoreV1().PersistentVolumes, - ) +func PersistentVolumeExtractor(resource *v1.PersistentVolume, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) - return source, err -} + sd, err := ParseScope(scope, false) -// MapPersistentVolumeList maps an interface that is underneath a -// *coreV1.PersistentVolumeList to a list of Items -func MapPersistentVolumeList(i interface{}) ([]*sdp.Item, error) { - var objectList *coreV1.PersistentVolumeList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*coreV1.PersistentVolumeList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.PersistentVolumeList", i) + if err != nil { + return nil, err } - for _, object := range objectList.Items { - if item, err = MapPersistentVolumeGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } + if resource.Spec.PersistentVolumeSource.AWSElasticBlockStore != nil { + // Link to EBS volume + queries = append(queries, &sdp.Query{ + Type: "ec2-volume", + Method: sdp.QueryMethod_GET, + Query: resource.Spec.PersistentVolumeSource.AWSElasticBlockStore.VolumeID, + Scope: "*", + }) } - return items, nil -} - -// MapPersistentVolumeGet maps an interface that is underneath a *coreV1.PersistentVolume to an item. If -// the interface isn't actually a *coreV1.PersistentVolume this will fail -func MapPersistentVolumeGet(i interface{}) (*sdp.Item, error) { - var object *coreV1.PersistentVolume - var ok bool - - // Expect this to be a *coreV1.PersistentVolume - if object, ok = i.(*coreV1.PersistentVolume); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.PersistentVolume", i) + if resource.Spec.ClaimRef != nil { + queries = append(queries, ObjectReferenceToQuery(resource.Spec.ClaimRef, sd)) } - item, err := mapK8sObject("persistentvolume", object) - - if err != nil { - return &sdp.Item{}, err + if resource.Spec.StorageClassName != "" { + queries = append(queries, &sdp.Query{ + Type: "StorageClass", + Method: sdp.QueryMethod_GET, + Query: resource.Spec.StorageClassName, + Scope: sd.ClusterName, + }) } - if claim := object.Spec.ClaimRef; claim != nil { - scope := strings.Join([]string{ClusterName, claim.Namespace}, ".") + return queries, nil +} - // Link to all items in the PersistentVolume - item.LinkedItemQueries = []*sdp.Query{ - // Search all types within the PersistentVolume's scope - { - Scope: scope, - Method: sdp.QueryMethod_GET, - Type: strings.ToLower(claim.Kind), - Query: claim.Name, - }, - } +func NewPersistentVolumeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.PersistentVolume, *v1.PersistentVolumeList] { + return KubeTypeSource[*v1.PersistentVolume, *v1.PersistentVolumeList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "PersistentVolume", + ClusterInterfaceBuilder: func() ItemInterface[*v1.PersistentVolume, *v1.PersistentVolumeList] { + return cs.CoreV1().PersistentVolumes() + }, + ListExtractor: func(list *v1.PersistentVolumeList) ([]*v1.PersistentVolume, error) { + extracted := make([]*v1.PersistentVolume, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + LinkedItemQueryExtractor: PersistentVolumeExtractor, } - - // Link to the storage class - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: ClusterName, - Method: sdp.QueryMethod_GET, - Query: object.Spec.StorageClassName, - Type: "storageclass", - }) - - return item, nil } diff --git a/internal/sources/persistentvolume_test.go b/internal/sources/persistentvolume_test.go new file mode 100644 index 0000000..bc26055 --- /dev/null +++ b/internal/sources/persistentvolume_test.go @@ -0,0 +1,39 @@ +package sources + +import ( + "testing" +) + +var persistentVolumeYAML = ` +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-test-pv +spec: + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /tmp/pv-test-pv +` + +func TestPersistentVolumeSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "", + } + + source := NewPersistentVolumeSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "pv-test-pv", + GetScope: sd.String(), + SetupYAML: persistentVolumeYAML, + GetQueryTests: QueryTests{}, + } + + st.Execute(t) +} diff --git a/internal/sources/persistentvolumeclaim.go b/internal/sources/persistentvolumeclaim.go new file mode 100644 index 0000000..620f17d --- /dev/null +++ b/internal/sources/persistentvolumeclaim.go @@ -0,0 +1,26 @@ +package sources + +import ( + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" +) + +func NewPersistentVolumeClaimSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.PersistentVolumeClaim, *v1.PersistentVolumeClaimList] { + return KubeTypeSource[*v1.PersistentVolumeClaim, *v1.PersistentVolumeClaimList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "PersistentVolumeClaim", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.PersistentVolumeClaim, *v1.PersistentVolumeClaimList] { + return cs.CoreV1().PersistentVolumeClaims(namespace) + }, + ListExtractor: func(list *v1.PersistentVolumeClaimList) ([]*v1.PersistentVolumeClaim, error) { + extracted := make([]*v1.PersistentVolumeClaim, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + } +} diff --git a/internal/sources/persistentvolumeclaim_test.go b/internal/sources/persistentvolumeclaim_test.go new file mode 100644 index 0000000..353a168 --- /dev/null +++ b/internal/sources/persistentvolumeclaim_test.go @@ -0,0 +1,65 @@ +package sources + +import ( + "testing" +) + +var persistentVolumeClaimYAML = ` +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-test-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc-test-pv +spec: + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /tmp/pvc-test-pv +--- +apiVersion: v1 +kind: Pod +metadata: + name: pvc-test-pod +spec: + containers: + - name: pvc-test-container + image: nginx + volumeMounts: + - name: pvc-test-volume + mountPath: /data + volumes: + - name: pvc-test-volume + persistentVolumeClaim: + claimName: pvc-test-pvc +` + +func TestPersistentVolumeClaimSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewPersistentVolumeClaimSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "pvc-test-pvc", + GetScope: sd.String(), + SetupYAML: persistentVolumeClaimYAML, + GetQueryTests: QueryTests{}, + } + + st.Execute(t) +} diff --git a/internal/sources/pods.go b/internal/sources/pods.go index bda4407..9e18a1e 100644 --- a/internal/sources/pods.go +++ b/internal/sources/pods.go @@ -95,7 +95,7 @@ func MapPodGet(i interface{}) (*sdp.Item, error) { Scope: item.Scope, Method: sdp.QueryMethod_GET, Query: vol.PersistentVolumeClaim.ClaimName, - Type: PVCType, + Type: "PersistentVolumeClaim", }) } diff --git a/internal/sources/pvc.go b/internal/sources/pvc.go deleted file mode 100644 index c334cb0..0000000 --- a/internal/sources/pvc.go +++ /dev/null @@ -1,76 +0,0 @@ -package sources - -import ( - "fmt" - - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" - "k8s.io/client-go/kubernetes" -) - -// PVCType is the name of the PVC type. I'm saving this as a const since it's a -// bit nasty and I might want to change it later -const PVCType = "persistentVolumeClaim" - -// PVCSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func PVCSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: PVCType, - MapGet: MapPVCGet, - MapList: MapPVCList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.CoreV1().PersistentVolumeClaims, - ) - - return source, err -} - -// MapPVCList maps an interface that is underneath a -// *coreV1.PersistentVolumeClaimList to a list of Items -func MapPVCList(i interface{}) ([]*sdp.Item, error) { - var pvcList *coreV1.PersistentVolumeClaimList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a pvcList - if pvcList, ok = i.(*coreV1.PersistentVolumeClaimList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.PersistentVolumeClaimList", i) - } - - for _, pvc := range pvcList.Items { - if item, err = MapPVCGet(&pvc); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} - -// MapPVCGet maps an interface that is underneath a *coreV1.PersistentVolumeClaim to -// an item. If the interface isn't actually a *coreV1.PersistentVolumeClaim this -// will fail -func MapPVCGet(i interface{}) (*sdp.Item, error) { - var pvc *coreV1.PersistentVolumeClaim - var ok bool - - // Expect this to be a pvc - if pvc, ok = i.(*coreV1.PersistentVolumeClaim); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.PersistentVolumeClaim", i) - } - - item, err := mapK8sObject(PVCType, pvc) - - if err != nil { - return &sdp.Item{}, err - } - - return item, nil -} diff --git a/internal/sources/shared_resourcesource.go b/internal/sources/shared_resourcesource.go index ceeec5e..8917cdb 100644 --- a/internal/sources/shared_resourcesource.go +++ b/internal/sources/shared_resourcesource.go @@ -45,21 +45,15 @@ type SourceFunction func(cs *kubernetes.Clientset) (ResourceSource, error) var SourceFunctions = []SourceFunction{ PodSource, ServiceSource, - PVCSource, SecretSource, ServiceAccountSource, - LimitRangeSource, ReplicationControllerSource, ResourceQuotaSource, ReplicaSetSource, StatefulSetSource, - NetworkPolicySource, PodDisruptionBudgetSource, RoleBindingSource, RoleSource, - NamespaceSource, - NodeSource, - PersistentVolumeSource, StorageClassSource, PriorityClassSource, } From 544d0f41f47048ec5b51f08b694e9cadf2450591 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Fri, 12 May 2023 16:49:44 +0000 Subject: [PATCH 11/24] Added node link to infra and EBS volumes --- internal/sources/node.go | 48 +++++++++++++++++++++++++++++++++++ internal/sources/node_test.go | 18 ++++++++++--- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/internal/sources/node.go b/internal/sources/node.go index 3b684bd..96fc20d 100644 --- a/internal/sources/node.go +++ b/internal/sources/node.go @@ -1,11 +1,58 @@ package sources import ( + "strings" + + "github.com/overmindtech/sdp-go" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) +func linkedItemExtractor(resource *v1.Node, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) + + for _, addr := range resource.Status.Addresses { + switch addr.Type { + case v1.NodeExternalDNS: + queries = append(queries, &sdp.Query{ + Type: "dns", + Method: sdp.QueryMethod_GET, + Query: addr.Address, + Scope: "global", + }) + case v1.NodeExternalIP, v1.NodeInternalIP: + queries = append(queries, &sdp.Query{ + Type: "ip", + Method: sdp.QueryMethod_GET, + Query: addr.Address, + Scope: "global", + }) + } + } + + for _, vol := range resource.Status.VolumesAttached { + // Look for EBS volumes since they follow the format: + // kubernetes.io/csi/ebs.csi.aws.com^vol-043e04d9cc6d72183 + if strings.HasPrefix(string(vol.Name), "kubernetes.io/csi/ebs.csi.aws.com") { + sections := strings.Split(string(vol.Name), "^") + + if len(sections) == 2 { + queries = append(queries, &sdp.Query{ + Type: "ec2-volume", + Method: sdp.QueryMethod_GET, + Query: sections[1], + Scope: "*", + }) + } + } + } + + return queries, nil +} + +// TODO: Should we try a DNS lookup for a node name? Is the hostname stored anywhere? + func NewNodeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Node, *v1.NodeList] { return KubeTypeSource[*v1.Node, *v1.NodeList]{ ClusterName: cluster, @@ -23,5 +70,6 @@ func NewNodeSource(cs *kubernetes.Clientset, cluster string, namespaces []string return extracted, nil }, + LinkedItemQueryExtractor: linkedItemExtractor, } } diff --git a/internal/sources/node_test.go b/internal/sources/node_test.go index ab10596..5896aac 100644 --- a/internal/sources/node_test.go +++ b/internal/sources/node_test.go @@ -1,7 +1,10 @@ package sources import ( + "regexp" "testing" + + "github.com/overmindtech/sdp-go" ) func TestNodeSource(t *testing.T) { @@ -13,10 +16,17 @@ func TestNodeSource(t *testing.T) { source := NewNodeSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, - GetQuery: "k8s-source-tests-control-plane", - GetScope: sd.String(), - GetQueryTests: QueryTests{}, + Source: &source, + GetQuery: "k8s-source-tests-control-plane", + GetScope: sd.String(), + GetQueryTests: QueryTests{ + { + ExpectedType: "ip", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedScope: "global", + ExpectedQueryMatches: regexp.MustCompile(`172\.`), + }, + }, } st.Execute(t) From a8628c22014db06d0e65c708c1d272bc81c14252 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Sat, 13 May 2023 10:02:44 +0000 Subject: [PATCH 12/24] Added loads more sources --- config.json | 6 + internal/sources/generic_source.go | 88 +++++---- internal/sources/generic_source_test.go | 38 +++- internal/sources/poddisruptionbudget.go | 95 +++------ internal/sources/poddisruptionbudget_test.go | 46 +++++ internal/sources/pods.go | 187 ++++++++---------- internal/sources/pods_test.go | 133 ++++++++++--- internal/sources/priorityclass.go | 81 ++------ internal/sources/priorityclass_test.go | 34 ++++ internal/sources/replicaset.go | 98 +++------ internal/sources/replicaset_test.go | 57 ++++++ internal/sources/replicationcontroller.go | 104 +++------- .../sources/replicationcontroller_test.go | 56 ++++++ internal/sources/resourcequota.go | 81 ++------ internal/sources/resourcequota_test.go | 37 ++++ internal/sources/role.go | 81 ++------ internal/sources/role_test.go | 53 ++--- internal/sources/rolebinding.go | 124 +++++------- internal/sources/rolebinding_test.go | 127 ++++++++++++ internal/sources/secret.go | 120 ++++------- internal/sources/secret_test.go | 38 ++-- internal/sources/service.go | 146 +++++++------- internal/sources/service_test.go | 81 ++++---- internal/sources/serviceaccount.go | 103 ++++------ internal/sources/serviceaccount_test.go | 67 +++++++ internal/sources/shared_resourcesource.go | 16 +- internal/sources/statefulset.go | 120 +++++------ internal/sources/statefulset_test.go | 90 +++++++++ internal/sources/storageclass.go | 82 ++------ internal/sources/storageclass_test.go | 34 ++++ internal/sources/volumeattachment.go | 59 ++++++ internal/sources/volumeattachment_test.go | 92 +++++++++ 32 files changed, 1479 insertions(+), 1095 deletions(-) create mode 100644 config.json create mode 100644 internal/sources/poddisruptionbudget_test.go create mode 100644 internal/sources/priorityclass_test.go create mode 100644 internal/sources/replicaset_test.go create mode 100644 internal/sources/replicationcontroller_test.go create mode 100644 internal/sources/resourcequota_test.go create mode 100644 internal/sources/rolebinding_test.go create mode 100644 internal/sources/serviceaccount_test.go create mode 100644 internal/sources/statefulset_test.go create mode 100644 internal/sources/storageclass_test.go create mode 100644 internal/sources/volumeattachment.go create mode 100644 internal/sources/volumeattachment_test.go diff --git a/config.json b/config.json new file mode 100644 index 0000000..0abb8d0 --- /dev/null +++ b/config.json @@ -0,0 +1,6 @@ +{ + "auths": { + "ghcr.io": {}, + }, + "credsStore": "desktop" +} diff --git a/internal/sources/generic_source.go b/internal/sources/generic_source.go index b33fe12..a54c1ca 100644 --- a/internal/sources/generic_source.go +++ b/internal/sources/generic_source.go @@ -46,6 +46,13 @@ type KubeTypeSource[Resource metav1.Object, ResourceList any] struct { // resource and scope LinkedItemQueryExtractor func(resource Resource, scope string) ([]*sdp.Query, error) + // A function that extracts health from the resource, this is optional + HealthExtractor func(resource Resource) *sdp.Health + + // A function that redacts sensitive data from the resource, this is + // optional + Redact func(resource Resource) Resource + // The type of items that this source should return. This should be the // "Kind" of the kubernetes resources, e.g. "Pod", "Node", "ServiceAccount" TypeName string @@ -146,21 +153,12 @@ func (k *KubeTypeSource[Resource, ResourceList]) Get(ctx context.Context, scope return nil, err } - item, err := resourceToItem(resource, k.ClusterName) + item, err := k.resourceToItem(resource) if err != nil { return nil, err } - if k.LinkedItemQueryExtractor != nil { - // Add linked items - item.LinkedItemQueries, err = k.LinkedItemQueryExtractor(resource, scope) - - if err != nil { - return nil, err - } - } - return item, nil } @@ -200,34 +198,6 @@ func (k *KubeTypeSource[Resource, ResourceList]) listWithOptions(ctx context.Con return items, nil } -// resourcesToItems Converts a slice of resources to a slice of items -func (k *KubeTypeSource[Resource, ResourceList]) resourcesToItems(resourceList []Resource, scope string) ([]*sdp.Item, error) { - items := make([]*sdp.Item, len(resourceList)) - - var err error - - for i := range resourceList { - items[i], err = resourceToItem(resourceList[i], k.ClusterName) - - if err != nil { - return nil, err - } - - if k.LinkedItemQueryExtractor != nil { - // Add linked items - newQueries, err := k.LinkedItemQueryExtractor(resourceList[i], scope) - - if err != nil { - return nil, err - } - - items[i].LinkedItemQueries = append(items[i].LinkedItemQueries, newQueries...) - } - } - - return items, nil -} - func (k *KubeTypeSource[Resource, ResourceList]) Search(ctx context.Context, scope string, query string) ([]*sdp.Item, error) { opts, err := QueryToListOptions(query) @@ -273,13 +243,36 @@ func ignored(key string) bool { return false } +// resourcesToItems Converts a slice of resources to a slice of items +func (k *KubeTypeSource[Resource, ResourceList]) resourcesToItems(resourceList []Resource, scope string) ([]*sdp.Item, error) { + items := make([]*sdp.Item, len(resourceList)) + + var err error + + for i := range resourceList { + items[i], err = k.resourceToItem(resourceList[i]) + + if err != nil { + return nil, err + } + + } + + return items, nil +} + // resourceToItem Converts a resource to an item -func resourceToItem(resource metav1.Object, cluster string) (*sdp.Item, error) { +func (k *KubeTypeSource[Resource, ResourceList]) resourceToItem(resource Resource) (*sdp.Item, error) { sd := ScopeDetails{ - ClusterName: cluster, + ClusterName: k.ClusterName, Namespace: resource.GetNamespace(), } + // Redact sensitive data if required + if k.Redact != nil { + resource = k.Redact(resource) + } + attributes, err := sdp.ToAttributesViaJson(resource) if err != nil { @@ -319,6 +312,21 @@ func resourceToItem(resource metav1.Object, cluster string) (*sdp.Item, error) { }) } + if k.LinkedItemQueryExtractor != nil { + // Add linked items + newQueries, err := k.LinkedItemQueryExtractor(resource, sd.String()) + + if err != nil { + return nil, err + } + + item.LinkedItemQueries = append(item.LinkedItemQueries, newQueries...) + } + + if k.HealthExtractor != nil { + item.Health = k.HealthExtractor(resource) + } + return item, nil } diff --git a/internal/sources/generic_source_test.go b/internal/sources/generic_source_test.go index 7892da0..2dd84ea 100644 --- a/internal/sources/generic_source_test.go +++ b/internal/sources/generic_source_test.go @@ -161,6 +161,9 @@ func createSource(namespaced bool) KubeTypeSource[*v1.Pod, *v1.PodList] { return queries, nil }, + HealthExtractor: func(resource *v1.Pod) *sdp.Health { + return sdp.Health_HEALTH_OK.Enum() + }, TypeName: "Pod", ClusterName: "minikube", Namespaces: []string{"default", "app1"}, @@ -321,6 +324,10 @@ func TestSourceGet(t *testing.T) { if item.UniqueAttributeValue() != "example" { t.Errorf("expected item with unique attribute value 'example', got %s", item.UniqueAttributeValue()) } + + if *item.Health != sdp.Health_HEALTH_OK { + t.Errorf("expected item with health HEALTH_OK, got %s", item.Health) + } }) t.Run("get non-existent item", func(t *testing.T) { @@ -359,7 +366,7 @@ func TestList(t *testing.T) { t.Run("when namespaced", func(t *testing.T) { source := createSource(true) - items, err := source.List(context.Background(), "foo") + items, err := source.List(context.Background(), "foo.bar") if err != nil { t.Errorf("expected no error, got %s", err) @@ -368,6 +375,10 @@ func TestList(t *testing.T) { if len(items) != 2 { t.Errorf("expected 2 items, got %d", len(items)) } + + if *items[0].Health != sdp.Health_HEALTH_OK { + t.Errorf("expected item with health HEALTH_OK, got %s", items[0].Health) + } }) t.Run("when not namespaced", func(t *testing.T) { @@ -437,6 +448,31 @@ func TestSearch(t *testing.T) { }) } +func TestRedact(t *testing.T) { + source := createSource(true) + source.Redact = func(resource *v1.Pod) *v1.Pod { + resource.Spec.Hostname = "redacted" + + return resource + } + + item, err := source.Get(context.Background(), "cluster.namespace", "test") + + if err != nil { + t.Error(err) + } + + hostname, err := item.Attributes.Get("spec.hostname") + + if err != nil { + t.Error(err) + } + + if hostname != "redacted" { + t.Errorf("expected hostname to be redacted, got %v", hostname) + } +} + type QueryTest struct { ExpectedType string ExpectedMethod sdp.QueryMethod diff --git a/internal/sources/poddisruptionbudget.go b/internal/sources/poddisruptionbudget.go index 8bf8dc6..24d96ef 100644 --- a/internal/sources/poddisruptionbudget.go +++ b/internal/sources/poddisruptionbudget.go @@ -1,80 +1,43 @@ package sources import ( - "fmt" - "github.com/overmindtech/sdp-go" - policyV1beta1 "k8s.io/api/policy/v1beta1" + v1 "k8s.io/api/policy/v1" "k8s.io/client-go/kubernetes" ) -// PodDisruptionBudgetSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func PodDisruptionBudgetSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "poddisruptionbudget", - MapGet: MapPodDisruptionBudgetGet, - MapList: MapPodDisruptionBudgetList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.PolicyV1beta1().PodDisruptionBudgets, - ) - - return source, err -} - -// MapPodDisruptionBudgetList maps an interface that is underneath a -// *policyV1beta1.PodDisruptionBudgetList to a list of Items -func MapPodDisruptionBudgetList(i interface{}) ([]*sdp.Item, error) { - var objectList *policyV1beta1.PodDisruptionBudgetList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*policyV1beta1.PodDisruptionBudgetList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *policyV1beta1.PodDisruptionBudgetList", i) - } +func podDisruptionBudgetExtractor(resource *v1.PodDisruptionBudget, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) - for _, object := range objectList.Items { - if item, err = MapPodDisruptionBudgetGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } + if resource.Spec.Selector != nil { + queries = append(queries, &sdp.Query{ + Type: "Pod", + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(resource.Spec.Selector), + Scope: scope, + }) } - return items, nil + return queries, nil } -// MapPodDisruptionBudgetGet maps an interface that is underneath a *policyV1beta1.PodDisruptionBudget to an item. If -// the interface isn't actually a *policyV1beta1.PodDisruptionBudget this will fail -func MapPodDisruptionBudgetGet(i interface{}) (*sdp.Item, error) { - var object *policyV1beta1.PodDisruptionBudget - var ok bool - - // Expect this to be a *policyV1beta1.PodDisruptionBudget - if object, ok = i.(*policyV1beta1.PodDisruptionBudget); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *policyV1beta1.PodDisruptionBudget", i) +func NewPodDisruptionBudgetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.PodDisruptionBudget, *v1.PodDisruptionBudgetList] { + return KubeTypeSource[*v1.PodDisruptionBudget, *v1.PodDisruptionBudgetList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "PodDisruptionBudget", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.PodDisruptionBudget, *v1.PodDisruptionBudgetList] { + return cs.PolicyV1().PodDisruptionBudgets(namespace) + }, + ListExtractor: func(list *v1.PodDisruptionBudgetList) ([]*v1.PodDisruptionBudget, error) { + extracted := make([]*v1.PodDisruptionBudget, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + LinkedItemQueryExtractor: podDisruptionBudgetExtractor, } - - item, err := mapK8sObject("poddisruptionbudget", object) - - if err != nil { - return &sdp.Item{}, err - } - - if selector := object.Spec.Selector; selector != nil { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(selector), - Type: "pod", - }) - } - - return item, nil } diff --git a/internal/sources/poddisruptionbudget_test.go b/internal/sources/poddisruptionbudget_test.go new file mode 100644 index 0000000..acee723 --- /dev/null +++ b/internal/sources/poddisruptionbudget_test.go @@ -0,0 +1,46 @@ +package sources + +import ( + "regexp" + "testing" + + "github.com/overmindtech/sdp-go" +) + +var PodDisruptionBudgetYAML = ` +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: example-pdb +spec: + minAvailable: 2 + selector: + matchLabels: + app: example-app +` + +func TestPodDisruptionBudgetSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewPodDisruptionBudgetSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "example-pdb", + GetScope: sd.String(), + SetupYAML: PodDisruptionBudgetYAML, + GetQueryTests: QueryTests{ + { + ExpectedQueryMatches: regexp.MustCompile("app=example-app"), + ExpectedType: "Pod", + ExpectedMethod: sdp.QueryMethod_SEARCH, + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/pods.go b/internal/sources/pods.go index 9e18a1e..cf15a25 100644 --- a/internal/sources/pods.go +++ b/internal/sources/pods.go @@ -1,98 +1,30 @@ package sources import ( - "fmt" - "strings" - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -// PodSource returns a ResourceSource for Pods for a given client and namespace -func PodSource(cs *kubernetes.Clientset) (ResourceSource, error) { - podsBackend := ResourceSource{ - ItemType: "pod", - MapGet: MapPodGet, - MapList: MapPodList, - Namespaced: true, - } - - err := podsBackend.LoadFunction( - cs.CoreV1().Pods, - ) - - return podsBackend, err -} - -// MapPodList maps an interface that is underneath a *coreV1.PodList to a list of -// Items -func MapPodList(i interface{}) ([]*sdp.Item, error) { - var podList *coreV1.PodList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a podList - if podList, ok = i.(*coreV1.PodList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.PodList", i) - } - - for _, pod := range podList.Items { - if item, err = MapPodGet(&pod); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} - -// MapPodGet maps an interface that is underneath a *coreV1.Pod to an item. If the -// interface isn't actually a *coreV1.Pod this will fail -func MapPodGet(i interface{}) (*sdp.Item, error) { - var pod *coreV1.Pod - var ok bool - - // Expect this to be a pod - if pod, ok = i.(*coreV1.Pod); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.Pod", i) - } - - item, err := mapK8sObject("pod", pod) - - if err != nil { - return &sdp.Item{}, err - } +func PodExtractor(resource *v1.Pod, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) // Link service accounts - if pod.Spec.ServiceAccountName != "" { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + if resource.Spec.ServiceAccountName != "" { + queries = append(queries, &sdp.Query{ + Scope: scope, Method: sdp.QueryMethod_GET, - Query: pod.Spec.ServiceAccountName, - Type: "serviceaccount", - }) - } - - // Link to the controller if relevant - for _, ref := range pod.GetOwnerReferences() { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, - Type: strings.ToLower(ref.Kind), - Method: sdp.QueryMethod_GET, - Query: ref.Name, + Query: resource.Spec.ServiceAccountName, + Type: "ServiceAccount", }) } // Link items from volumes - for _, vol := range pod.Spec.Volumes { + for _, vol := range resource.Spec.Volumes { // Link PVCs if vol.PersistentVolumeClaim != nil { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + queries = append(queries, &sdp.Query{ + Scope: scope, Method: sdp.QueryMethod_GET, Query: vol.PersistentVolumeClaim.ClaimName, Type: "PersistentVolumeClaim", @@ -101,46 +33,46 @@ func MapPodGet(i interface{}) (*sdp.Item, error) { // Link secrets if vol.Secret != nil { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + queries = append(queries, &sdp.Query{ + Scope: scope, Method: sdp.QueryMethod_GET, Query: vol.Secret.SecretName, - Type: "secret", + Type: "Secret", }) } // Link config map volumes if vol.ConfigMap != nil { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + queries = append(queries, &sdp.Query{ + Scope: scope, Method: sdp.QueryMethod_GET, Query: vol.ConfigMap.Name, - Type: "configMap", + Type: "ConfigMap", }) } } // Link items from containers - for _, container := range pod.Spec.Containers { + for _, container := range resource.Spec.Containers { // Loop over environment variables for _, env := range container.Env { if env.ValueFrom != nil { if env.ValueFrom.SecretKeyRef != nil { // Add linked item from spec.containers[].env[].valueFrom.secretKeyRef - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + queries = append(queries, &sdp.Query{ + Scope: scope, Method: sdp.QueryMethod_GET, Query: env.ValueFrom.SecretKeyRef.Name, - Type: "secret", + Type: "Secret", }) } if env.ValueFrom.ConfigMapKeyRef != nil { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + queries = append(queries, &sdp.Query{ + Scope: scope, Method: sdp.QueryMethod_GET, Query: env.ValueFrom.ConfigMapKeyRef.Name, - Type: "configMap", + Type: "ConfigMap", }) } } @@ -149,24 +81,79 @@ func MapPodGet(i interface{}) (*sdp.Item, error) { for _, envFrom := range container.EnvFrom { if envFrom.SecretRef != nil { // Add linked item from spec.containers[].EnvFrom[].secretKeyRef - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + queries = append(queries, &sdp.Query{ + Scope: scope, Method: sdp.QueryMethod_GET, Query: envFrom.SecretRef.Name, - Type: "secret", + Type: "Secret", }) } } } - if pod.Spec.PriorityClassName != "" { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ + if resource.Spec.PriorityClassName != "" { + queries = append(queries, &sdp.Query{ Scope: ClusterName, Method: sdp.QueryMethod_GET, - Query: pod.Spec.PriorityClassName, - Type: "priorityclassname", + Query: resource.Spec.PriorityClassName, + Type: "PriorityClass", + }) + } + + if len(resource.Status.PodIPs) > 0 { + for _, ip := range resource.Status.PodIPs { + queries = append(queries, &sdp.Query{ + Scope: "global", + Method: sdp.QueryMethod_GET, + Query: ip.IP, + Type: "ip", + }) + } + } else if resource.Status.PodIP != "" { + queries = append(queries, &sdp.Query{ + Type: "ip", + Method: sdp.QueryMethod_GET, + Query: resource.Status.PodIP, + Scope: "global", }) } - return item, nil + return queries, nil +} + +func NewPodSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Pod, *v1.PodList] { + return KubeTypeSource[*v1.Pod, *v1.PodList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "Pod", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.Pod, *v1.PodList] { + return cs.CoreV1().Pods(namespace) + }, + ListExtractor: func(list *v1.PodList) ([]*v1.Pod, error) { + extracted := make([]*v1.Pod, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + LinkedItemQueryExtractor: PodExtractor, + HealthExtractor: func(resource *v1.Pod) *sdp.Health { + switch resource.Status.Phase { + case v1.PodPending: + return sdp.Health_HEALTH_PENDING.Enum() + case v1.PodRunning: + return sdp.Health_HEALTH_OK.Enum() + case v1.PodSucceeded: + return sdp.Health_HEALTH_OK.Enum() + case v1.PodFailed: + return sdp.Health_HEALTH_ERROR.Enum() + case v1.PodUnknown: + return sdp.Health_HEALTH_UNKNOWN.Enum() + } + + return nil + }, + } } diff --git a/internal/sources/pods_test.go b/internal/sources/pods_test.go index 3af8b87..6544c95 100644 --- a/internal/sources/pods_test.go +++ b/internal/sources/pods_test.go @@ -1,47 +1,120 @@ package sources import ( + "regexp" "testing" + + "github.com/overmindtech/sdp-go" ) -var podYAML = ` -apiVersion: batch/v1 -kind: Job +var PodYAML = ` +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pod-test-serviceaccount +--- +apiVersion: v1 +kind: Secret +metadata: + name: pod-test-secret +type: Opaque +data: + username: dXNlcm5hbWU= + password: cGFzc3dvcmQ= +--- +apiVersion: v1 +kind: ConfigMap metadata: - name: hello - namespace: k8s-source-testing + name: pod-test-configmap +data: + config.ini: | + [database] + host=example.com + port=5432 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pod-test-pvc spec: - template: - # This is the pod template - spec: - containers: - - name: hello - image: busybox - command: ['sh', '-c', 'echo "Hello, Kubernetes!" && sleep 3600'] - restartPolicy: OnFailure - # The pod template ends here + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: pod-test-pod +spec: + serviceAccountName: pod-test-serviceaccount + volumes: + - name: pod-test-pvc-volume + persistentVolumeClaim: + claimName: pod-test-pvc + - name: database-config + configMap: + name: pod-test-configmap + containers: + - name: pod-test-container + image: nginx + volumeMounts: + - name: pod-test-pvc-volume + mountPath: /mnt/data + - name: database-config + mountPath: /etc/database + envFrom: + - secretRef: + name: pod-test-secret ` func TestPodSource(t *testing.T) { - var err error - var source ResourceSource - - // Create the required pod - err = CurrentCluster.Apply(podYAML) - - t.Cleanup(func() { - CurrentCluster.Delete(podYAML) - }) - - if err != nil { - t.Error(err) + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", } - source, err = PodSource(CurrentCluster.ClientSet) + source := NewPodSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) - if err != nil { - t.Error(err) + st := SourceTests{ + Source: &source, + GetQuery: "pod-test-pod", + GetScope: sd.String(), + SetupYAML: PodYAML, + GetQueryTests: QueryTests{ + { + ExpectedQueryMatches: regexp.MustCompile(`10\.`), + ExpectedType: "ip", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedScope: "global", + }, + { + ExpectedType: "ServiceAccount", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "pod-test-serviceaccount", + ExpectedScope: sd.String(), + }, + { + ExpectedType: "Secret", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "pod-test-secret", + ExpectedScope: sd.String(), + }, + { + ExpectedType: "ConfigMap", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "pod-test-configmap", + ExpectedScope: sd.String(), + }, + { + ExpectedType: "PersistentVolumeClaim", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "pod-test-pvc", + ExpectedScope: sd.String(), + }, + }, } - BasicGetListSearchTests(t, `{"labelSelector": "job-name=hello"}`, source) + st.Execute(t) } diff --git a/internal/sources/priorityclass.go b/internal/sources/priorityclass.go index 2996231..bdb37bd 100644 --- a/internal/sources/priorityclass.go +++ b/internal/sources/priorityclass.go @@ -1,72 +1,27 @@ package sources import ( - "fmt" + v1 "k8s.io/api/scheduling/v1" - coreV1 "k8s.io/api/scheduling/v1" - - "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// PriorityClassSource returns a ResourceSource for PriorityClassClaims for a given -// client -func PriorityClassSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "priorityclass", - MapGet: MapPriorityClassGet, - MapList: MapPriorityClassList, - Namespaced: false, - } - - err := source.LoadFunction( - cs.SchedulingV1().PriorityClasses, - ) - - return source, err -} - -// MapPriorityClassList maps an interface that is underneath a -// *coreV1.PriorityClassList to a list of Items -func MapPriorityClassList(i interface{}) ([]*sdp.Item, error) { - var objectList *coreV1.PriorityClassList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*coreV1.PriorityClassList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.ClusterRoleBindingList", i) - } - - for _, object := range objectList.Items { - if item, err = MapPriorityClassGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } +func NewPriorityClassSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.PriorityClass, *v1.PriorityClassList] { + return KubeTypeSource[*v1.PriorityClass, *v1.PriorityClassList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "PriorityClass", + ClusterInterfaceBuilder: func() ItemInterface[*v1.PriorityClass, *v1.PriorityClassList] { + return cs.SchedulingV1().PriorityClasses() + }, + ListExtractor: func(list *v1.PriorityClassList) ([]*v1.PriorityClass, error) { + extracted := make([]*v1.PriorityClass, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, } - - return items, nil -} - -// MapPriorityClassGet maps an interface that is underneath a *coreV1.PriorityClass to an item. If -// the interface isn't actually a *coreV1.PriorityClass this will fail -func MapPriorityClassGet(i interface{}) (*sdp.Item, error) { - var object *coreV1.PriorityClass - var ok bool - - // Expect this to be a *coreV1.PriorityClass - if object, ok = i.(*coreV1.PriorityClass); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.PriorityClass", i) - } - - item, err := mapK8sObject("priorityclass", object) - - if err != nil { - return &sdp.Item{}, err - } - - return item, nil } diff --git a/internal/sources/priorityclass_test.go b/internal/sources/priorityclass_test.go new file mode 100644 index 0000000..0a49554 --- /dev/null +++ b/internal/sources/priorityclass_test.go @@ -0,0 +1,34 @@ +package sources + +import ( + "testing" +) + +var priorityClassYAML = ` +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: ultra-mega-priority +value: 1000000 +globalDefault: false +description: "This priority class should be used for ultra-mega-priority workloads" +` + +func TestPriorityClassSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewPriorityClassSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "ultra-mega-priority", + GetScope: sd.String(), + SetupYAML: priorityClassYAML, + GetQueryTests: QueryTests{}, + } + + st.Execute(t) +} diff --git a/internal/sources/replicaset.go b/internal/sources/replicaset.go index 19b3b27..b359c19 100644 --- a/internal/sources/replicaset.go +++ b/internal/sources/replicaset.go @@ -1,84 +1,44 @@ package sources import ( - "fmt" - - appsV1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/apps/v1" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// ReplicaSetSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func ReplicaSetSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "replicaset", - MapGet: MapReplicaSetGet, - MapList: MapReplicaSetList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.AppsV1().ReplicaSets, - ) - - return source, err -} - -// MapReplicaSetList maps an interface that is underneath a -// *appsV1.ReplicaSetList to a list of Items -func MapReplicaSetList(i interface{}) ([]*sdp.Item, error) { - var objectList *appsV1.ReplicaSetList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*appsV1.ReplicaSetList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *appsV1.ReplicaSetList", i) - } +func replicaSetExtractor(resource *v1.ReplicaSet, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) - for _, object := range objectList.Items { - if item, err = MapReplicaSetGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } + if resource.Spec.Selector != nil { + queries = append(queries, &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(resource.Spec.Selector), + Type: "Pod", + }) } - return items, nil + return queries, nil } -// MapReplicaSetGet maps an interface that is underneath a *appsV1.ReplicaSet to an item. If -// the interface isn't actually a *appsV1.ReplicaSet this will fail -func MapReplicaSetGet(i interface{}) (*sdp.Item, error) { - var object *appsV1.ReplicaSet - var ok bool - - // Expect this to be a *appsV1.ReplicaSet - if object, ok = i.(*appsV1.ReplicaSet); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *appsV1.ReplicaSet", i) - } - - item, err := mapK8sObject("replicaset", object) - - if err != nil { - return &sdp.Item{}, err +func NewReplicaSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ReplicaSet, *v1.ReplicaSetList] { + return KubeTypeSource[*v1.ReplicaSet, *v1.ReplicaSetList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "ReplicaSet", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.ReplicaSet, *v1.ReplicaSetList] { + return cs.AppsV1().ReplicaSets(namespace) + }, + ListExtractor: func(list *v1.ReplicaSetList) ([]*v1.ReplicaSet, error) { + extracted := make([]*v1.ReplicaSet, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + LinkedItemQueryExtractor: replicaSetExtractor, } - - if object.Spec.Selector != nil { - item.LinkedItemQueries = []*sdp.Query{ - // Services are linked to pods via their selector - { - Scope: item.Scope, - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(object.Spec.Selector), - Type: "pod", - }, - } - } - - return item, nil } diff --git a/internal/sources/replicaset_test.go b/internal/sources/replicaset_test.go new file mode 100644 index 0000000..867fd12 --- /dev/null +++ b/internal/sources/replicaset_test.go @@ -0,0 +1,57 @@ +package sources + +import ( + "regexp" + "testing" + + "github.com/overmindtech/sdp-go" +) + +var replicaSetYAML = ` +apiVersion: apps/v1 +kind: ReplicaSet +metadata: + name: replica-set-test +spec: + replicas: 1 + selector: + matchLabels: + app: replica-set-test + template: + metadata: + labels: + app: replica-set-test + spec: + containers: + - name: replica-set-test + image: nginx:latest + ports: + - containerPort: 80 + +` + +func TestReplicaSetSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewReplicaSetSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "replica-set-test", + GetScope: sd.String(), + SetupYAML: replicaSetYAML, + GetQueryTests: QueryTests{ + { + ExpectedQueryMatches: regexp.MustCompile("app=replica-set-test"), + ExpectedType: "Pod", + ExpectedMethod: sdp.QueryMethod_SEARCH, + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/replicationcontroller.go b/internal/sources/replicationcontroller.go index 26d28bf..d8848a5 100644 --- a/internal/sources/replicationcontroller.go +++ b/internal/sources/replicationcontroller.go @@ -1,86 +1,46 @@ package sources import ( - "fmt" - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) -// ReplicationControllerSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func ReplicationControllerSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "replicationcontroller", - MapGet: MapReplicationControllerGet, - MapList: MapReplicationControllerList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.CoreV1().ReplicationControllers, - ) - - return source, err -} - -// MapReplicationControllerList maps an interface that is underneath a -// *coreV1.ReplicationControllerList to a list of Items -func MapReplicationControllerList(i interface{}) ([]*sdp.Item, error) { - var objectList *coreV1.ReplicationControllerList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*coreV1.ReplicationControllerList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.objectList", i) - } - - for _, object := range objectList.Items { - if item, err = MapReplicationControllerGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } +func replicationControllerExtractor(resource *v1.ReplicationController, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) + + if resource.Spec.Selector != nil { + queries = append(queries, &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(&metaV1.LabelSelector{ + MatchLabels: resource.Spec.Selector, + }), + Type: "Pod", + }) } - return items, nil + return queries, nil } -// MapReplicationControllerGet maps an interface that is underneath a *coreV1.ReplicationController to an item. If -// the interface isn't actually a *coreV1.ReplicationController this will fail -func MapReplicationControllerGet(i interface{}) (*sdp.Item, error) { - var object *coreV1.ReplicationController - var ok bool - - // Expect this to be a *coreV1.ReplicationController - if object, ok = i.(*coreV1.ReplicationController); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.ReplicationController", i) - } - - item, err := mapK8sObject("replicationcontroller", object) - - if err != nil { - return &sdp.Item{}, err - } - - if object.Spec.Selector != nil { - item.LinkedItemQueries = []*sdp.Query{ - // Replication controllers are linked to pods via their selector - { - Scope: item.Scope, - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(&metaV1.LabelSelector{ - MatchLabels: object.Spec.Selector, - }), - Type: "pod", - }, - } +func NewReplicationControllerSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ReplicationController, *v1.ReplicationControllerList] { + return KubeTypeSource[*v1.ReplicationController, *v1.ReplicationControllerList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "ReplicationController", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.ReplicationController, *v1.ReplicationControllerList] { + return cs.CoreV1().ReplicationControllers(namespace) + }, + ListExtractor: func(list *v1.ReplicationControllerList) ([]*v1.ReplicationController, error) { + extracted := make([]*v1.ReplicationController, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + LinkedItemQueryExtractor: replicationControllerExtractor, } - - return item, nil } diff --git a/internal/sources/replicationcontroller_test.go b/internal/sources/replicationcontroller_test.go new file mode 100644 index 0000000..5ad1299 --- /dev/null +++ b/internal/sources/replicationcontroller_test.go @@ -0,0 +1,56 @@ +package sources + +import ( + "regexp" + "testing" + + "github.com/overmindtech/sdp-go" +) + +var replicationControllerYAML = ` +apiVersion: v1 +kind: ReplicationController +metadata: + name: replication-controller-test +spec: + replicas: 1 + selector: + app: replication-controller-test + template: + metadata: + labels: + app: replication-controller-test + spec: + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 + +` + +func TestReplicationControllerSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewReplicationControllerSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "replication-controller-test", + GetScope: sd.String(), + SetupYAML: replicationControllerYAML, + GetQueryTests: QueryTests{ + { + ExpectedQueryMatches: regexp.MustCompile("app=replication-controller-test"), + ExpectedType: "Pod", + ExpectedMethod: sdp.QueryMethod_SEARCH, + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/resourcequota.go b/internal/sources/resourcequota.go index 7095902..b4b586f 100644 --- a/internal/sources/resourcequota.go +++ b/internal/sources/resourcequota.go @@ -1,71 +1,26 @@ package sources import ( - "fmt" - - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -// ResourceQuotaSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func ResourceQuotaSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "resourcequota", - MapGet: MapResourceQuotaGet, - MapList: MapResourceQuotaList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.CoreV1().ResourceQuotas, - ) - - return source, err -} - -// MapResourceQuotaList maps an interface that is underneath a -// *coreV1.ResourceQuotaList to a list of Items -func MapResourceQuotaList(i interface{}) ([]*sdp.Item, error) { - var objectList *coreV1.ResourceQuotaList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*coreV1.ResourceQuotaList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.ResourceQuotaList", i) - } - - for _, object := range objectList.Items { - if item, err = MapResourceQuotaGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } +func NewResourceQuotaSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ResourceQuota, *v1.ResourceQuotaList] { + return KubeTypeSource[*v1.ResourceQuota, *v1.ResourceQuotaList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "ResourceQuota", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.ResourceQuota, *v1.ResourceQuotaList] { + return cs.CoreV1().ResourceQuotas(namespace) + }, + ListExtractor: func(list *v1.ResourceQuotaList) ([]*v1.ResourceQuota, error) { + extracted := make([]*v1.ResourceQuota, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, } - - return items, nil -} - -// MapResourceQuotaGet maps an interface that is underneath a *coreV1.ResourceQuota to an item. If -// the interface isn't actually a *coreV1.ResourceQuota this will fail -func MapResourceQuotaGet(i interface{}) (*sdp.Item, error) { - var object *coreV1.ResourceQuota - var ok bool - - // Expect this to be a *coreV1.ResourceQuota - if object, ok = i.(*coreV1.ResourceQuota); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.ResourceQuota", i) - } - - item, err := mapK8sObject("resourcequota", object) - - if err != nil { - return &sdp.Item{}, err - } - - return item, nil } diff --git a/internal/sources/resourcequota_test.go b/internal/sources/resourcequota_test.go new file mode 100644 index 0000000..b2aa58c --- /dev/null +++ b/internal/sources/resourcequota_test.go @@ -0,0 +1,37 @@ +package sources + +import ( + "testing" +) + +var resourceQuotaYAML = ` +apiVersion: v1 +kind: ResourceQuota +metadata: + name: quota-example +spec: + hard: + pods: "10" + requests.cpu: "2" + requests.memory: 2Gi + limits.cpu: "4" + limits.memory: 4Gi +` + +func TestResourceQuotaSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewResourceQuotaSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "quota-example", + GetScope: sd.String(), + SetupYAML: resourceQuotaYAML, + } + + st.Execute(t) +} diff --git a/internal/sources/role.go b/internal/sources/role.go index 465991f..c20af07 100644 --- a/internal/sources/role.go +++ b/internal/sources/role.go @@ -1,72 +1,27 @@ package sources import ( - "fmt" + v1 "k8s.io/api/rbac/v1" - rbacV1 "k8s.io/api/rbac/v1" - - "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// RoleSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func RoleSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "role", - MapGet: MapRoleGet, - MapList: MapRoleList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.RbacV1().Roles, - ) - - return source, err -} - -// MapRoleList maps an interface that is underneath a -// *rbacV1.RoleList to a list of Items -func MapRoleList(i interface{}) ([]*sdp.Item, error) { - var objectList *rbacV1.RoleList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*rbacV1.RoleList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *rbacV1.RoleList", i) - } - - for _, object := range objectList.Items { - if item, err = MapRoleGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } +func NewRoleSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Role, *v1.RoleList] { + return KubeTypeSource[*v1.Role, *v1.RoleList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "Role", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.Role, *v1.RoleList] { + return cs.RbacV1().Roles(namespace) + }, + ListExtractor: func(list *v1.RoleList) ([]*v1.Role, error) { + extracted := make([]*v1.Role, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, } - - return items, nil -} - -// MapRoleGet maps an interface that is underneath a *rbacV1.Role to an item. If -// the interface isn't actually a *rbacV1.Role this will fail -func MapRoleGet(i interface{}) (*sdp.Item, error) { - var object *rbacV1.Role - var ok bool - - // Expect this to be a *rbacV1.Role - if object, ok = i.(*rbacV1.Role); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *rbacV1.Role", i) - } - - item, err := mapK8sObject("role", object) - - if err != nil { - return &sdp.Item{}, err - } - - return item, nil } diff --git a/internal/sources/role_test.go b/internal/sources/role_test.go index 5ef1e6d..5dd6e9f 100644 --- a/internal/sources/role_test.go +++ b/internal/sources/role_test.go @@ -4,38 +4,47 @@ import ( "testing" ) -var roleYAML = ` +var RoleYAML = ` apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - namespace: k8s-source-testing - name: pod-reader + name: role-test-role rules: -- apiGroups: [""] # "" indicates the core API group - resources: ["pods"] - verbs: ["get", "watch", "list"] + - apiGroups: + - "" + - "apps" + - "batch" + - "extensions" + resources: + - pods + - deployments + - jobs + - cronjobs + - configmaps + - secrets + verbs: + - get + - list + - watch + - create + - update + - delete ` func TestRoleSource(t *testing.T) { - var err error - var source ResourceSource - - // Create the required pod - err = CurrentCluster.Apply(roleYAML) - - t.Cleanup(func() { - CurrentCluster.Delete(roleYAML) - }) - - if err != nil { - t.Error(err) + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", } - source, err = RoleSource(CurrentCluster.ClientSet) + source := NewRoleSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) - if err != nil { - t.Error(err) + st := SourceTests{ + Source: &source, + GetQuery: "role-test-role", + GetScope: sd.String(), + SetupYAML: RoleYAML, } - BasicGetListSearchTests(t, `{"fieldSelector": "metadata.name=pod-reader"}`, source) + st.Execute(t) } diff --git a/internal/sources/rolebinding.go b/internal/sources/rolebinding.go index 20bce83..c7de6a1 100644 --- a/internal/sources/rolebinding.go +++ b/internal/sources/rolebinding.go @@ -1,109 +1,75 @@ package sources import ( - "fmt" - "strings" - - rbacV1 "k8s.io/api/rbac/v1" + v1 "k8s.io/api/rbac/v1" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// RoleBindingSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func RoleBindingSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "rolebinding", - MapGet: MapRoleBindingGet, - MapList: MapRoleBindingList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.RbacV1().RoleBindings, - ) +func roleBindingExtractor(resource *v1.RoleBinding, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) - return source, err -} - -// MapRoleBindingList maps an interface that is underneath a -// *rbacV1.RoleBindingList to a list of Items -func MapRoleBindingList(i interface{}) ([]*sdp.Item, error) { - var objectList *rbacV1.RoleBindingList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error + sd, err := ParseScope(scope, true) - // Expect this to be a objectList - if objectList, ok = i.(*rbacV1.RoleBindingList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *rbacV1.RoleBindingList", i) - } - - for _, object := range objectList.Items { - if item, err = MapRoleBindingGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } + if err != nil { + return nil, err } - return items, nil -} - -// MapRoleBindingGet maps an interface that is underneath a *rbacV1.RoleBinding to an item. If -// the interface isn't actually a *rbacV1.RoleBinding this will fail -func MapRoleBindingGet(i interface{}) (*sdp.Item, error) { - var object *rbacV1.RoleBinding - var ok bool - - // Expect this to be a *rbacV1.RoleBinding - if object, ok = i.(*rbacV1.RoleBinding); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *rbacV1.RoleBinding", i) + for _, subject := range resource.Subjects { + queries = append(queries, &sdp.Query{ + Method: sdp.QueryMethod_GET, + Query: subject.Name, + Type: subject.Kind, + Scope: ScopeDetails{ + ClusterName: sd.ClusterName, + Namespace: subject.Namespace, + }.String(), + }) } - item, err := mapK8sObject("rolebinding", object) - - if err != nil { - return &sdp.Item{}, err + refSD := ScopeDetails{ + ClusterName: sd.ClusterName, } - // Link the referenced role - var scope string - - switch object.RoleRef.Name { + switch resource.RoleRef.Kind { case "Role": // If this binding is linked to a role then it's in the same namespace - scope = item.Scope + refSD.Namespace = sd.Namespace case "ClusterRole": // If this is linked to a ClusterRole (which is not namespaced) we need // to make sure that we are querying the root scope i.e. the // non-namespaced scope - scope = ClusterName + refSD.Namespace = "" } - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: scope, + queries = append(queries, &sdp.Query{ + Scope: refSD.String(), Method: sdp.QueryMethod_GET, - Query: object.RoleRef.Name, - Type: strings.ToLower(object.RoleRef.Kind), + Query: resource.RoleRef.Name, + Type: resource.RoleRef.Kind, }) - for _, subject := range object.Subjects { - if subject.Namespace == "" { - scope = ClusterName - } else { - scope = ClusterName + "." + subject.Namespace - } + return queries, nil +} - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_GET, - Query: subject.Name, - Type: strings.ToLower(subject.Kind), - }) +func NewRoleBindingSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.RoleBinding, *v1.RoleBindingList] { + return KubeTypeSource[*v1.RoleBinding, *v1.RoleBindingList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "RoleBinding", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.RoleBinding, *v1.RoleBindingList] { + return cs.RbacV1().RoleBindings(namespace) + }, + ListExtractor: func(list *v1.RoleBindingList) ([]*v1.RoleBinding, error) { + extracted := make([]*v1.RoleBinding, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + LinkedItemQueryExtractor: roleBindingExtractor, } - - return item, nil } diff --git a/internal/sources/rolebinding_test.go b/internal/sources/rolebinding_test.go new file mode 100644 index 0000000..28deaef --- /dev/null +++ b/internal/sources/rolebinding_test.go @@ -0,0 +1,127 @@ +package sources + +import ( + "testing" + + "github.com/overmindtech/sdp-go" +) + +var roleBindingYAML = ` +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rb-test-service-account +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rb-test-role +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rb-test-role-binding +subjects: +- kind: ServiceAccount + name: rb-test-service-account + namespace: default +roleRef: + kind: Role + name: rb-test-role + apiGroup: rbac.authorization.k8s.io +--- +` + +var roleBindingYAML2 = ` +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rb-test-service-account2 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rb-test-cluster-role +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rb-test-role-binding-cluster + namespace: default +roleRef: + kind: ClusterRole + name: rb-test-cluster-role + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: rb-test-service-account2 + namespace: default +` + +func TestRoleBindingSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewRoleBindingSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + t.Run("With a Role", func(t *testing.T) { + st := SourceTests{ + Source: &source, + GetQuery: "rb-test-role-binding", + GetScope: sd.String(), + SetupYAML: roleBindingYAML, + GetQueryTests: QueryTests{ + { + ExpectedType: "Role", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "rb-test-role", + ExpectedScope: sd.String(), + }, + { + ExpectedType: "ServiceAccount", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "rb-test-service-account", + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) + }) + + t.Run("With a ClusterRole", func(t *testing.T) { + st := SourceTests{ + Source: &source, + GetQuery: "rb-test-role-binding-cluster", + GetScope: sd.String(), + SetupYAML: roleBindingYAML2, + GetQueryTests: QueryTests{ + { + ExpectedType: "ClusterRole", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "rb-test-cluster-role", + ExpectedScope: sd.ClusterName, + }, + { + ExpectedType: "ServiceAccount", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "rb-test-service-account2", + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) + }) + +} diff --git a/internal/sources/secret.go b/internal/sources/secret.go index bfda104..62ed930 100644 --- a/internal/sources/secret.go +++ b/internal/sources/secret.go @@ -1,91 +1,47 @@ package sources import ( - "fmt" + "crypto/sha512" - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -// SecretSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -// -// Secret Sauce -// -// * 1/4 cup salad dressing (like Miracle Whip) -// * 1/4 cup mayonnaise -// * 3 tablespoons French salad dressing (Wishbone brand) -// * 1/2 tablespoon sweet pickle relish (Heinz brand) -// * 1 1/2 tablespoons dill pickle relish (Vlasic or Heinz brand) -// * 1 teaspoon sugar -// * 1 teaspoon dried minced onion -// * 1 teaspoon white vinegar -// * 1 teaspoon ketchup -// * 1/8 teaspoon salt -func SecretSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "secret", - MapGet: MapSecretGet, - MapList: MapSecretList, - Namespaced: true, +// TODO: Configure redaction +func NewSecretSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Secret, *v1.SecretList] { + return KubeTypeSource[*v1.Secret, *v1.SecretList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "Secret", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.Secret, *v1.SecretList] { + return cs.CoreV1().Secrets(namespace) + }, + ListExtractor: func(list *v1.SecretList) ([]*v1.Secret, error) { + extracted := make([]*v1.Secret, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + Redact: func(resource *v1.Secret) *v1.Secret { + // We want to redact the data from a secret, but we also went to + // show people when it has changed, to that end we will hash all of + // the data in the secret and return the hash + hash := sha512.New() + + for k, v := range resource.Data { + // Write the data into the hash + hash.Write([]byte(k)) + hash.Write(v) + } + + resource.Data = map[string][]byte{ + "data-redacted": hash.Sum(nil), + } + + return resource + }, } - - err := source.LoadFunction( - cs.CoreV1().Secrets, - ) - - return source, err -} - -// MapSecretList maps an interface that is underneath a -// *coreV1.SecretList to a list of Items -func MapSecretList(i interface{}) ([]*sdp.Item, error) { - var objectList *coreV1.SecretList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*coreV1.SecretList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.SecretList", i) - } - - for _, object := range objectList.Items { - if item, err = MapSecretGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} - -// MapSecretGet maps an interface that is underneath a *coreV1.Secret to an item. If -// the interface isn't actually a *coreV1.Secret this will fail -func MapSecretGet(i interface{}) (*sdp.Item, error) { - var object *coreV1.Secret - var ok bool - - // Expect this to be a *coreV1.Secret - if object, ok = i.(*coreV1.Secret); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.Secret", i) - } - - // Redact data - for k := range object.Data { - // Set the data to some binary which base64 encodes to the word: - // REDACTED. Is this dumb or smart...? - object.Data[k] = []byte{68, 64, 192, 9, 49, 3} - } - - item, err := mapK8sObject("secret", object) - - if err != nil { - return &sdp.Item{}, err - } - - return item, nil } diff --git a/internal/sources/secret_test.go b/internal/sources/secret_test.go index 74648fa..20723a7 100644 --- a/internal/sources/secret_test.go +++ b/internal/sources/secret_test.go @@ -8,34 +8,28 @@ var secretYAML = ` apiVersion: v1 kind: Secret metadata: - name: secret-basic-auth - namespace: k8s-source-testing -type: kubernetes.io/basic-auth -stringData: - username: admin - password: t0p-Secret + name: secret-test-secret +type: Opaque +data: + username: dXNlcm5hbWUx # base64-encoded "username1" + password: cGFzc3dvcmQx # base64-encoded "password1" + ` func TestSecretSource(t *testing.T) { - var err error - var source ResourceSource - - // Create the required pod - err = CurrentCluster.Apply(secretYAML) - - t.Cleanup(func() { - CurrentCluster.Delete(secretYAML) - }) - - if err != nil { - t.Error(err) + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", } - source, err = SecretSource(CurrentCluster.ClientSet) + source := NewSecretSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) - if err != nil { - t.Error(err) + st := SourceTests{ + Source: &source, + GetQuery: "secret-test-secret", + GetScope: sd.String(), + SetupYAML: secretYAML, } - BasicGetListSearchTests(t, `{"fieldSelector": "metadata.name=secret-basic-auth"}`, source) + st.Execute(t) } diff --git a/internal/sources/service.go b/internal/sources/service.go index 113977e..3b54add 100644 --- a/internal/sources/service.go +++ b/internal/sources/service.go @@ -1,91 +1,105 @@ package sources import ( - "fmt" - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) -// ServiceSource returns a ResourceSource for Pods for a given client and namespace -func ServiceSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "service", - MapGet: MapServiceGet, - MapList: MapServiceList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.CoreV1().Services, - ) +func serviceExtractor(resource *v1.Service, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) - return source, err -} + if resource.Spec.Selector != nil { + queries = append(queries, &sdp.Query{ + Type: "Pod", + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(&metaV1.LabelSelector{ + MatchLabels: resource.Spec.Selector, + }), + Scope: scope, + }) + } -// MapServiceList Maps the interface output of our list function to a list of -// items -func MapServiceList(i interface{}) ([]*sdp.Item, error) { - var services *coreV1.ServiceList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error + ips := make([]string, 0) - if services, ok = i.(*coreV1.ServiceList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.ServiceList", i) + if len(resource.Spec.ClusterIPs) > 0 { + ips = append(ips, resource.Spec.ClusterIPs...) + } else if resource.Spec.ClusterIP != "" { + ips = append(ips, resource.Spec.ClusterIP) } - for _, service := range services.Items { - if item, err = MapServiceGet(&service); err == nil { - items = append(items, item) - } else { - return items, err + ips = append(ips, resource.Spec.ExternalIPs...) + ips = append(ips, resource.Spec.LoadBalancerIP) + + for _, ip := range ips { + if ip != "" { + queries = append(queries, &sdp.Query{ + Type: "ip", + Method: sdp.QueryMethod_GET, + Query: ip, + Scope: "global", + }) } } - return items, nil -} - -// MapServiceGet Maps an interface (which is the result of out Get function) to -// a service item -func MapServiceGet(i interface{}) (*sdp.Item, error) { - var s *coreV1.Service - var ok bool - - // Assert that i is a Service - if s, ok = i.(*coreV1.Service); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.Service", i) + if resource.Spec.ExternalName != "" { + queries = append(queries, &sdp.Query{ + Type: "dns", + Method: sdp.QueryMethod_GET, + Query: resource.Spec.ExternalName, + Scope: "global", + }) } - item, err := mapK8sObject("service", s) + // Services also generate an endpoint with the same name + queries = append(queries, &sdp.Query{ + Type: "Endpoint", + Method: sdp.QueryMethod_GET, + Query: resource.Name, + Scope: scope, + }) + + for _, ingress := range resource.Status.LoadBalancer.Ingress { + if ingress.IP != "" { + queries = append(queries, &sdp.Query{ + Type: "ip", + Method: sdp.QueryMethod_GET, + Query: ingress.IP, + Scope: "global", + }) + } - if err != nil { - return &sdp.Item{}, err + if ingress.Hostname != "" { + queries = append(queries, &sdp.Query{ + Type: "dns", + Method: sdp.QueryMethod_GET, + Query: ingress.Hostname, + Scope: "global", + }) + } } - item.LinkedItemQueries = []*sdp.Query{ - { - Scope: item.Scope, - Method: sdp.QueryMethod_GET, - Query: fmt.Sprint(item.UniqueAttributeValue()), - Type: "endpoint", + return queries, nil +} + +func NewServiceSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Service, *v1.ServiceList] { + return KubeTypeSource[*v1.Service, *v1.ServiceList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "Service", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.Service, *v1.ServiceList] { + return cs.CoreV1().Services(namespace) }, - } + ListExtractor: func(list *v1.ServiceList) ([]*v1.Service, error) { + extracted := make([]*v1.Service, len(list.Items)) - if sel := s.Spec.Selector; sel != nil { - // Services are linked to pods via their selector - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(&metaV1.LabelSelector{ - MatchLabels: sel, - }), - Type: "pod", - }) - } + for i := range list.Items { + extracted[i] = &list.Items[i] + } - return item, nil + return extracted, nil + }, + LinkedItemQueryExtractor: serviceExtractor, + } } diff --git a/internal/sources/service_test.go b/internal/sources/service_test.go index a294ad8..cb5bb92 100644 --- a/internal/sources/service_test.go +++ b/internal/sources/service_test.go @@ -1,68 +1,83 @@ package sources import ( + "regexp" "testing" + + "github.com/overmindtech/sdp-go" ) var serviceYAML = ` ---- apiVersion: apps/v1 kind: Deployment metadata: - name: nginx-deployment - namespace: k8s-source-testing - labels: - app: nginx + name: service-test-deployment spec: - replicas: 3 selector: matchLabels: - app: nginx + app: service-test + replicas: 1 template: metadata: labels: - app: nginx + app: service-test spec: containers: - - name: nginx - image: nginx:1.14.2 + - name: my-container + image: nginx ports: - - containerPort: 80 + - containerPort: 8080 --- apiVersion: v1 kind: Service metadata: - name: nginx-service - namespace: k8s-source-testing + name: service-test-service spec: selector: - app: nginx + app: service-test ports: - - protocol: TCP - port: 80 - targetPort: 80 + - name: http + protocol: TCP + port: 80 + targetPort: 8080 + type: LoadBalancer + externalName: service-test-external ` func TestServiceSource(t *testing.T) { - var err error - var source ResourceSource - - // Create the required pod - err = CurrentCluster.Apply(serviceYAML) - - t.Cleanup(func() { - CurrentCluster.Delete(serviceYAML) - }) - - if err != nil { - t.Error(err) + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", } - source, err = ServiceSource(CurrentCluster.ClientSet) + source := NewServiceSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) - if err != nil { - t.Error(err) + st := SourceTests{ + Source: &source, + GetQuery: "service-test-service", + GetScope: sd.String(), + SetupYAML: serviceYAML, + GetQueryTests: QueryTests{ + { + ExpectedType: "Pod", + ExpectedMethod: sdp.QueryMethod_SEARCH, + ExpectedScope: sd.String(), + ExpectedQueryMatches: regexp.MustCompile(`app=service-test`), + }, + { + ExpectedType: "Endpoint", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "service-test-service", + ExpectedScope: sd.String(), + }, + { + ExpectedType: "dns", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "service-test-external", + ExpectedScope: "global", + }, + }, } - BasicGetListSearchTests(t, `{"fieldSelector": "metadata.name=nginx-service"}`, source) + st.Execute(t) } diff --git a/internal/sources/serviceaccount.go b/internal/sources/serviceaccount.go index b3f2f18..0af38cb 100644 --- a/internal/sources/serviceaccount.go +++ b/internal/sources/serviceaccount.go @@ -1,89 +1,52 @@ package sources import ( - "fmt" - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -// ServiceAccountSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func ServiceAccountSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "serviceaccount", - MapGet: MapServiceAccountGet, - MapList: MapServiceAccountList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.CoreV1().ServiceAccounts, - ) - - return source, err -} - -// MapServiceAccountList maps an interface that is underneath a -// *coreV1.ServiceAccountList to a list of Items -func MapServiceAccountList(i interface{}) ([]*sdp.Item, error) { - var objectList *coreV1.ServiceAccountList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error +func serviceAccountExtractor(resource *v1.ServiceAccount, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) - // Expect this to be a objectList - if objectList, ok = i.(*coreV1.ServiceAccountList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *coreV1.ServiceAccountList", i) - } - - for _, object := range objectList.Items { - if item, err = MapServiceAccountGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} - -// MapServiceAccountGet maps an interface that is underneath a *coreV1.ServiceAccount to an item. If -// the interface isn't actually a *coreV1.ServiceAccount this will fail -func MapServiceAccountGet(i interface{}) (*sdp.Item, error) { - var object *coreV1.ServiceAccount - var ok bool - - // Expect this to be a *coreV1.ServiceAccount - if object, ok = i.(*coreV1.ServiceAccount); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *coreV1.ServiceAccount", i) - } - - item, err := mapK8sObject("serviceaccount", object) - - if err != nil { - return &sdp.Item{}, err - } - - for _, secret := range object.Secrets { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + for _, secret := range resource.Secrets { + queries = append(queries, &sdp.Query{ + Scope: scope, Method: sdp.QueryMethod_GET, Query: secret.Name, - Type: "secret", + Type: "Secret", }) } - for _, ipSecret := range object.ImagePullSecrets { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + for _, ipSecret := range resource.ImagePullSecrets { + queries = append(queries, &sdp.Query{ + Scope: scope, Method: sdp.QueryMethod_GET, Query: ipSecret.Name, - Type: "secret", + Type: "Secret", }) } - return item, nil + return queries, nil +} + +func NewServiceAccountSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ServiceAccount, *v1.ServiceAccountList] { + return KubeTypeSource[*v1.ServiceAccount, *v1.ServiceAccountList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "ServiceAccount", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.ServiceAccount, *v1.ServiceAccountList] { + return cs.CoreV1().ServiceAccounts(namespace) + }, + ListExtractor: func(list *v1.ServiceAccountList) ([]*v1.ServiceAccount, error) { + extracted := make([]*v1.ServiceAccount, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + LinkedItemQueryExtractor: serviceAccountExtractor, + } } diff --git a/internal/sources/serviceaccount_test.go b/internal/sources/serviceaccount_test.go new file mode 100644 index 0000000..7089724 --- /dev/null +++ b/internal/sources/serviceaccount_test.go @@ -0,0 +1,67 @@ +package sources + +import ( + "testing" + + "github.com/overmindtech/sdp-go" +) + +var serviceAccountYAML = ` +apiVersion: v1 +kind: Secret +metadata: + name: service-account-secret +type: Opaque +data: + username: Zm9vCg== + password: Zm9vCg== +--- +apiVersion: v1 +kind: Secret +metadata: + name: service-account-secret-pull +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: eyJhdXRocyI6eyJnaGNyLmlvIjp7InVzZXJuYW1lIjoiaHVudGVyIiwicGFzc3dvcmQiOiJodW50ZXIyIiwiZW1haWwiOiJmb29AYmFyLmNvbSIsImF1dGgiOiJhSFZ1ZEdWeU9taDFiblJsY2pJPSJ9fX0= +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-service-account +secrets: +- name: service-account-secret +imagePullSecrets: +- name: service-account-secret-pull +` + +func TestServiceAccountSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewServiceAccountSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "test-service-account", + GetScope: sd.String(), + SetupYAML: serviceAccountYAML, + GetQueryTests: QueryTests{ + { + ExpectedType: "Secret", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "service-account-secret", + ExpectedScope: sd.String(), + }, + { + ExpectedType: "Secret", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "service-account-secret-pull", + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/shared_resourcesource.go b/internal/sources/shared_resourcesource.go index 8917cdb..376ba77 100644 --- a/internal/sources/shared_resourcesource.go +++ b/internal/sources/shared_resourcesource.go @@ -42,21 +42,7 @@ type NonNamespacedSourceFunction func(cs *kubernetes.Clientset) ResourceSource type SourceFunction func(cs *kubernetes.Clientset) (ResourceSource, error) // SourceFunctions is the list of functions to load -var SourceFunctions = []SourceFunction{ - PodSource, - ServiceSource, - SecretSource, - ServiceAccountSource, - ReplicationControllerSource, - ResourceQuotaSource, - ReplicaSetSource, - StatefulSetSource, - PodDisruptionBudgetSource, - RoleBindingSource, - RoleSource, - StorageClassSource, - PriorityClassSource, -} +var SourceFunctions = []SourceFunction{} // ResourceSource represents a source of Kubernetes resources. one of these // sources needs to be created, and then have its get and list functions diff --git a/internal/sources/statefulset.go b/internal/sources/statefulset.go index b8587a5..ade78c5 100644 --- a/internal/sources/statefulset.go +++ b/internal/sources/statefulset.go @@ -1,99 +1,69 @@ package sources import ( - "fmt" - - appsV1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/apps/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// StatefulSetSource returns a ResourceSource for PersistentVolumeClaims for a given -// client and namespace -func StatefulSetSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "statefulset", - MapGet: MapStatefulSetGet, - MapList: MapStatefulSetList, - Namespaced: true, - } - - err := source.LoadFunction( - cs.AppsV1().StatefulSets, - ) - - return source, err -} - -// MapStatefulSetList maps an interface that is underneath a -// *appsV1.StatefulSetList to a list of Items -func MapStatefulSetList(i interface{}) ([]*sdp.Item, error) { - var objectList *appsV1.StatefulSetList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*appsV1.StatefulSetList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *appsV1.StatefulSetList", i) - } - - for _, object := range objectList.Items { - if item, err = MapStatefulSetGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } - } - - return items, nil -} +func statefulSetExtractor(resource *v1.StatefulSet, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) -// MapStatefulSetGet maps an interface that is underneath a *appsV1.StatefulSet to an item. If -// the interface isn't actually a *appsV1.StatefulSet this will fail -func MapStatefulSetGet(i interface{}) (*sdp.Item, error) { - var object *appsV1.StatefulSet - var ok bool - - // Expect this to be a *appsV1.StatefulSet - if object, ok = i.(*appsV1.StatefulSet); !ok { - return &sdp.Item{}, fmt.Errorf("could not assert %v as a *appsV1.StatefulSet", i) - } - - item, err := mapK8sObject("statefulset", object) - - if err != nil { - return &sdp.Item{}, err - } - - item.LinkedItemQueries = make([]*sdp.Query, 0) - - if object.Spec.Selector != nil { + if resource.Spec.Selector != nil { // Stateful sets are linked to pods via their selector - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + queries = append(queries, &sdp.Query{ + Type: "Pod", Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(object.Spec.Selector), - Type: "pod", + Query: LabelSelectorToQuery(resource.Spec.Selector), + Scope: scope, }) + + if len(resource.Spec.VolumeClaimTemplates) > 0 { + queries = append(queries, &sdp.Query{ + Type: "PersistentVolumeClaim", + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(resource.Spec.Selector), + Scope: scope, + }) + } } - if object.Spec.ServiceName != "" { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Scope: item.Scope, + if resource.Spec.ServiceName != "" { + queries = append(queries, &sdp.Query{ + Scope: scope, Method: sdp.QueryMethod_SEARCH, Query: ListOptionsToQuery(&metaV1.ListOptions{ FieldSelector: Selector{ - "metadata.name": object.Spec.ServiceName, - "metadata.namespace": object.Namespace, + "metadata.name": resource.Spec.ServiceName, + "metadata.namespace": resource.Namespace, }.String(), }), - Type: "service", + Type: "Service", }) } - return item, nil + return queries, nil +} + +func NewStatefulSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.StatefulSet, *v1.StatefulSetList] { + return KubeTypeSource[*v1.StatefulSet, *v1.StatefulSetList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "StatefulSet", + NamespacedInterfaceBuilder: func(namespace string) ItemInterface[*v1.StatefulSet, *v1.StatefulSetList] { + return cs.AppsV1().StatefulSets(namespace) + }, + ListExtractor: func(list *v1.StatefulSetList) ([]*v1.StatefulSet, error) { + extracted := make([]*v1.StatefulSet, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + LinkedItemQueryExtractor: statefulSetExtractor, + } } diff --git a/internal/sources/statefulset_test.go b/internal/sources/statefulset_test.go new file mode 100644 index 0000000..0e7a939 --- /dev/null +++ b/internal/sources/statefulset_test.go @@ -0,0 +1,90 @@ +package sources + +import ( + "regexp" + "testing" + + "github.com/overmindtech/sdp-go" +) + +var statefulSetYAML = ` +apiVersion: v1 +kind: PersistentVolume +metadata: + name: stateful-set-test-pv +spec: + capacity: + storage: 5Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: nginx-sc + hostPath: + path: /data/nginx +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: stateful-set-test +spec: + serviceName: nginx + replicas: 1 + selector: + matchLabels: + app: stateful-set-test + template: + metadata: + labels: + app: stateful-set-test + spec: + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 + volumeMounts: + - name: stateful-set-test-storage + mountPath: /usr/share/nginx/html + volumeClaimTemplates: + - metadata: + name: stateful-set-test-storage + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + storageClassName: nginx-sc + +` + +func TestStatefulSetSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewStatefulSetSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "stateful-set-test", + GetScope: sd.String(), + SetupYAML: statefulSetYAML, + GetQueryTests: QueryTests{ + { + ExpectedType: "Pod", + ExpectedMethod: sdp.QueryMethod_SEARCH, + ExpectedQueryMatches: regexp.MustCompile(`app=stateful-set-test`), + ExpectedScope: sd.String(), + }, + { + ExpectedType: "PersistentVolumeClaim", + ExpectedMethod: sdp.QueryMethod_SEARCH, + ExpectedQueryMatches: regexp.MustCompile(`app=stateful-set-test`), + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) +} diff --git a/internal/sources/storageclass.go b/internal/sources/storageclass.go index a8485bb..22dc94c 100644 --- a/internal/sources/storageclass.go +++ b/internal/sources/storageclass.go @@ -1,73 +1,27 @@ package sources import ( - "fmt" + v1 "k8s.io/api/storage/v1" - storageV1 "k8s.io/api/storage/v1" - - "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -// StorageClassSource returns a ResourceSource for StorageClassClaims for a given -// client -func StorageClassSource(cs *kubernetes.Clientset) (ResourceSource, error) { - source := ResourceSource{ - ItemType: "storageclass", - MapGet: MapStorageClassGet, - MapList: MapStorageClassList, - Namespaced: false, - } - - err := source.LoadFunction( - cs.StorageV1().StorageClasses, - ) - - return source, err -} - -// MapStorageClassList maps an interface that is underneath a -// *storageV1.StorageClassList to a list of Items -func MapStorageClassList(i interface{}) ([]*sdp.Item, error) { - var objectList *storageV1.StorageClassList - var ok bool - var items []*sdp.Item - var item *sdp.Item - var err error - - // Expect this to be a objectList - if objectList, ok = i.(*storageV1.StorageClassList); !ok { - return make([]*sdp.Item, 0), fmt.Errorf("could not convert %v to *storageV1.StorageClassList", i) - } - - for _, object := range objectList.Items { - if item, err = MapStorageClassGet(&object); err == nil { - items = append(items, item) - } else { - return items, err - } +func NewStorageClassSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.StorageClass, *v1.StorageClassList] { + return KubeTypeSource[*v1.StorageClass, *v1.StorageClassList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "StorageClass", + ClusterInterfaceBuilder: func() ItemInterface[*v1.StorageClass, *v1.StorageClassList] { + return cs.StorageV1().StorageClasses() + }, + ListExtractor: func(list *v1.StorageClassList) ([]*v1.StorageClass, error) { + extracted := make([]*v1.StorageClass, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, } - - return items, nil -} - -// MapStorageClassGet maps an interface that is underneath a *storageV1.StorageClass to an item. If -// the interface isn't actually a *storageV1.StorageClass this will fail -func MapStorageClassGet(i interface{}) (*sdp.Item, error) { - var object *storageV1.StorageClass - var item *sdp.Item - var ok bool - - // Expect this to be a *storageV1.StorageClass - if object, ok = i.(*storageV1.StorageClass); !ok { - return item, fmt.Errorf("could not assert %v as a *storageV1.StorageClass", i) - } - - item, err := mapK8sObject("storageclass", object) - - if err != nil { - return item, err - } - - return item, nil } diff --git a/internal/sources/storageclass_test.go b/internal/sources/storageclass_test.go new file mode 100644 index 0000000..72229cd --- /dev/null +++ b/internal/sources/storageclass_test.go @@ -0,0 +1,34 @@ +package sources + +import ( + "testing" +) + +var storageClassYAML = ` +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: storage-class-test +provisioner: kubernetes.io/aws-ebs +parameters: + type: gp2 + +` + +func TestStorageClassSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + Namespace: "default", + } + + source := NewStorageClassSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "storage-class-test", + GetScope: sd.String(), + SetupYAML: storageClassYAML, + } + + st.Execute(t) +} diff --git a/internal/sources/volumeattachment.go b/internal/sources/volumeattachment.go new file mode 100644 index 0000000..551c7d4 --- /dev/null +++ b/internal/sources/volumeattachment.go @@ -0,0 +1,59 @@ +package sources + +import ( + "github.com/overmindtech/sdp-go" + v1 "k8s.io/api/storage/v1" + "k8s.io/client-go/kubernetes" +) + +func volumeAttachmentExtractor(resource *v1.VolumeAttachment, scope string) ([]*sdp.Query, error) { + queries := make([]*sdp.Query, 0) + + if resource.Spec.Source.PersistentVolumeName != nil { + queries = append(queries, &sdp.Query{ + Type: "PersistentVolume", + Method: sdp.QueryMethod_GET, + Query: *resource.Spec.Source.PersistentVolumeName, + Scope: scope, + }) + } + + if resource.Spec.NodeName != "" { + queries = append(queries, &sdp.Query{ + Type: "Node", + Method: sdp.QueryMethod_GET, + Query: resource.Spec.NodeName, + Scope: scope, + }) + } + + return queries, nil +} + +func NewVolumeAttachmentSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.VolumeAttachment, *v1.VolumeAttachmentList] { + return KubeTypeSource[*v1.VolumeAttachment, *v1.VolumeAttachmentList]{ + ClusterName: cluster, + Namespaces: namespaces, + TypeName: "VolumeAttachment", + ClusterInterfaceBuilder: func() ItemInterface[*v1.VolumeAttachment, *v1.VolumeAttachmentList] { + return cs.StorageV1().VolumeAttachments() + }, + ListExtractor: func(list *v1.VolumeAttachmentList) ([]*v1.VolumeAttachment, error) { + extracted := make([]*v1.VolumeAttachment, len(list.Items)) + + for i := range list.Items { + extracted[i] = &list.Items[i] + } + + return extracted, nil + }, + LinkedItemQueryExtractor: volumeAttachmentExtractor, + HealthExtractor: func(resource *v1.VolumeAttachment) *sdp.Health { + if resource.Status.AttachError != nil || resource.Status.DetachError != nil { + return sdp.Health_HEALTH_ERROR.Enum() + } + + return sdp.Health_HEALTH_OK.Enum() + }, + } +} diff --git a/internal/sources/volumeattachment_test.go b/internal/sources/volumeattachment_test.go new file mode 100644 index 0000000..8db564f --- /dev/null +++ b/internal/sources/volumeattachment_test.go @@ -0,0 +1,92 @@ +package sources + +import ( + "testing" + + "github.com/overmindtech/sdp-go" +) + +var volumeAttachmentYAML = ` +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: volume-attachment-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: volume-attachment-pv +spec: + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + hostPath: + path: /data +--- +apiVersion: v1 +kind: Pod +metadata: + name: volume-attachment-pod +spec: + containers: + - name: volume-attachment-container + image: nginx + volumeMounts: + - name: volume-attachment-volume + mountPath: /data + volumes: + - name: volume-attachment-volume + persistentVolumeClaim: + claimName: volume-attachment-pvc +--- +apiVersion: storage.k8s.io/v1 +kind: VolumeAttachment +metadata: + name: volume-attachment-attachment +spec: + nodeName: k8s-source-tests-control-plane + attacher: kubernetes.io + source: + persistentVolumeName: volume-attachment-pv + +` + +func TestVolumeAttachmentSource(t *testing.T) { + sd := ScopeDetails{ + ClusterName: CurrentCluster.Name, + } + + source := NewVolumeAttachmentSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + + st := SourceTests{ + Source: &source, + GetQuery: "volume-attachment-attachment", + GetScope: sd.String(), + SetupYAML: volumeAttachmentYAML, + GetQueryTests: QueryTests{ + { + ExpectedType: "PersistentVolume", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "volume-attachment-pv", + ExpectedScope: sd.String(), + }, + { + ExpectedType: "Node", + ExpectedMethod: sdp.QueryMethod_GET, + ExpectedQuery: "k8s-source-tests-control-plane", + ExpectedScope: sd.String(), + }, + }, + } + + st.Execute(t) +} From fcdbfe254070c0ebea15410846c77fe0ddeaf164 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Sat, 13 May 2023 10:08:39 +0000 Subject: [PATCH 13/24] Moved files to sources folder --- internal/sources/shared_api.go | 65 --- internal/sources/shared_namespacestorage.go | 57 --- internal/sources/shared_resourcesource.go | 413 ------------------ internal/sources/shared_tests.go | 172 -------- {internal/sources => sources}/clusterrole.go | 0 .../sources => sources}/clusterrole_test.go | 0 .../sources => sources}/clusterrolebinding.go | 0 .../clusterrolebinding_test.go | 0 {internal/sources => sources}/configmap.go | 0 .../sources => sources}/configmap_test.go | 0 {internal/sources => sources}/cronjob.go | 0 {internal/sources => sources}/cronjob_test.go | 0 {internal/sources => sources}/daemonset.go | 0 .../sources => sources}/daemonset_test.go | 0 {internal/sources => sources}/deployment.go | 0 .../sources => sources}/deployment_test.go | 0 .../endpoint.go => sources/endpoints.go | 0 .../sources => sources}/endpoints_test.go | 0 .../sources => sources}/endpointslice.go | 0 .../sources => sources}/endpointslice_test.go | 0 .../sources => sources}/generic_source.go | 10 +- .../generic_source_test.go | 0 .../horizontalpodautoscaler.go | 0 .../horizontalpodautoscaler_test.go | 0 {internal/sources => sources}/ingress.go | 0 {internal/sources => sources}/ingress_test.go | 0 {internal/sources => sources}/job.go | 0 {internal/sources => sources}/job_test.go | 0 {internal/sources => sources}/limitrange.go | 0 .../sources => sources}/limitrange_test.go | 0 .../sources => sources}/networkpolicy.go | 0 .../sources => sources}/networkpolicy_test.go | 0 {internal/sources => sources}/node.go | 0 {internal/sources => sources}/node_test.go | 0 .../sources => sources}/persistentvolume.go | 0 .../persistentvolume_test.go | 0 .../persistentvolumeclaim.go | 0 .../persistentvolumeclaim_test.go | 0 .../poddisruptionbudget.go | 0 .../poddisruptionbudget_test.go | 0 {internal/sources => sources}/pods.go | 8 +- {internal/sources => sources}/pods_test.go | 0 .../sources => sources}/priorityclass.go | 0 .../sources => sources}/priorityclass_test.go | 0 {internal/sources => sources}/replicaset.go | 0 .../sources => sources}/replicaset_test.go | 0 .../replicationcontroller.go | 0 .../replicationcontroller_test.go | 0 .../sources => sources}/resourcequota.go | 0 .../sources => sources}/resourcequota_test.go | 0 {internal/sources => sources}/role.go | 0 {internal/sources => sources}/role_test.go | 0 {internal/sources => sources}/rolebinding.go | 0 .../sources => sources}/rolebinding_test.go | 0 {internal/sources => sources}/secret.go | 0 {internal/sources => sources}/secret_test.go | 0 {internal/sources => sources}/service.go | 0 {internal/sources => sources}/service_test.go | 0 .../sources => sources}/serviceaccount.go | 0 .../serviceaccount_test.go | 0 {internal/sources => sources}/shared_test.go | 0 {internal/sources => sources}/shared_util.go | 70 --- .../sources => sources}/shared_util_test.go | 0 {internal/sources => sources}/statefulset.go | 0 .../sources => sources}/statefulset_test.go | 0 {internal/sources => sources}/storageclass.go | 0 .../sources => sources}/storageclass_test.go | 0 .../sources => sources}/volumeattachment.go | 0 .../volumeattachment_test.go | 0 69 files changed, 12 insertions(+), 783 deletions(-) delete mode 100644 internal/sources/shared_api.go delete mode 100644 internal/sources/shared_namespacestorage.go delete mode 100644 internal/sources/shared_resourcesource.go delete mode 100644 internal/sources/shared_tests.go rename {internal/sources => sources}/clusterrole.go (100%) rename {internal/sources => sources}/clusterrole_test.go (100%) rename {internal/sources => sources}/clusterrolebinding.go (100%) rename {internal/sources => sources}/clusterrolebinding_test.go (100%) rename {internal/sources => sources}/configmap.go (100%) rename {internal/sources => sources}/configmap_test.go (100%) rename {internal/sources => sources}/cronjob.go (100%) rename {internal/sources => sources}/cronjob_test.go (100%) rename {internal/sources => sources}/daemonset.go (100%) rename {internal/sources => sources}/daemonset_test.go (100%) rename {internal/sources => sources}/deployment.go (100%) rename {internal/sources => sources}/deployment_test.go (100%) rename internal/sources/endpoint.go => sources/endpoints.go (100%) rename {internal/sources => sources}/endpoints_test.go (100%) rename {internal/sources => sources}/endpointslice.go (100%) rename {internal/sources => sources}/endpointslice_test.go (100%) rename {internal/sources => sources}/generic_source.go (97%) rename {internal/sources => sources}/generic_source_test.go (100%) rename {internal/sources => sources}/horizontalpodautoscaler.go (100%) rename {internal/sources => sources}/horizontalpodautoscaler_test.go (100%) rename {internal/sources => sources}/ingress.go (100%) rename {internal/sources => sources}/ingress_test.go (100%) rename {internal/sources => sources}/job.go (100%) rename {internal/sources => sources}/job_test.go (100%) rename {internal/sources => sources}/limitrange.go (100%) rename {internal/sources => sources}/limitrange_test.go (100%) rename {internal/sources => sources}/networkpolicy.go (100%) rename {internal/sources => sources}/networkpolicy_test.go (100%) rename {internal/sources => sources}/node.go (100%) rename {internal/sources => sources}/node_test.go (100%) rename {internal/sources => sources}/persistentvolume.go (100%) rename {internal/sources => sources}/persistentvolume_test.go (100%) rename {internal/sources => sources}/persistentvolumeclaim.go (100%) rename {internal/sources => sources}/persistentvolumeclaim_test.go (100%) rename {internal/sources => sources}/poddisruptionbudget.go (100%) rename {internal/sources => sources}/poddisruptionbudget_test.go (100%) rename {internal/sources => sources}/pods.go (97%) rename {internal/sources => sources}/pods_test.go (100%) rename {internal/sources => sources}/priorityclass.go (100%) rename {internal/sources => sources}/priorityclass_test.go (100%) rename {internal/sources => sources}/replicaset.go (100%) rename {internal/sources => sources}/replicaset_test.go (100%) rename {internal/sources => sources}/replicationcontroller.go (100%) rename {internal/sources => sources}/replicationcontroller_test.go (100%) rename {internal/sources => sources}/resourcequota.go (100%) rename {internal/sources => sources}/resourcequota_test.go (100%) rename {internal/sources => sources}/role.go (100%) rename {internal/sources => sources}/role_test.go (100%) rename {internal/sources => sources}/rolebinding.go (100%) rename {internal/sources => sources}/rolebinding_test.go (100%) rename {internal/sources => sources}/secret.go (100%) rename {internal/sources => sources}/secret_test.go (100%) rename {internal/sources => sources}/service.go (100%) rename {internal/sources => sources}/service_test.go (100%) rename {internal/sources => sources}/serviceaccount.go (100%) rename {internal/sources => sources}/serviceaccount_test.go (100%) rename {internal/sources => sources}/shared_test.go (100%) rename {internal/sources => sources}/shared_util.go (78%) rename {internal/sources => sources}/shared_util_test.go (100%) rename {internal/sources => sources}/statefulset.go (100%) rename {internal/sources => sources}/statefulset_test.go (100%) rename {internal/sources => sources}/storageclass.go (100%) rename {internal/sources => sources}/storageclass_test.go (100%) rename {internal/sources => sources}/volumeattachment.go (100%) rename {internal/sources => sources}/volumeattachment_test.go (100%) diff --git a/internal/sources/shared_api.go b/internal/sources/shared_api.go deleted file mode 100644 index 9c4b3cd..0000000 --- a/internal/sources/shared_api.go +++ /dev/null @@ -1,65 +0,0 @@ -package sources - -import ( - "context" - - log "github.com/sirupsen/logrus" - - coreV1 "k8s.io/api/core/v1" - metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" -) - -// APITimeoutContext Returns a context representing the configured timeout for -// API calls. This is configured in the "apitimeout" setting and is represented -// in seconds -func APITimeoutContext() (context.Context, context.CancelFunc) { - if apiTimeoutSet { - return context.WithTimeout(context.Background(), apiTimeout) - } - - // If the config hasn't been set yet then load it - apiTimeout = apiTimeoutDefault - - // TODO: Reimplement after refactor - // if t := sources.ConfigGetInt("apitimeout", BackendPackage); t != 0 { - // // If a timeout has been set then use that - // apiTimeout = time.Duration(t) * time.Second - // } - - // Note that we have set it so we don't have to go through this again - apiTimeoutSet = true - - return APITimeoutContext() -} - -// GetAllNamespaceNames gets the names of all namespaces that can be found for a -// given connection -func GetAllNamespaceNames(cs *kubernetes.Clientset) ([]string, error) { - var opts metaV1.ListOptions - var namespaces []string - var namespaceList *coreV1.NamespaceList - var err error - - // Use the configured API timeout - ctx, cancel := APITimeoutContext() - defer cancel() - - log.WithFields(log.Fields{ - "APIVersion": cs.CoreV1().RESTClient().APIVersion().Version, - "APIURL": cs.CoreV1().RESTClient().Get().URL(), - }).Trace("Getting list of namepaces") - - // Get the list of namespaces - namespaceList, err = cs.CoreV1().Namespaces().List(ctx, opts) - - if err != nil { - return []string{}, err - } - - for _, item := range namespaceList.Items { - namespaces = append(namespaces, item.Name) - } - - return namespaces, nil -} diff --git a/internal/sources/shared_namespacestorage.go b/internal/sources/shared_namespacestorage.go deleted file mode 100644 index e8269c6..0000000 --- a/internal/sources/shared_namespacestorage.go +++ /dev/null @@ -1,57 +0,0 @@ -package sources - -import ( - "errors" - "sync" - "time" - - "k8s.io/client-go/kubernetes" -) - -// NamespaceStorage is an object that is used for storing the list of -// namespaces. Many of the non-namespaced resources will need to know what thee -// list of namespaces is in order for them to create LinkedItemQueries. This -// object stores them to ensure that we don't unnecessarily spam the API -type NamespaceStorage struct { - CS *kubernetes.Clientset - CacheDuration time.Duration - - lastUpdate time.Time - namespaces []string - namespacesMutex sync.RWMutex -} - -// Namespaces returns the list of namespaces, updating if required -func (ns *NamespaceStorage) Namespaces() ([]string, error) { - if ns == nil { - // Protect against getting called on a nil pointer - return []string{}, errors.New("namespaces called on nil NamespaceStorage object") - } - - ns.namespacesMutex.RLock() - - // Check that the cache is up to date - if time.Since(ns.lastUpdate) < ns.CacheDuration { - defer ns.namespacesMutex.RUnlock() - return ns.namespaces, nil - } - - ns.namespacesMutex.RUnlock() - ns.namespacesMutex.Lock() - - var err error - - // Call the API - ns.namespaces, err = GetAllNamespaceNames(ns.CS) - - if err != nil { - ns.namespacesMutex.Unlock() - return ns.namespaces, err - } - - // Update dates - ns.lastUpdate = time.Now() - ns.namespacesMutex.Unlock() - - return ns.Namespaces() -} diff --git a/internal/sources/shared_resourcesource.go b/internal/sources/shared_resourcesource.go deleted file mode 100644 index 376ba77..0000000 --- a/internal/sources/shared_resourcesource.go +++ /dev/null @@ -1,413 +0,0 @@ -package sources - -import ( - "context" - "errors" - "fmt" - "reflect" - "time" - - "github.com/overmindtech/sdp-go" - - metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - - log "github.com/sirupsen/logrus" -) - -// defaultAPITimeout is the default amount of time to wait per for each API -// query to K8s. This is passed as a context to API request functions -var apiTimeoutDefault = (10 * time.Second) -var apiTimeoutSet = false -var apiTimeout time.Duration - -// ClusterName stores the name of the cluster, this is also used as the scope -// for non-namespaced items. This designed to be user by namespaced items to -// create linked item requests on non-namespaced items -var ClusterName string - -// NamespacedSourceFunction is a function that accepts a kubernetes client and -// namespace, and returns a ResourceSource for a given type. This also satisfies -// the Backend interface -type NamespacedSourceFunction func(cs *kubernetes.Clientset) ResourceSource - -// NonNamespacedSourceFunction is a function that accepts a kubernetes client and -// returns a ResourceSource for a given type. This also satisfies the Backend -// interface -type NonNamespacedSourceFunction func(cs *kubernetes.Clientset) ResourceSource - -// SourceFunction is a function that accepts a kubernetes client and returns a -// ResourceSource for a given type. This also satisfies the discovery.Source -// interface -type SourceFunction func(cs *kubernetes.Clientset) (ResourceSource, error) - -// SourceFunctions is the list of functions to load -var SourceFunctions = []SourceFunction{} - -// ResourceSource represents a source of Kubernetes resources. one of these -// sources needs to be created, and then have its get and list functions -// registered by calling the LoadFunctions method. Note that in order for this -// to be able to discover any time of Kubernetes resource it uses as significant -// amount of reflection. The LoadFunctions method should do enough error -// checking to ensure that the methods on this struct don't cause any panics, -// but there is still a very real chance that there will be panics so be -// careful doing anything non-standard with this struct -type ResourceSource struct { - // The type of items that will be returned from this source - ItemType string - // A function that will accept an interface and return a list of items. The - // interface that is passed will be the first item returned from - // "listFunction", as an interface. The function should covert this to - // whatever format it is expecting, the proceed to map to Items - MapList func(interface{}) ([]*sdp.Item, error) - // A function that will accept an interface and return an item. The - // interface that is passed will be the first item returned from - // "getFunction", as an interface. The function should covert this to - // whatever format it is expecting, the proceed to map to an item - MapGet func(interface{}) (*sdp.Item, error) - - // Whether or not this source is for namespaced resources - Namespaced bool - - // NSS Namespace storage for when backends need to lookup the list of - // namespaces - NSS *NamespaceStorage - - // interfaceFunction is used to store the function which, when called, - // returns an interface that we can call Get() and List() against in order - // to get item details - interfaceFunction reflect.Value -} - -// LoadFunctions performs validation on the supplied interface function. This -// function should retrun an interface which has Get() and List() methods -// -// A Get should be: -// -// func(ctx context.Context, name string, opts metaV1.GetOptions) -// -// A List should be: -// -// List(ctx context.Context, opts metaV1.ListOptions) -func (rs *ResourceSource) LoadFunction(interfaceFunction interface{}) error { - // Reflect to values - interfaceFunctionValue := reflect.ValueOf(interfaceFunction) - interfaceFunctionType := reflect.TypeOf(interfaceFunction) - - switch interfaceFunctionType.NumIn() { - case 0: - // Do nothing - case 1: - if interfaceFunctionType.In(0).Kind() != reflect.String { - return errors.New("interfaceFunction first argument must be a string") - } - default: - return errors.New("interfaceFunction should have 0 or 1 parameters") - } - - if interfaceFunctionType.Out(0).Kind() != reflect.Interface { - return errors.New("interfaceFunction return value should be an interface") - } - - // This is the value that is going ot be returned when the interface - // function is called. We need to check that this has the methods that we - // expect and is therefore going to work when we try to interact with it - returnInterface := interfaceFunctionType.Out(0) - - getMethod, getFound := returnInterface.MethodByName("Get") - - if !getFound { - return errors.New("interfaceFunction does not have a 'Get' method") - } - - listMethod, listFound := returnInterface.MethodByName("List") - - if !listFound { - return errors.New("interfaceFunction does not have a 'List' method") - } - - getFunctionType := getMethod.Type - listFunctionType := listMethod.Type - - if getFunctionType.NumIn() != 3 { - return errors.New("getFunction must accept 3 arguments") - } - - // Check that parameters are as expected - if getFunctionType.In(0).Kind() != reflect.Interface { - return errors.New("getFunction first argument must be a context.Context") - } - - if getFunctionType.In(1).Kind() != reflect.String { - return errors.New("getFunction second argument must be a string") - } - - if getFunctionType.In(2).Kind() != reflect.Struct { - return errors.New("getFunction third argument must be a metaV1.GetOptions") - } - - if getFunctionType.NumOut() != 2 { - return errors.New("getFunction must return 2 values") - } - - if listFunctionType.NumIn() != 2 { - return errors.New("listFunction must accept 2 arguments") - } - - if listFunctionType.In(0).Kind() != reflect.Interface { - return errors.New("listFunction first argument must be a context.Context") - } - - if listFunctionType.In(1).Kind() != reflect.Struct { - return errors.New("getFunction second argument must be a metaV1.ListOptions") - } - - if listFunctionType.NumOut() != 2 { - return errors.New("listFunction must return 2 values") - } - - // Save values for later use - rs.interfaceFunction = interfaceFunctionValue - - return nil -} - -// Get takes the UniqueAttribute value as a parameter (also referred to as the -// "name" of the item) and returns a full item will all details. This function -// must return an item whose UniqueAttribute value exactly matches the supplied -// parameter. If the item cannot be found it should return an ItemNotFoundError -// (Required) -func (rs *ResourceSource) Get(ctx context.Context, itemScope string, name string) (*sdp.Item, error) { - var ctxValue reflect.Value - var opts metaV1.GetOptions - var optsValue reflect.Value - var nameValue reflect.Value - var params []reflect.Value - var returns []reflect.Value - var function reflect.Value - var err error - - opts = metaV1.GetOptions{} - - // TODO: Logging - - ctxValue = reflect.ValueOf(ctx) - nameValue = reflect.ValueOf(name) - optsValue = reflect.ValueOf(opts) - params = []reflect.Value{ - ctxValue, - nameValue, - optsValue, - } - - // Call the function - function, err = rs.getFunction(itemScope) - - if err != nil { - return nil, err - } - - returns = function.Call(params) - - if e := returns[1].Interface(); e != nil { - if err, ok := e.(error); ok { - return &sdp.Item{}, err - } - return &sdp.Item{}, errors.New("unknown error occurred") - } - - // Map results and return - return rs.MapGet(returns[0].Interface()) -} - -// List finds all items that the backend possibly can. It maybe be possible that -// this might not be an exhaustive list though in the case of kubernetes it is -// unlikely -func (rs *ResourceSource) List(ctx context.Context, itemScope string) ([]*sdp.Item, error) { - var ctxValue reflect.Value - var opts metaV1.ListOptions - var optsValue reflect.Value - var params []reflect.Value - var function reflect.Value - var returns []reflect.Value - var err error - - opts = metaV1.ListOptions{} - - // TODO: Logging - - ctxValue = reflect.ValueOf(ctx) - optsValue = reflect.ValueOf(opts) - params = []reflect.Value{ - ctxValue, - optsValue, - } - - // TODO: The below relies on being able to parse out the scope from the - // query. However it's entirely possible that the scope could be '*', so - // we need to be able to handle that - - // Call the function - function, err = rs.listFunction(itemScope) - - if err != nil { - return nil, err - } - - returns = function.Call(params) - - // Check if the error is nil. If it's nil then we know there wasn't an - // error. If not then we know there was an error - if returns[1].Interface() != nil { - return make([]*sdp.Item, 0), returns[1].Interface().(error) - } - - return rs.MapList(returns[0].Interface()) -} - -// Search This search for items that match a given ListOptions. The query must -// be provided as a JSON object that can be cast to a -// [ListOptions](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#ListOptions) -// object. -// -// *Note:* Additional changes will be made to the ListOptions object after -// deserialization such as limiting the scope to items of the same type as the -// current ResourceSource, and drooping any options such as "Watch" -func (rs *ResourceSource) Search(ctx context.Context, itemScope string, query string) ([]*sdp.Item, error) { - var ctxValue reflect.Value - var opts metaV1.ListOptions - var optsValue reflect.Value - var params []reflect.Value - var returns []reflect.Value - var function reflect.Value - var err error - - opts, err = QueryToListOptions(query) - - if err != nil { - log.WithFields(log.Fields{ - "query": query, - "type": rs.ItemType, - "scope": itemScope, - "parseError": err.Error(), - }).Error("error while parsing query") - - return nil, err - } - - // TODO: Logging - - ctxValue = reflect.ValueOf(ctx) - optsValue = reflect.ValueOf(opts) - params = []reflect.Value{ - ctxValue, - optsValue, - } - - // Call the function - function, err = rs.listFunction(itemScope) - - if err != nil { - return nil, err - } - - returns = function.Call(params) - - // Check for an error - if returns[1].Interface() != nil { - return make([]*sdp.Item, 0), returns[1].Interface().(error) - } - - return rs.MapList(returns[0].Interface()) -} - -// Type is the type of items that this returns (Required) -func (rs *ResourceSource) Type() string { - return rs.ItemType -} - -// Name returns a descriptive name for the source, used in logging and metadata -func (rs *ResourceSource) Name() string { - return fmt.Sprintf("k8s-%v", rs.ItemType) -} - -// Scopes Returns the list of scops that this source is capable of finding -// items for. This is usually the name of the cluster, plus any namespaces in -// the format {clusterName}.{namespace} -func (rs *ResourceSource) Scopes() []string { - contexts := make([]string, 0) - - if rs.Namespaced { - namespaces, _ := rs.NSS.Namespaces() - - for _, namespace := range namespaces { - contexts = append(contexts, ClusterName+"."+namespace) - } - } else { - contexts = append(contexts, ClusterName) - } - - return contexts -} - -// Weight The weight of this source, used for conflict resolution. Currently -// returns a static value of 100 -func (rs *ResourceSource) Weight() int { - return 100 -} - -// interactionInterface Calls the interface function to return an interface that -// will allow us to call Get and List functions which will in turn actually -// execute API queries against K8s -func (rs *ResourceSource) interactionInterface(itemScope string) (reflect.Value, error) { - contextDetails, _ := ParseScope(itemScope, rs.Namespaced) - interfaceFunctionArgs := make([]reflect.Value, 0) - - if rs.Namespaced { - // If the interface function is namespaced we need to pass in the namespace that we want to query - interfaceFunctionArgs = append(interfaceFunctionArgs, reflect.ValueOf(contextDetails.Namespace)) - } - - // Call the interface function in order to return a list function for the - // given namespace (or not, if the source isn't namespaced) - results := rs.interfaceFunction.Call(interfaceFunctionArgs) - - // Validate the results before sending them back - if len(results) != 1 { - return reflect.Value{}, errors.New("could not load list function, loading returned too many results") - } - - return results[0], nil -} - -func (rs *ResourceSource) getFunction(itemScope string) (reflect.Value, error) { - var getMethod reflect.Value - var iFace reflect.Value - var err error - - iFace, err = rs.interactionInterface(itemScope) - - if err != nil { - return reflect.Value{}, err - } - - getMethod = iFace.MethodByName("Get") - - return getMethod, nil -} - -func (rs *ResourceSource) listFunction(itemScope string) (reflect.Value, error) { - var listMethod reflect.Value - var iFace reflect.Value - var err error - - iFace, err = rs.interactionInterface(itemScope) - - if err != nil { - return reflect.Value{}, err - } - - listMethod = iFace.MethodByName("List") - - return listMethod, nil -} diff --git a/internal/sources/shared_tests.go b/internal/sources/shared_tests.go deleted file mode 100644 index 49faac3..0000000 --- a/internal/sources/shared_tests.go +++ /dev/null @@ -1,172 +0,0 @@ -// Reusable testing libraries for testing backends -package sources - -import ( - "context" - "testing" - "time" - - "github.com/overmindtech/sdp-go" -) - -// BasicGetListSearchTests Executes a series of basic tests against a kubernetes -// source. These tests include: -// -// - Searching with a given query for items -// - Grabbing the name of one of the found items and making sure that we can -// Get() it -// - Running a List() and ensuring that there is > 0 results -func BasicGetListSearchTests(t *testing.T, query string, source ResourceSource) { - itemScope := "testcluster:443.k8s-source-testing" - - var getName string - - t.Run("Testing basic Search()", func(t *testing.T) { - var items []*sdp.Item - var err error - - // Give it some time for the pod to come up - for i := 0; i < 30; i++ { - items, err = source.Search(context.Background(), itemScope, query) - - if len(items) > 0 { - break - } - - time.Sleep(1 * time.Second) - } - - if err != nil { - t.Fatal(err) - } - - if l := len(items); l != 1 { - t.Fatalf("Expected 1 item, got %v", l) - } - - item := items[0] - - TestValidateItem(t, item) - - // Populate this so that the get test can work - getName = item.UniqueAttributeValue() - }) - - t.Run("Testing basic Get()", func(t *testing.T) { - if getName == "" { - t.Skip("Nothing found from Search(), skipping") - } - - item, err := source.Get(context.Background(), itemScope, getName) - - if err != nil { - t.Error(err) - } - - if x := item.UniqueAttributeValue(); x != getName { - t.Errorf("expected pod name hello, got %v", x) - } - - TestValidateItem(t, item) - }) - - t.Run("Testing basic List()", func(t *testing.T) { - if getName == "" { - t.Skip("Nothing found from Search(), skipping") - } - - items, err := source.List(context.Background(), itemScope) - - if err != nil { - t.Fatal(err) - } - - found := false - - // Make sure the item from the get was there - for _, item := range items { - TestValidateItem(t, item) - - if item.UniqueAttributeValue() == getName { - found = true - break - } - } - - if !found { - t.Fatalf("List() did not return pod %v", getName) - } - }) -} - -// TestValidateItem Checks an item to ensure it is a valid SDP item. This includes -// checking that all required attributes are populated -func TestValidateItem(t *testing.T, i *sdp.Item) { - // Ensure that the item has the required fields set i.e. - // - // * Type - // * UniqueAttribute - // * Attributes - if i.GetType() == "" { - t.Errorf("Item %v has an empty Type", i.GloballyUniqueName()) - } - - if i.GetUniqueAttribute() == "" { - t.Errorf("Item %v has an empty UniqueAttribute", i.GloballyUniqueName()) - } - - if i.GetScope() == "" { - t.Errorf("Item %v has an empty Scope", i.GetScope()) - } - - attrMap := i.GetAttributes().AttrStruct.AsMap() - - if len(attrMap) == 0 { - t.Errorf("Attributes for item %v are empty", i.GloballyUniqueName()) - } - - // Check the attributes themselves for validity - for k := range attrMap { - if k == "" { - t.Errorf("Item %v has an attribute with an empty name", i.GloballyUniqueName()) - } - } - - // Make sure that the UniqueAttributeValue is populated - if i.UniqueAttributeValue() == "" { - t.Errorf("UniqueAttribute %v for item %v is empty", i.GetUniqueAttribute(), i.GloballyUniqueName()) - } - - for index, linkedItem := range i.GetLinkedItems() { - if linkedItem.GetType() == "" { - t.Errorf("LinkedItem %v of item %v has empty type", index, i.GloballyUniqueName()) - } - - if linkedItem.GetUniqueAttributeValue() == "" { - t.Errorf("LinkedItem %v of item %v has empty UniqueAttributeValue", index, i.GloballyUniqueName()) - } - - // We don't need to check for an empty scope here since if it's empty - // it will just inherit the scope of the parent - } - - for index, linkedQuery := range i.GetLinkedItemQueries() { - if linkedQuery.GetType() == "" { - t.Errorf("LinkedQuery %v of item %v has empty type", index, i.GloballyUniqueName()) - - } - - if linkedQuery.GetMethod() != sdp.QueryMethod_LIST { - if linkedQuery.GetQuery() == "" { - t.Errorf("LinkedQuery %v of item %v has empty query. This is not allowed unless the method is LIST", index, i.GloballyUniqueName()) - } - } - } -} - -// TestValidateItems Runs TestValidateItem on many items -func TestValidateItems(t *testing.T, is []*sdp.Item) { - for _, i := range is { - TestValidateItem(t, i) - } -} diff --git a/internal/sources/clusterrole.go b/sources/clusterrole.go similarity index 100% rename from internal/sources/clusterrole.go rename to sources/clusterrole.go diff --git a/internal/sources/clusterrole_test.go b/sources/clusterrole_test.go similarity index 100% rename from internal/sources/clusterrole_test.go rename to sources/clusterrole_test.go diff --git a/internal/sources/clusterrolebinding.go b/sources/clusterrolebinding.go similarity index 100% rename from internal/sources/clusterrolebinding.go rename to sources/clusterrolebinding.go diff --git a/internal/sources/clusterrolebinding_test.go b/sources/clusterrolebinding_test.go similarity index 100% rename from internal/sources/clusterrolebinding_test.go rename to sources/clusterrolebinding_test.go diff --git a/internal/sources/configmap.go b/sources/configmap.go similarity index 100% rename from internal/sources/configmap.go rename to sources/configmap.go diff --git a/internal/sources/configmap_test.go b/sources/configmap_test.go similarity index 100% rename from internal/sources/configmap_test.go rename to sources/configmap_test.go diff --git a/internal/sources/cronjob.go b/sources/cronjob.go similarity index 100% rename from internal/sources/cronjob.go rename to sources/cronjob.go diff --git a/internal/sources/cronjob_test.go b/sources/cronjob_test.go similarity index 100% rename from internal/sources/cronjob_test.go rename to sources/cronjob_test.go diff --git a/internal/sources/daemonset.go b/sources/daemonset.go similarity index 100% rename from internal/sources/daemonset.go rename to sources/daemonset.go diff --git a/internal/sources/daemonset_test.go b/sources/daemonset_test.go similarity index 100% rename from internal/sources/daemonset_test.go rename to sources/daemonset_test.go diff --git a/internal/sources/deployment.go b/sources/deployment.go similarity index 100% rename from internal/sources/deployment.go rename to sources/deployment.go diff --git a/internal/sources/deployment_test.go b/sources/deployment_test.go similarity index 100% rename from internal/sources/deployment_test.go rename to sources/deployment_test.go diff --git a/internal/sources/endpoint.go b/sources/endpoints.go similarity index 100% rename from internal/sources/endpoint.go rename to sources/endpoints.go diff --git a/internal/sources/endpoints_test.go b/sources/endpoints_test.go similarity index 100% rename from internal/sources/endpoints_test.go rename to sources/endpoints_test.go diff --git a/internal/sources/endpointslice.go b/sources/endpointslice.go similarity index 100% rename from internal/sources/endpointslice.go rename to sources/endpointslice.go diff --git a/internal/sources/endpointslice_test.go b/sources/endpointslice_test.go similarity index 100% rename from internal/sources/endpointslice_test.go rename to sources/endpointslice_test.go diff --git a/internal/sources/generic_source.go b/sources/generic_source.go similarity index 97% rename from internal/sources/generic_source.go rename to sources/generic_source.go index a54c1ca..6b56674 100644 --- a/internal/sources/generic_source.go +++ b/sources/generic_source.go @@ -66,23 +66,23 @@ type KubeTypeSource[Resource metav1.Object, ResourceList any] struct { // validate Validates that the source is correctly set up func (k *KubeTypeSource[Resource, ResourceList]) Validate() error { if k.NamespacedInterfaceBuilder == nil && k.ClusterInterfaceBuilder == nil { - return fmt.Errorf("either NamespacedInterfaceBuilder or ClusterInterfaceBuilder must be specified") + return errors.New("either NamespacedInterfaceBuilder or ClusterInterfaceBuilder must be specified") } if k.ListExtractor == nil { - return fmt.Errorf("ListExtractor must be specified") + return errors.New("listExtractor must be specified") } if k.TypeName == "" { - return fmt.Errorf("TypeName must be specified") + return errors.New("typeName must be specified") } if k.namespaced() && len(k.Namespaces) == 0 { - return fmt.Errorf("Namespaces must be specified when NamespacedInterfaceBuilder is specified") + return errors.New("namespaces must be specified when NamespacedInterfaceBuilder is specified") } if k.ClusterName == "" { - return fmt.Errorf("ClusterName must be specified") + return errors.New("clusterName must be specified") } return nil diff --git a/internal/sources/generic_source_test.go b/sources/generic_source_test.go similarity index 100% rename from internal/sources/generic_source_test.go rename to sources/generic_source_test.go diff --git a/internal/sources/horizontalpodautoscaler.go b/sources/horizontalpodautoscaler.go similarity index 100% rename from internal/sources/horizontalpodautoscaler.go rename to sources/horizontalpodautoscaler.go diff --git a/internal/sources/horizontalpodautoscaler_test.go b/sources/horizontalpodautoscaler_test.go similarity index 100% rename from internal/sources/horizontalpodautoscaler_test.go rename to sources/horizontalpodautoscaler_test.go diff --git a/internal/sources/ingress.go b/sources/ingress.go similarity index 100% rename from internal/sources/ingress.go rename to sources/ingress.go diff --git a/internal/sources/ingress_test.go b/sources/ingress_test.go similarity index 100% rename from internal/sources/ingress_test.go rename to sources/ingress_test.go diff --git a/internal/sources/job.go b/sources/job.go similarity index 100% rename from internal/sources/job.go rename to sources/job.go diff --git a/internal/sources/job_test.go b/sources/job_test.go similarity index 100% rename from internal/sources/job_test.go rename to sources/job_test.go diff --git a/internal/sources/limitrange.go b/sources/limitrange.go similarity index 100% rename from internal/sources/limitrange.go rename to sources/limitrange.go diff --git a/internal/sources/limitrange_test.go b/sources/limitrange_test.go similarity index 100% rename from internal/sources/limitrange_test.go rename to sources/limitrange_test.go diff --git a/internal/sources/networkpolicy.go b/sources/networkpolicy.go similarity index 100% rename from internal/sources/networkpolicy.go rename to sources/networkpolicy.go diff --git a/internal/sources/networkpolicy_test.go b/sources/networkpolicy_test.go similarity index 100% rename from internal/sources/networkpolicy_test.go rename to sources/networkpolicy_test.go diff --git a/internal/sources/node.go b/sources/node.go similarity index 100% rename from internal/sources/node.go rename to sources/node.go diff --git a/internal/sources/node_test.go b/sources/node_test.go similarity index 100% rename from internal/sources/node_test.go rename to sources/node_test.go diff --git a/internal/sources/persistentvolume.go b/sources/persistentvolume.go similarity index 100% rename from internal/sources/persistentvolume.go rename to sources/persistentvolume.go diff --git a/internal/sources/persistentvolume_test.go b/sources/persistentvolume_test.go similarity index 100% rename from internal/sources/persistentvolume_test.go rename to sources/persistentvolume_test.go diff --git a/internal/sources/persistentvolumeclaim.go b/sources/persistentvolumeclaim.go similarity index 100% rename from internal/sources/persistentvolumeclaim.go rename to sources/persistentvolumeclaim.go diff --git a/internal/sources/persistentvolumeclaim_test.go b/sources/persistentvolumeclaim_test.go similarity index 100% rename from internal/sources/persistentvolumeclaim_test.go rename to sources/persistentvolumeclaim_test.go diff --git a/internal/sources/poddisruptionbudget.go b/sources/poddisruptionbudget.go similarity index 100% rename from internal/sources/poddisruptionbudget.go rename to sources/poddisruptionbudget.go diff --git a/internal/sources/poddisruptionbudget_test.go b/sources/poddisruptionbudget_test.go similarity index 100% rename from internal/sources/poddisruptionbudget_test.go rename to sources/poddisruptionbudget_test.go diff --git a/internal/sources/pods.go b/sources/pods.go similarity index 97% rename from internal/sources/pods.go rename to sources/pods.go index cf15a25..3cac80a 100644 --- a/internal/sources/pods.go +++ b/sources/pods.go @@ -9,6 +9,12 @@ import ( func PodExtractor(resource *v1.Pod, scope string) ([]*sdp.Query, error) { queries := make([]*sdp.Query, 0) + sd, err := ParseScope(scope, true) + + if err != nil { + return nil, err + } + // Link service accounts if resource.Spec.ServiceAccountName != "" { queries = append(queries, &sdp.Query{ @@ -93,7 +99,7 @@ func PodExtractor(resource *v1.Pod, scope string) ([]*sdp.Query, error) { if resource.Spec.PriorityClassName != "" { queries = append(queries, &sdp.Query{ - Scope: ClusterName, + Scope: sd.ClusterName, Method: sdp.QueryMethod_GET, Query: resource.Spec.PriorityClassName, Type: "PriorityClass", diff --git a/internal/sources/pods_test.go b/sources/pods_test.go similarity index 100% rename from internal/sources/pods_test.go rename to sources/pods_test.go diff --git a/internal/sources/priorityclass.go b/sources/priorityclass.go similarity index 100% rename from internal/sources/priorityclass.go rename to sources/priorityclass.go diff --git a/internal/sources/priorityclass_test.go b/sources/priorityclass_test.go similarity index 100% rename from internal/sources/priorityclass_test.go rename to sources/priorityclass_test.go diff --git a/internal/sources/replicaset.go b/sources/replicaset.go similarity index 100% rename from internal/sources/replicaset.go rename to sources/replicaset.go diff --git a/internal/sources/replicaset_test.go b/sources/replicaset_test.go similarity index 100% rename from internal/sources/replicaset_test.go rename to sources/replicaset_test.go diff --git a/internal/sources/replicationcontroller.go b/sources/replicationcontroller.go similarity index 100% rename from internal/sources/replicationcontroller.go rename to sources/replicationcontroller.go diff --git a/internal/sources/replicationcontroller_test.go b/sources/replicationcontroller_test.go similarity index 100% rename from internal/sources/replicationcontroller_test.go rename to sources/replicationcontroller_test.go diff --git a/internal/sources/resourcequota.go b/sources/resourcequota.go similarity index 100% rename from internal/sources/resourcequota.go rename to sources/resourcequota.go diff --git a/internal/sources/resourcequota_test.go b/sources/resourcequota_test.go similarity index 100% rename from internal/sources/resourcequota_test.go rename to sources/resourcequota_test.go diff --git a/internal/sources/role.go b/sources/role.go similarity index 100% rename from internal/sources/role.go rename to sources/role.go diff --git a/internal/sources/role_test.go b/sources/role_test.go similarity index 100% rename from internal/sources/role_test.go rename to sources/role_test.go diff --git a/internal/sources/rolebinding.go b/sources/rolebinding.go similarity index 100% rename from internal/sources/rolebinding.go rename to sources/rolebinding.go diff --git a/internal/sources/rolebinding_test.go b/sources/rolebinding_test.go similarity index 100% rename from internal/sources/rolebinding_test.go rename to sources/rolebinding_test.go diff --git a/internal/sources/secret.go b/sources/secret.go similarity index 100% rename from internal/sources/secret.go rename to sources/secret.go diff --git a/internal/sources/secret_test.go b/sources/secret_test.go similarity index 100% rename from internal/sources/secret_test.go rename to sources/secret_test.go diff --git a/internal/sources/service.go b/sources/service.go similarity index 100% rename from internal/sources/service.go rename to sources/service.go diff --git a/internal/sources/service_test.go b/sources/service_test.go similarity index 100% rename from internal/sources/service_test.go rename to sources/service_test.go diff --git a/internal/sources/serviceaccount.go b/sources/serviceaccount.go similarity index 100% rename from internal/sources/serviceaccount.go rename to sources/serviceaccount.go diff --git a/internal/sources/serviceaccount_test.go b/sources/serviceaccount_test.go similarity index 100% rename from internal/sources/serviceaccount_test.go rename to sources/serviceaccount_test.go diff --git a/internal/sources/shared_test.go b/sources/shared_test.go similarity index 100% rename from internal/sources/shared_test.go rename to sources/shared_test.go diff --git a/internal/sources/shared_util.go b/sources/shared_util.go similarity index 78% rename from internal/sources/shared_util.go rename to sources/shared_util.go index cb9a90f..f9a9b66 100644 --- a/internal/sources/shared_util.go +++ b/sources/shared_util.go @@ -228,73 +228,3 @@ func ObjectReferenceToLIR(ref *coreV1.ObjectReference, parentScope string) *sdp. Scope: scope, } } - -// mapK8sObject Converts a kubernetes object in to an item that we understand. -// This includes extracting certain useful metadata using GetK8sMeta() and then -// extracting all other useful information, with some exceptions -func mapK8sObject(typ string, object metaV1.Object) (*sdp.Item, error) { - var attributes map[string]interface{} - var item *sdp.Item - var v reflect.Value - var t reflect.Type - var err error - - item = &sdp.Item{ - Type: typ, - UniqueAttribute: "name", - } - attributes = GetK8sMeta(object) - - // Assign the scope - if ns, ok := attributes["namespace"]; ok { - item.Scope = fmt.Sprintf("%v.%v", ClusterName, ns) - } else { - item.Scope = ClusterName - } - - // Get the reflected details of the object - v = reflect.Indirect(reflect.ValueOf(object)) - t = v.Type() - - // Ignore the following fields - ignore := []string{ - "TypeMeta", - "ObjectMeta", - "BinaryData", - "StringData", - "Immutable", - } - - if v.Kind() == reflect.Struct { - // Range over fields - n := t.NumField() - for i := 0; i < n; i++ { - field := t.Field(i) - - // Check if the field is meant to be ignored - var shouldIgnore bool - for _, f := range ignore { - if f == field.Name { - shouldIgnore = true - } - } - if shouldIgnore { - continue - } - - // Get the zero value for this field - zero := reflect.Zero(reflect.TypeOf(field)).Interface() - - // Check if the field is it's nil value - // Check if there actually was a field with that name - if !reflect.DeepEqual(field, zero) { - attributes[strings.ToLower(field.Name)] = v.Field(i).Interface() - } - } - } - - item.Attributes, err = sdp.ToAttributes(attributes) - item.LinkedItemQueries = []*sdp.Query{} - - return item, err -} diff --git a/internal/sources/shared_util_test.go b/sources/shared_util_test.go similarity index 100% rename from internal/sources/shared_util_test.go rename to sources/shared_util_test.go diff --git a/internal/sources/statefulset.go b/sources/statefulset.go similarity index 100% rename from internal/sources/statefulset.go rename to sources/statefulset.go diff --git a/internal/sources/statefulset_test.go b/sources/statefulset_test.go similarity index 100% rename from internal/sources/statefulset_test.go rename to sources/statefulset_test.go diff --git a/internal/sources/storageclass.go b/sources/storageclass.go similarity index 100% rename from internal/sources/storageclass.go rename to sources/storageclass.go diff --git a/internal/sources/storageclass_test.go b/sources/storageclass_test.go similarity index 100% rename from internal/sources/storageclass_test.go rename to sources/storageclass_test.go diff --git a/internal/sources/volumeattachment.go b/sources/volumeattachment.go similarity index 100% rename from internal/sources/volumeattachment.go rename to sources/volumeattachment.go diff --git a/internal/sources/volumeattachment_test.go b/sources/volumeattachment_test.go similarity index 100% rename from internal/sources/volumeattachment_test.go rename to sources/volumeattachment_test.go From 8a3c1268803a7b375b6342035723cb3aca53896c Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Sat, 13 May 2023 10:11:09 +0000 Subject: [PATCH 14/24] Remove unused code --- sources/shared_test.go | 409 ----------------------------------------- sources/shared_util.go | 133 +------------- 2 files changed, 8 insertions(+), 534 deletions(-) diff --git a/sources/shared_test.go b/sources/shared_test.go index 8333f0d..9953675 100644 --- a/sources/shared_test.go +++ b/sources/shared_test.go @@ -205,412 +205,3 @@ func TestMain(m *testing.M) { os.Exit(code) } - -// const ClusterBaseline string = ` -// apiVersion: v1 -// kind: PersistentVolume -// metadata: -// name: d1 -// labels: -// type: local -// spec: -// storageClassName: manual -// capacity: -// storage: 20Gi -// accessModes: -// - ReadWriteOnce -// hostPath: -// path: "/mnt/d1" -// --- -// apiVersion: v1 -// kind: PersistentVolume -// metadata: -// name: d2 -// labels: -// type: local -// spec: -// storageClassName: manual -// capacity: -// storage: 20Gi -// accessModes: -// - ReadWriteOnce -// hostPath: -// path: "/mnt/d2" -// --- -// apiVersion: v1 -// kind: Service -// metadata: -// name: wordpress-mysql -// labels: -// app: wordpress -// spec: -// ports: -// - port: 3306 -// selector: -// app: wordpress -// tier: mysql -// clusterIP: None -// --- -// apiVersion: v1 -// kind: LimitRange -// metadata: -// name: test-lr2 -// spec: -// limits: -// - max: -// cpu: "200m" -// min: -// cpu: "50m" -// type: Container -// --- -// apiVersion: v1 -// kind: PersistentVolumeClaim -// metadata: -// name: mysql-pv-claim -// labels: -// app: wordpress -// spec: -// accessModes: -// - ReadWriteOnce -// resources: -// requests: -// storage: 1Gi -// --- -// apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 -// kind: Deployment -// metadata: -// name: wordpress-mysql -// labels: -// app: wordpress -// spec: -// selector: -// matchLabels: -// app: wordpress -// tier: mysql -// strategy: -// type: Recreate -// template: -// metadata: -// labels: -// app: wordpress -// tier: mysql -// spec: -// containers: -// - image: mysql:5.6 -// name: mysql -// env: -// - name: MYSQL_ROOT_PASSWORD -// valueFrom: -// secretKeyRef: -// name: mysql-pass -// key: password -// ports: -// - containerPort: 3306 -// name: mysql -// volumeMounts: -// - name: mysql-persistent-storage -// mountPath: /var/lib/mysql -// volumes: -// - name: mysql-persistent-storage -// persistentVolumeClaim: -// claimName: mysql-pv-claim -// --- -// apiVersion: autoscaling/v1 -// kind: HorizontalPodAutoscaler -// metadata: -// name: wordpress-mysql-as -// spec: -// scaleTargetRef: -// apiVersion: apps/v1 -// kind: Deployment -// name: wordpress-mysql -// minReplicas: 1 -// maxReplicas: 3 -// targetCPUUtilizationPercentage: 50 -// --- -// apiVersion: v1 -// kind: Service -// metadata: -// name: wordpress -// labels: -// app: wordpress -// spec: -// ports: -// - port: 8088 -// selector: -// app: wordpress -// tier: frontend -// type: LoadBalancer -// --- -// apiVersion: networking.k8s.io/v1 -// kind: Ingress -// metadata: -// name: wordpress-ingress -// annotations: -// nginx.ingress.kubernetes.io/rewrite-target: / -// spec: -// rules: -// - http: -// paths: -// - path: /foo -// pathType: Prefix -// backend: -// service: -// name: wordpress -// port: -// number: 8088 -// --- -// apiVersion: v1 -// kind: PersistentVolumeClaim -// metadata: -// name: wp-pv-claim -// labels: -// app: wordpress -// spec: -// accessModes: -// - ReadWriteOnce -// resources: -// requests: -// storage: 1Gi -// --- -// apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 -// kind: Deployment -// metadata: -// name: wordpress -// labels: -// app: wordpress -// spec: -// selector: -// matchLabels: -// app: wordpress -// tier: frontend -// strategy: -// type: Recreate -// template: -// metadata: -// labels: -// app: wordpress -// tier: frontend -// spec: -// containers: -// - image: wordpress:4.8-apache -// name: wordpress -// env: -// - name: WORDPRESS_DB_HOST -// value: wordpress-mysql -// - name: WORDPRESS_DB_PASSWORD -// valueFrom: -// secretKeyRef: -// name: mysql-pass -// key: password -// resources: -// limits: -// cpu: 200m -// requests: -// cpu: 200m -// ports: -// - containerPort: 80 -// name: wordpress -// volumeMounts: -// - name: wordpress-persistent-storage -// mountPath: /var/www/html -// volumes: -// - name: wordpress-persistent-storage -// persistentVolumeClaim: -// claimName: wp-pv-claim -// --- -// # Example replication controller. These are old school and basically replaced -// # by deployments -// apiVersion: v1 -// kind: ReplicationController -// metadata: -// name: nginx -// spec: -// replicas: 1 -// selector: -// app: nginx -// template: -// metadata: -// name: nginx -// labels: -// app: nginx -// spec: -// containers: -// - name: nginx -// image: nginx -// ports: -// - containerPort: 80 -// --- -// apiVersion: v1 -// kind: ResourceQuota -// metadata: -// name: pods-high -// spec: -// hard: -// cpu: "1000" -// memory: 200Gi -// pods: "10" -// scopeSelector: -// matchExpressions: -// - operator : In -// scopeName: PriorityClass -// values: ["high"] -// --- -// apiVersion: v1 -// kind: ResourceQuota -// metadata: -// name: pods-medium -// spec: -// hard: -// cpu: "10" -// memory: 20Gi -// pods: "10" -// scopeSelector: -// matchExpressions: -// - operator : In -// scopeName: PriorityClass -// values: ["medium"] -// --- -// apiVersion: v1 -// kind: ResourceQuota -// metadata: -// name: pods-low -// spec: -// hard: -// cpu: "5" -// memory: 10Gi -// pods: "10" -// scopeSelector: -// matchExpressions: -// - operator : In -// scopeName: PriorityClass -// values: ["low"] -// --- -// apiVersion: apps/v1 -// kind: DaemonSet -// metadata: -// name: fluentd-elasticsearch -// labels: -// k8s-app: fluentd-logging -// spec: -// selector: -// matchLabels: -// name: fluentd-elasticsearch -// template: -// metadata: -// labels: -// name: fluentd-elasticsearch -// spec: -// containers: -// - name: fluentd-elasticsearch -// image: quay.io/fluentd_elasticsearch/fluentd:v2.5.2 -// resources: -// limits: -// memory: 200Mi -// requests: -// cpu: 50m -// memory: 200Mi -// volumeMounts: -// - name: varlog -// mountPath: /var/log -// - name: varlibdockercontainers -// mountPath: /var/lib/docker/containers -// readOnly: true -// terminationGracePeriodSeconds: 30 -// volumes: -// - name: varlog -// hostPath: -// path: /var/log -// - name: varlibdockercontainers -// hostPath: -// path: /var/lib/docker/containers -// --- -// # Example stateful set -// apiVersion: v1 -// kind: Service -// metadata: -// name: nginx -// labels: -// app: nginx -// spec: -// ports: -// - port: 8089 -// name: web -// clusterIP: None -// selector: -// app: nginx -// --- -// apiVersion: apps/v1 -// kind: StatefulSet -// metadata: -// name: web -// spec: -// serviceName: "nginx" -// replicas: 1 -// selector: -// matchLabels: -// app: nginx -// template: -// metadata: -// labels: -// app: nginx -// spec: -// containers: -// - name: nginx -// image: k8s.gcr.io/nginx-slim:0.8 -// ports: -// - containerPort: 80 -// name: web -// volumeMounts: -// - name: www -// mountPath: /usr/share/nginx/html -// resources: -// limits: -// cpu: 50m -// requests: -// cpu: 50m -// volumeClaimTemplates: -// - metadata: -// name: www -// spec: -// accessModes: [ "ReadWriteOnce" ] -// resources: -// requests: -// storage: 1Gi -// --- -// # job example -// apiVersion: batch/v1 -// kind: Job -// metadata: -// name: pi -// spec: -// template: -// spec: -// containers: -// - name: pi -// image: perl -// command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"] -// restartPolicy: Never -// backoffLimit: 4 -// parallelism: 3 -// --- -// apiVersion: batch/v1beta1 -// kind: CronJob -// metadata: -// name: hello -// spec: -// schedule: "*/1 * * * *" -// jobTemplate: -// spec: -// template: -// spec: -// containers: -// - name: hello -// image: busybox -// args: -// - /bin/sh -// - -c -// - date; echo Hello from the Kubernetes cluster -// restartPolicy: OnFailure - -// ` diff --git a/sources/shared_util.go b/sources/shared_util.go index f9a9b66..def4736 100644 --- a/sources/shared_util.go +++ b/sources/shared_util.go @@ -3,12 +3,9 @@ package sources import ( "encoding/json" "fmt" - "reflect" "strings" - "github.com/overmindtech/sdp-go" - coreV1 "k8s.io/api/core/v1" - metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type ScopeDetails struct { @@ -77,94 +74,7 @@ func (l Selector) String() string { return strings.Join(conditions, ",") } -// ThroughJSON Converts the object though JSON and back returning an interface -// with all other type data stripped -func ThroughJSON(i interface{}) (interface{}, error) { - var ri interface{} - var jsonData []byte - var err error - - // Marshall to JSON - if jsonData, err = json.Marshal(i); err != nil { - return ri, err - } - - // Convert back - err = json.Unmarshal(jsonData, &ri) - - return ri, err -} - -// GetK8sMeta will assign attributes to an existing attributes hash that are -// taken from Kubernetes ObjectMetadata -func GetK8sMeta(s metaV1.Object) map[string]interface{} { - a := make(map[string]interface{}) - - // TODO: I could do this with even more dynamic reflection... - - if v := reflect.ValueOf(s.GetNamespace()); !v.IsZero() { - a["namespace"] = s.GetNamespace() - } - - if v := reflect.ValueOf(s.GetName()); !v.IsZero() { - a["name"] = s.GetName() - } - - if v := reflect.ValueOf(s.GetGenerateName()); !v.IsZero() { - a["generateName"] = s.GetGenerateName() - } - - if v := reflect.ValueOf(s.GetUID()); !v.IsZero() { - a["uID"] = s.GetUID() - } - - if v := reflect.ValueOf(s.GetResourceVersion()); !v.IsZero() { - a["resourceVersion"] = s.GetResourceVersion() - } - - if v := reflect.ValueOf(s.GetGeneration()); !v.IsZero() { - a["generation"] = s.GetGeneration() - } - - if v := reflect.ValueOf(s.GetSelfLink()); !v.IsZero() { - a["selfLink"] = s.GetSelfLink() - } - - if v := reflect.ValueOf(s.GetCreationTimestamp()); !v.IsZero() { - a["creationTimestamp"] = s.GetCreationTimestamp() - } - - if v := reflect.ValueOf(s.GetDeletionTimestamp()); !v.IsZero() { - a["deletionTimestamp"] = s.GetDeletionTimestamp() - } - - if v := reflect.ValueOf(s.GetDeletionGracePeriodSeconds()); !v.IsZero() { - a["deletionGracePeriodSeconds"] = s.GetDeletionGracePeriodSeconds() - } - - if v := reflect.ValueOf(s.GetLabels()); !v.IsZero() { - a["labels"] = s.GetLabels() - } - - if v := reflect.ValueOf(s.GetAnnotations()); !v.IsZero() { - a["annotations"] = s.GetAnnotations() - } - - if v := reflect.ValueOf(s.GetFinalizers()); !v.IsZero() { - a["finalizers"] = s.GetFinalizers() - } - - if v := reflect.ValueOf(s.GetOwnerReferences()); !v.IsZero() { - a["ownerReferences"] = s.GetOwnerReferences() - } - - // Note that we are deliberately ignoring ManagedFields here since it's a - // lot of data and I'm not sure if its value - - return a -} - -func ListOptionsToQuery(lo *metaV1.ListOptions) string { +func ListOptionsToQuery(lo *metav1.ListOptions) string { jsonData, err := json.Marshal(lo) if err == nil { @@ -174,22 +84,20 @@ func ListOptionsToQuery(lo *metaV1.ListOptions) string { return "" } -// SelectorToQuery converts a LabelSelector to JSON so that it can be passed to -// a SEARCH query - -// TODO: Rename to LabelSelectorToQuery -func LabelSelectorToQuery(labelSelector *metaV1.LabelSelector) string { - return ListOptionsToQuery(&metaV1.ListOptions{ +// LabelSelectorToQuery converts a LabelSelector to JSON so that it can be +// passed to a SEARCH query +func LabelSelectorToQuery(labelSelector *metav1.LabelSelector) string { + return ListOptionsToQuery(&metav1.ListOptions{ LabelSelector: Selector(labelSelector.MatchLabels).String(), }) } // QueryToListOptions converts a Search() query string to a ListOptions object that can // be used to query the API -func QueryToListOptions(query string) (metaV1.ListOptions, error) { +func QueryToListOptions(query string) (metav1.ListOptions, error) { var queryBytes []byte var err error - var listOptions metaV1.ListOptions + var listOptions metav1.ListOptions queryBytes = []byte(query) @@ -203,28 +111,3 @@ func QueryToListOptions(query string) (metaV1.ListOptions, error) { return listOptions, nil } - -// ObjectReferenceToLIR Converts a K8s ObjectReference to a linked item request. -// Note that you must provide the parent scope (the name of the cluster) since -// the reference could be an object in a different namespace and therefore -// scope. If the parent scope is empty, the scope will be assumed to be -// the same as the current object -func ObjectReferenceToLIR(ref *coreV1.ObjectReference, parentScope string) *sdp.Query { - if ref == nil { - return nil - } - - var scope string - - // If we have a namespace then calculate the full scope name - if ref.Namespace != "" && parentScope != "" { - scope = fmt.Sprintf("%v.%v", parentScope, ref.Namespace) - } - - return &sdp.Query{ - Type: strings.ToLower(ref.Kind), // Lowercase as per convention - Method: sdp.QueryMethod_GET, // Object references are to a specific object - Query: ref.Name, - Scope: scope, - } -} From 3fd168046195880db636d3e7d446db5517c9218a Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Sat, 13 May 2023 10:43:30 +0000 Subject: [PATCH 15/24] Fixed tests --- sources/cronjob_test.go | 2 +- sources/endpoints_test.go | 3 ++ sources/generic_source_test.go | 50 ++++++++++++++++++++++++---------- sources/ingress_test.go | 2 +- sources/pods_test.go | 3 ++ sources/statefulset_test.go | 17 +----------- 6 files changed, 45 insertions(+), 32 deletions(-) diff --git a/sources/cronjob_test.go b/sources/cronjob_test.go index ef21f96..3405ec5 100644 --- a/sources/cronjob_test.go +++ b/sources/cronjob_test.go @@ -49,7 +49,7 @@ func TestCronJobSource(t *testing.T) { jobSource := NewJobSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) // Wait for the job to be created - err := WaitFor(10*time.Second, func() bool { + err := WaitFor(30*time.Second, func() bool { jobs, err := jobSource.List(context.Background(), sd.String()) if err != nil { diff --git a/sources/endpoints_test.go b/sources/endpoints_test.go index 43aff53..1224855 100644 --- a/sources/endpoints_test.go +++ b/sources/endpoints_test.go @@ -77,6 +77,9 @@ func TestEndpointsSource(t *testing.T) { ExpectedScope: sd.String(), }, }, + Wait: func(item *sdp.Item) bool { + return len(item.LinkedItemQueries) > 0 + }, } st.Execute(t) diff --git a/sources/generic_source_test.go b/sources/generic_source_test.go index 2dd84ea..bfcd27e 100644 --- a/sources/generic_source_test.go +++ b/sources/generic_source_test.go @@ -533,10 +533,15 @@ type SourceTests struct { // YAML to apply before testing, it will be removed after SetupYAML string + + // An optional function to wait to return true before running the tests. It + // is passed the current item that Get tests will be run against, and should + // return a boolean indicating whether the tests should continue or wait + Wait func(item *sdp.Item) bool } func (s SourceTests) Execute(t *testing.T) { - t.Parallel() + // t.Parallel() if s.SetupYAML != "" { err := CurrentCluster.Apply(s.SetupYAML) @@ -550,26 +555,43 @@ func (s SourceTests) Execute(t *testing.T) { }) } - t.Run(s.Source.Name(), func(t *testing.T) { - var getQuery string + var getQuery string - if s.GetQueryRegexp != nil { - items, err := s.Source.List(context.Background(), s.GetScope) + if s.GetQueryRegexp != nil { + items, err := s.Source.List(context.Background(), s.GetScope) - if err != nil { - t.Fatal(err) + if err != nil { + t.Fatal(err) + } + + for _, item := range items { + if s.GetQueryRegexp.MatchString(item.UniqueAttributeValue()) { + getQuery = item.UniqueAttributeValue() + break } + } + } else { + getQuery = s.GetQuery + } - for _, item := range items { - if s.GetQueryRegexp.MatchString(item.UniqueAttributeValue()) { - getQuery = item.UniqueAttributeValue() - break - } + if s.Wait != nil { + t.Log("waiting before executing tests") + err := WaitFor(20*time.Second, func() bool { + item, err := s.Source.Get(context.Background(), s.GetScope, getQuery) + + if err != nil { + return false } - } else { - getQuery = s.GetQuery + + return s.Wait(item) + }) + + if err != nil { + t.Fatalf("timed out waiting before starting tests: %v", err) } + } + t.Run(s.Source.Name(), func(t *testing.T) { if getQuery != "" { t.Run(fmt.Sprintf("GET:%v", getQuery), func(t *testing.T) { item, err := s.Source.Get(context.Background(), s.GetScope, getQuery) diff --git a/sources/ingress_test.go b/sources/ingress_test.go index 4273334..29d0f7d 100644 --- a/sources/ingress_test.go +++ b/sources/ingress_test.go @@ -12,7 +12,7 @@ kind: Deployment metadata: name: ingress-app spec: - replicas: 3 + replicas: 1 selector: matchLabels: app: ingress-app diff --git a/sources/pods_test.go b/sources/pods_test.go index 6544c95..a3b2bbe 100644 --- a/sources/pods_test.go +++ b/sources/pods_test.go @@ -114,6 +114,9 @@ func TestPodSource(t *testing.T) { ExpectedScope: sd.String(), }, }, + Wait: func(item *sdp.Item) bool { + return len(item.LinkedItemQueries) >= 5 + }, } st.Execute(t) diff --git a/sources/statefulset_test.go b/sources/statefulset_test.go index 0e7a939..60119fb 100644 --- a/sources/statefulset_test.go +++ b/sources/statefulset_test.go @@ -8,20 +8,6 @@ import ( ) var statefulSetYAML = ` -apiVersion: v1 -kind: PersistentVolume -metadata: - name: stateful-set-test-pv -spec: - capacity: - storage: 5Gi - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Delete - storageClassName: nginx-sc - hostPath: - path: /data/nginx ---- apiVersion: apps/v1 kind: StatefulSet metadata: @@ -53,8 +39,7 @@ spec: resources: requests: storage: 1Gi - storageClassName: nginx-sc - + storageClassName: standard ` func TestStatefulSetSource(t *testing.T) { From 994b3357f0a8acb0ee80cd6c816d6a7bdcbb1487 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Sat, 13 May 2023 10:48:29 +0000 Subject: [PATCH 16/24] More test fixes --- sources/cronjob_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/cronjob_test.go b/sources/cronjob_test.go index 3405ec5..350d303 100644 --- a/sources/cronjob_test.go +++ b/sources/cronjob_test.go @@ -49,7 +49,7 @@ func TestCronJobSource(t *testing.T) { jobSource := NewJobSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) // Wait for the job to be created - err := WaitFor(30*time.Second, func() bool { + err := WaitFor(60*time.Second, func() bool { jobs, err := jobSource.List(context.Background(), sd.String()) if err != nil { From b4b8cb1a792235f9fdee8974c205c27b0f016703 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Mon, 15 May 2023 14:05:02 +0000 Subject: [PATCH 17/24] Refactor engine start process --- .vscode/launch.json | 23 ++++ cmd/root.go | 172 +++++++++++++++--------- go.mod | 10 +- go.sum | 20 +-- sources/clusterrole.go | 9 +- sources/clusterrole_test.go | 4 +- sources/clusterrolebinding.go | 34 ++--- sources/clusterrolebinding_test.go | 4 +- sources/configmap.go | 8 +- sources/configmap_test.go | 4 +- sources/cronjob.go | 4 +- sources/cronjob_test.go | 12 +- sources/daemonset.go | 4 +- sources/daemonset_test.go | 4 +- sources/deployment.go | 4 +- sources/deployment_test.go | 4 +- sources/endpoints.go | 44 +++--- sources/endpoints_test.go | 4 +- sources/endpointslice.go | 56 ++++---- sources/endpointslice_test.go | 4 +- sources/generic_source.go | 28 ++-- sources/generic_source_test.go | 69 ++++++---- sources/horizontalpodautoscaler.go | 22 +-- sources/horizontalpodautoscaler_test.go | 4 +- sources/ingress.go | 80 ++++++----- sources/ingress_test.go | 4 +- sources/job.go | 20 +-- sources/job_test.go | 4 +- sources/limitrange.go | 4 +- sources/limitrange_test.go | 4 +- sources/main.go | 41 ++++++ sources/networkpolicy.go | 32 +++-- sources/networkpolicy_test.go | 4 +- sources/node.go | 44 +++--- sources/node_test.go | 4 +- sources/persistentvolume.go | 32 +++-- sources/persistentvolume_test.go | 4 +- sources/persistentvolumeclaim.go | 4 +- sources/persistentvolumeclaim_test.go | 4 +- sources/poddisruptionbudget.go | 20 +-- sources/poddisruptionbudget_test.go | 4 +- sources/pods.go | 128 ++++++++++-------- sources/pods_test.go | 4 +- sources/priorityclass.go | 4 +- sources/priorityclass_test.go | 4 +- sources/replicaset.go | 20 +-- sources/replicaset_test.go | 4 +- sources/replicationcontroller.go | 24 ++-- sources/replicationcontroller_test.go | 4 +- sources/resourcequota.go | 4 +- sources/resourcequota_test.go | 4 +- sources/role.go | 4 +- sources/role_test.go | 4 +- sources/rolebinding.go | 38 +++--- sources/rolebinding_test.go | 6 +- sources/secret.go | 4 +- sources/secret_test.go | 4 +- sources/service.go | 84 +++++++----- sources/service_test.go | 4 +- sources/serviceaccount.go | 32 +++-- sources/serviceaccount_test.go | 4 +- sources/statefulset.go | 54 ++++---- sources/statefulset_test.go | 4 +- sources/storageclass.go | 4 +- sources/storageclass_test.go | 4 +- sources/volumeattachment.go | 32 +++-- sources/volumeattachment_test.go | 4 +- 67 files changed, 787 insertions(+), 559 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 sources/main.go diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..5192675 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,23 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/main.go", + "args": [ + "--kubeconfig", + "/home/vscode/.kube/config", + "--log", + "debug", + "--nats-servers", + "nats://localhost:4222" + ] + } + ] +} \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index cd851e6..786462b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "errors" "fmt" "net/http" @@ -16,9 +17,11 @@ import ( "github.com/nats-io/nkeys" "github.com/overmindtech/connect" "github.com/overmindtech/discovery" - "github.com/overmindtech/k8s-source/internal/sources" + "github.com/overmindtech/k8s-source/sources" "github.com/spf13/cobra" "github.com/spf13/pflag" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -32,17 +35,8 @@ var cfgFile string // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "k8s-source", - Short: "Remote primary source for kubernetes", - Long: `This is designed to be run as part of srcman -(https://github.com/overmindtech/srcman) - -It responds to requests for items relating to kubernetes clusters. -Each namespace is a separate scope, as are non-namespaced resources -within each cluster. - -This can be configured using a yaml file and the --config flag, or by -using appropriately named environment variables, for example "nats-name-prefix" -can be set using an environment variable named "NATS_NAME_PREFIX" + Short: "Kubernetes source", + Long: `Gathers details from existing kubernetes clusters `, Run: func(cmd *cobra.Command, args []string) { natsServers := viper.GetStringSlice("nats-servers") @@ -108,8 +102,6 @@ can be set using an environment variable named "NATS_NAME_PREFIX" // Now that we have a connection to the kubernetes cluster we need to go // about generating some sources. var k8sURL *url.URL - var nss sources.NamespaceStorage - var sourceList []discovery.Source k8sURL, err = url.Parse(rc.Host) @@ -131,32 +123,6 @@ can be set using an environment variable named "NATS_NAME_PREFIX" } } - sources.ClusterName = k8sURL.Host - - // Get list of namspaces - nss = sources.NamespaceStorage{ - CS: clientSet, - CacheDuration: (10 * time.Second), - } - - // Load all sources - for _, srcFunction := range sources.SourceFunctions { - src, err := srcFunction(clientSet) - - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "sourceName": src.Name(), - }).Error("Failed loading source") - - continue - } - - src.NSS = &nss - - sourceList = append(sourceList, &src) - } - // Validate the auth params and create a token client if we are using // auth if natsJWT != "" || natsNKeySeed != "" { @@ -177,7 +143,7 @@ can be set using an environment variable named "NATS_NAME_PREFIX" "error": err.Error(), }).Fatal("Error initializing Engine") } - e.Name = "source-template" + e.Name = "k8s-source" e.NATSOptions = &connect.NATSOptions{ NumRetries: -1, RetryDelay: 5 * time.Second, @@ -189,11 +155,9 @@ can be set using an environment variable named "NATS_NAME_PREFIX" ReconnectJitter: 2 * time.Second, TokenClient: tokenClient, } - e.NATSQueueName = "source-template" // This should be the same as your engine name + e.NATSQueueName = "k8s-source" // This should be the same as your engine name e.MaxParallelExecutions = maxParallel - e.AddSources(sourceList...) - // Start HTTP server for status healthCheckPort := 8080 healthCheckPath := "/healthz" @@ -223,37 +187,121 @@ can be set using an environment variable named "NATS_NAME_PREFIX" os.Exit(1) } - err = e.Start() + // Create channels for interrupts + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + restart := make(chan watch.Event, 1024) + + // Get the initial starting point + list, err := clientSet.CoreV1().Namespaces().List(context.Background(), v1.ListOptions{}) if err != nil { - log.WithFields(log.Fields{ - "error": err, - }).Error("Could not start engine") + log.Fatalf("Could not list namespaces: %v", err) + } - os.Exit(1) + // Watch namespaces from here + sendInitialEvents := false + wi, err := clientSet.CoreV1().Namespaces().Watch(context.Background(), v1.ListOptions{ + SendInitialEvents: &sendInitialEvents, + ResourceVersion: list.ResourceVersion, + }) + + if err != nil { + log.Fatalf("Could not start watching namespaces: %v", err) } - sigs := make(chan os.Signal, 1) + watchCtx, watchCancel := context.WithCancel(context.Background()) + defer watchCancel() + + go func() { + for { + select { + case event := <-wi.ResultChan(): + // Restart the engine + restart <- event + case <-watchCtx.Done(): + return + } + } + }() + + for { + // Query all namespaces + log.Info("Listing namespaces") + list, err := clientSet.CoreV1().Namespaces().List(context.Background(), v1.ListOptions{}) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + if err != nil { + log.Fatal(err) + } - <-sigs + namespaces := make([]string, len(list.Items)) - log.Info("Stopping engine") + for i := range list.Items { + namespaces[i] = list.Items[i].Name + } - err = e.Stop() + log.Infof("got %v namespaces", len(namespaces)) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - }).Error("Could not stop engine") + // Create the sources + sourceList := sources.LoadAllSources(clientSet, k8sURL.Host, namespaces) - os.Exit(1) - } + // Add sources to the engine + e.AddSources(sourceList...) - log.Info("Stopped") + // Start the engine + err = e.Start() + + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Error("Could not start engine") + + os.Exit(1) + } - os.Exit(0) + // Start waiting for either an interrupt or a restart + select { + case <-quit: + log.Info("Stopping engine") + + err = e.Stop() + + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Error("Could not stop engine") + + os.Exit(1) + } + + log.Info("Stopped") + + os.Exit(0) + case event := <-restart: + log.Infof("Restarting engine due to namespace event: %v", event.Type) + + // Stop the engine + err = e.Stop() + + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Error("Could not stop engine") + + os.Exit(1) + } + + // Clear the sources + e.ClearSources() + + // Stop the engine + err = e.Stop() + + if err != nil { + log.Fatal(err) + } + } + } }, } diff --git a/go.mod b/go.mod index 90cf843..3d520da 100644 --- a/go.mod +++ b/go.mod @@ -4,15 +4,15 @@ go 1.20 // Direct dependencies of my codebase require ( + github.com/google/uuid v1.3.0 github.com/nats-io/jwt/v2 v2.4.1 github.com/nats-io/nkeys v0.4.4 - github.com/overmindtech/connect v0.9.0 - github.com/overmindtech/discovery v0.19.0 - github.com/overmindtech/sdp-go v0.29.1 + github.com/overmindtech/connect v0.10.0 + github.com/overmindtech/discovery v0.20.2 + github.com/overmindtech/sdp-go v0.30.0 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - github.com/google/uuid v1.3.0 github.com/spf13/viper v1.15.0 k8s.io/api v0.27.1 k8s.io/apimachinery v0.27.1 @@ -58,7 +58,7 @@ require ( github.com/nats-io/nats.go v1.25.0 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/overmindtech/api-client v0.14.0 // indirect - github.com/overmindtech/sdpcache v1.3.0 // indirect + github.com/overmindtech/sdpcache v1.4.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index cb00542..1a5e0be 100644 --- a/go.sum +++ b/go.sum @@ -199,7 +199,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -228,7 +228,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4= github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= -github.com/nats-io/nats-server/v2 v2.9.15 h1:MuwEJheIwpvFgqvbs20W8Ish2azcygjf4Z0liVu2I4c= +github.com/nats-io/nats-server/v2 v2.9.16 h1:SuNe6AyCcVy0g5326wtyU8TdqYmcPqzTjhkHojAjprc= github.com/nats-io/nats.go v1.25.0 h1:t5/wCPGciR7X3Mu8QOi4jiJaXaWM8qtkLu4lzGZvYHE= github.com/nats-io/nats.go v1.25.0/go.mod h1:D2WALIhz7V8M0pH8Scx8JZXlg6Oqz5VG+nQkK8nJdvg= github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= @@ -239,14 +239,14 @@ github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= github.com/overmindtech/api-client v0.14.0 h1:zXyjJsIeawNqoWv7FqOjwcqgFpLrDYz7l9MWqh1G9ZQ= github.com/overmindtech/api-client v0.14.0/go.mod h1:msdkTAQFlvDGOU4tQk2adk2P8j23uaMWkJ9YRX4wGWI= -github.com/overmindtech/connect v0.9.0 h1:laork6I4Ww+4dYv1WHRXhmfRFQKKUORWUSdTMnyMlrI= -github.com/overmindtech/connect v0.9.0/go.mod h1:CgP8Wko8JBqTM6klTv75Q3kCWagvPXOjD9dvyQRAtHE= -github.com/overmindtech/discovery v0.19.0 h1:XbEET1tumhKgXw7kDgr6DUyolC7zIN1EtdscCFEkVFY= -github.com/overmindtech/discovery v0.19.0/go.mod h1:uvScd6AXzX9h4icJAl21rtIuweaaZ64nQxAxzF9ljlY= -github.com/overmindtech/sdp-go v0.29.1 h1:TCdHCmzU5NvFdQhr0Qb1QXiWjwrT9VQMlhh7jPJ5kiw= -github.com/overmindtech/sdp-go v0.29.1/go.mod h1:aAnIhq9x6RExxxPfgYYCvGQE5lu7vwr904wFF/GbwcI= -github.com/overmindtech/sdpcache v1.3.0 h1:58kmqw1l+ZyZafxOaXW5ccDeapdx4E7QHOMRPpWZ5VA= -github.com/overmindtech/sdpcache v1.3.0/go.mod h1:tE2Qtssr4rZykNctJnS5dCw5fd8JVbBfCh6zUGPRrSI= +github.com/overmindtech/connect v0.10.0 h1:4EYZle2YZf9IbfpYiz4qoBgdBMDzTxQ5fGyAslPm9hM= +github.com/overmindtech/connect v0.10.0/go.mod h1:C3FVTWt9GC3ApaT+qF7+Jo9M7sVezysrXFRgsMrHT3M= +github.com/overmindtech/discovery v0.20.2 h1:6oWW/KOE1TNVu3wznRRVde1fgzUN5phVkK7gdKJX/Z0= +github.com/overmindtech/discovery v0.20.2/go.mod h1:nVy//9Tgg0HI0V92GGWifkfA6iI1+avI8mpeWkvkA9s= +github.com/overmindtech/sdp-go v0.30.0 h1:N+wjSQsnXao2l1n5+MAReA0KHyYpH6uXlIcbEkEuzPI= +github.com/overmindtech/sdp-go v0.30.0/go.mod h1:aAnIhq9x6RExxxPfgYYCvGQE5lu7vwr904wFF/GbwcI= +github.com/overmindtech/sdpcache v1.4.0 h1:8u+KBn7PcgIrWnmQ1yzgtxDyVulDIdocb5BS0Kw3Jrw= +github.com/overmindtech/sdpcache v1.4.0/go.mod h1:57BnPdDWIZlNrzoP+pqqMPeZd3JgGuM9eZuRN9BhL7k= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= diff --git a/sources/clusterrole.go b/sources/clusterrole.go index eb9ffb4..be2290e 100644 --- a/sources/clusterrole.go +++ b/sources/clusterrole.go @@ -3,12 +3,11 @@ package sources import ( v1 "k8s.io/api/rbac/v1" - "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) -func NewClusterRoleSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ClusterRole, *v1.ClusterRoleList] { - return KubeTypeSource[*v1.ClusterRole, *v1.ClusterRoleList]{ +func newClusterRoleSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ClusterRole, *v1.ClusterRoleList] { + return &KubeTypeSource[*v1.ClusterRole, *v1.ClusterRoleList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "ClusterRole", @@ -24,9 +23,5 @@ func NewClusterRoleSource(cs *kubernetes.Clientset, cluster string, namespaces [ return bindings, nil }, - LinkedItemQueryExtractor: func(resource *v1.ClusterRole, scope string) ([]*sdp.Query, error) { - // No linked items - return []*sdp.Query{}, nil - }, } } diff --git a/sources/clusterrole_test.go b/sources/clusterrole_test.go index 33dbf5d..46d058d 100644 --- a/sources/clusterrole_test.go +++ b/sources/clusterrole_test.go @@ -22,10 +22,10 @@ roleRef: ` func TestClusterRoleBindingSource(t *testing.T) { - source := NewClusterRoleBindingSource(CurrentCluster.ClientSet, CurrentCluster.Name, []string{}) + source := newClusterRoleBindingSource(CurrentCluster.ClientSet, CurrentCluster.Name, []string{}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "admin-binding", GetScope: CurrentCluster.Name, SetupYAML: clusterRoleBindingYAML, diff --git a/sources/clusterrolebinding.go b/sources/clusterrolebinding.go index 9741bd9..93c7106 100644 --- a/sources/clusterrolebinding.go +++ b/sources/clusterrolebinding.go @@ -7,14 +7,16 @@ import ( "k8s.io/client-go/kubernetes" ) -func clusterRoleBindingExtractor(resource *v1.ClusterRoleBinding, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) - - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_GET, - Query: resource.RoleRef.Name, - Type: resource.RoleRef.Kind, +func clusterRoleBindingExtractor(resource *v1.ClusterRoleBinding, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) + + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_GET, + Query: resource.RoleRef.Name, + Type: resource.RoleRef.Kind, + }, }) for _, subject := range resource.Subjects { @@ -26,19 +28,21 @@ func clusterRoleBindingExtractor(resource *v1.ClusterRoleBinding, scope string) sd.Namespace = subject.Namespace } - queries = append(queries, &sdp.Query{ - Scope: sd.String(), - Method: sdp.QueryMethod_GET, - Query: subject.Name, - Type: subject.Kind, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: sd.String(), + Method: sdp.QueryMethod_GET, + Query: subject.Name, + Type: subject.Kind, + }, }) } return queries, nil } -func NewClusterRoleBindingSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ClusterRoleBinding, *v1.ClusterRoleBindingList] { - return KubeTypeSource[*v1.ClusterRoleBinding, *v1.ClusterRoleBindingList]{ +func newClusterRoleBindingSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ClusterRoleBinding, *v1.ClusterRoleBindingList] { + return &KubeTypeSource[*v1.ClusterRoleBinding, *v1.ClusterRoleBindingList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "ClusterRoleBinding", diff --git a/sources/clusterrolebinding_test.go b/sources/clusterrolebinding_test.go index 26feebf..2186bb4 100644 --- a/sources/clusterrolebinding_test.go +++ b/sources/clusterrolebinding_test.go @@ -17,10 +17,10 @@ rules: ` func TestClusterRoleSource(t *testing.T) { - source := NewClusterRoleSource(CurrentCluster.ClientSet, CurrentCluster.Name, []string{}) + source := newClusterRoleSource(CurrentCluster.ClientSet, CurrentCluster.Name, []string{}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "read-only", GetScope: CurrentCluster.Name, SetupYAML: clusterRoleYAML, diff --git a/sources/configmap.go b/sources/configmap.go index dc2145a..fde8837 100644 --- a/sources/configmap.go +++ b/sources/configmap.go @@ -1,13 +1,12 @@ package sources import ( - "github.com/overmindtech/sdp-go" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -func NewConfigMapSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ConfigMap, *v1.ConfigMapList] { - return KubeTypeSource[*v1.ConfigMap, *v1.ConfigMapList]{ +func newConfigMapSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ConfigMap, *v1.ConfigMapList] { + return &KubeTypeSource[*v1.ConfigMap, *v1.ConfigMapList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "ConfigMap", @@ -23,8 +22,5 @@ func NewConfigMapSource(cs *kubernetes.Clientset, cluster string, namespaces []s return bindings, nil }, - LinkedItemQueryExtractor: func(resource *v1.ConfigMap, scope string) ([]*sdp.Query, error) { - return []*sdp.Query{}, nil - }, } } diff --git a/sources/configmap_test.go b/sources/configmap_test.go index 5934b97..0dccfab 100644 --- a/sources/configmap_test.go +++ b/sources/configmap_test.go @@ -18,10 +18,10 @@ func TestConfigMapSource(t *testing.T) { Namespace: "default", } - source := NewConfigMapSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newConfigMapSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "my-configmap", GetScope: sd.String(), SetupYAML: configMapYAML, diff --git a/sources/cronjob.go b/sources/cronjob.go index 1da57bb..7f120d3 100644 --- a/sources/cronjob.go +++ b/sources/cronjob.go @@ -6,8 +6,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func NewCronJobSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.CronJob, *v1.CronJobList] { - return KubeTypeSource[*v1.CronJob, *v1.CronJobList]{ +func newCronJobSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.CronJob, *v1.CronJobList] { + return &KubeTypeSource[*v1.CronJob, *v1.CronJobList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "CronJob", diff --git a/sources/cronjob_test.go b/sources/cronjob_test.go index 350d303..fac927d 100644 --- a/sources/cronjob_test.go +++ b/sources/cronjob_test.go @@ -32,10 +32,10 @@ func TestCronJobSource(t *testing.T) { Namespace: "default", } - source := NewCronJobSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newCronJobSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "my-cronjob", GetScope: sd.String(), SetupYAML: cronJobYAML, @@ -46,7 +46,7 @@ func TestCronJobSource(t *testing.T) { // Additionally, make sure that the job has a link back to the cronjob that // created it - jobSource := NewJobSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + jobSource := newJobSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) // Wait for the job to be created err := WaitFor(60*time.Second, func() bool { @@ -60,8 +60,10 @@ func TestCronJobSource(t *testing.T) { // Ensure that the job has a link back to the cronjob for _, job := range jobs { for _, q := range job.LinkedItemQueries { - if q.Query == "my-cronjob" { - return true + if q.Query != nil { + if q.Query.Query == "my-cronjob" { + return true + } } } diff --git a/sources/daemonset.go b/sources/daemonset.go index 68a0f7a..3cb2dae 100644 --- a/sources/daemonset.go +++ b/sources/daemonset.go @@ -6,8 +6,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func NewDaemonSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.DaemonSet, *v1.DaemonSetList] { - return KubeTypeSource[*v1.DaemonSet, *v1.DaemonSetList]{ +func newDaemonSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.DaemonSet, *v1.DaemonSetList] { + return &KubeTypeSource[*v1.DaemonSet, *v1.DaemonSetList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "DaemonSet", diff --git a/sources/daemonset_test.go b/sources/daemonset_test.go index ed790b4..810dc70 100644 --- a/sources/daemonset_test.go +++ b/sources/daemonset_test.go @@ -32,10 +32,10 @@ func TestDaemonSetSource(t *testing.T) { Namespace: "default", } - source := NewDaemonSetSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newDaemonSetSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "my-daemonset", GetScope: sd.String(), SetupYAML: daemonSetYAML, diff --git a/sources/deployment.go b/sources/deployment.go index 2f4c308..65f3d3d 100644 --- a/sources/deployment.go +++ b/sources/deployment.go @@ -6,8 +6,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func NewDeploymentSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Deployment, *v1.DeploymentList] { - return KubeTypeSource[*v1.Deployment, *v1.DeploymentList]{ +func newDeploymentSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Deployment, *v1.DeploymentList] { + return &KubeTypeSource[*v1.Deployment, *v1.DeploymentList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "Deployment", diff --git a/sources/deployment_test.go b/sources/deployment_test.go index 331a833..fd64848 100644 --- a/sources/deployment_test.go +++ b/sources/deployment_test.go @@ -32,10 +32,10 @@ func TestDeploymentSource(t *testing.T) { Namespace: "default", } - source := NewDeploymentSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newDeploymentSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "my-deployment", GetScope: sd.String(), SetupYAML: deploymentYAML, diff --git a/sources/endpoints.go b/sources/endpoints.go index 3f3e1fa..6836e55 100644 --- a/sources/endpoints.go +++ b/sources/endpoints.go @@ -6,8 +6,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func EndpointsExtractor(resource *v1.Endpoints, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func EndpointsExtractor(resource *v1.Endpoints, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) sd, err := ParseScope(scope, true) @@ -18,29 +18,35 @@ func EndpointsExtractor(resource *v1.Endpoints, scope string) ([]*sdp.Query, err for _, subset := range resource.Subsets { for _, address := range subset.Addresses { if address.Hostname != "" { - queries = append(queries, &sdp.Query{ - Scope: "global", - Method: sdp.QueryMethod_GET, - Query: address.Hostname, - Type: "dns", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: "global", + Method: sdp.QueryMethod_GET, + Query: address.Hostname, + Type: "dns", + }, }) } if address.NodeName != nil { - queries = append(queries, &sdp.Query{ - Type: "Node", - Scope: sd.ClusterName, - Method: sdp.QueryMethod_GET, - Query: *address.NodeName, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "Node", + Scope: sd.ClusterName, + Method: sdp.QueryMethod_GET, + Query: *address.NodeName, + }, }) } if address.IP != "" { - queries = append(queries, &sdp.Query{ - Type: "ip", - Method: sdp.QueryMethod_GET, - Query: address.IP, - Scope: "global", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "ip", + Method: sdp.QueryMethod_GET, + Query: address.IP, + Scope: "global", + }, }) } @@ -54,8 +60,8 @@ func EndpointsExtractor(resource *v1.Endpoints, scope string) ([]*sdp.Query, err return queries, nil } -func NewEndpointsSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Endpoints, *v1.EndpointsList] { - return KubeTypeSource[*v1.Endpoints, *v1.EndpointsList]{ +func newEndpointsSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Endpoints, *v1.EndpointsList] { + return &KubeTypeSource[*v1.Endpoints, *v1.EndpointsList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "Endpoints", diff --git a/sources/endpoints_test.go b/sources/endpoints_test.go index 1224855..94215c4 100644 --- a/sources/endpoints_test.go +++ b/sources/endpoints_test.go @@ -50,10 +50,10 @@ func TestEndpointsSource(t *testing.T) { Namespace: "default", } - source := NewEndpointsSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newEndpointsSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "endpoint-service", GetScope: sd.String(), SetupYAML: endpointsYAML, diff --git a/sources/endpointslice.go b/sources/endpointslice.go index 655857e..510736d 100644 --- a/sources/endpointslice.go +++ b/sources/endpointslice.go @@ -7,8 +7,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func endpointSliceExtractor(resource *v1.EndpointSlice, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func endpointSliceExtractor(resource *v1.EndpointSlice, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) sd, err := ParseScope(scope, true) @@ -18,20 +18,24 @@ func endpointSliceExtractor(resource *v1.EndpointSlice, scope string) ([]*sdp.Qu for _, endpoint := range resource.Endpoints { if endpoint.Hostname != nil { - queries = append(queries, &sdp.Query{ - Type: "dns", - Method: sdp.QueryMethod_GET, - Query: *endpoint.Hostname, - Scope: "global", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "dns", + Method: sdp.QueryMethod_GET, + Query: *endpoint.Hostname, + Scope: "global", + }, }) } if endpoint.NodeName != nil { - queries = append(queries, &sdp.Query{ - Type: "Node", - Method: sdp.QueryMethod_GET, - Query: *endpoint.NodeName, - Scope: sd.ClusterName, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "Node", + Method: sdp.QueryMethod_GET, + Query: *endpoint.NodeName, + Scope: sd.ClusterName, + }, }) } @@ -43,18 +47,22 @@ func endpointSliceExtractor(resource *v1.EndpointSlice, scope string) ([]*sdp.Qu for _, address := range endpoint.Addresses { switch resource.AddressType { case v1.AddressTypeIPv4, v1.AddressTypeIPv6: - queries = append(queries, &sdp.Query{ - Type: "ip", - Method: sdp.QueryMethod_GET, - Query: address, - Scope: "global", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "ip", + Method: sdp.QueryMethod_GET, + Query: address, + Scope: "global", + }, }) case v1.AddressTypeFQDN: - queries = append(queries, &sdp.Query{ - Type: "dns", - Method: sdp.QueryMethod_GET, - Query: address, - Scope: "global", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "dns", + Method: sdp.QueryMethod_GET, + Query: address, + Scope: "global", + }, }) } } @@ -63,8 +71,8 @@ func endpointSliceExtractor(resource *v1.EndpointSlice, scope string) ([]*sdp.Qu return queries, nil } -func NewEndpointSliceSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.EndpointSlice, *v1.EndpointSliceList] { - return KubeTypeSource[*v1.EndpointSlice, *v1.EndpointSliceList]{ +func newEndpointSliceSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.EndpointSlice, *v1.EndpointSliceList] { + return &KubeTypeSource[*v1.EndpointSlice, *v1.EndpointSliceList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "EndpointSlice", diff --git a/sources/endpointslice_test.go b/sources/endpointslice_test.go index 6155d32..79c7e25 100644 --- a/sources/endpointslice_test.go +++ b/sources/endpointslice_test.go @@ -50,10 +50,10 @@ func TestEndpointSliceSource(t *testing.T) { Namespace: "default", } - source := NewEndpointSliceSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newEndpointSliceSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQueryRegexp: regexp.MustCompile("endpoint-service"), GetScope: sd.String(), SetupYAML: endpointSliceYAML, diff --git a/sources/generic_source.go b/sources/generic_source.go index 6b56674..6386e1e 100644 --- a/sources/generic_source.go +++ b/sources/generic_source.go @@ -44,7 +44,7 @@ type KubeTypeSource[Resource metav1.Object, ResourceList any] struct { // A function that returns a list of linked item queries for a given // resource and scope - LinkedItemQueryExtractor func(resource Resource, scope string) ([]*sdp.Query, error) + LinkedItemQueryExtractor func(resource Resource, scope string) ([]*sdp.LinkedItemQuery, error) // A function that extracts health from the resource, this is optional HealthExtractor func(resource Resource) *sdp.Health @@ -304,11 +304,13 @@ func (k *KubeTypeSource[Resource, ResourceList]) resourceToItem(resource Resourc // Automatically create links to owner references for _, ref := range resource.GetOwnerReferences() { - item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.Query{ - Type: ref.Kind, - Method: sdp.QueryMethod_GET, - Query: ref.Name, - Scope: sd.String(), + item.LinkedItemQueries = append(item.LinkedItemQueries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: ref.Kind, + Method: sdp.QueryMethod_GET, + Query: ref.Name, + Scope: sd.String(), + }, }) } @@ -334,7 +336,7 @@ func (k *KubeTypeSource[Resource, ResourceList]) resourceToItem(resource Resourc // request. Note that you must provide the parent scope since the reference // could be an object in a different namespace, if it is we need to re-use the // cluster name from the parent scope -func ObjectReferenceToQuery(ref *corev1.ObjectReference, parentScope ScopeDetails) *sdp.Query { +func ObjectReferenceToQuery(ref *corev1.ObjectReference, parentScope ScopeDetails) *sdp.LinkedItemQuery { if ref == nil { return nil } @@ -342,10 +344,12 @@ func ObjectReferenceToQuery(ref *corev1.ObjectReference, parentScope ScopeDetail // Update the namespace, but keep the cluster the same parentScope.Namespace = ref.Namespace - return &sdp.Query{ - Type: ref.Kind, - Method: sdp.QueryMethod_GET, // Object references are to a specific object - Query: ref.Name, - Scope: parentScope.String(), + return &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: ref.Kind, + Method: sdp.QueryMethod_GET, // Object references are to a specific object + Query: ref.Name, + Scope: parentScope.String(), + }, } } diff --git a/sources/generic_source_test.go b/sources/generic_source_test.go index bfcd27e..4ee2070 100644 --- a/sources/generic_source_test.go +++ b/sources/generic_source_test.go @@ -121,7 +121,7 @@ func (p PodClient) List(ctx context.Context, opts metav1.ListOptions) (*v1.PodLi }, nil } -func createSource(namespaced bool) KubeTypeSource[*v1.Pod, *v1.PodList] { +func createSource(namespaced bool) *KubeTypeSource[*v1.Pod, *v1.PodList] { var clusterInterfaceBuilder ClusterInterfaceBuilder[*v1.Pod, *v1.PodList] var namespacedInterfaceBuilder NamespacedInterfaceBuilder[*v1.Pod, *v1.PodList] @@ -135,7 +135,7 @@ func createSource(namespaced bool) KubeTypeSource[*v1.Pod, *v1.PodList] { } } - return KubeTypeSource[*v1.Pod, *v1.PodList]{ + return &KubeTypeSource[*v1.Pod, *v1.PodList]{ ClusterInterfaceBuilder: clusterInterfaceBuilder, NamespacedInterfaceBuilder: namespacedInterfaceBuilder, ListExtractor: func(p *v1.PodList) ([]*v1.Pod, error) { @@ -147,15 +147,17 @@ func createSource(namespaced bool) KubeTypeSource[*v1.Pod, *v1.PodList] { return pods, nil }, - LinkedItemQueryExtractor: func(p *v1.Pod, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) + LinkedItemQueryExtractor: func(p *v1.Pod, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) if p.Spec.NodeName == "" { - queries = append(queries, &sdp.Query{ - Type: "node", - Method: sdp.QueryMethod_GET, - Query: p.Spec.NodeName, - Scope: scope, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "node", + Method: sdp.QueryMethod_GET, + Query: p.Spec.NodeName, + Scope: scope, + }, }) } @@ -351,7 +353,7 @@ func TestSourceGet(t *testing.T) { func TestFailingQueryExtractor(t *testing.T) { source := createSource(false) - source.LinkedItemQueryExtractor = func(_ *v1.Pod, _ string) ([]*sdp.Query, error) { + source.LinkedItemQueryExtractor = func(_ *v1.Pod, _ string) ([]*sdp.LinkedItemQuery, error) { return nil, errors.New("failed to extract queries") } @@ -410,7 +412,7 @@ func TestList(t *testing.T) { t.Run("with failing query extractor", func(t *testing.T) { source := createSource(false) - source.LinkedItemQueryExtractor = func(_ *v1.Pod, _ string) ([]*sdp.Query, error) { + source.LinkedItemQueryExtractor = func(_ *v1.Pod, _ string) ([]*sdp.LinkedItemQuery, error) { return nil, errors.New("failed to extract queries") } @@ -503,19 +505,32 @@ func (i QueryTests) Execute(t *testing.T, item *sdp.Item) { } } -func lirMatches(test QueryTest, req *sdp.Query) bool { - methodOK := test.ExpectedMethod == req.Method - scopeOK := test.ExpectedScope == req.Scope - typeOK := test.ExpectedType == req.Type - var queryOK bool +func lirMatches(test QueryTest, req *sdp.LinkedItemQuery) bool { + if req.Query != nil { + if test.ExpectedMethod != req.Query.Method { + return false + } + if test.ExpectedScope != req.Query.Scope { + return false + } + if test.ExpectedType != req.Query.Type { + return false + } - if test.ExpectedQueryMatches != nil { - queryOK = test.ExpectedQueryMatches.MatchString(req.Query) - } else { - queryOK = test.ExpectedQuery == req.Query + if test.ExpectedQueryMatches != nil { + if !test.ExpectedQueryMatches.MatchString(req.Query.Query) { + return false + } + } else { + if test.ExpectedQuery != req.Query.Query { + return false + } + } } - return methodOK && scopeOK && typeOK && queryOK + // TODO: check for blast radius differences + + return true } type SourceTests struct { @@ -670,16 +685,16 @@ func TestObjectReferenceToQuery(t *testing.T) { Namespace: "default", }) - if query.Type != "Pod" { - t.Errorf("expected type Pod, got %s", query.Type) + if query.Query.Type != "Pod" { + t.Errorf("expected type Pod, got %s", query.Query.Type) } - if query.Query != "foo" { - t.Errorf("expected query to be foo, got %s", query.Query) + if query.Query.Query != "foo" { + t.Errorf("expected query to be foo, got %s", query.Query.Query) } - if query.Scope != "test-cluster.default" { - t.Errorf("expected scope to be test-cluster.default, got %s", query.Scope) + if query.Query.Scope != "test-cluster.default" { + t.Errorf("expected scope to be test-cluster.default, got %s", query.Query.Scope) } }) diff --git a/sources/horizontalpodautoscaler.go b/sources/horizontalpodautoscaler.go index dc4fb0a..2ce2d74 100644 --- a/sources/horizontalpodautoscaler.go +++ b/sources/horizontalpodautoscaler.go @@ -7,21 +7,23 @@ import ( "k8s.io/client-go/kubernetes" ) -func horizontalPodAutoscalerExtractor(resource *v2.HorizontalPodAutoscaler, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) - - queries = append(queries, &sdp.Query{ - Type: resource.Spec.ScaleTargetRef.Kind, - Method: sdp.QueryMethod_GET, - Query: resource.Spec.ScaleTargetRef.Name, - Scope: scope, +func horizontalPodAutoscalerExtractor(resource *v2.HorizontalPodAutoscaler, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) + + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: resource.Spec.ScaleTargetRef.Kind, + Method: sdp.QueryMethod_GET, + Query: resource.Spec.ScaleTargetRef.Name, + Scope: scope, + }, }) return queries, nil } -func NewHorizontalPodAutoscalerSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v2.HorizontalPodAutoscaler, *v2.HorizontalPodAutoscalerList] { - return KubeTypeSource[*v2.HorizontalPodAutoscaler, *v2.HorizontalPodAutoscalerList]{ +func newHorizontalPodAutoscalerSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v2.HorizontalPodAutoscaler, *v2.HorizontalPodAutoscalerList] { + return &KubeTypeSource[*v2.HorizontalPodAutoscaler, *v2.HorizontalPodAutoscalerList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "HorizontalPodAutoscaler", diff --git a/sources/horizontalpodautoscaler_test.go b/sources/horizontalpodautoscaler_test.go index 6facd93..c725c5c 100644 --- a/sources/horizontalpodautoscaler_test.go +++ b/sources/horizontalpodautoscaler_test.go @@ -53,10 +53,10 @@ func TestHorizontalPodAutoscalerSource(t *testing.T) { Namespace: "default", } - source := NewHorizontalPodAutoscalerSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newHorizontalPodAutoscalerSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "my-hpa", GetScope: sd.String(), SetupYAML: horizontalPodAutoscalerYAML, diff --git a/sources/ingress.go b/sources/ingress.go index 1ef96ac..f5aefab 100644 --- a/sources/ingress.go +++ b/sources/ingress.go @@ -7,65 +7,77 @@ import ( "k8s.io/client-go/kubernetes" ) -func ingressExtractor(resource *v1.Ingress, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func ingressExtractor(resource *v1.Ingress, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) if resource.Spec.IngressClassName != nil { - queries = append(queries, &sdp.Query{ - Type: "IngressClass", - Method: sdp.QueryMethod_GET, - Query: *resource.Spec.IngressClassName, - Scope: scope, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "IngressClass", + Method: sdp.QueryMethod_GET, + Query: *resource.Spec.IngressClassName, + Scope: scope, + }, }) } if resource.Spec.DefaultBackend != nil { if resource.Spec.DefaultBackend.Service != nil { - queries = append(queries, &sdp.Query{ - Type: "Service", - Method: sdp.QueryMethod_GET, - Query: resource.Spec.DefaultBackend.Service.Name, - Scope: scope, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "Service", + Method: sdp.QueryMethod_GET, + Query: resource.Spec.DefaultBackend.Service.Name, + Scope: scope, + }, }) } if linkRes := resource.Spec.DefaultBackend.Resource; linkRes != nil { - queries = append(queries, &sdp.Query{ - Type: linkRes.Kind, - Method: sdp.QueryMethod_GET, - Query: linkRes.Name, - Scope: scope, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: linkRes.Kind, + Method: sdp.QueryMethod_GET, + Query: linkRes.Name, + Scope: scope, + }, }) } } for _, rule := range resource.Spec.Rules { if rule.Host != "" { - queries = append(queries, &sdp.Query{ - Type: "dns", - Method: sdp.QueryMethod_GET, - Query: rule.Host, - Scope: "global", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "dns", + Method: sdp.QueryMethod_GET, + Query: rule.Host, + Scope: "global", + }, }) } if rule.HTTP != nil { for _, path := range rule.HTTP.Paths { if path.Backend.Service != nil { - queries = append(queries, &sdp.Query{ - Type: "Service", - Method: sdp.QueryMethod_GET, - Query: path.Backend.Service.Name, - Scope: scope, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "Service", + Method: sdp.QueryMethod_GET, + Query: path.Backend.Service.Name, + Scope: scope, + }, }) } if path.Backend.Resource != nil { - queries = append(queries, &sdp.Query{ - Type: path.Backend.Resource.Kind, - Method: sdp.QueryMethod_GET, - Query: path.Backend.Resource.Name, - Scope: scope, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: path.Backend.Resource.Kind, + Method: sdp.QueryMethod_GET, + Query: path.Backend.Resource.Name, + Scope: scope, + }, }) } } @@ -75,8 +87,8 @@ func ingressExtractor(resource *v1.Ingress, scope string) ([]*sdp.Query, error) return queries, nil } -func NewIngressSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Ingress, *v1.IngressList] { - return KubeTypeSource[*v1.Ingress, *v1.IngressList]{ +func newIngressSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Ingress, *v1.IngressList] { + return &KubeTypeSource[*v1.Ingress, *v1.IngressList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "Ingress", diff --git a/sources/ingress_test.go b/sources/ingress_test.go index 29d0f7d..5ea277c 100644 --- a/sources/ingress_test.go +++ b/sources/ingress_test.go @@ -64,10 +64,10 @@ func TestIngressSource(t *testing.T) { Namespace: "default", } - source := NewIngressSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newIngressSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "ingress-app", GetScope: sd.String(), SetupYAML: ingressYAML, diff --git a/sources/job.go b/sources/job.go index 7b5f48c..eb47e2d 100644 --- a/sources/job.go +++ b/sources/job.go @@ -7,23 +7,25 @@ import ( "k8s.io/client-go/kubernetes" ) -func jobExtractor(resource *v1.Job, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func jobExtractor(resource *v1.Job, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) if resource.Spec.Selector != nil { - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(resource.Spec.Selector), - Type: "Pod", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(resource.Spec.Selector), + Type: "Pod", + }, }) } return queries, nil } -func NewJobSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Job, *v1.JobList] { - return KubeTypeSource[*v1.Job, *v1.JobList]{ +func newJobSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Job, *v1.JobList] { + return &KubeTypeSource[*v1.Job, *v1.JobList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "Job", diff --git a/sources/job_test.go b/sources/job_test.go index 8d3f3fa..5ca535b 100644 --- a/sources/job_test.go +++ b/sources/job_test.go @@ -47,10 +47,10 @@ func TestJobSource(t *testing.T) { Namespace: "default", } - source := NewJobSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newJobSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "my-job", GetScope: sd.String(), SetupYAML: jobYAML, diff --git a/sources/limitrange.go b/sources/limitrange.go index c3f9640..630ded1 100644 --- a/sources/limitrange.go +++ b/sources/limitrange.go @@ -5,8 +5,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func NewLimitRangeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.LimitRange, *v1.LimitRangeList] { - return KubeTypeSource[*v1.LimitRange, *v1.LimitRangeList]{ +func newLimitRangeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.LimitRange, *v1.LimitRangeList] { + return &KubeTypeSource[*v1.LimitRange, *v1.LimitRangeList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "LimitRange", diff --git a/sources/limitrange_test.go b/sources/limitrange_test.go index 5270efa..9aa2455 100644 --- a/sources/limitrange_test.go +++ b/sources/limitrange_test.go @@ -37,10 +37,10 @@ func TestLimitRangeSource(t *testing.T) { Namespace: "default", } - source := NewLimitRangeSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newLimitRangeSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "example-limit-range", GetScope: sd.String(), SetupYAML: limitRangeYAML, diff --git a/sources/main.go b/sources/main.go new file mode 100644 index 0000000..c283f26 --- /dev/null +++ b/sources/main.go @@ -0,0 +1,41 @@ +package sources + +import ( + "github.com/overmindtech/discovery" + "k8s.io/client-go/kubernetes" +) + +func LoadAllSources(cs *kubernetes.Clientset, cluster string, namespaces []string) []discovery.Source { + return []discovery.Source{ + newClusterRoleSource(cs, cluster, namespaces), + newClusterRoleBindingSource(cs, cluster, namespaces), + newConfigMapSource(cs, cluster, namespaces), + newCronJobSource(cs, cluster, namespaces), + newDaemonSetSource(cs, cluster, namespaces), + newDeploymentSource(cs, cluster, namespaces), + newEndpointsSource(cs, cluster, namespaces), + newEndpointSliceSource(cs, cluster, namespaces), + newHorizontalPodAutoscalerSource(cs, cluster, namespaces), + newIngressSource(cs, cluster, namespaces), + newJobSource(cs, cluster, namespaces), + newLimitRangeSource(cs, cluster, namespaces), + newNetworkPolicySource(cs, cluster, namespaces), + newNodeSource(cs, cluster, namespaces), + newPersistentVolumeSource(cs, cluster, namespaces), + newPersistentVolumeClaimSource(cs, cluster, namespaces), + newPodDisruptionBudgetSource(cs, cluster, namespaces), + newPodSource(cs, cluster, namespaces), + newPriorityClassSource(cs, cluster, namespaces), + newReplicaSetSource(cs, cluster, namespaces), + newReplicationControllerSource(cs, cluster, namespaces), + newResourceQuotaSource(cs, cluster, namespaces), + newRoleSource(cs, cluster, namespaces), + newRoleBindingSource(cs, cluster, namespaces), + newSecretSource(cs, cluster, namespaces), + newServiceSource(cs, cluster, namespaces), + newServiceAccountSource(cs, cluster, namespaces), + newStatefulSetSource(cs, cluster, namespaces), + newStorageClassSource(cs, cluster, namespaces), + newVolumeAttachmentSource(cs, cluster, namespaces), + } +} diff --git a/sources/networkpolicy.go b/sources/networkpolicy.go index db168ef..3f03ed8 100644 --- a/sources/networkpolicy.go +++ b/sources/networkpolicy.go @@ -7,14 +7,16 @@ import ( "k8s.io/client-go/kubernetes" ) -func NetworkPolicyExtractor(resource *v1.NetworkPolicy, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func NetworkPolicyExtractor(resource *v1.NetworkPolicy, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) - queries = append(queries, &sdp.Query{ - Type: "Pod", - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(&resource.Spec.PodSelector), - Scope: scope, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "Pod", + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(&resource.Spec.PodSelector), + Scope: scope, + }, }) var peers []v1.NetworkPolicyPeer @@ -35,11 +37,13 @@ func NetworkPolicyExtractor(resource *v1.NetworkPolicy, scope string) ([]*sdp.Qu // matchLabels: // project: something - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_GET, - Query: LabelSelectorToQuery(ps), - Type: "Pod", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_GET, + Query: LabelSelectorToQuery(ps), + Type: "Pod", + }, }) } } @@ -47,8 +51,8 @@ func NetworkPolicyExtractor(resource *v1.NetworkPolicy, scope string) ([]*sdp.Qu return queries, nil } -func NewNetworkPolicySource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.NetworkPolicy, *v1.NetworkPolicyList] { - return KubeTypeSource[*v1.NetworkPolicy, *v1.NetworkPolicyList]{ +func newNetworkPolicySource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.NetworkPolicy, *v1.NetworkPolicyList] { + return &KubeTypeSource[*v1.NetworkPolicy, *v1.NetworkPolicyList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "NetworkPolicy", diff --git a/sources/networkpolicy_test.go b/sources/networkpolicy_test.go index 463b914..2ff068f 100644 --- a/sources/networkpolicy_test.go +++ b/sources/networkpolicy_test.go @@ -34,10 +34,10 @@ func TestNetworkPolicySource(t *testing.T) { Namespace: "default", } - source := NewNetworkPolicySource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newNetworkPolicySource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "allow-nginx", GetScope: sd.String(), SetupYAML: NetworkPolicyYAML, diff --git a/sources/node.go b/sources/node.go index 96fc20d..affe798 100644 --- a/sources/node.go +++ b/sources/node.go @@ -9,24 +9,28 @@ import ( "k8s.io/client-go/kubernetes" ) -func linkedItemExtractor(resource *v1.Node, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func linkedItemExtractor(resource *v1.Node, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) for _, addr := range resource.Status.Addresses { switch addr.Type { case v1.NodeExternalDNS: - queries = append(queries, &sdp.Query{ - Type: "dns", - Method: sdp.QueryMethod_GET, - Query: addr.Address, - Scope: "global", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "dns", + Method: sdp.QueryMethod_GET, + Query: addr.Address, + Scope: "global", + }, }) case v1.NodeExternalIP, v1.NodeInternalIP: - queries = append(queries, &sdp.Query{ - Type: "ip", - Method: sdp.QueryMethod_GET, - Query: addr.Address, - Scope: "global", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "ip", + Method: sdp.QueryMethod_GET, + Query: addr.Address, + Scope: "global", + }, }) } } @@ -38,11 +42,13 @@ func linkedItemExtractor(resource *v1.Node, scope string) ([]*sdp.Query, error) sections := strings.Split(string(vol.Name), "^") if len(sections) == 2 { - queries = append(queries, &sdp.Query{ - Type: "ec2-volume", - Method: sdp.QueryMethod_GET, - Query: sections[1], - Scope: "*", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "ec2-volume", + Method: sdp.QueryMethod_GET, + Query: sections[1], + Scope: "*", + }, }) } } @@ -53,8 +59,8 @@ func linkedItemExtractor(resource *v1.Node, scope string) ([]*sdp.Query, error) // TODO: Should we try a DNS lookup for a node name? Is the hostname stored anywhere? -func NewNodeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Node, *v1.NodeList] { - return KubeTypeSource[*v1.Node, *v1.NodeList]{ +func newNodeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Node, *v1.NodeList] { + return &KubeTypeSource[*v1.Node, *v1.NodeList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "Node", diff --git a/sources/node_test.go b/sources/node_test.go index 5896aac..b051672 100644 --- a/sources/node_test.go +++ b/sources/node_test.go @@ -13,10 +13,10 @@ func TestNodeSource(t *testing.T) { Namespace: "default", } - source := NewNodeSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newNodeSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "k8s-source-tests-control-plane", GetScope: sd.String(), GetQueryTests: QueryTests{ diff --git a/sources/persistentvolume.go b/sources/persistentvolume.go index b4264d9..6a542b6 100644 --- a/sources/persistentvolume.go +++ b/sources/persistentvolume.go @@ -6,8 +6,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func PersistentVolumeExtractor(resource *v1.PersistentVolume, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func PersistentVolumeExtractor(resource *v1.PersistentVolume, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) sd, err := ParseScope(scope, false) @@ -17,11 +17,13 @@ func PersistentVolumeExtractor(resource *v1.PersistentVolume, scope string) ([]* if resource.Spec.PersistentVolumeSource.AWSElasticBlockStore != nil { // Link to EBS volume - queries = append(queries, &sdp.Query{ - Type: "ec2-volume", - Method: sdp.QueryMethod_GET, - Query: resource.Spec.PersistentVolumeSource.AWSElasticBlockStore.VolumeID, - Scope: "*", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "ec2-volume", + Method: sdp.QueryMethod_GET, + Query: resource.Spec.PersistentVolumeSource.AWSElasticBlockStore.VolumeID, + Scope: "*", + }, }) } @@ -30,19 +32,21 @@ func PersistentVolumeExtractor(resource *v1.PersistentVolume, scope string) ([]* } if resource.Spec.StorageClassName != "" { - queries = append(queries, &sdp.Query{ - Type: "StorageClass", - Method: sdp.QueryMethod_GET, - Query: resource.Spec.StorageClassName, - Scope: sd.ClusterName, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "StorageClass", + Method: sdp.QueryMethod_GET, + Query: resource.Spec.StorageClassName, + Scope: sd.ClusterName, + }, }) } return queries, nil } -func NewPersistentVolumeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.PersistentVolume, *v1.PersistentVolumeList] { - return KubeTypeSource[*v1.PersistentVolume, *v1.PersistentVolumeList]{ +func newPersistentVolumeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.PersistentVolume, *v1.PersistentVolumeList] { + return &KubeTypeSource[*v1.PersistentVolume, *v1.PersistentVolumeList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "PersistentVolume", diff --git a/sources/persistentvolume_test.go b/sources/persistentvolume_test.go index bc26055..c15a5de 100644 --- a/sources/persistentvolume_test.go +++ b/sources/persistentvolume_test.go @@ -25,10 +25,10 @@ func TestPersistentVolumeSource(t *testing.T) { Namespace: "", } - source := NewPersistentVolumeSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newPersistentVolumeSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "pv-test-pv", GetScope: sd.String(), SetupYAML: persistentVolumeYAML, diff --git a/sources/persistentvolumeclaim.go b/sources/persistentvolumeclaim.go index 620f17d..7804284 100644 --- a/sources/persistentvolumeclaim.go +++ b/sources/persistentvolumeclaim.go @@ -5,8 +5,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func NewPersistentVolumeClaimSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.PersistentVolumeClaim, *v1.PersistentVolumeClaimList] { - return KubeTypeSource[*v1.PersistentVolumeClaim, *v1.PersistentVolumeClaimList]{ +func newPersistentVolumeClaimSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.PersistentVolumeClaim, *v1.PersistentVolumeClaimList] { + return &KubeTypeSource[*v1.PersistentVolumeClaim, *v1.PersistentVolumeClaimList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "PersistentVolumeClaim", diff --git a/sources/persistentvolumeclaim_test.go b/sources/persistentvolumeclaim_test.go index 353a168..0cb60c8 100644 --- a/sources/persistentvolumeclaim_test.go +++ b/sources/persistentvolumeclaim_test.go @@ -51,10 +51,10 @@ func TestPersistentVolumeClaimSource(t *testing.T) { Namespace: "default", } - source := NewPersistentVolumeClaimSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newPersistentVolumeClaimSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "pvc-test-pvc", GetScope: sd.String(), SetupYAML: persistentVolumeClaimYAML, diff --git a/sources/poddisruptionbudget.go b/sources/poddisruptionbudget.go index 24d96ef..6e0b42b 100644 --- a/sources/poddisruptionbudget.go +++ b/sources/poddisruptionbudget.go @@ -6,23 +6,25 @@ import ( "k8s.io/client-go/kubernetes" ) -func podDisruptionBudgetExtractor(resource *v1.PodDisruptionBudget, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func podDisruptionBudgetExtractor(resource *v1.PodDisruptionBudget, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) if resource.Spec.Selector != nil { - queries = append(queries, &sdp.Query{ - Type: "Pod", - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(resource.Spec.Selector), - Scope: scope, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "Pod", + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(resource.Spec.Selector), + Scope: scope, + }, }) } return queries, nil } -func NewPodDisruptionBudgetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.PodDisruptionBudget, *v1.PodDisruptionBudgetList] { - return KubeTypeSource[*v1.PodDisruptionBudget, *v1.PodDisruptionBudgetList]{ +func newPodDisruptionBudgetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.PodDisruptionBudget, *v1.PodDisruptionBudgetList] { + return &KubeTypeSource[*v1.PodDisruptionBudget, *v1.PodDisruptionBudgetList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "PodDisruptionBudget", diff --git a/sources/poddisruptionbudget_test.go b/sources/poddisruptionbudget_test.go index acee723..18fc1ee 100644 --- a/sources/poddisruptionbudget_test.go +++ b/sources/poddisruptionbudget_test.go @@ -25,10 +25,10 @@ func TestPodDisruptionBudgetSource(t *testing.T) { Namespace: "default", } - source := NewPodDisruptionBudgetSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newPodDisruptionBudgetSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "example-pdb", GetScope: sd.String(), SetupYAML: PodDisruptionBudgetYAML, diff --git a/sources/pods.go b/sources/pods.go index 3cac80a..8260ce0 100644 --- a/sources/pods.go +++ b/sources/pods.go @@ -6,8 +6,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func PodExtractor(resource *v1.Pod, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func PodExtractor(resource *v1.Pod, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) sd, err := ParseScope(scope, true) @@ -17,11 +17,13 @@ func PodExtractor(resource *v1.Pod, scope string) ([]*sdp.Query, error) { // Link service accounts if resource.Spec.ServiceAccountName != "" { - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_GET, - Query: resource.Spec.ServiceAccountName, - Type: "ServiceAccount", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_GET, + Query: resource.Spec.ServiceAccountName, + Type: "ServiceAccount", + }, }) } @@ -29,31 +31,37 @@ func PodExtractor(resource *v1.Pod, scope string) ([]*sdp.Query, error) { for _, vol := range resource.Spec.Volumes { // Link PVCs if vol.PersistentVolumeClaim != nil { - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_GET, - Query: vol.PersistentVolumeClaim.ClaimName, - Type: "PersistentVolumeClaim", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_GET, + Query: vol.PersistentVolumeClaim.ClaimName, + Type: "PersistentVolumeClaim", + }, }) } // Link secrets if vol.Secret != nil { - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_GET, - Query: vol.Secret.SecretName, - Type: "Secret", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_GET, + Query: vol.Secret.SecretName, + Type: "Secret", + }, }) } // Link config map volumes if vol.ConfigMap != nil { - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_GET, - Query: vol.ConfigMap.Name, - Type: "ConfigMap", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_GET, + Query: vol.ConfigMap.Name, + Type: "ConfigMap", + }, }) } } @@ -65,20 +73,24 @@ func PodExtractor(resource *v1.Pod, scope string) ([]*sdp.Query, error) { if env.ValueFrom != nil { if env.ValueFrom.SecretKeyRef != nil { // Add linked item from spec.containers[].env[].valueFrom.secretKeyRef - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_GET, - Query: env.ValueFrom.SecretKeyRef.Name, - Type: "Secret", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_GET, + Query: env.ValueFrom.SecretKeyRef.Name, + Type: "Secret", + }, }) } if env.ValueFrom.ConfigMapKeyRef != nil { - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_GET, - Query: env.ValueFrom.ConfigMapKeyRef.Name, - Type: "ConfigMap", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_GET, + Query: env.ValueFrom.ConfigMapKeyRef.Name, + Type: "ConfigMap", + }, }) } } @@ -87,48 +99,56 @@ func PodExtractor(resource *v1.Pod, scope string) ([]*sdp.Query, error) { for _, envFrom := range container.EnvFrom { if envFrom.SecretRef != nil { // Add linked item from spec.containers[].EnvFrom[].secretKeyRef - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_GET, - Query: envFrom.SecretRef.Name, - Type: "Secret", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_GET, + Query: envFrom.SecretRef.Name, + Type: "Secret", + }, }) } } } if resource.Spec.PriorityClassName != "" { - queries = append(queries, &sdp.Query{ - Scope: sd.ClusterName, - Method: sdp.QueryMethod_GET, - Query: resource.Spec.PriorityClassName, - Type: "PriorityClass", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: sd.ClusterName, + Method: sdp.QueryMethod_GET, + Query: resource.Spec.PriorityClassName, + Type: "PriorityClass", + }, }) } if len(resource.Status.PodIPs) > 0 { for _, ip := range resource.Status.PodIPs { - queries = append(queries, &sdp.Query{ - Scope: "global", - Method: sdp.QueryMethod_GET, - Query: ip.IP, - Type: "ip", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: "global", + Method: sdp.QueryMethod_GET, + Query: ip.IP, + Type: "ip", + }, }) } } else if resource.Status.PodIP != "" { - queries = append(queries, &sdp.Query{ - Type: "ip", - Method: sdp.QueryMethod_GET, - Query: resource.Status.PodIP, - Scope: "global", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "ip", + Method: sdp.QueryMethod_GET, + Query: resource.Status.PodIP, + Scope: "global", + }, }) } return queries, nil } -func NewPodSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Pod, *v1.PodList] { - return KubeTypeSource[*v1.Pod, *v1.PodList]{ +func newPodSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Pod, *v1.PodList] { + return &KubeTypeSource[*v1.Pod, *v1.PodList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "Pod", diff --git a/sources/pods_test.go b/sources/pods_test.go index a3b2bbe..bba3d51 100644 --- a/sources/pods_test.go +++ b/sources/pods_test.go @@ -75,10 +75,10 @@ func TestPodSource(t *testing.T) { Namespace: "default", } - source := NewPodSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newPodSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "pod-test-pod", GetScope: sd.String(), SetupYAML: PodYAML, diff --git a/sources/priorityclass.go b/sources/priorityclass.go index bdb37bd..15ece3c 100644 --- a/sources/priorityclass.go +++ b/sources/priorityclass.go @@ -6,8 +6,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func NewPriorityClassSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.PriorityClass, *v1.PriorityClassList] { - return KubeTypeSource[*v1.PriorityClass, *v1.PriorityClassList]{ +func newPriorityClassSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.PriorityClass, *v1.PriorityClassList] { + return &KubeTypeSource[*v1.PriorityClass, *v1.PriorityClassList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "PriorityClass", diff --git a/sources/priorityclass_test.go b/sources/priorityclass_test.go index 0a49554..db3fa41 100644 --- a/sources/priorityclass_test.go +++ b/sources/priorityclass_test.go @@ -20,10 +20,10 @@ func TestPriorityClassSource(t *testing.T) { Namespace: "default", } - source := NewPriorityClassSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newPriorityClassSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "ultra-mega-priority", GetScope: sd.String(), SetupYAML: priorityClassYAML, diff --git a/sources/replicaset.go b/sources/replicaset.go index b359c19..819007b 100644 --- a/sources/replicaset.go +++ b/sources/replicaset.go @@ -7,23 +7,25 @@ import ( "k8s.io/client-go/kubernetes" ) -func replicaSetExtractor(resource *v1.ReplicaSet, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func replicaSetExtractor(resource *v1.ReplicaSet, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) if resource.Spec.Selector != nil { - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(resource.Spec.Selector), - Type: "Pod", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(resource.Spec.Selector), + Type: "Pod", + }, }) } return queries, nil } -func NewReplicaSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ReplicaSet, *v1.ReplicaSetList] { - return KubeTypeSource[*v1.ReplicaSet, *v1.ReplicaSetList]{ +func newReplicaSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ReplicaSet, *v1.ReplicaSetList] { + return &KubeTypeSource[*v1.ReplicaSet, *v1.ReplicaSetList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "ReplicaSet", diff --git a/sources/replicaset_test.go b/sources/replicaset_test.go index 867fd12..7a271af 100644 --- a/sources/replicaset_test.go +++ b/sources/replicaset_test.go @@ -36,10 +36,10 @@ func TestReplicaSetSource(t *testing.T) { Namespace: "default", } - source := NewReplicaSetSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newReplicaSetSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "replica-set-test", GetScope: sd.String(), SetupYAML: replicaSetYAML, diff --git a/sources/replicationcontroller.go b/sources/replicationcontroller.go index d8848a5..0782e19 100644 --- a/sources/replicationcontroller.go +++ b/sources/replicationcontroller.go @@ -7,25 +7,27 @@ import ( "k8s.io/client-go/kubernetes" ) -func replicationControllerExtractor(resource *v1.ReplicationController, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func replicationControllerExtractor(resource *v1.ReplicationController, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) if resource.Spec.Selector != nil { - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(&metaV1.LabelSelector{ - MatchLabels: resource.Spec.Selector, - }), - Type: "Pod", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(&metaV1.LabelSelector{ + MatchLabels: resource.Spec.Selector, + }), + Type: "Pod", + }, }) } return queries, nil } -func NewReplicationControllerSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ReplicationController, *v1.ReplicationControllerList] { - return KubeTypeSource[*v1.ReplicationController, *v1.ReplicationControllerList]{ +func newReplicationControllerSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ReplicationController, *v1.ReplicationControllerList] { + return &KubeTypeSource[*v1.ReplicationController, *v1.ReplicationControllerList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "ReplicationController", diff --git a/sources/replicationcontroller_test.go b/sources/replicationcontroller_test.go index 5ad1299..c4549f6 100644 --- a/sources/replicationcontroller_test.go +++ b/sources/replicationcontroller_test.go @@ -35,10 +35,10 @@ func TestReplicationControllerSource(t *testing.T) { Namespace: "default", } - source := NewReplicationControllerSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newReplicationControllerSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "replication-controller-test", GetScope: sd.String(), SetupYAML: replicationControllerYAML, diff --git a/sources/resourcequota.go b/sources/resourcequota.go index b4b586f..f5e3d13 100644 --- a/sources/resourcequota.go +++ b/sources/resourcequota.go @@ -5,8 +5,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func NewResourceQuotaSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ResourceQuota, *v1.ResourceQuotaList] { - return KubeTypeSource[*v1.ResourceQuota, *v1.ResourceQuotaList]{ +func newResourceQuotaSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ResourceQuota, *v1.ResourceQuotaList] { + return &KubeTypeSource[*v1.ResourceQuota, *v1.ResourceQuotaList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "ResourceQuota", diff --git a/sources/resourcequota_test.go b/sources/resourcequota_test.go index b2aa58c..03fc8a7 100644 --- a/sources/resourcequota_test.go +++ b/sources/resourcequota_test.go @@ -24,10 +24,10 @@ func TestResourceQuotaSource(t *testing.T) { Namespace: "default", } - source := NewResourceQuotaSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newResourceQuotaSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "quota-example", GetScope: sd.String(), SetupYAML: resourceQuotaYAML, diff --git a/sources/role.go b/sources/role.go index c20af07..abe0515 100644 --- a/sources/role.go +++ b/sources/role.go @@ -6,8 +6,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func NewRoleSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Role, *v1.RoleList] { - return KubeTypeSource[*v1.Role, *v1.RoleList]{ +func newRoleSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Role, *v1.RoleList] { + return &KubeTypeSource[*v1.Role, *v1.RoleList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "Role", diff --git a/sources/role_test.go b/sources/role_test.go index 5dd6e9f..90a6bdc 100644 --- a/sources/role_test.go +++ b/sources/role_test.go @@ -37,10 +37,10 @@ func TestRoleSource(t *testing.T) { Namespace: "default", } - source := NewRoleSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newRoleSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "role-test-role", GetScope: sd.String(), SetupYAML: RoleYAML, diff --git a/sources/rolebinding.go b/sources/rolebinding.go index c7de6a1..cc4ee88 100644 --- a/sources/rolebinding.go +++ b/sources/rolebinding.go @@ -7,8 +7,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func roleBindingExtractor(resource *v1.RoleBinding, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func roleBindingExtractor(resource *v1.RoleBinding, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) sd, err := ParseScope(scope, true) @@ -17,14 +17,16 @@ func roleBindingExtractor(resource *v1.RoleBinding, scope string) ([]*sdp.Query, } for _, subject := range resource.Subjects { - queries = append(queries, &sdp.Query{ - Method: sdp.QueryMethod_GET, - Query: subject.Name, - Type: subject.Kind, - Scope: ScopeDetails{ - ClusterName: sd.ClusterName, - Namespace: subject.Namespace, - }.String(), + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Method: sdp.QueryMethod_GET, + Query: subject.Name, + Type: subject.Kind, + Scope: ScopeDetails{ + ClusterName: sd.ClusterName, + Namespace: subject.Namespace, + }.String(), + }, }) } @@ -43,18 +45,20 @@ func roleBindingExtractor(resource *v1.RoleBinding, scope string) ([]*sdp.Query, refSD.Namespace = "" } - queries = append(queries, &sdp.Query{ - Scope: refSD.String(), - Method: sdp.QueryMethod_GET, - Query: resource.RoleRef.Name, - Type: resource.RoleRef.Kind, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: refSD.String(), + Method: sdp.QueryMethod_GET, + Query: resource.RoleRef.Name, + Type: resource.RoleRef.Kind, + }, }) return queries, nil } -func NewRoleBindingSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.RoleBinding, *v1.RoleBindingList] { - return KubeTypeSource[*v1.RoleBinding, *v1.RoleBindingList]{ +func newRoleBindingSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.RoleBinding, *v1.RoleBindingList] { + return &KubeTypeSource[*v1.RoleBinding, *v1.RoleBindingList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "RoleBinding", diff --git a/sources/rolebinding_test.go b/sources/rolebinding_test.go index 28deaef..76cc42f 100644 --- a/sources/rolebinding_test.go +++ b/sources/rolebinding_test.go @@ -72,11 +72,11 @@ func TestRoleBindingSource(t *testing.T) { Namespace: "default", } - source := NewRoleBindingSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newRoleBindingSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) t.Run("With a Role", func(t *testing.T) { st := SourceTests{ - Source: &source, + Source: source, GetQuery: "rb-test-role-binding", GetScope: sd.String(), SetupYAML: roleBindingYAML, @@ -101,7 +101,7 @@ func TestRoleBindingSource(t *testing.T) { t.Run("With a ClusterRole", func(t *testing.T) { st := SourceTests{ - Source: &source, + Source: source, GetQuery: "rb-test-role-binding-cluster", GetScope: sd.String(), SetupYAML: roleBindingYAML2, diff --git a/sources/secret.go b/sources/secret.go index 62ed930..d522fd2 100644 --- a/sources/secret.go +++ b/sources/secret.go @@ -8,8 +8,8 @@ import ( ) // TODO: Configure redaction -func NewSecretSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Secret, *v1.SecretList] { - return KubeTypeSource[*v1.Secret, *v1.SecretList]{ +func newSecretSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Secret, *v1.SecretList] { + return &KubeTypeSource[*v1.Secret, *v1.SecretList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "Secret", diff --git a/sources/secret_test.go b/sources/secret_test.go index 20723a7..692e314 100644 --- a/sources/secret_test.go +++ b/sources/secret_test.go @@ -22,10 +22,10 @@ func TestSecretSource(t *testing.T) { Namespace: "default", } - source := NewSecretSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newSecretSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "secret-test-secret", GetScope: sd.String(), SetupYAML: secretYAML, diff --git a/sources/service.go b/sources/service.go index 3b54add..2773964 100644 --- a/sources/service.go +++ b/sources/service.go @@ -7,17 +7,19 @@ import ( "k8s.io/client-go/kubernetes" ) -func serviceExtractor(resource *v1.Service, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func serviceExtractor(resource *v1.Service, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) if resource.Spec.Selector != nil { - queries = append(queries, &sdp.Query{ - Type: "Pod", - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(&metaV1.LabelSelector{ - MatchLabels: resource.Spec.Selector, - }), - Scope: scope, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "Pod", + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(&metaV1.LabelSelector{ + MatchLabels: resource.Spec.Selector, + }), + Scope: scope, + }, }) } @@ -34,48 +36,58 @@ func serviceExtractor(resource *v1.Service, scope string) ([]*sdp.Query, error) for _, ip := range ips { if ip != "" { - queries = append(queries, &sdp.Query{ - Type: "ip", - Method: sdp.QueryMethod_GET, - Query: ip, - Scope: "global", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "ip", + Method: sdp.QueryMethod_GET, + Query: ip, + Scope: "global", + }, }) } } if resource.Spec.ExternalName != "" { - queries = append(queries, &sdp.Query{ - Type: "dns", - Method: sdp.QueryMethod_GET, - Query: resource.Spec.ExternalName, - Scope: "global", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "dns", + Method: sdp.QueryMethod_GET, + Query: resource.Spec.ExternalName, + Scope: "global", + }, }) } // Services also generate an endpoint with the same name - queries = append(queries, &sdp.Query{ - Type: "Endpoint", - Method: sdp.QueryMethod_GET, - Query: resource.Name, - Scope: scope, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "Endpoint", + Method: sdp.QueryMethod_GET, + Query: resource.Name, + Scope: scope, + }, }) for _, ingress := range resource.Status.LoadBalancer.Ingress { if ingress.IP != "" { - queries = append(queries, &sdp.Query{ - Type: "ip", - Method: sdp.QueryMethod_GET, - Query: ingress.IP, - Scope: "global", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "ip", + Method: sdp.QueryMethod_GET, + Query: ingress.IP, + Scope: "global", + }, }) } if ingress.Hostname != "" { - queries = append(queries, &sdp.Query{ - Type: "dns", - Method: sdp.QueryMethod_GET, - Query: ingress.Hostname, - Scope: "global", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "dns", + Method: sdp.QueryMethod_GET, + Query: ingress.Hostname, + Scope: "global", + }, }) } } @@ -83,8 +95,8 @@ func serviceExtractor(resource *v1.Service, scope string) ([]*sdp.Query, error) return queries, nil } -func NewServiceSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.Service, *v1.ServiceList] { - return KubeTypeSource[*v1.Service, *v1.ServiceList]{ +func newServiceSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Service, *v1.ServiceList] { + return &KubeTypeSource[*v1.Service, *v1.ServiceList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "Service", diff --git a/sources/service_test.go b/sources/service_test.go index cb5bb92..5c733d3 100644 --- a/sources/service_test.go +++ b/sources/service_test.go @@ -50,10 +50,10 @@ func TestServiceSource(t *testing.T) { Namespace: "default", } - source := NewServiceSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newServiceSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "service-test-service", GetScope: sd.String(), SetupYAML: serviceYAML, diff --git a/sources/serviceaccount.go b/sources/serviceaccount.go index 0af38cb..3425273 100644 --- a/sources/serviceaccount.go +++ b/sources/serviceaccount.go @@ -6,32 +6,36 @@ import ( "k8s.io/client-go/kubernetes" ) -func serviceAccountExtractor(resource *v1.ServiceAccount, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func serviceAccountExtractor(resource *v1.ServiceAccount, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) for _, secret := range resource.Secrets { - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_GET, - Query: secret.Name, - Type: "Secret", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_GET, + Query: secret.Name, + Type: "Secret", + }, }) } for _, ipSecret := range resource.ImagePullSecrets { - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_GET, - Query: ipSecret.Name, - Type: "Secret", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_GET, + Query: ipSecret.Name, + Type: "Secret", + }, }) } return queries, nil } -func NewServiceAccountSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.ServiceAccount, *v1.ServiceAccountList] { - return KubeTypeSource[*v1.ServiceAccount, *v1.ServiceAccountList]{ +func newServiceAccountSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ServiceAccount, *v1.ServiceAccountList] { + return &KubeTypeSource[*v1.ServiceAccount, *v1.ServiceAccountList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "ServiceAccount", diff --git a/sources/serviceaccount_test.go b/sources/serviceaccount_test.go index 7089724..a4aeb36 100644 --- a/sources/serviceaccount_test.go +++ b/sources/serviceaccount_test.go @@ -40,10 +40,10 @@ func TestServiceAccountSource(t *testing.T) { Namespace: "default", } - source := NewServiceAccountSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newServiceAccountSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "test-service-account", GetScope: sd.String(), SetupYAML: serviceAccountYAML, diff --git a/sources/statefulset.go b/sources/statefulset.go index ade78c5..4337f14 100644 --- a/sources/statefulset.go +++ b/sources/statefulset.go @@ -8,47 +8,53 @@ import ( "k8s.io/client-go/kubernetes" ) -func statefulSetExtractor(resource *v1.StatefulSet, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func statefulSetExtractor(resource *v1.StatefulSet, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) if resource.Spec.Selector != nil { // Stateful sets are linked to pods via their selector - queries = append(queries, &sdp.Query{ - Type: "Pod", - Method: sdp.QueryMethod_SEARCH, - Query: LabelSelectorToQuery(resource.Spec.Selector), - Scope: scope, - }) - - if len(resource.Spec.VolumeClaimTemplates) > 0 { - queries = append(queries, &sdp.Query{ - Type: "PersistentVolumeClaim", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "Pod", Method: sdp.QueryMethod_SEARCH, Query: LabelSelectorToQuery(resource.Spec.Selector), Scope: scope, + }, + }) + + if len(resource.Spec.VolumeClaimTemplates) > 0 { + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "PersistentVolumeClaim", + Method: sdp.QueryMethod_SEARCH, + Query: LabelSelectorToQuery(resource.Spec.Selector), + Scope: scope, + }, }) } } if resource.Spec.ServiceName != "" { - queries = append(queries, &sdp.Query{ - Scope: scope, - Method: sdp.QueryMethod_SEARCH, - Query: ListOptionsToQuery(&metaV1.ListOptions{ - FieldSelector: Selector{ - "metadata.name": resource.Spec.ServiceName, - "metadata.namespace": resource.Namespace, - }.String(), - }), - Type: "Service", + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Scope: scope, + Method: sdp.QueryMethod_SEARCH, + Query: ListOptionsToQuery(&metaV1.ListOptions{ + FieldSelector: Selector{ + "metadata.name": resource.Spec.ServiceName, + "metadata.namespace": resource.Namespace, + }.String(), + }), + Type: "Service", + }, }) } return queries, nil } -func NewStatefulSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.StatefulSet, *v1.StatefulSetList] { - return KubeTypeSource[*v1.StatefulSet, *v1.StatefulSetList]{ +func newStatefulSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.StatefulSet, *v1.StatefulSetList] { + return &KubeTypeSource[*v1.StatefulSet, *v1.StatefulSetList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "StatefulSet", diff --git a/sources/statefulset_test.go b/sources/statefulset_test.go index 60119fb..376f0d5 100644 --- a/sources/statefulset_test.go +++ b/sources/statefulset_test.go @@ -48,10 +48,10 @@ func TestStatefulSetSource(t *testing.T) { Namespace: "default", } - source := NewStatefulSetSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newStatefulSetSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "stateful-set-test", GetScope: sd.String(), SetupYAML: statefulSetYAML, diff --git a/sources/storageclass.go b/sources/storageclass.go index 22dc94c..0de80a9 100644 --- a/sources/storageclass.go +++ b/sources/storageclass.go @@ -6,8 +6,8 @@ import ( "k8s.io/client-go/kubernetes" ) -func NewStorageClassSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.StorageClass, *v1.StorageClassList] { - return KubeTypeSource[*v1.StorageClass, *v1.StorageClassList]{ +func newStorageClassSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.StorageClass, *v1.StorageClassList] { + return &KubeTypeSource[*v1.StorageClass, *v1.StorageClassList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "StorageClass", diff --git a/sources/storageclass_test.go b/sources/storageclass_test.go index 72229cd..aa4dd54 100644 --- a/sources/storageclass_test.go +++ b/sources/storageclass_test.go @@ -21,10 +21,10 @@ func TestStorageClassSource(t *testing.T) { Namespace: "default", } - source := NewStorageClassSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newStorageClassSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "storage-class-test", GetScope: sd.String(), SetupYAML: storageClassYAML, diff --git a/sources/volumeattachment.go b/sources/volumeattachment.go index 551c7d4..3e6816e 100644 --- a/sources/volumeattachment.go +++ b/sources/volumeattachment.go @@ -6,32 +6,36 @@ import ( "k8s.io/client-go/kubernetes" ) -func volumeAttachmentExtractor(resource *v1.VolumeAttachment, scope string) ([]*sdp.Query, error) { - queries := make([]*sdp.Query, 0) +func volumeAttachmentExtractor(resource *v1.VolumeAttachment, scope string) ([]*sdp.LinkedItemQuery, error) { + queries := make([]*sdp.LinkedItemQuery, 0) if resource.Spec.Source.PersistentVolumeName != nil { - queries = append(queries, &sdp.Query{ - Type: "PersistentVolume", - Method: sdp.QueryMethod_GET, - Query: *resource.Spec.Source.PersistentVolumeName, - Scope: scope, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "PersistentVolume", + Method: sdp.QueryMethod_GET, + Query: *resource.Spec.Source.PersistentVolumeName, + Scope: scope, + }, }) } if resource.Spec.NodeName != "" { - queries = append(queries, &sdp.Query{ - Type: "Node", - Method: sdp.QueryMethod_GET, - Query: resource.Spec.NodeName, - Scope: scope, + queries = append(queries, &sdp.LinkedItemQuery{ + Query: &sdp.Query{ + Type: "Node", + Method: sdp.QueryMethod_GET, + Query: resource.Spec.NodeName, + Scope: scope, + }, }) } return queries, nil } -func NewVolumeAttachmentSource(cs *kubernetes.Clientset, cluster string, namespaces []string) KubeTypeSource[*v1.VolumeAttachment, *v1.VolumeAttachmentList] { - return KubeTypeSource[*v1.VolumeAttachment, *v1.VolumeAttachmentList]{ +func newVolumeAttachmentSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.VolumeAttachment, *v1.VolumeAttachmentList] { + return &KubeTypeSource[*v1.VolumeAttachment, *v1.VolumeAttachmentList]{ ClusterName: cluster, Namespaces: namespaces, TypeName: "VolumeAttachment", diff --git a/sources/volumeattachment_test.go b/sources/volumeattachment_test.go index 8db564f..68fa18a 100644 --- a/sources/volumeattachment_test.go +++ b/sources/volumeattachment_test.go @@ -65,10 +65,10 @@ func TestVolumeAttachmentSource(t *testing.T) { ClusterName: CurrentCluster.Name, } - source := NewVolumeAttachmentSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) + source := newVolumeAttachmentSource(CurrentCluster.ClientSet, sd.ClusterName, []string{sd.Namespace}) st := SourceTests{ - Source: &source, + Source: source, GetQuery: "volume-attachment-attachment", GetScope: sd.String(), SetupYAML: volumeAttachmentYAML, From 3c7e3b0cdbc97df30cb4466ae7b7fb89b8bb6e8a Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Mon, 15 May 2023 14:17:46 +0000 Subject: [PATCH 18/24] Added verbosity to tests --- .github/workflows/test-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 6778adc..6da1aa7 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -18,7 +18,7 @@ jobs: run: go vet ./... - name: Test - run: go test ./... + run: go test -v ./... build: name: Build From f39210642c47b4f4bab7123389eedeaf9797bd90 Mon Sep 17 00:00:00 2001 From: Dylan Date: Mon, 15 May 2023 17:45:37 +0100 Subject: [PATCH 19/24] Add race detection to tests Co-authored-by: David Schmitt <118179693+DavidS-om@users.noreply.github.com> --- .github/workflows/test-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 6da1aa7..080eab0 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -18,7 +18,7 @@ jobs: run: go vet ./... - name: Test - run: go test -v ./... + run: go test -v -race -timeout 3m ./... build: name: Build From 16a424d4932486ad608eea2e3ebbed6ec2aae2bb Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Mon, 15 May 2023 18:14:06 +0100 Subject: [PATCH 20/24] Standardise API import name --- cmd/root.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 786462b..7b139db 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,7 +20,7 @@ import ( "github.com/overmindtech/k8s-source/sources" "github.com/spf13/cobra" "github.com/spf13/pflag" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -193,7 +193,7 @@ var rootCmd = &cobra.Command{ restart := make(chan watch.Event, 1024) // Get the initial starting point - list, err := clientSet.CoreV1().Namespaces().List(context.Background(), v1.ListOptions{}) + list, err := clientSet.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{}) if err != nil { log.Fatalf("Could not list namespaces: %v", err) @@ -201,7 +201,7 @@ var rootCmd = &cobra.Command{ // Watch namespaces from here sendInitialEvents := false - wi, err := clientSet.CoreV1().Namespaces().Watch(context.Background(), v1.ListOptions{ + wi, err := clientSet.CoreV1().Namespaces().Watch(context.Background(), metav1.ListOptions{ SendInitialEvents: &sendInitialEvents, ResourceVersion: list.ResourceVersion, }) @@ -228,7 +228,7 @@ var rootCmd = &cobra.Command{ for { // Query all namespaces log.Info("Listing namespaces") - list, err := clientSet.CoreV1().Namespaces().List(context.Background(), v1.ListOptions{}) + list, err := clientSet.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{}) if err != nil { log.Fatal(err) From 29a93d5da8290bbd4536e21744a613047f0e014f Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Mon, 15 May 2023 17:35:11 +0000 Subject: [PATCH 21/24] Added NATS task --- .vscode/tasks.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .vscode/tasks.json diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..a341f7e --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,12 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "Run NATS Server", + "type": "shell", + "command": "docker run -p 4222:4222 -p 8222:8222 nats" + } + ] +} \ No newline at end of file From 7a6adab260e69c8067bbf174b287230295a83b68 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Mon, 15 May 2023 17:36:23 +0000 Subject: [PATCH 22/24] Added config hash --- cmd/root.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 7b139db..9564ae1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "crypto/sha1" "errors" "fmt" "net/http" @@ -137,6 +138,12 @@ var rootCmd = &cobra.Command{ } } + // Calculate the SHA-1 hash of the config to use as the queue name. This + // means that sources with the same config will be in the same queue. + // Note that the config object implements redaction in the String() + // method so we don't have to worry about leaking secrets + configHash := fmt.Sprintf("%x", sha1.Sum([]byte(rc.String()))) + e, err := discovery.NewEngine() if err != nil { log.WithFields(log.Fields{ @@ -155,7 +162,7 @@ var rootCmd = &cobra.Command{ ReconnectJitter: 2 * time.Second, TokenClient: tokenClient, } - e.NATSQueueName = "k8s-source" // This should be the same as your engine name + e.NATSQueueName = fmt.Sprintf("k8s-source-%v", configHash) e.MaxParallelExecutions = maxParallel // Start HTTP server for status From 2fb219626e085c89ef5594e194402bf6dfb2f31f Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Tue, 16 May 2023 07:08:18 +0000 Subject: [PATCH 23/24] Fixed double-stop --- cmd/root.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 9564ae1..1788d29 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -300,13 +300,6 @@ var rootCmd = &cobra.Command{ // Clear the sources e.ClearSources() - - // Stop the engine - err = e.Stop() - - if err != nil { - log.Fatal(err) - } } } }, From 08c8dd9d2253e68328edf20fae0b53f848e898dc Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Tue, 16 May 2023 07:20:10 +0000 Subject: [PATCH 24/24] Moved to self-registration of sources --- sources/clusterrole.go | 7 ++++- sources/clusterrolebinding.go | 7 ++++- sources/configmap.go | 7 ++++- sources/cronjob.go | 7 ++++- sources/daemonset.go | 7 ++++- sources/deployment.go | 7 ++++- sources/endpoints.go | 7 ++++- sources/endpointslice.go | 7 ++++- sources/horizontalpodautoscaler.go | 7 ++++- sources/ingress.go | 7 ++++- sources/job.go | 7 ++++- sources/limitrange.go | 7 ++++- sources/main.go | 45 ++++++++++-------------------- sources/networkpolicy.go | 7 ++++- sources/node.go | 7 ++++- sources/persistentvolume.go | 7 ++++- sources/persistentvolumeclaim.go | 7 ++++- sources/poddisruptionbudget.go | 7 ++++- sources/pods.go | 7 ++++- sources/priorityclass.go | 7 ++++- sources/replicaset.go | 7 ++++- sources/replicationcontroller.go | 7 ++++- sources/resourcequota.go | 7 ++++- sources/role.go | 7 ++++- sources/rolebinding.go | 7 ++++- sources/secret.go | 8 ++++-- sources/service.go | 7 ++++- sources/serviceaccount.go | 7 ++++- sources/statefulset.go | 7 ++++- sources/storageclass.go | 7 ++++- sources/volumeattachment.go | 7 ++++- 31 files changed, 194 insertions(+), 62 deletions(-) diff --git a/sources/clusterrole.go b/sources/clusterrole.go index be2290e..861943d 100644 --- a/sources/clusterrole.go +++ b/sources/clusterrole.go @@ -1,12 +1,13 @@ package sources import ( + "github.com/overmindtech/discovery" v1 "k8s.io/api/rbac/v1" "k8s.io/client-go/kubernetes" ) -func newClusterRoleSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ClusterRole, *v1.ClusterRoleList] { +func newClusterRoleSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.ClusterRole, *v1.ClusterRoleList]{ ClusterName: cluster, Namespaces: namespaces, @@ -25,3 +26,7 @@ func newClusterRoleSource(cs *kubernetes.Clientset, cluster string, namespaces [ }, } } + +func init() { + registerSourceLoader(newClusterRoleSource) +} diff --git a/sources/clusterrolebinding.go b/sources/clusterrolebinding.go index 93c7106..9341d4b 100644 --- a/sources/clusterrolebinding.go +++ b/sources/clusterrolebinding.go @@ -3,6 +3,7 @@ package sources import ( v1 "k8s.io/api/rbac/v1" + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) @@ -41,7 +42,7 @@ func clusterRoleBindingExtractor(resource *v1.ClusterRoleBinding, scope string) return queries, nil } -func newClusterRoleBindingSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ClusterRoleBinding, *v1.ClusterRoleBindingList] { +func newClusterRoleBindingSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.ClusterRoleBinding, *v1.ClusterRoleBindingList]{ ClusterName: cluster, Namespaces: namespaces, @@ -61,3 +62,7 @@ func newClusterRoleBindingSource(cs *kubernetes.Clientset, cluster string, names LinkedItemQueryExtractor: clusterRoleBindingExtractor, } } + +func init() { + registerSourceLoader(newClusterRoleBindingSource) +} diff --git a/sources/configmap.go b/sources/configmap.go index fde8837..4ab4ace 100644 --- a/sources/configmap.go +++ b/sources/configmap.go @@ -1,11 +1,12 @@ package sources import ( + "github.com/overmindtech/discovery" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -func newConfigMapSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ConfigMap, *v1.ConfigMapList] { +func newConfigMapSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.ConfigMap, *v1.ConfigMapList]{ ClusterName: cluster, Namespaces: namespaces, @@ -24,3 +25,7 @@ func newConfigMapSource(cs *kubernetes.Clientset, cluster string, namespaces []s }, } } + +func init() { + registerSourceLoader(newConfigMapSource) +} diff --git a/sources/cronjob.go b/sources/cronjob.go index 7f120d3..3a21239 100644 --- a/sources/cronjob.go +++ b/sources/cronjob.go @@ -1,12 +1,13 @@ package sources import ( + "github.com/overmindtech/discovery" v1 "k8s.io/api/batch/v1" "k8s.io/client-go/kubernetes" ) -func newCronJobSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.CronJob, *v1.CronJobList] { +func newCronJobSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.CronJob, *v1.CronJobList]{ ClusterName: cluster, Namespaces: namespaces, @@ -27,3 +28,7 @@ func newCronJobSource(cs *kubernetes.Clientset, cluster string, namespaces []str // automatically } } + +func init() { + registerSourceLoader(newCronJobSource) +} diff --git a/sources/daemonset.go b/sources/daemonset.go index 3cb2dae..160afe7 100644 --- a/sources/daemonset.go +++ b/sources/daemonset.go @@ -1,12 +1,13 @@ package sources import ( + "github.com/overmindtech/discovery" v1 "k8s.io/api/apps/v1" "k8s.io/client-go/kubernetes" ) -func newDaemonSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.DaemonSet, *v1.DaemonSetList] { +func newDaemonSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.DaemonSet, *v1.DaemonSetList]{ ClusterName: cluster, Namespaces: namespaces, @@ -26,3 +27,7 @@ func newDaemonSetSource(cs *kubernetes.Clientset, cluster string, namespaces []s // Pods are linked automatically } } + +func init() { + registerSourceLoader(newDaemonSetSource) +} diff --git a/sources/deployment.go b/sources/deployment.go index 65f3d3d..fb287f3 100644 --- a/sources/deployment.go +++ b/sources/deployment.go @@ -1,12 +1,13 @@ package sources import ( + "github.com/overmindtech/discovery" v1 "k8s.io/api/apps/v1" "k8s.io/client-go/kubernetes" ) -func newDeploymentSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Deployment, *v1.DeploymentList] { +func newDeploymentSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.Deployment, *v1.DeploymentList]{ ClusterName: cluster, Namespaces: namespaces, @@ -26,3 +27,7 @@ func newDeploymentSource(cs *kubernetes.Clientset, cluster string, namespaces [] // Replicasets are linked automatically } } + +func init() { + registerSourceLoader(newDeploymentSource) +} diff --git a/sources/endpoints.go b/sources/endpoints.go index 6836e55..0d12493 100644 --- a/sources/endpoints.go +++ b/sources/endpoints.go @@ -1,6 +1,7 @@ package sources import ( + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" @@ -60,7 +61,7 @@ func EndpointsExtractor(resource *v1.Endpoints, scope string) ([]*sdp.LinkedItem return queries, nil } -func newEndpointsSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Endpoints, *v1.EndpointsList] { +func newEndpointsSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.Endpoints, *v1.EndpointsList]{ ClusterName: cluster, Namespaces: namespaces, @@ -80,3 +81,7 @@ func newEndpointsSource(cs *kubernetes.Clientset, cluster string, namespaces []s LinkedItemQueryExtractor: EndpointsExtractor, } } + +func init() { + registerSourceLoader(newEndpointsSource) +} diff --git a/sources/endpointslice.go b/sources/endpointslice.go index 510736d..fa24675 100644 --- a/sources/endpointslice.go +++ b/sources/endpointslice.go @@ -3,6 +3,7 @@ package sources import ( v1 "k8s.io/api/discovery/v1" + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) @@ -71,7 +72,7 @@ func endpointSliceExtractor(resource *v1.EndpointSlice, scope string) ([]*sdp.Li return queries, nil } -func newEndpointSliceSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.EndpointSlice, *v1.EndpointSliceList] { +func newEndpointSliceSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.EndpointSlice, *v1.EndpointSliceList]{ ClusterName: cluster, Namespaces: namespaces, @@ -91,3 +92,7 @@ func newEndpointSliceSource(cs *kubernetes.Clientset, cluster string, namespaces LinkedItemQueryExtractor: endpointSliceExtractor, } } + +func init() { + registerSourceLoader(newEndpointSliceSource) +} diff --git a/sources/horizontalpodautoscaler.go b/sources/horizontalpodautoscaler.go index 2ce2d74..a15cd6e 100644 --- a/sources/horizontalpodautoscaler.go +++ b/sources/horizontalpodautoscaler.go @@ -3,6 +3,7 @@ package sources import ( v2 "k8s.io/api/autoscaling/v2" + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) @@ -22,7 +23,7 @@ func horizontalPodAutoscalerExtractor(resource *v2.HorizontalPodAutoscaler, scop return queries, nil } -func newHorizontalPodAutoscalerSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v2.HorizontalPodAutoscaler, *v2.HorizontalPodAutoscalerList] { +func newHorizontalPodAutoscalerSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v2.HorizontalPodAutoscaler, *v2.HorizontalPodAutoscalerList]{ ClusterName: cluster, Namespaces: namespaces, @@ -42,3 +43,7 @@ func newHorizontalPodAutoscalerSource(cs *kubernetes.Clientset, cluster string, LinkedItemQueryExtractor: horizontalPodAutoscalerExtractor, } } + +func init() { + registerSourceLoader(newHorizontalPodAutoscalerSource) +} diff --git a/sources/ingress.go b/sources/ingress.go index f5aefab..69eead4 100644 --- a/sources/ingress.go +++ b/sources/ingress.go @@ -3,6 +3,7 @@ package sources import ( v1 "k8s.io/api/networking/v1" + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) @@ -87,7 +88,7 @@ func ingressExtractor(resource *v1.Ingress, scope string) ([]*sdp.LinkedItemQuer return queries, nil } -func newIngressSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Ingress, *v1.IngressList] { +func newIngressSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.Ingress, *v1.IngressList]{ ClusterName: cluster, Namespaces: namespaces, @@ -107,3 +108,7 @@ func newIngressSource(cs *kubernetes.Clientset, cluster string, namespaces []str LinkedItemQueryExtractor: ingressExtractor, } } + +func init() { + registerSourceLoader(newIngressSource) +} diff --git a/sources/job.go b/sources/job.go index eb47e2d..7e66dba 100644 --- a/sources/job.go +++ b/sources/job.go @@ -3,6 +3,7 @@ package sources import ( v1 "k8s.io/api/batch/v1" + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) @@ -24,7 +25,7 @@ func jobExtractor(resource *v1.Job, scope string) ([]*sdp.LinkedItemQuery, error return queries, nil } -func newJobSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Job, *v1.JobList] { +func newJobSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.Job, *v1.JobList]{ ClusterName: cluster, Namespaces: namespaces, @@ -44,3 +45,7 @@ func newJobSource(cs *kubernetes.Clientset, cluster string, namespaces []string) LinkedItemQueryExtractor: jobExtractor, } } + +func init() { + registerSourceLoader(newJobSource) +} diff --git a/sources/limitrange.go b/sources/limitrange.go index 630ded1..20856f3 100644 --- a/sources/limitrange.go +++ b/sources/limitrange.go @@ -1,11 +1,12 @@ package sources import ( + "github.com/overmindtech/discovery" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -func newLimitRangeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.LimitRange, *v1.LimitRangeList] { +func newLimitRangeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.LimitRange, *v1.LimitRangeList]{ ClusterName: cluster, Namespaces: namespaces, @@ -24,3 +25,7 @@ func newLimitRangeSource(cs *kubernetes.Clientset, cluster string, namespaces [] }, } } + +func init() { + registerSourceLoader(newLimitRangeSource) +} diff --git a/sources/main.go b/sources/main.go index c283f26..5d2cf49 100644 --- a/sources/main.go +++ b/sources/main.go @@ -5,37 +5,20 @@ import ( "k8s.io/client-go/kubernetes" ) +type SourceLoader func(clientSet *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source + +var sourceLoaders []SourceLoader + +func registerSourceLoader(loader SourceLoader) { + sourceLoaders = append(sourceLoaders, loader) +} + func LoadAllSources(cs *kubernetes.Clientset, cluster string, namespaces []string) []discovery.Source { - return []discovery.Source{ - newClusterRoleSource(cs, cluster, namespaces), - newClusterRoleBindingSource(cs, cluster, namespaces), - newConfigMapSource(cs, cluster, namespaces), - newCronJobSource(cs, cluster, namespaces), - newDaemonSetSource(cs, cluster, namespaces), - newDeploymentSource(cs, cluster, namespaces), - newEndpointsSource(cs, cluster, namespaces), - newEndpointSliceSource(cs, cluster, namespaces), - newHorizontalPodAutoscalerSource(cs, cluster, namespaces), - newIngressSource(cs, cluster, namespaces), - newJobSource(cs, cluster, namespaces), - newLimitRangeSource(cs, cluster, namespaces), - newNetworkPolicySource(cs, cluster, namespaces), - newNodeSource(cs, cluster, namespaces), - newPersistentVolumeSource(cs, cluster, namespaces), - newPersistentVolumeClaimSource(cs, cluster, namespaces), - newPodDisruptionBudgetSource(cs, cluster, namespaces), - newPodSource(cs, cluster, namespaces), - newPriorityClassSource(cs, cluster, namespaces), - newReplicaSetSource(cs, cluster, namespaces), - newReplicationControllerSource(cs, cluster, namespaces), - newResourceQuotaSource(cs, cluster, namespaces), - newRoleSource(cs, cluster, namespaces), - newRoleBindingSource(cs, cluster, namespaces), - newSecretSource(cs, cluster, namespaces), - newServiceSource(cs, cluster, namespaces), - newServiceAccountSource(cs, cluster, namespaces), - newStatefulSetSource(cs, cluster, namespaces), - newStorageClassSource(cs, cluster, namespaces), - newVolumeAttachmentSource(cs, cluster, namespaces), + sources := make([]discovery.Source, len(sourceLoaders)) + + for i, loader := range sourceLoaders { + sources[i] = loader(cs, cluster, namespaces) } + + return sources } diff --git a/sources/networkpolicy.go b/sources/networkpolicy.go index 3f03ed8..4f9e8e4 100644 --- a/sources/networkpolicy.go +++ b/sources/networkpolicy.go @@ -3,6 +3,7 @@ package sources import ( v1 "k8s.io/api/networking/v1" + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) @@ -51,7 +52,7 @@ func NetworkPolicyExtractor(resource *v1.NetworkPolicy, scope string) ([]*sdp.Li return queries, nil } -func newNetworkPolicySource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.NetworkPolicy, *v1.NetworkPolicyList] { +func newNetworkPolicySource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.NetworkPolicy, *v1.NetworkPolicyList]{ ClusterName: cluster, Namespaces: namespaces, @@ -71,3 +72,7 @@ func newNetworkPolicySource(cs *kubernetes.Clientset, cluster string, namespaces LinkedItemQueryExtractor: NetworkPolicyExtractor, } } + +func init() { + registerSourceLoader(newNetworkPolicySource) +} diff --git a/sources/node.go b/sources/node.go index affe798..bd0e727 100644 --- a/sources/node.go +++ b/sources/node.go @@ -3,6 +3,7 @@ package sources import ( "strings" + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" v1 "k8s.io/api/core/v1" @@ -59,7 +60,7 @@ func linkedItemExtractor(resource *v1.Node, scope string) ([]*sdp.LinkedItemQuer // TODO: Should we try a DNS lookup for a node name? Is the hostname stored anywhere? -func newNodeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Node, *v1.NodeList] { +func newNodeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.Node, *v1.NodeList]{ ClusterName: cluster, Namespaces: namespaces, @@ -79,3 +80,7 @@ func newNodeSource(cs *kubernetes.Clientset, cluster string, namespaces []string LinkedItemQueryExtractor: linkedItemExtractor, } } + +func init() { + registerSourceLoader(newNodeSource) +} diff --git a/sources/persistentvolume.go b/sources/persistentvolume.go index 6a542b6..48fdbc1 100644 --- a/sources/persistentvolume.go +++ b/sources/persistentvolume.go @@ -1,6 +1,7 @@ package sources import ( + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" @@ -45,7 +46,7 @@ func PersistentVolumeExtractor(resource *v1.PersistentVolume, scope string) ([]* return queries, nil } -func newPersistentVolumeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.PersistentVolume, *v1.PersistentVolumeList] { +func newPersistentVolumeSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.PersistentVolume, *v1.PersistentVolumeList]{ ClusterName: cluster, Namespaces: namespaces, @@ -65,3 +66,7 @@ func newPersistentVolumeSource(cs *kubernetes.Clientset, cluster string, namespa LinkedItemQueryExtractor: PersistentVolumeExtractor, } } + +func init() { + registerSourceLoader(newPersistentVolumeSource) +} diff --git a/sources/persistentvolumeclaim.go b/sources/persistentvolumeclaim.go index 7804284..ee0118b 100644 --- a/sources/persistentvolumeclaim.go +++ b/sources/persistentvolumeclaim.go @@ -1,11 +1,12 @@ package sources import ( + "github.com/overmindtech/discovery" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -func newPersistentVolumeClaimSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.PersistentVolumeClaim, *v1.PersistentVolumeClaimList] { +func newPersistentVolumeClaimSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.PersistentVolumeClaim, *v1.PersistentVolumeClaimList]{ ClusterName: cluster, Namespaces: namespaces, @@ -24,3 +25,7 @@ func newPersistentVolumeClaimSource(cs *kubernetes.Clientset, cluster string, na }, } } + +func init() { + registerSourceLoader(newPersistentVolumeClaimSource) +} diff --git a/sources/poddisruptionbudget.go b/sources/poddisruptionbudget.go index 6e0b42b..58c9025 100644 --- a/sources/poddisruptionbudget.go +++ b/sources/poddisruptionbudget.go @@ -1,6 +1,7 @@ package sources import ( + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" v1 "k8s.io/api/policy/v1" "k8s.io/client-go/kubernetes" @@ -23,7 +24,7 @@ func podDisruptionBudgetExtractor(resource *v1.PodDisruptionBudget, scope string return queries, nil } -func newPodDisruptionBudgetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.PodDisruptionBudget, *v1.PodDisruptionBudgetList] { +func newPodDisruptionBudgetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.PodDisruptionBudget, *v1.PodDisruptionBudgetList]{ ClusterName: cluster, Namespaces: namespaces, @@ -43,3 +44,7 @@ func newPodDisruptionBudgetSource(cs *kubernetes.Clientset, cluster string, name LinkedItemQueryExtractor: podDisruptionBudgetExtractor, } } + +func init() { + registerSourceLoader(newPodDisruptionBudgetSource) +} diff --git a/sources/pods.go b/sources/pods.go index 8260ce0..ce5cacb 100644 --- a/sources/pods.go +++ b/sources/pods.go @@ -1,6 +1,7 @@ package sources import ( + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" @@ -147,7 +148,7 @@ func PodExtractor(resource *v1.Pod, scope string) ([]*sdp.LinkedItemQuery, error return queries, nil } -func newPodSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Pod, *v1.PodList] { +func newPodSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.Pod, *v1.PodList]{ ClusterName: cluster, Namespaces: namespaces, @@ -183,3 +184,7 @@ func newPodSource(cs *kubernetes.Clientset, cluster string, namespaces []string) }, } } + +func init() { + registerSourceLoader(newPodSource) +} diff --git a/sources/priorityclass.go b/sources/priorityclass.go index 15ece3c..8f8a3da 100644 --- a/sources/priorityclass.go +++ b/sources/priorityclass.go @@ -1,12 +1,13 @@ package sources import ( + "github.com/overmindtech/discovery" v1 "k8s.io/api/scheduling/v1" "k8s.io/client-go/kubernetes" ) -func newPriorityClassSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.PriorityClass, *v1.PriorityClassList] { +func newPriorityClassSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.PriorityClass, *v1.PriorityClassList]{ ClusterName: cluster, Namespaces: namespaces, @@ -25,3 +26,7 @@ func newPriorityClassSource(cs *kubernetes.Clientset, cluster string, namespaces }, } } + +func init() { + registerSourceLoader(newPriorityClassSource) +} diff --git a/sources/replicaset.go b/sources/replicaset.go index 819007b..b3476f5 100644 --- a/sources/replicaset.go +++ b/sources/replicaset.go @@ -3,6 +3,7 @@ package sources import ( v1 "k8s.io/api/apps/v1" + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) @@ -24,7 +25,7 @@ func replicaSetExtractor(resource *v1.ReplicaSet, scope string) ([]*sdp.LinkedIt return queries, nil } -func newReplicaSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ReplicaSet, *v1.ReplicaSetList] { +func newReplicaSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.ReplicaSet, *v1.ReplicaSetList]{ ClusterName: cluster, Namespaces: namespaces, @@ -44,3 +45,7 @@ func newReplicaSetSource(cs *kubernetes.Clientset, cluster string, namespaces [] LinkedItemQueryExtractor: replicaSetExtractor, } } + +func init() { + registerSourceLoader(newReplicaSetSource) +} diff --git a/sources/replicationcontroller.go b/sources/replicationcontroller.go index 0782e19..0dfaae6 100644 --- a/sources/replicationcontroller.go +++ b/sources/replicationcontroller.go @@ -1,6 +1,7 @@ package sources import ( + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" v1 "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,7 +27,7 @@ func replicationControllerExtractor(resource *v1.ReplicationController, scope st return queries, nil } -func newReplicationControllerSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ReplicationController, *v1.ReplicationControllerList] { +func newReplicationControllerSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.ReplicationController, *v1.ReplicationControllerList]{ ClusterName: cluster, Namespaces: namespaces, @@ -46,3 +47,7 @@ func newReplicationControllerSource(cs *kubernetes.Clientset, cluster string, na LinkedItemQueryExtractor: replicationControllerExtractor, } } + +func init() { + registerSourceLoader(newReplicationControllerSource) +} diff --git a/sources/resourcequota.go b/sources/resourcequota.go index f5e3d13..818b82d 100644 --- a/sources/resourcequota.go +++ b/sources/resourcequota.go @@ -1,11 +1,12 @@ package sources import ( + "github.com/overmindtech/discovery" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -func newResourceQuotaSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ResourceQuota, *v1.ResourceQuotaList] { +func newResourceQuotaSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.ResourceQuota, *v1.ResourceQuotaList]{ ClusterName: cluster, Namespaces: namespaces, @@ -24,3 +25,7 @@ func newResourceQuotaSource(cs *kubernetes.Clientset, cluster string, namespaces }, } } + +func init() { + registerSourceLoader(newResourceQuotaSource) +} diff --git a/sources/role.go b/sources/role.go index abe0515..4e1f535 100644 --- a/sources/role.go +++ b/sources/role.go @@ -1,12 +1,13 @@ package sources import ( + "github.com/overmindtech/discovery" v1 "k8s.io/api/rbac/v1" "k8s.io/client-go/kubernetes" ) -func newRoleSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Role, *v1.RoleList] { +func newRoleSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.Role, *v1.RoleList]{ ClusterName: cluster, Namespaces: namespaces, @@ -25,3 +26,7 @@ func newRoleSource(cs *kubernetes.Clientset, cluster string, namespaces []string }, } } + +func init() { + registerSourceLoader(newRoleSource) +} diff --git a/sources/rolebinding.go b/sources/rolebinding.go index cc4ee88..774dd4c 100644 --- a/sources/rolebinding.go +++ b/sources/rolebinding.go @@ -3,6 +3,7 @@ package sources import ( v1 "k8s.io/api/rbac/v1" + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) @@ -57,7 +58,7 @@ func roleBindingExtractor(resource *v1.RoleBinding, scope string) ([]*sdp.Linked return queries, nil } -func newRoleBindingSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.RoleBinding, *v1.RoleBindingList] { +func newRoleBindingSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.RoleBinding, *v1.RoleBindingList]{ ClusterName: cluster, Namespaces: namespaces, @@ -77,3 +78,7 @@ func newRoleBindingSource(cs *kubernetes.Clientset, cluster string, namespaces [ LinkedItemQueryExtractor: roleBindingExtractor, } } + +func init() { + registerSourceLoader(newRoleBindingSource) +} diff --git a/sources/secret.go b/sources/secret.go index d522fd2..c1e35b1 100644 --- a/sources/secret.go +++ b/sources/secret.go @@ -3,12 +3,12 @@ package sources import ( "crypto/sha512" + "github.com/overmindtech/discovery" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) -// TODO: Configure redaction -func newSecretSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Secret, *v1.SecretList] { +func newSecretSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.Secret, *v1.SecretList]{ ClusterName: cluster, Namespaces: namespaces, @@ -45,3 +45,7 @@ func newSecretSource(cs *kubernetes.Clientset, cluster string, namespaces []stri }, } } + +func init() { + registerSourceLoader(newSecretSource) +} diff --git a/sources/service.go b/sources/service.go index 2773964..d1261e1 100644 --- a/sources/service.go +++ b/sources/service.go @@ -1,6 +1,7 @@ package sources import ( + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" v1 "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -95,7 +96,7 @@ func serviceExtractor(resource *v1.Service, scope string) ([]*sdp.LinkedItemQuer return queries, nil } -func newServiceSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.Service, *v1.ServiceList] { +func newServiceSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.Service, *v1.ServiceList]{ ClusterName: cluster, Namespaces: namespaces, @@ -115,3 +116,7 @@ func newServiceSource(cs *kubernetes.Clientset, cluster string, namespaces []str LinkedItemQueryExtractor: serviceExtractor, } } + +func init() { + registerSourceLoader(newServiceSource) +} diff --git a/sources/serviceaccount.go b/sources/serviceaccount.go index 3425273..7043774 100644 --- a/sources/serviceaccount.go +++ b/sources/serviceaccount.go @@ -1,6 +1,7 @@ package sources import ( + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" @@ -34,7 +35,7 @@ func serviceAccountExtractor(resource *v1.ServiceAccount, scope string) ([]*sdp. return queries, nil } -func newServiceAccountSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.ServiceAccount, *v1.ServiceAccountList] { +func newServiceAccountSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.ServiceAccount, *v1.ServiceAccountList]{ ClusterName: cluster, Namespaces: namespaces, @@ -54,3 +55,7 @@ func newServiceAccountSource(cs *kubernetes.Clientset, cluster string, namespace LinkedItemQueryExtractor: serviceAccountExtractor, } } + +func init() { + registerSourceLoader(newServiceAccountSource) +} diff --git a/sources/statefulset.go b/sources/statefulset.go index 4337f14..3a68f45 100644 --- a/sources/statefulset.go +++ b/sources/statefulset.go @@ -4,6 +4,7 @@ import ( v1 "k8s.io/api/apps/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" "k8s.io/client-go/kubernetes" ) @@ -53,7 +54,7 @@ func statefulSetExtractor(resource *v1.StatefulSet, scope string) ([]*sdp.Linked return queries, nil } -func newStatefulSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.StatefulSet, *v1.StatefulSetList] { +func newStatefulSetSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.StatefulSet, *v1.StatefulSetList]{ ClusterName: cluster, Namespaces: namespaces, @@ -73,3 +74,7 @@ func newStatefulSetSource(cs *kubernetes.Clientset, cluster string, namespaces [ LinkedItemQueryExtractor: statefulSetExtractor, } } + +func init() { + registerSourceLoader(newStatefulSetSource) +} diff --git a/sources/storageclass.go b/sources/storageclass.go index 0de80a9..b379ee0 100644 --- a/sources/storageclass.go +++ b/sources/storageclass.go @@ -1,12 +1,13 @@ package sources import ( + "github.com/overmindtech/discovery" v1 "k8s.io/api/storage/v1" "k8s.io/client-go/kubernetes" ) -func newStorageClassSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.StorageClass, *v1.StorageClassList] { +func newStorageClassSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.StorageClass, *v1.StorageClassList]{ ClusterName: cluster, Namespaces: namespaces, @@ -25,3 +26,7 @@ func newStorageClassSource(cs *kubernetes.Clientset, cluster string, namespaces }, } } + +func init() { + registerSourceLoader(newStorageClassSource) +} diff --git a/sources/volumeattachment.go b/sources/volumeattachment.go index 3e6816e..a8ea996 100644 --- a/sources/volumeattachment.go +++ b/sources/volumeattachment.go @@ -1,6 +1,7 @@ package sources import ( + "github.com/overmindtech/discovery" "github.com/overmindtech/sdp-go" v1 "k8s.io/api/storage/v1" "k8s.io/client-go/kubernetes" @@ -34,7 +35,7 @@ func volumeAttachmentExtractor(resource *v1.VolumeAttachment, scope string) ([]* return queries, nil } -func newVolumeAttachmentSource(cs *kubernetes.Clientset, cluster string, namespaces []string) *KubeTypeSource[*v1.VolumeAttachment, *v1.VolumeAttachmentList] { +func newVolumeAttachmentSource(cs *kubernetes.Clientset, cluster string, namespaces []string) discovery.Source { return &KubeTypeSource[*v1.VolumeAttachment, *v1.VolumeAttachmentList]{ ClusterName: cluster, Namespaces: namespaces, @@ -61,3 +62,7 @@ func newVolumeAttachmentSource(cs *kubernetes.Clientset, cluster string, namespa }, } } + +func init() { + registerSourceLoader(newVolumeAttachmentSource) +}