From b85c5a0f7bc6a49fddb28585eb429fcf2c6a9c62 Mon Sep 17 00:00:00 2001 From: Ken Sipe Date: Tue, 2 Jun 2020 10:36:28 -0500 Subject: [PATCH] teststep apply, assert and errors can now take urls, files or folder (#123) Signed-off-by: Ken Sipe --- pkg/file/files.go | 2 +- pkg/http/http.go | 57 +++++++++++++++++++++++++++++++ pkg/http/http_test.go | 43 +++++++++++++++++++++++ pkg/kuttlctl/cmd/test.go | 2 +- pkg/test/step.go | 50 ++++++++++++++++----------- pkg/test/utils/kubernetes.go | 21 +++++++++--- pkg/test/utils/kubernetes_test.go | 4 +-- 7 files changed, 150 insertions(+), 29 deletions(-) create mode 100644 pkg/http/http.go create mode 100644 pkg/http/http_test.go diff --git a/pkg/file/files.go b/pkg/file/files.go index adfd9bba..162f4677 100644 --- a/pkg/file/files.go +++ b/pkg/file/files.go @@ -16,7 +16,7 @@ func ToRuntimeObjects(paths []string) ([]runtime.Object, error) { apply := []runtime.Object{} for _, path := range paths { - objs, err := testutils.LoadYAML(path) + objs, err := testutils.LoadYAMLFromFile(path) if err != nil { return nil, fmt.Errorf("file %q load yaml error", path) } diff --git a/pkg/http/http.go b/pkg/http/http.go new file mode 100644 index 00000000..cb72922c --- /dev/null +++ b/pkg/http/http.go @@ -0,0 +1,57 @@ +package http + +import ( + "bytes" + "fmt" + "io" + coreHTTP "net/http" + "net/url" + + "k8s.io/apimachinery/pkg/runtime" + + testutils "github.com/kudobuilder/kuttl/pkg/test/utils" +) + +// IsURL returns true if string is an URL +func IsURL(str string) bool { + u, err := url.Parse(str) + return err == nil && u.Scheme != "" && u.Host != "" +} + +// ToRuntimeObjects takes a url, pulls the file and returns []runtime.Object +// url must be a full path to a manifest file. that file can have multiple runtime objects. +func ToRuntimeObjects(urlPath string) ([]runtime.Object, error) { + apply := []runtime.Object{} + + buf, err := Read(urlPath) + if err != nil { + return nil, err + } + + objs, err := testutils.LoadYAML(urlPath, buf) + if err != nil { + return nil, fmt.Errorf("url %q load yaml error", urlPath) + } + apply = append(apply, objs...) + + return apply, nil +} + +// Read returns a buffer for the file at the url +func Read(urlPath string) (*bytes.Buffer, error) { + + response, err := coreHTTP.Get(urlPath) // nolint:gosec + if err != nil { + return nil, err + } + if response != nil { + defer response.Body.Close() + } + var buf bytes.Buffer + _, err = io.Copy(&buf, response.Body) + if err != nil { + return nil, err + } + + return &buf, nil +} diff --git a/pkg/http/http_test.go b/pkg/http/http_test.go new file mode 100644 index 00000000..e55c269c --- /dev/null +++ b/pkg/http/http_test.go @@ -0,0 +1,43 @@ +package http + +import ( + "testing" +) + +func TestIsURL(t *testing.T) { + + tests := []struct { + name string + path string + want bool + }{ + { + "path to folder", + "/opt/foo", + false, + }, + { + "path to file", + "/opt/foo.txt", + false, + }, + { + "http to file", + "http://kuttl.dev/foo.txt", + true, + }, + { + "https to file", + "https://kuttl.dev/foo.txt", + true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + if got := IsURL(tt.path); got != tt.want { + t.Errorf("IsURL() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/kuttlctl/cmd/test.go b/pkg/kuttlctl/cmd/test.go index 35adee9b..d5d43e1a 100644 --- a/pkg/kuttlctl/cmd/test.go +++ b/pkg/kuttlctl/cmd/test.go @@ -80,7 +80,7 @@ For more detailed documentation, visit: https://kudo.dev/docs/testing`, // Load the configuration YAML into options. if configPath != "" { - objects, err := testutils.LoadYAML(configPath) + objects, err := testutils.LoadYAMLFromFile(configPath) if err != nil { return err } diff --git a/pkg/test/step.go b/pkg/test/step.go index 47c28c21..351e25be 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -17,6 +17,7 @@ import ( harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" kfile "github.com/kudobuilder/kuttl/pkg/file" + "github.com/kudobuilder/kuttl/pkg/http" testutils "github.com/kudobuilder/kuttl/pkg/test/utils" ) @@ -415,14 +416,14 @@ func (s *Step) String() string { return fmt.Sprintf("%d-%s", s.Index, s.Name) } -// LoadYAML loads the resources from a YAML file for a test step: +// LoadYAMLFromFile loads the resources from a YAML file for a test step: // * If the YAML file is called "assert", then it contains objects to // add to the test step's list of assertions. // * If the YAML file is called "errors", then it contains objects that, // if seen, mark a test immediately failed. // * All other YAML files are considered resources to create. func (s *Step) LoadYAML(file string) error { - objects, err := testutils.LoadYAML(file) + objects, err := testutils.LoadYAMLFromFile(file) if err != nil { return fmt.Errorf("loading %s: %s", file, err) } @@ -481,37 +482,25 @@ func (s *Step) LoadYAML(file string) error { if s.Step != nil { // process configured step applies for _, applyPath := range s.Step.Apply { - paths, err := kfile.FromPath(filepath.Join(s.Dir, applyPath), "*.yaml") + apply, err := runtimeObjectsFromPath(s, applyPath) if err != nil { - return fmt.Errorf("step %q apply %w", s.Name, err) - } - apply, err := kfile.ToRuntimeObjects(paths) - if err != nil { - return fmt.Errorf("step %q apply %w", s.Name, err) + return err } applies = append(applies, apply...) } // process configured step asserts for _, assertPath := range s.Step.Assert { - paths, err := kfile.FromPath(filepath.Join(s.Dir, assertPath), "*.yaml") - if err != nil { - return fmt.Errorf("step %q assert %w", s.Name, err) - } - assert, err := kfile.ToRuntimeObjects(paths) + assert, err := runtimeObjectsFromPath(s, assertPath) if err != nil { - return fmt.Errorf("step %q assert %w", s.Name, err) + return err } asserts = append(asserts, assert...) } // process configured errors for _, errorPath := range s.Step.Error { - paths, err := kfile.FromPath(filepath.Join(s.Dir, errorPath), "*.yaml") + errObjs, err := runtimeObjectsFromPath(s, errorPath) if err != nil { - return fmt.Errorf("step %q error %w", s.Name, err) - } - errObjs, err := kfile.ToRuntimeObjects(paths) - if err != nil { - return fmt.Errorf("step %q error %w", s.Name, err) + return err } s.Errors = append(s.Errors, errObjs...) } @@ -521,3 +510,24 @@ func (s *Step) LoadYAML(file string) error { s.Asserts = asserts return nil } + +func runtimeObjectsFromPath(s *Step, path string) ([]runtime.Object, error) { + if http.IsURL(path) { + apply, err := http.ToRuntimeObjects(path) + if err != nil { + return nil, fmt.Errorf("step %q apply %w", s.Name, err) + } + return apply, nil + } + + // it's a directory or file + paths, err := kfile.FromPath(filepath.Join(s.Dir, path), "*.yaml") + if err != nil { + return nil, fmt.Errorf("step %q apply %w", s.Name, err) + } + apply, err := kfile.ToRuntimeObjects(paths) + if err != nil { + return nil, fmt.Errorf("step %q apply %w", s.Name, err) + } + return apply, nil +} diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index 55fa4595..dad620ec 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -454,15 +454,19 @@ func MarshalObjectJSON(o runtime.Object, w io.Writer) error { return json.NewSerializer(json.DefaultMetaFactory, nil, nil, false).Encode(copied, w) } -// LoadYAML loads all objects from a YAML file. -func LoadYAML(path string) ([]runtime.Object, error) { +// LoadYAMLFromFile loads all objects from a YAML file. +func LoadYAMLFromFile(path string) ([]runtime.Object, error) { opened, err := os.Open(path) if err != nil { return nil, err } defer opened.Close() - yamlReader := yaml.NewYAMLReader(bufio.NewReader(opened)) + return LoadYAML(path, opened) +} + +func LoadYAML(path string, r io.Reader) ([]runtime.Object, error) { + yamlReader := yaml.NewYAMLReader(bufio.NewReader(r)) objects := []runtime.Object{} @@ -486,11 +490,18 @@ func LoadYAML(path string) ([]runtime.Object, error) { if err != nil { return nil, fmt.Errorf("error converting unstructured object %s (%s): %w", ResourceID(unstructuredObj), path, err) } + // discovered reader will return empty objects if a number of lines are preceding a yaml separator (---) + // this detects that, logs and continues + if obj.GetObjectKind().GroupVersionKind().Kind == "" { + log.Println("object detected with no GVK Kind for path", path) + } else { + objects = append(objects, obj) + } - objects = append(objects, obj) } return objects, nil + } // MatchesKind returns true if the Kubernetes kind of obj matches any of kinds. @@ -532,7 +543,7 @@ func InstallManifests(ctx context.Context, client client.Client, dClient discove return nil } - objs, err := LoadYAML(path) + objs, err := LoadYAMLFromFile(path) if err != nil { return err } diff --git a/pkg/test/utils/kubernetes_test.go b/pkg/test/utils/kubernetes_test.go index b2872745..c35adaee 100644 --- a/pkg/test/utils/kubernetes_test.go +++ b/pkg/test/utils/kubernetes_test.go @@ -158,7 +158,7 @@ spec: t.Fatal(err) } - objs, err := LoadYAML(tmpfile.Name()) + objs, err := LoadYAMLFromFile(tmpfile.Name()) assert.Nil(t, err) assert.Equal(t, &unstructured.Unstructured{ @@ -227,7 +227,7 @@ metadata: t.Fatal(err) } - objs, err := LoadYAML(tmpfile.Name()) + objs, err := LoadYAMLFromFile(tmpfile.Name()) assert.Nil(t, err) crd := NewResource("apiextensions.k8s.io/v1beta1", "CustomResourceDefinition", "", "")