chore(yaml): add go-yaml types package (#1225)

* enhance(yaml): allow for users to parse pipelines using old library

* testing file for internal yaml

* chore(compiler): convert unmarshaled buildkite to go-yaml

* remove tests used in later PRs

* lintfix

* fix schema

* gci

api/pipeline/template.go

@@ -13,7 +13,7 @@ import (
 	"github.com/go-vela/server/api/types"
 	"github.com/go-vela/server/compiler"
 	"github.com/go-vela/server/compiler/registry/github"
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/internal"
 	"github.com/go-vela/server/router/middleware/org"
 	"github.com/go-vela/server/router/middleware/pipeline"

api/types/string.go

@@ -7,8 +7,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/buildkite/yaml"
 	json "github.com/ghodss/yaml"
+	"gopkg.in/yaml.v3"
 )
 
 // ToString is a helper function to convert

compiler/engine.go

@@ -9,7 +9,7 @@ import (
 	"github.com/go-vela/server/api/types/settings"
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/compiler/types/raw"
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/database"
 	"github.com/go-vela/server/internal"
 	"github.com/go-vela/server/scm"

compiler/native/clone.go

@@ -3,7 +3,7 @@
 package native
 
 import (
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/constants"
 )
 

compiler/native/clone_test.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/urfave/cli/v2"
 
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 )
 
 const defaultCloneImage = "target/vela-git-slim:latest"

compiler/native/compile.go

@@ -12,14 +12,14 @@ import (
 	"strings"
 	"time"
 
-	yml "github.com/buildkite/yaml"
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/go-retryablehttp"
+	yml "gopkg.in/yaml.v3"
 
 	api "github.com/go-vela/server/api/types"
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/compiler/types/raw"
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/constants"
 )
 

compiler/native/compile_test.go

@@ -13,16 +13,16 @@ import (
 	"testing"
 	"time"
 
-	yml "github.com/buildkite/yaml"
 	"github.com/gin-gonic/gin"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-github/v65/github"
 	"github.com/urfave/cli/v2"
+	yml "gopkg.in/yaml.v3"
 
 	api "github.com/go-vela/server/api/types"
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/compiler/types/raw"
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/constants"
 	"github.com/go-vela/server/internal"
 )
@@ -2056,7 +2056,7 @@ func Test_client_modifyConfig(t *testing.T) {
 				Name:        "docker",
 				Pull:        "always",
 				Parameters: map[string]interface{}{
-					"init_options": map[interface{}]interface{}{
+					"init_options": map[string]interface{}{
 						"get_plugins": "true",
 					},
 				},
@@ -2089,7 +2089,7 @@ func Test_client_modifyConfig(t *testing.T) {
 				Name:        "docker",
 				Pull:        "always",
 				Parameters: map[string]interface{}{
-					"init_options": map[interface{}]interface{}{
+					"init_options": map[string]interface{}{
 						"get_plugins": "true",
 					},
 				},

compiler/native/environment.go

@@ -9,7 +9,7 @@ import (
 
 	api "github.com/go-vela/server/api/types"
 	"github.com/go-vela/server/compiler/types/raw"
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/constants"
 	"github.com/go-vela/server/internal"
 )

compiler/native/environment_test.go

@@ -13,7 +13,7 @@ import (
 
 	api "github.com/go-vela/server/api/types"
 	"github.com/go-vela/server/compiler/types/raw"
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/internal"
 )
 

compiler/native/expand.go

@@ -15,7 +15,7 @@ import (
 	"github.com/go-vela/server/compiler/template/starlark"
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/compiler/types/raw"
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/constants"
 )
 

compiler/native/expand_test.go

@@ -17,7 +17,7 @@ import (
 	api "github.com/go-vela/server/api/types"
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/compiler/types/raw"
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 )
 
 func TestNative_ExpandStages(t *testing.T) {

compiler/native/initialize.go

@@ -3,7 +3,7 @@
 package native
 
 import (
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/constants"
 )
 

compiler/native/initialize_test.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/urfave/cli/v2"
 
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 )
 
 func TestNative_InitStage(t *testing.T) {

compiler/native/parse.go

@@ -7,12 +7,13 @@ import (
 	"io"
 	"os"
 
-	"github.com/buildkite/yaml"
+	bkYaml "github.com/buildkite/yaml"
 
 	"github.com/go-vela/server/compiler/template/native"
 	"github.com/go-vela/server/compiler/template/starlark"
 	typesRaw "github.com/go-vela/server/compiler/types/raw"
-	types "github.com/go-vela/server/compiler/types/yaml"
+	bkYamlTypes "github.com/go-vela/server/compiler/types/yaml/buildkite"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/constants"
 )
 
@@ -41,9 +42,9 @@ func (c *client) ParseRaw(v interface{}) (string, error) {
 }
 
 // Parse converts an object to a yaml configuration.
-func (c *client) Parse(v interface{}, pipelineType string, template *types.Template) (*types.Build, []byte, error) {
+func (c *client) Parse(v interface{}, pipelineType string, template *yaml.Template) (*yaml.Build, []byte, error) {
 	var (
-		p   *types.Build
+		p   *yaml.Build
 		raw []byte
 	)
 
@@ -112,11 +113,11 @@ func (c *client) Parse(v interface{}, pipelineType string, template *types.Templ
 }
 
 // ParseBytes converts a byte slice to a yaml configuration.
-func ParseBytes(data []byte) (*types.Build, []byte, error) {
-	config := new(types.Build)
+func ParseBytes(data []byte) (*yaml.Build, []byte, error) {
+	config := new(bkYamlTypes.Build)
 
 	// unmarshal the bytes into the yaml configuration
-	err := yaml.Unmarshal(data, config)
+	err := bkYaml.Unmarshal(data, config)
 	if err != nil {
 		return nil, data, fmt.Errorf("unable to unmarshal yaml: %w", err)
 	}
@@ -128,11 +129,11 @@ func ParseBytes(data []byte) (*types.Build, []byte, error) {
 		config.Environment = typesRaw.StringSliceMap{}
 	}
 
-	return config, data, nil
+	return config.ToYAML(), data, nil
 }
 
 // ParseFile converts an os.File into a yaml configuration.
-func ParseFile(f *os.File) (*types.Build, []byte, error) {
+func ParseFile(f *os.File) (*yaml.Build, []byte, error) {
 	return ParseReader(f)
 }
 
@@ -142,7 +143,7 @@ func ParseFileRaw(f *os.File) (string, error) {
 }
 
 // ParsePath converts a file path into a yaml configuration.
-func ParsePath(p string) (*types.Build, []byte, error) {
+func ParsePath(p string) (*yaml.Build, []byte, error) {
 	// open the file for reading
 	f, err := os.Open(p)
 	if err != nil {
@@ -168,7 +169,7 @@ func ParsePathRaw(p string) (string, error) {
 }
 
 // ParseReader converts an io.Reader into a yaml configuration.
-func ParseReader(r io.Reader) (*types.Build, []byte, error) {
+func ParseReader(r io.Reader) (*yaml.Build, []byte, error) {
 	// read all the bytes from the reader
 	data, err := io.ReadAll(r)
 	if err != nil {
@@ -190,6 +191,6 @@ func ParseReaderRaw(r io.Reader) (string, error) {
 }
 
 // ParseString converts a string into a yaml configuration.
-func ParseString(s string) (*types.Build, []byte, error) {
+func ParseString(s string) (*yaml.Build, []byte, error) {
 	return ParseBytes([]byte(s))
 }

compiler/native/parse_test.go

@@ -15,7 +15,7 @@ import (
 
 	api "github.com/go-vela/server/api/types"
 	"github.com/go-vela/server/compiler/types/raw"
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/constants"
 )
 

compiler/native/script.go

@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 )
 
 // ScriptStages injects the script for each step in every stage in a yaml configuration.

compiler/native/script_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/urfave/cli/v2"
 
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 )
 
 func TestNative_ScriptStages(t *testing.T) {

compiler/native/substitute.go

@@ -6,10 +6,10 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/buildkite/yaml"
 	"github.com/drone/envsubst"
+	"gopkg.in/yaml.v3"
 
-	types "github.com/go-vela/server/compiler/types/yaml"
+	types "github.com/go-vela/server/compiler/types/yaml/yaml"
 )
 
 // SubstituteStages replaces every declared environment

compiler/native/substitute_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/urfave/cli/v2"
 
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 )
 
 func Test_client_SubstituteStages(t *testing.T) {

compiler/native/transform.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 
 	"github.com/go-vela/server/compiler/types/pipeline"
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 )
 
 const (
@@ -65,12 +65,13 @@ func (c *client) TransformStages(r *pipeline.RuleData, p *yaml.Build) (*pipeline
 
 	// create new executable pipeline
 	pipeline := &pipeline.Build{
-		Version:  p.Version,
-		Metadata: *p.Metadata.ToPipeline(),
-		Stages:   *p.Stages.ToPipeline(),
-		Secrets:  *p.Secrets.ToPipeline(),
-		Services: *p.Services.ToPipeline(),
-		Worker:   *p.Worker.ToPipeline(),
+		Version:    p.Version,
+		Metadata:   *p.Metadata.ToPipeline(),
+		Stages:     *p.Stages.ToPipeline(),
+		Secrets:    *p.Secrets.ToPipeline(),
+		Services:   *p.Services.ToPipeline(),
+		Worker:     *p.Worker.ToPipeline(),
+		Deployment: *p.Deployment.ToPipeline(),
 	}
 
 	// set the unique ID for the executable pipeline

compiler/native/transform_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/go-vela/server/compiler/types/pipeline"
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/internal"
 )
 

compiler/native/validate.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/hashicorp/go-multierror"
 
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 	"github.com/go-vela/server/constants"
 )
 

compiler/native/validate_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/go-vela/server/compiler/types/raw"
-	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/compiler/types/yaml/yaml"
 )
 
 func TestNative_Validate_NoVersion(t *testing.T) {

compiler/template/native/convert.go

@@ -5,7 +5,7 @@ package native
 import (
 	"strings"
 
-	"github.com/buildkite/yaml"
+	"gopkg.in/yaml.v3"
 
 	"github.com/go-vela/server/compiler/types/raw"
 )

compiler/template/native/render.go

@@ -11,13 +11,14 @@ import (
 	"github.com/buildkite/yaml"
 
 	"github.com/go-vela/server/compiler/types/raw"
-	types "github.com/go-vela/server/compiler/types/yaml"
+	bkTypes "github.com/go-vela/server/compiler/types/yaml/buildkite"
+	types "github.com/go-vela/server/compiler/types/yaml/yaml"
 )
 
 // Render combines the template with the step in the yaml pipeline.
 func Render(tmpl string, name string, tName string, environment raw.StringSliceMap, variables map[string]interface{}) (*types.Build, error) {
 	buffer := new(bytes.Buffer)
-	config := new(types.Build)
+	config := new(bkTypes.Build)
 
 	velaFuncs := funcHandler{envs: convertPlatformVars(environment, name)}
 	templateFuncMap := map[string]interface{}{
@@ -57,13 +58,13 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM
 		config.Steps[index].Name = fmt.Sprintf("%s_%s", name, newStep.Name)
 	}
 
-	return &types.Build{Metadata: config.Metadata, Deployment: config.Deployment, Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment, Templates: config.Templates}, nil
+	return &types.Build{Metadata: *config.Metadata.ToYAML(), Steps: *config.Steps.ToYAML(), Secrets: *config.Secrets.ToYAML(), Services: *config.Services.ToYAML(), Environment: config.Environment, Templates: *config.Templates.ToYAML(), Deployment: *config.Deployment.ToYAML()}, nil
 }
 
 // RenderBuild renders the templated build.
 func RenderBuild(tmpl string, b string, envs map[string]string, variables map[string]interface{}) (*types.Build, error) {
 	buffer := new(bytes.Buffer)
-	config := new(types.Build)
+	config := new(bkTypes.Build)
 
 	velaFuncs := funcHandler{envs: convertPlatformVars(envs, tmpl)}
 	templateFuncMap := map[string]interface{}{
@@ -98,5 +99,5 @@ func RenderBuild(tmpl string, b string, envs map[string]string, variables map[st
 		return nil, fmt.Errorf("unable to unmarshal yaml: %w", err)
 	}
 
-	return config, nil
+	return config.ToYAML(), nil
 }