From 88e4ac658ab28ccf0d27df2495700fa282ce7f30 Mon Sep 17 00:00:00 2001 From: Murad Biashimov Date: Fri, 8 Nov 2024 09:31:06 +0100 Subject: [PATCH] feat: use new schema endpoint (#206) --- .gitignore | 1 + Taskfile.yml | 24 +- cmd/root.go | 37 +- go.mod | 9 +- go.sum | 39 +- internal/convert/convert.go | 184 -------- internal/convert/convert_test.go | 458 -------------------- internal/gen/gen.go | 336 +++++++++++---- internal/gen/legacy.go | 43 ++ internal/pkg/util/util.go | 50 --- internal/reader/reader.go | 23 +- internal/writer/writer.go | 26 +- pkg/dist/integration_endpoint_types.yml | 62 +-- pkg/dist/integration_types.yml | 8 +- pkg/dist/service_types.yml | 547 +++++++----------------- 15 files changed, 534 insertions(+), 1313 deletions(-) delete mode 100644 internal/convert/convert.go delete mode 100644 internal/convert/convert_test.go create mode 100644 internal/gen/legacy.go diff --git a/.gitignore b/.gitignore index 514338a..0324596 100644 --- a/.gitignore +++ b/.gitignore @@ -210,3 +210,4 @@ $RECYCLE.BIN/ # Project specific ./go-api-schemas +*.json diff --git a/Taskfile.yml b/Taskfile.yml index c96a367..a160e56 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,12 +1,34 @@ version: "3" tasks: + curlSpec: + cmds: + - curl -so service_types.json https://api.aiven.io/v1/service_types + curlSpecSensitive: + requires: + vars: + - AIVEN_TOKEN + - AIVEN_PROJECT_NAME + cmds: + - for: [integration_types, integration_endpoint_types] + cmd: 'curl -so {{.ITEM}}.json https://api.aiven.io/v1/project/{{.AIVEN_PROJECT_NAME}}/{{.ITEM}} --header "Authorization: aivenv1 {{.AIVEN_TOKEN}}"' + silent: true + generate: + cmds: + - go run main.go service_types.json integration_types.json integration_endpoint_types.json run: cmds: - - go run main.go + - task: curlSpec + - task: curlSpecSensitive + - task: generate build: cmds: - go build test: cmds: - go test -v ./... + curlDist: + desc: During the development, the diff might change multiple times. Downloads files from main + cmds: + - for: [service_types, integration_types, integration_endpoint_types] + cmd: "curl -so ./pkg/dist/{{.ITEM}}.yml https://raw.githubusercontent.com/aiven/go-api-schemas/refs/heads/main/pkg/dist/{{.ITEM}}.yml" diff --git a/cmd/root.go b/cmd/root.go index 776413c..a7e4d55 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,12 +2,10 @@ package cmd import ( - "context" "errors" "os" "strings" - "github.com/aiven/aiven-go-client/v2" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -22,21 +20,14 @@ import ( // logger is the logger of the application. var logger = &util.Logger{} -// env is a map of environment variables. -var env = util.EnvMap{ - util.EnvAivenProjectName: "", -} - -// client is a pointer to the Aiven client. -var client = &aiven.Client{} - // NewCmdRoot returns a pointer to the root command. func NewCmdRoot(l *util.Logger) *cobra.Command { cmd := &cobra.Command{ - Use: "go-api-schemas", + Use: "go-api-schemas foo.json bar.json baz.json", Short: "go-api-schemas is a tool for generating and persisting user configuration option schemas from " + "Aiven APIs.", - Run: run, + Run: run, + Args: cobra.MinimumNArgs(1), } cmd.Flags().StringP("output-dir", "o", "", "the output directory for the generated files") @@ -94,20 +85,10 @@ func setup(flags *pflag.FlagSet) { } logger.Info.Println("setting up environment variables") - - if err := util.SetupEnv(env); err != nil { - logger.Error.Fatalf("error setting up environment variables: %s", err) - } - - logger.Info.Println("setting up client") - - if err := util.SetupClient(client); err != nil { - logger.Error.Fatalf("error setting up client: %s", err) - } } // run is the function that is called when the rootCmd is executed. -func run(cmd *cobra.Command, _ []string) { +func run(cmd *cobra.Command, args []string) { flags := cmd.Flags() setup(flags) @@ -117,11 +98,9 @@ func run(cmd *cobra.Command, _ []string) { logger.Error.Fatalf("error getting regeneration flag: %s", err) } - ctx := context.Background() - logger.Info.Println("generating") - gr, err := gen.Run(ctx, logger, env, client) + gr, err := gen.Run(args...) if err != nil { logger.Error.Fatalf("error generating: %s", err) } @@ -131,22 +110,20 @@ func run(cmd *cobra.Command, _ []string) { if !shouldRegenerate { logger.Info.Println("reading files") - rr, err = reader.Run(ctx, logger, flags) + rr, err = reader.Run(logger, flags) if err != nil && !os.IsNotExist(err) { logger.Error.Fatalf("error reading files: %s", err) } } logger.Info.Println("diffing") - dr, err := diff.Run(rr, gr) if err != nil { logger.Error.Fatalf("error diffing: %s", err) } logger.Info.Println("writing files") - - if err = writer.Run(ctx, logger, flags, dr); err != nil { + if err = writer.Run(logger, flags, dr); err != nil { logger.Error.Fatalf("error writing files: %s", err) } diff --git a/go.mod b/go.mod index 0076814..7c6cb13 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,17 @@ module github.com/aiven/go-api-schemas go 1.22 require ( - github.com/aiven/aiven-go-client/v2 v2.30.0 github.com/google/go-cmp v0.6.0 - github.com/mitchellh/copystructure v1.2.0 + github.com/huandu/xstrings v1.5.0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 - golang.org/x/net v0.30.0 - golang.org/x/sync v0.8.0 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 193f282..2ef7870 100644 --- a/go.sum +++ b/go.sum @@ -1,36 +1,12 @@ -github.com/aiven/aiven-go-client/v2 v2.30.0 h1:dj1nRuO1XglnwH2IwKmqEl3SGaqKebDoxnd+SRjxQRg= -github.com/aiven/aiven-go-client/v2 v2.30.0/go.mod h1:Eyxa+fNgayObmUBW94uJuEkyOe1646cEpjFzhm/NETY= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= -github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= -github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= -github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= -github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= -github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -42,18 +18,7 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/convert/convert.go b/internal/convert/convert.go deleted file mode 100644 index 768f997..0000000 --- a/internal/convert/convert.go +++ /dev/null @@ -1,184 +0,0 @@ -// Package convert is the package that contains the convert functionality. -package convert - -import ( - "errors" - "fmt" - - "github.com/aiven/aiven-go-client/v2" - "github.com/mitchellh/copystructure" - "golang.org/x/exp/slices" - - "github.com/aiven/go-api-schemas/internal/pkg/types" -) - -// errUnexpected is the error that is returned when an unexpected error occurs. -var errUnexpected = errors.New("unexpected conversion error") - -// maxSafeNumber the last number in double precision floating point that can make valid comparison -const maxSafeNumber = float64(1<<53 - 1) - -// UserConfigSchema converts aiven.UserConfigSchema to UserConfigSchema. -// nolint:funlen,gocognit // This function is long, but it's a conversion function. -// // This function is complex, but it's a conversion function. -func UserConfigSchema(v aiven.UserConfigSchema) (*types.UserConfigSchema, error) { - var r []string - r = append(r, v.Required...) - - var cnp map[string]types.UserConfigSchema - - if len(v.Properties) != 0 { - cnp = make(map[string]types.UserConfigSchema, len(v.Properties)) - - p, err := copystructure.Copy(v.Properties) - if err != nil { - return nil, err - } - - ap, ok := p.(map[string]aiven.UserConfigSchema) - if !ok { - return nil, errUnexpected - } - - var cv *types.UserConfigSchema - - for k, v := range ap { - if isImmutableObject(v) { - continue - } - - cv, err = UserConfigSchema(v) - if err != nil { - return nil, err - } - - cnp[k] = *cv - } - } - - var cni *types.UserConfigSchema - - if v.Items != nil { - var err error - - cni, err = UserConfigSchema(*v.Items) - if err != nil { - return nil, err - } - } - - var cno []types.UserConfigSchema - - if len(v.OneOf) != 0 { - cno = make([]types.UserConfigSchema, len(v.OneOf)) - - o, err := copystructure.Copy(v.OneOf) - if err != nil { - return nil, err - } - - ao, ok := o.([]aiven.UserConfigSchema) - if !ok { - return nil, errUnexpected - } - - var cv *types.UserConfigSchema - - for i, v := range ao { - cv, err = UserConfigSchema(v) - if err != nil { - return nil, err - } - - cno[i] = *cv - } - } - - e := make([]types.UserConfigSchemaEnumValue, 0, len(v.Enum)) - - for _, v := range v.Enum { - if v != "" { - e = append(e, types.UserConfigSchemaEnumValue{Value: v}) - } - } - - // YAML uses scientific notation for floats, they won't change that - // https://github.com/go-yaml/yaml/issues/669 - var max *float64 - - if v.Maximum != nil { - // If this is an integer it has to be lte maxSafeNumber - // Otherwise, uses it as is - if !slices.Contains(normalizeTypes(v.Type), "integer") || *v.Maximum <= maxSafeNumber { - max = v.Maximum - } - } - - // Removes empty examples - var example any - if v.Example != nil && fmt.Sprintf("%v", v.Example) != "" { - example = v.Example - } - - return &types.UserConfigSchema{ - Title: v.Title, - Description: v.Description, - Type: v.Type, - Default: v.Default, - Required: r, - Properties: cnp, - Items: cni, - OneOf: cno, - Enum: e, - Minimum: v.Minimum, - Maximum: max, - MinLength: v.MinLength, - MaxLength: v.MaxLength, - MaxItems: v.MaxItems, - CreateOnly: v.CreateOnly, - Pattern: v.Pattern, - Example: example, - UserError: v.UserError, - Secure: v.Secure, - }, nil -} - -// normalizeTypes json type field can be a string or list of strings. -// Turns into a slice of strings -func normalizeTypes(t any) []string { - s, ok := t.(string) - if ok { - return []string{s} - } - - typeList := make([]string, 0) - - a, ok := t.([]any) - if !ok { - return typeList - } - - for _, v := range a { - if vv, ok := v.(string); ok { - typeList = append(typeList, vv) - } - } - - return typeList -} - -// isImmutableObject Ignores immutable objects. -// Returns true if the object has no properties and does not allow additional properties. -// Ignores `patternProperties`. -func isImmutableObject(u aiven.UserConfigSchema) bool { - // An object with empty properties - t, ok := u.Type.(string) - if !(ok && t == "object" && len(u.Properties) == 0) { - return false - } - - // Either no additional properties allowed or it is nil - allowed, ok := u.AdditionalProperties.(bool) - - return ok != allowed || u.AdditionalProperties == nil -} diff --git a/internal/convert/convert_test.go b/internal/convert/convert_test.go deleted file mode 100644 index 4a72e71..0000000 --- a/internal/convert/convert_test.go +++ /dev/null @@ -1,458 +0,0 @@ -// Package convert is the package that contains the convert functionality. -package convert - -import ( - "testing" - - "github.com/aiven/aiven-go-client/v2" - "github.com/google/go-cmp/cmp" - - "github.com/aiven/go-api-schemas/internal/pkg/types" - "github.com/aiven/go-api-schemas/internal/pkg/util" -) - -// TestUserConfigSchema tests the UserConfigSchema function. -// nolint:funlen,lll // This function is long, but it's a test function. -// // These lines are long, but they're test data. -func TestUserConfigSchema(t *testing.T) { - type args struct { - v aiven.UserConfigSchema - } - - tests := []struct { - name string - args args - want *types.UserConfigSchema - wantErr error - }{ - { - name: "basic", - args: args{ - v: aiven.UserConfigSchema{ - Title: "", - Description: "", - Type: nil, - Default: nil, - Required: nil, - Properties: nil, - AdditionalProperties: nil, - Items: nil, - OneOf: nil, - Enum: nil, - Minimum: util.Ref(0.0), - Maximum: util.Ref(0.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: nil, - UserError: "", - }, - }, - want: &types.UserConfigSchema{ - UserConfigSchemaDeprecationInfo: types.UserConfigSchemaDeprecationInfo{}, - Title: "", - Description: "", - Type: nil, - Default: nil, - Required: nil, - Properties: nil, - Items: nil, - OneOf: nil, - Enum: []types.UserConfigSchemaEnumValue{}, - Minimum: util.Ref(0.0), - Maximum: util.Ref(0.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: nil, - UserError: "", - }, - wantErr: nil, - }, - { - name: "filled", - args: args{ - v: aiven.UserConfigSchema{ - Title: "", - Description: "", - Type: "object", - Default: nil, - Required: []string{ - "datadog_api_key", - }, - Properties: map[string]aiven.UserConfigSchema{ - "max_partition_contexts": { - Title: "Maximum number of partition contexts to send", - Description: "", - Type: "integer", - Default: nil, - Required: nil, - Properties: nil, - AdditionalProperties: nil, - Items: nil, - OneOf: nil, - Enum: nil, - Minimum: util.Ref(200.0), - Maximum: util.Ref(200000.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: "32000", - UserError: "", - }, - "site": { - Title: "Datadog intake site. Defaults to datadoghq.com", - Description: "", - Type: "string", - Default: nil, - Required: nil, - Properties: nil, - AdditionalProperties: nil, - Items: nil, - OneOf: nil, - Enum: []interface{}{ - "datadoghq.com", - "datadoghq.eu", - }, - Minimum: util.Ref(0.0), - Maximum: util.Ref(0.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: "datadoghq.com", - UserError: "", - }, - "datadog_api_key": { - Title: "Datadog API key", - Description: "", - Type: "string", - Default: nil, - Required: nil, - Properties: nil, - AdditionalProperties: nil, - Items: nil, - OneOf: nil, - Enum: nil, - Minimum: util.Ref(0.0), - Maximum: util.Ref(0.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "^[A-Za-z0-9]{32}$", - Example: "848f30907c15c55d601fe45487cce9b6", - UserError: "Must consist of alpha-numeric characters and contain 32 characters", - }, - "datadog_tags": { - Title: "Custom tags provided by user", - Description: "", - Type: "array", - Default: nil, - Required: nil, - Properties: nil, - AdditionalProperties: nil, - Items: nil, - OneOf: nil, - Enum: nil, - Minimum: util.Ref(0.0), - Maximum: util.Ref(0.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: []interface{}{ - map[string]interface{}{"tag": "foo"}, - map[string]interface{}{ - "comment": "Useful tag", - "tag": "bar:buzz", - }, - }, - UserError: "", - }, - "disable_consumer_stats": { - Title: "Disable consumer group metrics", - Description: "", - Type: "boolean", - Default: nil, - Required: nil, - Properties: nil, - AdditionalProperties: nil, - Items: nil, - OneOf: nil, - Enum: nil, - Minimum: util.Ref(0.0), - Maximum: util.Ref(0.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: true, - UserError: "", - }, - "kafka_consumer_check_instances": { - Title: "Number of separate instances to fetch kafka consumer statistics with", - Description: "", - Type: "integer", - Default: nil, - Required: nil, - Properties: nil, - AdditionalProperties: nil, - Items: nil, - OneOf: nil, - Enum: nil, - Minimum: util.Ref(1.0), - Maximum: util.Ref(100.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: "8", - UserError: "", - }, - "kafka_consumer_stats_timeout": { - Title: "Number of seconds that datadog will wait to get consumer statistics from brokers", - Description: "", - Type: "integer", - Default: nil, - Required: nil, - Properties: nil, - AdditionalProperties: nil, - Items: nil, - OneOf: nil, - Enum: nil, - Minimum: util.Ref(2.0), - Maximum: util.Ref(600.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: "60", - UserError: "", - }, - }, - AdditionalProperties: false, - Items: nil, - OneOf: nil, - Enum: nil, - Minimum: util.Ref(0.0), - Maximum: util.Ref(0.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: nil, - UserError: "", - }, - }, - want: &types.UserConfigSchema{ - UserConfigSchemaDeprecationInfo: types.UserConfigSchemaDeprecationInfo{}, - Title: "", - Description: "", - Type: "object", - Default: nil, - Required: []string{ - "datadog_api_key", - }, - Properties: map[string]types.UserConfigSchema{ - "datadog_api_key": { - UserConfigSchemaDeprecationInfo: types.UserConfigSchemaDeprecationInfo{}, - Title: "Datadog API key", - Description: "", - Type: "string", - Default: nil, - Required: nil, - Properties: nil, - Items: nil, - OneOf: nil, - Enum: []types.UserConfigSchemaEnumValue{}, - Minimum: util.Ref(0.0), - Maximum: util.Ref(0.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "^[A-Za-z0-9]{32}$", - Example: "848f30907c15c55d601fe45487cce9b6", - UserError: "Must consist of alpha-numeric characters and contain 32 characters", - }, - "datadog_tags": { - UserConfigSchemaDeprecationInfo: types.UserConfigSchemaDeprecationInfo{}, - Title: "Custom tags provided by user", - Description: "", - Type: "array", - Default: nil, - Required: nil, - Properties: nil, - Items: nil, - OneOf: nil, - Enum: []types.UserConfigSchemaEnumValue{}, - Minimum: util.Ref(0.0), - Maximum: util.Ref(0.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: []interface{}{ - map[string]interface{}{"tag": "foo"}, - map[string]interface{}{ - "comment": "Useful tag", - "tag": "bar:buzz", - }, - }, - UserError: "", - }, - "disable_consumer_stats": { - UserConfigSchemaDeprecationInfo: types.UserConfigSchemaDeprecationInfo{}, - Title: "Disable consumer group metrics", - Description: "", - Type: "boolean", - Default: nil, - Required: nil, - Properties: nil, - Items: nil, - OneOf: nil, - Enum: []types.UserConfigSchemaEnumValue{}, - Minimum: util.Ref(0.0), - Maximum: util.Ref(0.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: true, - UserError: "", - }, - "kafka_consumer_check_instances": { - UserConfigSchemaDeprecationInfo: types.UserConfigSchemaDeprecationInfo{}, - Title: "Number of separate instances to fetch kafka consumer statistics with", - Description: "", - Type: "integer", - Default: nil, - Required: nil, - Properties: nil, - Items: nil, - OneOf: nil, - Enum: []types.UserConfigSchemaEnumValue{}, - Minimum: util.Ref(1.0), - Maximum: util.Ref(100.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: "8", - UserError: "", - }, - "kafka_consumer_stats_timeout": { - UserConfigSchemaDeprecationInfo: types.UserConfigSchemaDeprecationInfo{}, - Title: "Number of seconds that datadog will wait to get consumer statistics from brokers", - Description: "", - Type: "integer", - Default: nil, - Required: nil, - Properties: nil, - Items: nil, - OneOf: nil, - Enum: []types.UserConfigSchemaEnumValue{}, - Minimum: util.Ref(2.0), - Maximum: util.Ref(600.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: "60", - UserError: "", - }, - "max_partition_contexts": { - UserConfigSchemaDeprecationInfo: types.UserConfigSchemaDeprecationInfo{}, - Title: "Maximum number of partition contexts to send", - Description: "", - Type: "integer", - Default: nil, - Required: nil, - Properties: nil, - Items: nil, - OneOf: nil, - Enum: []types.UserConfigSchemaEnumValue{}, - Minimum: util.Ref(200.0), - Maximum: util.Ref(200000.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: "32000", - UserError: "", - }, - "site": { - UserConfigSchemaDeprecationInfo: types.UserConfigSchemaDeprecationInfo{}, - Title: "Datadog intake site. Defaults to datadoghq.com", - Description: "", - Type: "string", - Default: nil, - Required: nil, - Properties: nil, - Items: nil, - OneOf: nil, - Enum: []types.UserConfigSchemaEnumValue{ - {Value: "datadoghq.com"}, - {Value: "datadoghq.eu"}, - }, - Minimum: util.Ref(0.0), - Maximum: util.Ref(0.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: "datadoghq.com", - UserError: "", - }, - }, - Items: nil, - OneOf: nil, - Enum: []types.UserConfigSchemaEnumValue{}, - Minimum: util.Ref(0.0), - Maximum: util.Ref(0.0), - MinLength: util.Ref(0), - MaxLength: util.Ref(0), - MaxItems: util.Ref(0), - CreateOnly: false, - Pattern: "", - Example: nil, - UserError: "", - }, - wantErr: nil, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := UserConfigSchema(tt.args.v) - if !cmp.Equal(err, tt.wantErr) { - t.Errorf("UserConfigSchema() error = %v, wantErr %v", err, tt.wantErr) - - return - } - - if !cmp.Equal(got, tt.want) { - t.Error(cmp.Diff(tt.want, got)) - } - }) - } -} diff --git a/internal/gen/gen.go b/internal/gen/gen.go index 45b4133..66c15c1 100644 --- a/internal/gen/gen.go +++ b/internal/gen/gen.go @@ -2,136 +2,314 @@ package gen import ( - "github.com/aiven/aiven-go-client/v2" - "golang.org/x/net/context" - "golang.org/x/sync/errgroup" + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + + "github.com/huandu/xstrings" - "github.com/aiven/go-api-schemas/internal/convert" "github.com/aiven/go-api-schemas/internal/pkg/types" - "github.com/aiven/go-api-schemas/internal/pkg/util" ) -const ( - // generating is a part of the message that is printed when the generation process starts. - generating = "generating %s" -) +type doc struct { + // openapi-uc.json + Components struct { + Schemas map[string]*schema `json:"schemas"` + } `json:"components"` -// logger is a pointer to the logger. -var logger *util.Logger + // Legacy user config files + legacyDoc +} -// env is a map of environment variables. -var env util.EnvMap +type schema struct { + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + Type any `json:"type,omitempty"` + Default interface{} `json:"default,omitempty"` + Required []string `json:"required,omitempty"` + Properties map[string]*schema `json:"properties,omitempty"` + Items *schema `json:"items,omitempty"` + AnyOf []*schema `json:"anyOf,omitempty"` + OneOf []*schema `json:"oneOf,omitempty"` + Enum []any `json:"enum,omitempty"` + Minimum *float64 `json:"minimum,omitempty"` + Maximum *float64 `json:"maximum,omitempty"` + MinLength *int `json:"minLength,omitempty"` + MaxLength *int `json:"maxLength,omitempty"` + MaxItems *int `json:"maxItems,omitempty"` + Pattern string `json:"pattern,omitempty"` + Example any `json:"example,omitempty"` + Nullable bool `json:"nullable,omitempty"` + UserError string `json:"user_error,omitempty"` + CreateOnly bool `json:"createOnly,omitempty"` + Secure bool `json:"_secure,omitempty"` + + // Openapi-uc + XUserError string `json:"x-user_error,omitempty"` + XCreateOnly bool `json:"x-createOnly,omitempty"` + XSecure bool `json:"x-_secure,omitempty"` + + // Internal + isRequired bool // true, when a field is on the parent's Required list +} -// client is a pointer to the Aiven client. -var client *aiven.Client +// maxSafeInteger +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER +const maxSafeInteger = float64(1<<53 - 1) // Or 9007199254740991.0 -// result is the result of the generation process. -var result types.GenerationResult +func isSafeInt(v float64) bool { + // Maximum is float64, it can't fit more that 2^53-1 + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER + return v < maxSafeInteger +} -// serviceTypes generates the service types. -func serviceTypes(ctx context.Context) error { - defer util.MeasureExecutionTime(logger)() +// reUserConfigKey finds user config keys. +var reUserConfigKey = regexp.MustCompile(`^(Service|IntegrationEndpoint|Integration)([0-9a-zA-Z_]+)UserConfig$`) - logger.Info.Printf(generating, "service types") +func fromFile(fileName string) (types.GenerationResult, error) { + b, err := os.ReadFile(filepath.Clean(fileName)) + if err != nil { + return nil, err + } - r, err := client.Projects.ServiceTypes(ctx, env[util.EnvAivenProjectName]) + d := new(doc) + err = json.Unmarshal(b, &d) if err != nil { - return err + return nil, err } - out := make(map[string]types.UserConfigSchema, len(r)) + legacyToComponents(d) + + kinds := map[string]int{ + "Service": types.KeyServiceTypes, + "Integration": types.KeyIntegrationTypes, + "IntegrationEndpoint": types.KeyIntegrationEndpointTypes, + } + + // New openapi-uc schema file + result := make(types.GenerationResult) + for _, v := range kinds { + result[v] = make(map[string]types.UserConfigSchema) + } + + for k, v := range d.Components.Schemas { + match := reUserConfigKey.FindStringSubmatch(k) + if len(match) == 0 { + continue + } + + kind := kinds[match[1]] + name := match[2] + if !strings.HasPrefix(name, "m3") { + // m3 is a special case + name = xstrings.ToSnakeCase(match[2]) + } - for k, v := range r { - cv, err := convert.UserConfigSchema(v.UserConfigSchema) + uc, err := toUserConfig(v) if err != nil { - return err + return nil, fmt.Errorf("failed to convert %s %s: %w", match[1], match[2], err) } - out[k] = *cv + result[kind][name] = *uc } - result[types.KeyServiceTypes] = out - - return nil + return result, nil } -// integrationTypes generates the integration types. -func integrationTypes(ctx context.Context) error { - defer util.MeasureExecutionTime(logger)() +func toUserConfig(src *schema) (*types.UserConfigSchema, error) { // nolint: funlen + normTypes, err := normalizeType(src) + if err != nil { + return nil, err + } - logger.Info.Printf(generating, "integration types") + uc := types.UserConfigSchema{ + Properties: make(map[string]types.UserConfigSchema), + Title: src.Title, + Description: src.Description, + Required: src.Required, + Minimum: src.Minimum, + MinLength: src.MinLength, + MaxLength: src.MaxLength, + MaxItems: src.MaxItems, + Pattern: src.Pattern, + UserError: or(src.XUserError, src.UserError), + Secure: or(src.XSecure, src.Secure), + CreateOnly: or(src.XCreateOnly, src.CreateOnly), + Example: formatValue(normTypes[0], src.Example), + Default: formatValue(normTypes[0], src.Default), + } - r, err := client.Projects.IntegrationTypes(ctx, env[util.EnvAivenProjectName]) - if err != nil { - return err + if src.Maximum != nil { + if isSafeInt(*src.Maximum) { + uc.Maximum = src.Maximum + } + } + + // Collects all the types + kinds := make([]string, 0, 1) + if normTypes != nil { + kinds = append(kinds, normTypes...) + } + + switch len(kinds) { + case 1: + uc.Type = kinds[0] + default: + uc.Type = kinds + } + + for _, v := range src.Enum { + if fmt.Sprintf("%v", v) != "" { + uc.Enum = append(uc.Enum, types.UserConfigSchemaEnumValue{Value: v}) + } } - out := make(map[string]types.UserConfigSchema, len(r)) + if src.Items != nil { + item, err := toUserConfig(src.Items) + if err != nil { + return nil, err + } + uc.Items = item + } - for _, v := range r { - cv, err := convert.UserConfigSchema(v.UserConfigSchema) + for k, v := range src.Properties { + v.isRequired = slices.Contains(src.Required, k) + child, err := toUserConfig(v) if err != nil { - return err + return nil, err } + uc.Properties[k] = *child + } - out[v.IntegrationType] = *cv + if len(src.OneOf) != 0 { + // Must remove the type if there is a oneOf + uc.Type = nil } - result[types.KeyIntegrationTypes] = out + for _, v := range src.OneOf { + child, err := toUserConfig(v) + if err != nil { + return nil, err + } + uc.OneOf = append(uc.OneOf, *child) + } - return nil + return &uc, nil } -// integrationEndpointTypes generates the integration endpoint types. -func integrationEndpointTypes(ctx context.Context) error { - defer util.MeasureExecutionTime(logger)() +func distinct[T comparable](list []T) []T { + seen := make(map[T]bool) + result := make([]T, 0, len(list)) + for _, v := range list { + if !seen[v] { + seen[v] = true + result = append(result, v) + } + } + return result +} - logger.Info.Printf(generating, "integration endpoint types") +// or returns the first non-zero value. +func or[T comparable](args ...T) T { + var zero T + for _, a := range args { + if a != zero { + return a + } + } + return zero +} - r, err := client.Projects.IntegrationEndpointTypes(ctx, env[util.EnvAivenProjectName]) - if err != nil { - return err +func normalizeType(s *schema) ([]string, error) { + result := make([]string, 0) + switch t := s.Type.(type) { + case string: + return []string{t}, nil + case []any: + for _, v := range t { + if v == "null" { + s.Nullable = true + } else { + result = append(result, fmt.Sprintf("%v", v)) + } + } + case nil: + default: + return nil, fmt.Errorf("unknown type %T", s.Type) } - out := make(map[string]types.UserConfigSchema, len(r)) + // Golang supports null for required fields only + if s.isRequired && s.Nullable { + result = append(result, "null") + } - for _, v := range r { - cv, err := convert.UserConfigSchema(v.UserConfigSchema) - if err != nil { - return err - } + for _, v := range s.AnyOf { + if v.Type != nil { + vType, err := normalizeType(v) + if err != nil { + return nil, err + } - out[v.EndpointType] = *cv + result = append(result, vType...) + } } - result[types.KeyIntegrationEndpointTypes] = out + if len(result) == 0 { + result = append(result, "string") + } - return nil + return distinct(result), nil } -// setup sets up the generation process. -func setup(l *util.Logger, e util.EnvMap, c *aiven.Client) { - logger = l - env = e - client = c +func formatValue(t string, v any) any { + s := fmt.Sprintf("%v", v) + if v == nil || s == "" { + return nil + } + + switch t { + case "integer": + i, ok := v.(float64) + if ok && isSafeInt(i) { + return fmt.Sprintf("%d", int(i)) + } + return nil + case "number": + return fmt.Sprintf("%.1f", v) + case "boolean", "array", "object": + return v + } - result = types.GenerationResult{} + return s } // Run executes the generation process. -func Run( - ctx context.Context, - logger *util.Logger, - env util.EnvMap, - client *aiven.Client, -) (types.GenerationResult, error) { - setup(logger, env, client) +func Run(fileNames ...string) (types.GenerationResult, error) { + result := make(types.GenerationResult) + for _, fileName := range fileNames { + subResult, err := fromFile(fileName) + if err != nil { + return nil, err + } - g, ctx := errgroup.WithContext(ctx) + for kind, value := range subResult { + if len(value) == 0 { + continue + } - g.Go(func() error { return serviceTypes(ctx) }) - g.Go(func() error { return integrationTypes(ctx) }) - g.Go(func() error { return integrationEndpointTypes(ctx) }) + if _, ok := result[kind]; !ok { + result[kind] = value + continue + } - return result, g.Wait() + for k, v := range value { + result[kind][k] = v + } + } + } + return result, nil } diff --git a/internal/gen/legacy.go b/internal/gen/legacy.go new file mode 100644 index 0000000..ed5f4b7 --- /dev/null +++ b/internal/gen/legacy.go @@ -0,0 +1,43 @@ +package gen + +import ( + "fmt" +) + +type legacyDoc struct { + ServiceTypes map[string]legacySchema `json:"service_types"` + IntegrationTypes []struct { + legacySchema + Key string `json:"integration_type"` + } `json:"integration_types"` + IntegrationEndpointTypes []struct { + legacySchema + Key string `json:"endpoint_type"` + } `json:"endpoint_types"` +} + +type legacySchema struct { + UserConfigSchema *schema `json:"user_config_schema"` +} + +// legacyToComponents adapts legacy schemas to the new format. +func legacyToComponents(d *doc) { + if d.Components.Schemas == nil { + d.Components.Schemas = make(map[string]*schema) + } + + for k, v := range d.ServiceTypes { + key := fmt.Sprintf("Service%sUserConfig", k) + d.Components.Schemas[key] = v.UserConfigSchema + } + + for _, v := range d.IntegrationTypes { + key := fmt.Sprintf("Integration%sUserConfig", v.Key) + d.Components.Schemas[key] = v.UserConfigSchema + } + + for _, v := range d.IntegrationEndpointTypes { + key := fmt.Sprintf("IntegrationEndpoint%sUserConfig", v.Key) + d.Components.Schemas[key] = v.UserConfigSchema + } +} diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index 19a12dc..3459ef6 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -2,13 +2,8 @@ package util import ( - "fmt" "log" "os" - "runtime" - "time" - - "github.com/aiven/aiven-go-client/v2" ) const ( @@ -19,9 +14,6 @@ const ( logFlag = log.LstdFlags | log.LUTC | log.Lmsgprefix ) -// EnvAivenProjectName is the environment variable for the Aiven project name. -const EnvAivenProjectName = "AIVEN_PROJECT_NAME" - const ( // ServiceTypesFilename is the name of the service types file. ServiceTypesFilename = "service_types.yml" @@ -51,48 +43,6 @@ func SetupLogger(logger *Logger) { logger.Error = log.New(os.Stderr, "[ERROR]"+logSeparator, logFlag) } -// SetupEnv populates the provided environment variables map. -func SetupEnv(env EnvMap) error { - for k := range env { - ev, ok := os.LookupEnv(k) - if !ok { - return fmt.Errorf("environment variable is not set: %s", k) - } - - env[k] = ev - } - - return nil -} - -// SetupClient sets up the Aiven client. -func SetupClient(client *aiven.Client) error { - c, err := aiven.SetupEnvClient("go-api-schemas") - if err != nil { - return err - } - - *client = *c - - return nil -} - -// MeasureExecutionTime prints the execution time of the caller when deferred. -func MeasureExecutionTime(logger *Logger) func() { - start := time.Now() - - pc, _, _, ok := runtime.Caller(1) - if !ok { - panic("runtime.Caller failed") - } - - fn := runtime.FuncForPC(pc) - - return func() { - logger.Info.Printf("%s took %dms", fn.Name(), time.Since(start).Milliseconds()) - } -} - // Ref returns the reference (pointer) of the provided value. func Ref[T any](v T) *T { return &v diff --git a/internal/reader/reader.go b/internal/reader/reader.go index 413adc5..0b91329 100644 --- a/internal/reader/reader.go +++ b/internal/reader/reader.go @@ -2,12 +2,10 @@ package reader import ( - "context" "os" "strings" "github.com/spf13/pflag" - "golang.org/x/sync/errgroup" "gopkg.in/yaml.v3" "github.com/aiven/go-api-schemas/internal/pkg/types" @@ -52,22 +50,16 @@ func read(filename string, schema map[string]types.UserConfigSchema) error { // readServiceTypes reads the service types from a file. func readServiceTypes() error { - defer util.MeasureExecutionTime(logger)() - return read(util.ServiceTypesFilename, result[types.KeyServiceTypes]) } // readIntegrationTypes reads the integration types from a file. func readIntegrationTypes() error { - defer util.MeasureExecutionTime(logger)() - return read(util.IntegrationTypesFilename, result[types.KeyIntegrationTypes]) } // readIntegrationEndpointTypes reads the integration endpoint types from a file. func readIntegrationEndpointTypes() error { - defer util.MeasureExecutionTime(logger)() - return read(util.IntegrationEndpointTypesFilename, result[types.KeyIntegrationEndpointTypes]) } @@ -84,17 +76,18 @@ func setup(l *util.Logger, f *pflag.FlagSet) { } // Run runs the reader. -func Run(ctx context.Context, logger *util.Logger, flags *pflag.FlagSet) (types.ReadResult, error) { +func Run(logger *util.Logger, flags *pflag.FlagSet) (types.ReadResult, error) { setup(logger, flags) - errs, _ := errgroup.WithContext(ctx) + if err := readServiceTypes(); err != nil { + return nil, err + } - errs.Go(readServiceTypes) - errs.Go(readIntegrationTypes) - errs.Go(readIntegrationEndpointTypes) + if err := readIntegrationTypes(); err != nil { + return nil, err + } - err := errs.Wait() - if err != nil { + if err := readIntegrationEndpointTypes(); err != nil { return nil, err } diff --git a/internal/writer/writer.go b/internal/writer/writer.go index 257e0ae..0f840f7 100644 --- a/internal/writer/writer.go +++ b/internal/writer/writer.go @@ -2,12 +2,10 @@ package writer import ( - "context" "os" "strings" "github.com/spf13/pflag" - "golang.org/x/sync/errgroup" "gopkg.in/yaml.v3" "github.com/aiven/go-api-schemas/internal/pkg/types" @@ -61,22 +59,16 @@ func write(filename string, schema map[string]types.UserConfigSchema) error { // writeServiceTypes writes the service types to a file. func writeServiceTypes() error { - defer util.MeasureExecutionTime(logger)() - return write(util.ServiceTypesFilename, result[types.KeyServiceTypes]) } // writeIntegrationTypes writes the integration types to a file. func writeIntegrationTypes() error { - defer util.MeasureExecutionTime(logger)() - return write(util.IntegrationTypesFilename, result[types.KeyIntegrationTypes]) } // writeIntegrationEndpointTypes writes the integration endpoint types to a file. func writeIntegrationEndpointTypes() error { - defer util.MeasureExecutionTime(logger)() - return write(util.IntegrationEndpointTypesFilename, result[types.KeyIntegrationEndpointTypes]) } @@ -88,14 +80,20 @@ func setup(l *util.Logger, f *pflag.FlagSet, r types.DiffResult) { } // Run runs the writer. -func Run(ctx context.Context, logger *util.Logger, flags *pflag.FlagSet, result types.DiffResult) error { +func Run(logger *util.Logger, flags *pflag.FlagSet, result types.DiffResult) error { setup(logger, flags, result) - errs, _ := errgroup.WithContext(ctx) + if err := writeServiceTypes(); err != nil { + return err + } + + if err := writeIntegrationTypes(); err != nil { + return err + } - errs.Go(writeServiceTypes) - errs.Go(writeIntegrationTypes) - errs.Go(writeIntegrationEndpointTypes) + if err := writeIntegrationEndpointTypes(); err != nil { + return err + } - return errs.Wait() + return nil } diff --git a/pkg/dist/integration_endpoint_types.yml b/pkg/dist/integration_endpoint_types.yml index fe9353c..1d91f2a 100644 --- a/pkg/dist/integration_endpoint_types.yml +++ b/pkg/dist/integration_endpoint_types.yml @@ -233,9 +233,7 @@ external_elasticsearch_logs: properties: ca: title: PEM encoded CA certificate - type: - - string - - "null" + type: string max_length: 16384 example: | -----BEGIN CERTIFICATE----- @@ -330,8 +328,7 @@ external_kafka: example: 10.0.0.1:9092,10.0.0.2:9092 sasl_mechanism: title: SASL mechanism used for connections to the Kafka server. - type: - - string + type: string enum: - value: PLAIN - value: SCRAM-SHA-256 @@ -339,18 +336,14 @@ external_kafka: example: PLAIN sasl_plain_password: title: Password for SASL PLAIN mechanism in the Kafka server. - type: - - string - - "null" + type: string min_length: 1 max_length: 256 example: admin _secure: true sasl_plain_username: title: Username for SASL PLAIN mechanism in the Kafka server. - type: - - string - - "null" + type: string min_length: 1 max_length: 256 example: admin @@ -365,9 +358,7 @@ external_kafka: example: PLAINTEXT ssl_ca_cert: title: PEM-encoded CA certificate - type: - - string - - "null" + type: string max_length: 16384 example: | -----BEGIN CERTIFICATE----- @@ -375,9 +366,7 @@ external_kafka: -----END CERTIFICATE----- ssl_client_cert: title: PEM-encoded client certificate - type: - - string - - "null" + type: string max_length: 16384 example: | -----BEGIN CERTIFICATE----- @@ -385,9 +374,7 @@ external_kafka: -----END CERTIFICATE----- ssl_client_key: title: PEM-encoded client key - type: - - string - - "null" + type: string max_length: 16384 example: | -----BEGIN PRIVATE KEY----- @@ -396,8 +383,7 @@ external_kafka: _secure: true ssl_endpoint_identification_algorithm: title: The endpoint identification algorithm to validate server hostname using server certificate. - type: - - string + type: string enum: - value: https example: https @@ -436,7 +422,6 @@ external_mysql: ssl_root_cert: title: SSL Root Cert type: string - default: "" max_length: 16384 example: | -----BEGIN CERTIFICATE----- @@ -455,9 +440,7 @@ external_opensearch_logs: properties: ca: title: PEM encoded CA certificate - type: - - string - - "null" + type: string max_length: 16384 example: | -----BEGIN CERTIFICATE----- @@ -526,7 +509,6 @@ external_postgresql: ssl_client_certificate: title: Client certificate type: string - default: "" max_length: 16384 example: | -----BEGIN CERTIFICATE----- @@ -535,7 +517,6 @@ external_postgresql: ssl_client_key: title: Client key type: string - default: "" max_length: 16384 example: |- -----BEGIN PRIVATE KEY----- @@ -564,7 +545,6 @@ external_postgresql: ssl_root_cert: title: SSL Root Cert type: string - default: "" max_length: 16384 example: | -----BEGIN CERTIFICATE----- @@ -613,17 +593,13 @@ external_schema_registry: example: basic basic_auth_password: title: Basic authentication password - type: - - string - - "null" + type: string max_length: 256 example: Zm9vYg== _secure: true basic_auth_username: title: Basic authentication user name - type: - - string - - "null" + type: string max_length: 256 example: avnadmin url: @@ -677,9 +653,7 @@ rsyslog: properties: ca: title: PEM encoded CA certificate - type: - - string - - "null" + type: string max_length: 16384 example: | -----BEGIN CERTIFICATE----- @@ -687,9 +661,7 @@ rsyslog: -----END CERTIFICATE----- cert: title: PEM encoded client certificate - type: - - string - - "null" + type: string max_length: 16384 example: | -----BEGIN CERTIFICATE----- @@ -706,9 +678,7 @@ rsyslog: example: rfc5424 key: title: PEM encoded client key - type: - - string - - "null" + type: string max_length: 16384 example: | -----BEGIN PRIVATE KEY----- @@ -738,9 +708,7 @@ rsyslog: example: "514" sd: title: Structured data block for log message - type: - - string - - "null" + type: string max_length: 1024 example: TOKEN tag="LiteralValue" server: diff --git a/pkg/dist/integration_types.yml b/pkg/dist/integration_types.yml index c30b505..0a2721c 100644 --- a/pkg/dist/integration_types.yml +++ b/pkg/dist/integration_types.yml @@ -554,9 +554,7 @@ kafka_mirrormaker: consumer_auto_offset_reset: title: Set the auto.offset.reset to consumer. description: 'Set where consumer starts to consume data. Value `earliest`: Start replication from the earliest offset. Value `latest`: Start replication from the latest offset. Default is `earliest`.' - type: - - string - - "null" + type: string enum: - value: earliest - value: latest @@ -570,9 +568,7 @@ kafka_mirrormaker: consumer_max_poll_records: title: Set the max.poll.records to consumer. description: Set consumer max.poll.records. The default is 500. - type: - - integer - - "null" + type: integer minimum: 100 maximum: 20000 example: "500" diff --git a/pkg/dist/service_types.yml b/pkg/dist/service_types.yml index ef1be57..26c33e8 100644 --- a/pkg/dist/service_types.yml +++ b/pkg/dist/service_types.yml @@ -14,9 +14,7 @@ alloydbomni: max_items: 1 admin_password: title: Custom password for admin user. Defaults to random string. This must be set only when a new service is being created. - type: - - string - - "null" + type: string min_length: 8 max_length: 256 create_only: true @@ -26,9 +24,7 @@ alloydbomni: _secure: true admin_username: title: Custom username for admin user. This must be set only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[_A-Za-z0-9][-._A-Za-z0-9]{0,63}$ @@ -36,24 +32,18 @@ alloydbomni: user_error: Must consist of alpha-numeric characters, dots, underscores or dashes, may not start with dash or dot, max 64 characters alloydbomni_version: title: PostgreSQL major version - type: - - string - - "null" + type: string enum: - value: "15" backup_hour: title: The hour of day (in UTC) when backup for the service is started. New backup is only started if previous backup has already completed. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 23 example: "3" backup_minute: title: The minute of an hour when backup for the service is started. New backup is only started if previous backup has already completed. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 59 example: "30" @@ -352,8 +342,7 @@ alloydbomni: pg_stat_statements.track: title: pg_stat_statements.track description: Controls which statements are counted. Specify top to track top-level statements (those issued directly by clients), all to also track nested statements (such as statements invoked within functions), or none to disable statement statistics collection. The default value is top. - type: - - string + type: string enum: - value: all - value: none @@ -419,24 +408,18 @@ alloydbomni: pg_read_replica: title: Should the service which is being forked be a read replica (deprecated, use read_replica service integration instead). description: This setting is deprecated. Use read_replica service integration instead. - type: - - boolean - - "null" + type: boolean example: true pg_service_to_fork_from: title: Name of the PG Service from which to fork (deprecated, use service_to_fork_from). This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ example: anotherservicename pg_version: title: PostgreSQL major version - type: - - string - - "null" + type: string enum: - value: "15" pgbouncer: @@ -523,7 +506,7 @@ alloydbomni: description: System-wide settings for pglookout. type: object default: - max_failover_replication_time_lag: "60" + max_failover_replication_time_lag: 60 properties: max_failover_replication_time_lag: title: max_failover_replication_time_lag @@ -565,9 +548,7 @@ alloydbomni: example: true project_to_fork_from: title: Name of another project to fork a service from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 63 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -590,24 +571,18 @@ alloydbomni: example: true recovery_target_time: title: Recovery target time when forking a service. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 32 create_only: true example: "2019-01-01 23:34:45" service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true service_to_fork_from: title: Name of another service to fork from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -633,9 +608,7 @@ alloydbomni: example: "off" variant: title: Variant of the PostgreSQL service, may affect the features that are exposed by default - type: - - string - - "null" + type: string enum: - value: aiven - value: timescale @@ -663,17 +636,13 @@ cassandra: max_items: 1 backup_hour: title: The hour of day (in UTC) when backup for the service is started. New backup is only started if previous backup has already completed. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 23 example: "3" backup_minute: title: The minute of an hour when backup for the service is started. New backup is only started if previous backup has already completed. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 59 example: "30" @@ -717,14 +686,14 @@ cassandra: example: "2000" cassandra_version: title: Cassandra version - type: - - string - - "null" + type: string enum: - is_deprecated: true deprecation_notice: This value is deprecated. value: "3" - - value: "4" + - is_deprecated: true + deprecation_notice: This value is deprecated. + value: "4" - value: "4.1" pattern: ^[0-9]+(\.[0-9]+)?$ ip_filter: @@ -769,9 +738,7 @@ cassandra: example: true project_to_fork_from: title: Name of another project to fork a service from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 63 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -787,15 +754,11 @@ cassandra: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true service_to_fork_from: title: Name of another service to fork from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -897,9 +860,7 @@ clickhouse: example: true project_to_fork_from: title: Name of another project to fork a service from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 63 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -933,15 +894,11 @@ clickhouse: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true service_to_fork_from: title: Name of another service to fork from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -999,9 +956,7 @@ dragonfly: max_items: 1024 migration: title: Migrate data from existing server - type: - - object - - "null" + type: object required: - host - port @@ -1080,9 +1035,7 @@ dragonfly: example: true project_to_fork_from: title: Name of another project to fork a service from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 63 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -1108,15 +1061,11 @@ dragonfly: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true service_to_fork_from: title: Name of another service to fork from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -1735,11 +1684,11 @@ flink: max_items: 1 flink_version: title: Flink major version - type: - - string - - "null" + type: string enum: - - value: "1.16" + - is_deprecated: true + deprecation_notice: This value is deprecated. + value: "1.16" - value: "1.19" create_only: true ip_filter: @@ -1813,9 +1762,7 @@ flink: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true static_ips: title: Static IP addresses @@ -2205,9 +2152,7 @@ grafana: custom_domain: title: Custom domain description: Serve the web frontend using a custom CNAME pointing to the Aiven DNS name - type: - - string - - "null" + type: string max_length: 255 example: grafana.example.org dashboard_previews_enabled: @@ -2416,9 +2361,7 @@ grafana: example: true project_to_fork_from: title: Name of another project to fork a service from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 63 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -2440,15 +2383,11 @@ grafana: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true service_to_fork_from: title: Name of another service to fork from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -2472,9 +2411,7 @@ grafana: from_name: title: From name description: Name used in outgoing emails, defaults to Grafana - type: - - string - - "null" + type: string max_length: 128 pattern: ^[^\x00-\x1F]+$ example: Company Grafana @@ -2486,9 +2423,7 @@ grafana: example: smtp.example.com password: title: Password for SMTP authentication - type: - - string - - "null" + type: string max_length: 255 pattern: ^[^\x00-\x1F]+$ example: ein0eemeev5eeth3Ahfu @@ -2515,9 +2450,7 @@ grafana: example: NoStartTLS username: title: Username for SMTP authentication - type: - - string - - "null" + type: string max_length: 255 pattern: ^[^\x00-\x1F]+$ example: smtpuser @@ -2574,9 +2507,7 @@ influxdb: custom_domain: title: Custom domain description: Serve the web frontend using a custom CNAME pointing to the Aiven DNS name - type: - - string - - "null" + type: string max_length: 255 example: grafana.example.org influxdb: @@ -2619,9 +2550,7 @@ influxdb: maximum: 3600 influxdb_version: title: InfluxDB major version - type: - - string - - "null" + type: string default: "1.8" enum: - value: "1.8" @@ -2671,9 +2600,7 @@ influxdb: example: true project_to_fork_from: title: Name of another project to fork a service from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 63 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -2695,15 +2622,11 @@ influxdb: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true service_to_fork_from: title: Name of another service to fork from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -2736,9 +2659,7 @@ kafka: custom_domain: title: Custom domain description: Serve the web frontend using a custom CNAME pointing to the Aiven DNS name - type: - - string - - "null" + type: string max_length: 255 example: grafana.example.org follower_fetching: @@ -2870,7 +2791,6 @@ kafka: description: 'The number of messages accumulated on a log partition before messages are flushed to disk (Default: 9223372036854775807 (Long.MAX_VALUE))' type: integer minimum: 1 - example: "9223372036854775807" log_flush_interval_ms: title: log.flush.interval.ms description: 'The maximum time in ms that a message in any topic is kept in memory (page-cache) before flushed to disk. If not set, the value in log.flush.scheduler.interval.ms is used (Default: null)' @@ -3267,8 +3187,8 @@ kafka: description: An enumeration. type: integer enum: - - value: "1" - - value: "2" + - value: 1 + - value: 2 prefix_path_depth: title: Prefix path depth of the secrets Engine. Default is 1. If the secrets engine path has more than one segment it has to be increased to the number of segments. type: integer @@ -3307,9 +3227,9 @@ kafka: type: integer default: "1000" enum: - - value: "1000" - - value: "15000" - - value: "30000" + - value: 1000 + - value: 15000 + - value: 30000 minimum: 1000 maximum: 30000 name_strategy: @@ -3385,9 +3305,7 @@ kafka: default: true kafka_version: title: Kafka major version - type: - - string - - "null" + type: string enum: - is_deprecated: true deprecation_notice: This value is deprecated. @@ -3401,16 +3319,18 @@ kafka: - is_deprecated: true deprecation_notice: This value is deprecated. value: "3.4" - - value: "3.5" - - value: "3.6" + - is_deprecated: true + deprecation_notice: This value is deprecated. + value: "3.5" + - is_deprecated: true + deprecation_notice: This value is deprecated. + value: "3.6" - value: "3.7" - value: "3.8" letsencrypt_sasl_privatelink: title: Use Letsencrypt CA for Kafka SASL via Privatelink description: Use Letsencrypt CA for Kafka SASL via Privatelink - type: - - boolean - - "null" + type: boolean example: true private_access: title: Allow access to selected service ports from private networks @@ -3523,9 +3443,7 @@ kafka: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true single_zone: title: Single-zone configuration @@ -3833,8 +3751,8 @@ kafka_connect: description: An enumeration. type: integer enum: - - value: "1" - - value: "2" + - value: 1 + - value: 2 prefix_path_depth: title: Prefix path depth of the secrets Engine. Default is 1. If the secrets engine path has more than one segment it has to be increased to the number of segments. type: integer @@ -3846,9 +3764,7 @@ kafka_connect: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true static_ips: title: Static IP addresses @@ -3904,9 +3820,7 @@ kafka_mirrormaker: properties: admin_timeout_ms: description: Timeout for administrative tasks, e.g. detecting new topics, loading of consumer group and offsets. Defaults to 60000 milliseconds (1 minute). - type: - - integer - - "null" + type: integer minimum: 30000 maximum: 1.8e+06 example: "60000" @@ -3990,9 +3904,7 @@ kafka_mirrormaker: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true static_ips: title: Static IP addresses @@ -4005,9 +3917,7 @@ m3aggregator: custom_domain: title: Custom domain description: Serve the web frontend using a custom CNAME pointing to the Aiven DNS name - type: - - string - - "null" + type: string max_length: 255 example: grafana.example.org ip_filter: @@ -4039,9 +3949,7 @@ m3aggregator: max_items: 1024 m3aggregator_version: title: M3 major version (the minimum compatible version) - type: - - string - - "null" + type: string enum: - value: "1.1" - value: "1.2" @@ -4049,9 +3957,7 @@ m3aggregator: example: "1.1" m3_version: title: M3 major version (deprecated, use m3aggregator_version) - type: - - string - - "null" + type: string enum: - value: "1.1" - value: "1.2" @@ -4060,9 +3966,7 @@ m3aggregator: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true static_ips: title: Static IP addresses @@ -4086,9 +3990,7 @@ m3db: custom_domain: title: Custom domain description: Serve the web frontend using a custom CNAME pointing to the Aiven DNS name - type: - - string - - "null" + type: string max_length: 255 example: grafana.example.org ip_filter: @@ -4176,9 +4078,7 @@ m3db: example: true m3db_version: title: M3 major version (the minimum compatible version) - type: - - string - - "null" + type: string enum: - value: "1.1" - value: "1.2" @@ -4186,9 +4086,7 @@ m3db: example: "1.1" m3_version: title: M3 major version (deprecated, use m3db_version) - type: - - string - - "null" + type: string enum: - value: "1.1" - value: "1.2" @@ -4285,9 +4183,7 @@ m3db: example: true project_to_fork_from: title: Name of another project to fork a service from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 63 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -4424,15 +4320,11 @@ m3db: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true service_to_fork_from: title: Name of another service to fork from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -4458,9 +4350,7 @@ mysql: max_items: 1 admin_password: title: Custom password for admin user. Defaults to random string. This must be set only when a new service is being created. - type: - - string - - "null" + type: string min_length: 8 max_length: 256 create_only: true @@ -4470,9 +4360,7 @@ mysql: _secure: true admin_username: title: Custom username for admin user. This must be set only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[_A-Za-z0-9][-._A-Za-z0-9]{0,63}$ @@ -4480,17 +4368,13 @@ mysql: user_error: Must consist of alpha-numeric characters, dots, underscores or dashes, may not start with dash or dot, max 64 characters backup_hour: title: The hour of day (in UTC) when backup for the service is started. New backup is only started if previous backup has already completed. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 23 example: "3" backup_minute: title: The minute of an hour when backup for the service is started. New backup is only started if previous backup has already completed. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 59 example: "30" @@ -4529,9 +4413,7 @@ mysql: max_items: 1024 migration: title: Migrate data from existing server - type: - - object - - "null" + type: object required: - host - port @@ -4640,9 +4522,7 @@ mysql: innodb_ft_server_stopword_table: title: innodb_ft_server_stopword_table description: This option is used to specify your own InnoDB FULLTEXT index stopword list for all InnoDB tables. - type: - - "null" - - string + type: string max_length: 1024 pattern: ^.+/.+$ example: db_name/table_name @@ -4729,7 +4609,7 @@ mysql: type: number minimum: 0 maximum: 3600 - example: "10" + example: "10.0" max_allowed_packet: title: max_allowed_packet description: Size of the largest message in bytes that can be received by the server. Default is 67108864 (64M) @@ -4806,9 +4686,7 @@ mysql: example: "28800" mysql_version: title: MySQL major version - type: - - string - - "null" + type: string enum: - value: "8" private_access: @@ -4845,9 +4723,7 @@ mysql: example: true project_to_fork_from: title: Name of another project to fork a service from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 63 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -4870,24 +4746,18 @@ mysql: example: true recovery_target_time: title: Recovery target time when forking a service. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 32 create_only: true example: "2019-01-01 23:34:45" service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true service_to_fork_from: title: Name of another service to fork from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -4983,17 +4853,13 @@ opensearch: custom_domain: title: Custom domain description: Serve the web frontend using a custom CNAME pointing to the Aiven DNS name - type: - - string - - "null" + type: string max_length: 255 example: grafana.example.org disable_replication_factor_adjustment: title: Disable replication factor adjustment description: 'DEPRECATED: Disable automatic replication factor adjustment for multi-node services. By default, Aiven ensures all indexes are replicated at least to two nodes. Note: Due to potential data loss in case of losing a service node, this setting can no longer be activated.' - type: - - boolean - - "null" + type: boolean example: false gcs_migration: description: Google Cloud Storage migration settings @@ -5112,27 +4978,21 @@ opensearch: mapping_nested_objects_limit: title: index.mapping.nested_objects.limit description: The maximum number of nested JSON objects that a single document can contain across all nested types. This limit helps to prevent out of memory errors when a document contains too many nested objects. Default is 10000. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 100000 example: "10000" number_of_replicas: title: index.number_of_replicas description: The number of replicas each primary shard has. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 29 example: "1" number_of_shards: title: index.number_of_shards description: The number of primary shards that an index should have. - type: - - integer - - "null" + type: integer minimum: 1 maximum: 1024 example: "1" @@ -5221,9 +5081,7 @@ opensearch: jwt_header: title: The HTTP header that stores the token description: 'The HTTP header that stores the token. Typically the Authorization header with the Bearer schema: Authorization: Bearer . Optional. Default is Authorization.' - type: - - string - - "null" + type: string min_length: 1 max_length: 1024 pattern: ^[^\r\n]*$ @@ -5231,9 +5089,7 @@ opensearch: jwt_url_parameter: title: URL JWT token. description: If the token is not transmitted in the HTTP header, but as an URL parameter, define the name of the parameter here. Optional. - type: - - string - - "null" + type: string min_length: 1 max_length: 1024 pattern: ^[^\r\n]*$ @@ -5241,27 +5097,19 @@ opensearch: refresh_rate_limit_count: title: The maximum number of unknown key IDs in the time frame description: The maximum number of unknown key IDs in the time frame. Default is 10. Optional. - type: - - integer - - "null" + type: integer default: "10" minimum: 10 - example: preferred_username refresh_rate_limit_time_window_ms: title: The time frame to use when checking the maximum number of unknown key IDs, in milliseconds description: The time frame to use when checking the maximum number of unknown key IDs, in milliseconds. Optional.Default is 10000 (10 seconds). - type: - - integer - - "null" + type: integer default: "10000" minimum: 10000 - example: preferred_username roles_key: title: The key in the JSON payload that stores the user’s roles description: The key in the JSON payload that stores the user’s roles. The value of this key must be a comma-separated list of roles. Required only if you want to use roles in the JWT - type: - - string - - "null" + type: string min_length: 1 max_length: 1024 pattern: ^[^\r\n]*$ @@ -5276,9 +5124,7 @@ opensearch: subject_key: title: The key in the JSON payload that stores the user’s name description: The key in the JSON payload that stores the user’s name. If not defined, the subject registered claim is used. Most IdP providers use the preferred_username claim. Optional. - type: - - string - - "null" + type: string min_length: 1 max_length: 1024 pattern: ^[^\r\n]*$ @@ -5294,9 +5140,7 @@ opensearch: example: false action_destructive_requires_name: title: Require explicit index names when deleting - type: - - boolean - - "null" + type: boolean example: true auth_failure_listeners: title: Opensearch Security Plugin Settings @@ -5465,9 +5309,7 @@ opensearch: indices_fielddata_cache_size: title: indices.fielddata.cache.size description: Relative amount. Maximum amount of heap memory used for field data cache. This is an expert setting; decreasing the value too much will increase overhead of loading field data; too much memory used for field data cache will decrease amount of heap available for other operations. - type: - - integer - - "null" + type: integer minimum: 3 maximum: 100 indices_memory_index_buffer_size: @@ -5568,14 +5410,10 @@ opensearch: reindex_remote_whitelist: title: reindex_remote_whitelist description: Whitelisted addresses for reindexing. Changing this value will cause all OpenSearch instances to restart. - type: - - array - - "null" + type: array items: title: Address (hostname:port or IP:port) - type: - - string - - "null" + type: string max_length: 261 pattern: ^[^\r\n]*$ example: anotherservice.aivencloud.com:12398 @@ -5727,9 +5565,7 @@ opensearch: search_max_buckets: title: search.max_buckets description: Maximum number of aggregation buckets allowed in a single response. OpenSearch default value is used when this is not defined. - type: - - integer - - "null" + type: integer minimum: 1 maximum: 1e+06 example: "10000" @@ -5897,9 +5733,7 @@ opensearch: maximum: 120000 opensearch_version: title: OpenSearch major version - type: - - string - - "null" + type: string enum: - value: "1" - value: "2" @@ -5937,9 +5771,7 @@ opensearch: example: true project_to_fork_from: title: Name of another project to fork a service from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 63 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -6073,9 +5905,7 @@ opensearch: idp_pemtrustedcas_content: title: PEM-encoded root CA Content for SAML IdP server verification description: This parameter specifies the PEM-encoded root certificate authority (CA) content for the SAML identity provider (IdP) server verification. The root CA content is used to verify the SSL/TLS certificate presented by the server. - type: - - string - - "null" + type: string max_length: 16384 example: | -----BEGIN CERTIFICATE----- @@ -6084,9 +5914,7 @@ opensearch: roles_key: title: SAML response role attribute description: Optional. Specifies the attribute in the SAML response where role information is stored, if available. Role attributes are not required for SAML authentication, but can be included in SAML assertions by most Identity Providers (IdPs) to determine user access levels or permissions. - type: - - string - - "null" + type: string min_length: 1 max_length: 256 pattern: ^[^\r\n]*$ @@ -6102,9 +5930,7 @@ opensearch: subject_key: title: SAML response subject attribute description: Optional. Specifies the attribute in the SAML response where the subject identifier is stored. If not configured, the NameID attribute is used by default. - type: - - string - - "null" + type: string min_length: 1 max_length: 256 pattern: ^[^\r\n]*$ @@ -6112,15 +5938,11 @@ opensearch: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true service_to_fork_from: title: Name of another service to fork from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -6148,9 +5970,7 @@ pg: max_items: 1 admin_password: title: Custom password for admin user. Defaults to random string. This must be set only when a new service is being created. - type: - - string - - "null" + type: string min_length: 8 max_length: 256 create_only: true @@ -6160,9 +5980,7 @@ pg: _secure: true admin_username: title: Custom username for admin user. This must be set only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[_A-Za-z0-9][-._A-Za-z0-9]{0,63}$ @@ -6170,17 +5988,13 @@ pg: user_error: Must consist of alpha-numeric characters, dots, underscores or dashes, may not start with dash or dot, max 64 characters backup_hour: title: The hour of day (in UTC) when backup for the service is started. New backup is only started if previous backup has already completed. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 23 example: "3" backup_minute: title: The minute of an hour when backup for the service is started. New backup is only started if previous backup has already completed. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 59 example: "30" @@ -6218,9 +6032,7 @@ pg: max_items: 1024 migration: title: Migrate data from existing server - type: - - object - - "null" + type: object required: - host - port @@ -6535,8 +6347,7 @@ pg: pg_stat_statements.track: title: pg_stat_statements.track description: Controls which statements are counted. Specify top to track top-level statements (those issued directly by clients), all to also track nested statements (such as statements invoked within functions), or none to disable statement statistics collection. The default value is top. - type: - - string + type: string enum: - value: all - value: none @@ -6646,15 +6457,11 @@ pg: pg_read_replica: title: Should the service which is being forked be a read replica (deprecated, use read_replica service integration instead). description: This setting is deprecated. Use read_replica service integration instead. - type: - - boolean - - "null" + type: boolean example: true pg_service_to_fork_from: title: Name of the PG Service from which to fork (deprecated, use service_to_fork_from). This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -6666,9 +6473,7 @@ pg: default: false pg_version: title: PostgreSQL major version - type: - - string - - "null" + type: string enum: - is_deprecated: true deprecation_notice: This value is deprecated. @@ -6676,7 +6481,9 @@ pg: - is_deprecated: true deprecation_notice: This value is deprecated. value: "11" - - value: "12" + - is_deprecated: true + deprecation_notice: This value is deprecated. + value: "12" - value: "13" - value: "14" - value: "15" @@ -6899,7 +6706,7 @@ pg: description: System-wide settings for pglookout. type: object default: - max_failover_replication_time_lag: "60" + max_failover_replication_time_lag: 60 properties: max_failover_replication_time_lag: title: max_failover_replication_time_lag @@ -6941,9 +6748,7 @@ pg: example: true project_to_fork_from: title: Name of another project to fork a service from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 63 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -6966,24 +6771,18 @@ pg: example: true recovery_target_time: title: Recovery target time when forking a service. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 32 create_only: true example: "2019-01-01 23:34:45" service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true service_to_fork_from: title: Name of another service to fork from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -7022,9 +6821,7 @@ pg: example: "8" variant: title: Variant of the PostgreSQL service, may affect the features that are exposed by default - type: - - string - - "null" + type: string enum: - value: aiven - value: timescale @@ -7052,17 +6849,13 @@ redis: max_items: 1 backup_hour: title: The hour of day (in UTC) when backup for the service is started. New backup is only started if previous backup has already completed. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 23 example: "3" backup_minute: title: The minute of an hour when backup for the service is started. New backup is only started if previous backup has already completed. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 59 example: "30" @@ -7095,9 +6888,7 @@ redis: max_items: 1024 migration: title: Migrate data from existing server - type: - - object - - "null" + type: object required: - host - port @@ -7176,9 +6967,7 @@ redis: example: true project_to_fork_from: title: Name of another project to fork a service from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 63 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -7229,9 +7018,7 @@ redis: maximum: 100 redis_maxmemory_policy: title: Redis maxmemory-policy - type: - - string - - "null" + type: string default: noeviction enum: - value: allkeys-lfu @@ -7245,7 +7032,6 @@ redis: redis_notify_keyspace_events: title: Set notify-keyspace-events option type: string - default: "" max_length: 32 pattern: ^[KEg\$lshzxentdmA]*$ redis_number_of_databases: @@ -7281,23 +7067,17 @@ redis: maximum: 2.0736e+06 redis_version: title: Redis major version - type: - - string - - "null" + type: string enum: - value: "7.0" service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true service_to_fork_from: title: Name of another service to fork from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -7442,18 +7222,32 @@ thanos: description: Whether to align the query range boundaries with the step. If enabled, the query range boundaries will be aligned to the step, providing more accurate results for queries with high-resolution data. type: boolean default: true + receiver-ingesting: + title: CommonReceiveUserConfig + description: Generic model + type: object + receiver-routing: + title: ThanosReceiveRoutingUserConfig + description: Generic model + type: object + ruler: + title: ThanosRulerUserConfig + description: Generic model + type: object service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true static_ips: title: Static IP addresses description: Use static public IP addresses type: boolean example: true + store: + title: ThanosStoreUserConfig + description: Generic model + type: object valkey: type: object properties: @@ -7470,17 +7264,13 @@ valkey: max_items: 1 backup_hour: title: The hour of day (in UTC) when backup for the service is started. New backup is only started if previous backup has already completed. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 23 example: "3" backup_minute: title: The minute of an hour when backup for the service is started. New backup is only started if previous backup has already completed. - type: - - integer - - "null" + type: integer minimum: 0 maximum: 59 example: "30" @@ -7513,9 +7303,7 @@ valkey: max_items: 1024 migration: title: Migrate data from existing server - type: - - object - - "null" + type: object required: - host - port @@ -7594,9 +7382,7 @@ valkey: example: true project_to_fork_from: title: Name of another project to fork a service from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 63 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -7622,15 +7408,11 @@ valkey: service_log: title: Service logging description: Store logs for the service so that they are available in the HTTP API and console. - type: - - boolean - - "null" + type: boolean example: true service_to_fork_from: title: Name of another service to fork from. This has effect only when a new service is being created. - type: - - string - - "null" + type: string max_length: 64 create_only: true pattern: ^[a-z][-a-z0-9]{0,63}$|^$ @@ -7668,9 +7450,7 @@ valkey: maximum: 100 valkey_maxmemory_policy: title: Valkey maxmemory-policy - type: - - string - - "null" + type: string default: noeviction enum: - value: allkeys-lfu @@ -7684,7 +7464,6 @@ valkey: valkey_notify_keyspace_events: title: Set notify-keyspace-events option type: string - default: "" max_length: 32 pattern: ^[KEg\$lshzxentdmA]*$ valkey_number_of_databases: