From b021d546bc7ec228036bcb8995915a5c5ac7aa1f Mon Sep 17 00:00:00 2001 From: Murad Biashimov Date: Tue, 14 Nov 2023 10:35:53 +0100 Subject: [PATCH] feat(release): crd changelog generator --- CHANGELOG.md | 35 ++ docs/docs/contributing/resource-generation.md | 5 +- generators/charts/changelog.go | 349 ++++++++++++++++++ generators/charts/changelog_test.go | 84 +++++ generators/charts/crds.go | 10 +- generators/charts/main.go | 9 +- generators/charts/version.go | 6 +- 7 files changed, 489 insertions(+), 9 deletions(-) create mode 100644 generators/charts/changelog.go create mode 100644 generators/charts/changelog_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ae2002b5..b55fa0dc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,41 @@ - Upgrade to Go 1.21 - Add a format for `string` type fields to the documentation +- Generate CRD changelog +- Add `Clickhouse` field `userConfig.private_access.clickhouse_mysql`, type `boolean`: Allow clients + to connect to clickhouse_mysql with a DNS name that always resolves to the service's private IP addresses +- Add `Clickhouse` field `userConfig.privatelink_access.clickhouse_mysql`, type `boolean`: Enable clickhouse_mysql +- Add `Clickhouse` field `userConfig.public_access.clickhouse_mysql`, type `boolean`: Allow clients to + connect to clickhouse_mysql from the public internet for service nodes that are in a project VPC + or another type of private network +- Add `Grafana` field `userConfig.unified_alerting_enabled`, type `boolean`: Enable or disable Grafana + unified alerting functionality +- Add `Kafka` field `userConfig.aiven_kafka_topic_messages`, type `boolean`: Allow access to read Kafka + topic messages in the Aiven Console and REST API +- Add `Kafka` field `userConfig.kafka.sasl_oauthbearer_expected_audience`, type `string`: The (optional) + comma-delimited setting for the broker to use to verify that the JWT was issued for one of the + expected audiences +- Add `Kafka` field `userConfig.kafka.sasl_oauthbearer_expected_issuer`, type `string`: Optional setting + for the broker to use to verify that the JWT was created by the expected issuer +- Add `Kafka` field `userConfig.kafka.sasl_oauthbearer_jwks_endpoint_url`, type `string`: OIDC JWKS endpoint + URL. By setting this the SASL SSL OAuth2/OIDC authentication is enabled +- Add `Kafka` field `userConfig.kafka.sasl_oauthbearer_sub_claim_name`, type `string`: Name of the scope + from which to extract the subject claim from the JWT. Defaults to sub +- Change `Kafka` field `userConfig.kafka_version`: enum ~`[3.1, 3.3, 3.4, 3.5]`~ → `[3.1, 3.3, 3.4, + 3.5, 3.6]` +- Change `Kafka` field `userConfig.tiered_storage.local_cache.size`: deprecated +- Add `OpenSearch` field `userConfig.opensearch.indices_memory_max_index_buffer_size`, type `integer`: + Absolute value. Default is unbound. Doesn't work without indices.memory.index_buffer_size +- Add `OpenSearch` field `userConfig.opensearch.indices_memory_min_index_buffer_size`, type `integer`: + Absolute value. Default is 48mb. Doesn't work without indices.memory.index_buffer_size +- Change `OpenSearch` field `userConfig.opensearch.auth_failure_listeners.internal_authentication_backend_limiting.authentication_backend`: + enum `[internal]` +- Change `OpenSearch` field `userConfig.opensearch.auth_failure_listeners.internal_authentication_backend_limiting.type`: + enum `[username]` +- Change `OpenSearch` field `userConfig.opensearch.auth_failure_listeners.ip_rate_limiting.type`: enum `[ip]` +- Change `OpenSearch` field `userConfig.opensearch.search_max_buckets`: maximum ~`65536`~ → `1000000` +- Change `ServiceIntegration` field `kafkaMirrormaker.kafka_mirrormaker.producer_max_request_size`: maximum + ~`67108864`~ → `268435456` ## v0.14.0 - 2023-09-21 diff --git a/docs/docs/contributing/resource-generation.md b/docs/docs/contributing/resource-generation.md index 93128f7aa..0a599736d 100644 --- a/docs/docs/contributing/resource-generation.md +++ b/docs/docs/contributing/resource-generation.md @@ -51,7 +51,8 @@ Here how it goes in the details: 2. generates full spec reference out of the schema 3. creates a markdown file with spec and example (if exists) 4. Charts generator - updates CRDs, webhooks and cluster roles charts + updates CRDs, webhooks and cluster roles charts, + adds all changes to the changelog [go-api-schemas]: https://github.com/aiven/go-api-schemas [service-types]: https://api.aiven.io/doc/#tag/Service/operation/ListPublicServiceTypes @@ -71,7 +72,7 @@ flowchart TB Examples--> Markdown(creates docs out of CRDs, adds examples) Reference-->Markdown(kafka.md) end - CRD-->|yaml files|Charts(charts generator
updates helm charts) + CRD-->|yaml files|Charts(charts generator
updates helm charts
and the changelog) Charts-->ToRelease("Ready to release 🎉") Markdown-->ToRelease ``` diff --git a/generators/charts/changelog.go b/generators/charts/changelog.go new file mode 100644 index 000000000..7a03e438e --- /dev/null +++ b/generators/charts/changelog.go @@ -0,0 +1,349 @@ +package main + +import ( + "fmt" + "os" + "path" + "sort" + "strconv" + "strings" + + "github.com/google/go-cmp/cmp" + "golang.org/x/exp/constraints" + "golang.org/x/exp/slices" + "gopkg.in/yaml.v3" +) + +const ( + lineWidth = 100 + deprecatedMark = "deprecated" + changelogFile = "CHANGELOG.md" +) + +// crdType kubernetes CRD representation +type crdType struct { + Spec struct { + Names struct { + Kind string `yaml:"kind"` + } `yaml:"names"` + Versions []struct { + Schema struct { + OpenAPIV3Schema *schema `yaml:"openAPIV3Schema"` + } `yaml:"schema"` + } `yaml:"versions"` + } `yaml:"spec"` +} + +type schema struct { + Kind string `yaml:"-"` + Properties map[string]*schema `yaml:"properties"` + Type string `yaml:"type"` + Description string `yaml:"description"` + Enum []any `yaml:"enum"` + Pattern string `yaml:"pattern"` + Format string `yaml:"format"` + MinItems *int `yaml:"minItems"` + MaxItems *int `yaml:"maxItems"` + MinLength *int `yaml:"minLength"` + MaxLength *int `yaml:"maxLength"` + Minimum *float64 `yaml:"minimum"` + Maximum *float64 `yaml:"maximum"` +} + +func loadSchema(b []byte) (*schema, error) { + crd := new(crdType) + err := yaml.Unmarshal(b, crd) + if err != nil { + return nil, err + } + + if crd.Spec.Versions == nil { + return nil, fmt.Errorf("empty schema for kind %s", crd.Spec.Names.Kind) + } + + s := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + s.Kind = crd.Spec.Names.Kind + return s, nil +} + +// genChangelog generates changelog for given yamls +func genChangelog(wasBytes, hasBytes []byte) ([]string, error) { + wasSchema, err := loadSchema(wasBytes) + if err != nil { + return nil, err + } + + hasSchema, err := loadSchema(hasBytes) + if err != nil { + return nil, err + } + + changes := cmpSchemas(hasSchema.Kind, "", wasSchema, hasSchema) + sort.Slice(changes, func(i, j int) bool { + return changes[i][0] < changes[j][0] + }) + + return changes, nil +} + +func cmpSchemas(kind, parent string, wasSpec, hasSpec *schema) []string { + if cmp.Equal(wasSpec, hasSpec) { + return nil + } + + changes := make([]string, 0) + for _, k := range mergedKeys(wasSpec.Properties, hasSpec.Properties) { + ov, oOk := wasSpec.Properties[k] + nv, nOk := hasSpec.Properties[k] + + fieldPath := k + if parent != "" { + fieldPath = fmt.Sprintf("%s.%s", parent, k) + } + + switch { + case !nOk: + changes = append(changes, fmt.Sprintf("Remove `%s` field `%s`, type `%s`: %s", kind, fieldPath, ov.Type, shortDescription(ov.Description))) + case !oOk: + changes = append(changes, fmt.Sprintf("Add `%s` field `%s`, type `%s`: %s", kind, fieldPath, nv.Type, shortDescription(nv.Description))) + case !cmp.Equal(ov, nv): + switch ov.Type { + case "object": + changes = append(changes, cmpSchemas(kind, fieldPath, ov, nv)...) + default: + c := fmtChanges(ov, nv) + if c != "" { + changes = append(changes, fmt.Sprintf("Change `%s` field `%s`: %s", kind, fieldPath, c)) + } + } + } + } + return changes +} + +func fmtChanges(was, has *schema) string { + changes := make(map[string]bool) + changes[fmtChange("pattern", &was.Pattern, &has.Pattern)] = true + changes[fmtChange("format", &was.Format, &has.Format)] = true + changes[fmtChange("minItems", was.MinItems, has.MinItems)] = true + changes[fmtChange("maxItems", was.MaxItems, has.MaxItems)] = true + changes[fmtChange("minLength", was.MinLength, has.MinLength)] = true + changes[fmtChange("maxLength", was.MaxLength, has.MaxLength)] = true + changes[fmtChange("minimum", was.Minimum, has.Minimum)] = true + changes[fmtChange("maximum", was.Maximum, has.Maximum)] = true + changes[fmtChange("enum", &was.Enum, &has.Enum)] = true + + if !isDeprecated(was.Description) && isDeprecated(has.Description) { + changes[deprecatedMark] = true + } + + delete(changes, "") + return strings.Join(sortedKeys(changes), ", ") +} + +// fmtChange returns a string like: foo ~`0`~ → `1` or empty string +func fmtChange[T any](title string, was, has *T) string { + if cmp.Equal(was, has) { + return "" + } + + var w, h string + if was != nil { + if v := strAny(*was); v != "" { + w = fmt.Sprintf("`%v`", v) + } + } + if has != nil { + if v := strAny(*has); v != "" { + h = fmt.Sprintf("`%v`", v) + } + } + + return fmt.Sprintf("%s %s", title, fmtWasHas(" → ", w, h)) +} + +func fmtWasHas(sep, was, has string) string { + switch "" { + case was: + return has + case has: + return fmt.Sprintf("~%s~", was) + } + return fmt.Sprintf("~%s~%s%s", was, sep, has) +} + +func strSlice(src []any) string { + result := make([]string, len(src)) + for i, v := range src { + result[i] = fmt.Sprint(v) + } + slices.Sort(result) + s := strings.Join(result, ", ") + if s != "" { + s = fmt.Sprintf("[%s]", s) + } + return s +} + +func strAny(a any) string { + switch v := a.(type) { + case int: + return fmt.Sprintf("%d", v) + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) + case []any: + return strSlice(v) + default: + return fmt.Sprint(v) + } +} + +func isDeprecated(s string) bool { + return strings.HasPrefix(strings.ToLower(s), deprecatedMark) +} + +// shortDescription returns a string shorter than lineWidth when possible. +func shortDescription(s string) string { + chunks := strings.Split(s, ". ") + description := chunks[0] + for i := 1; i < len(chunks); i++ { + d := fmt.Sprintf("%s. %s", description, chunks[i]) + if len(d) > lineWidth { + break + } + description = d + } + return strings.TrimSuffix(description, ".") +} + +// softWrapLineToleranceFactor a line shorter than this factor won't be wrapped +// to not break it right before a small word +const softWrapLineToleranceFactor = 1.1 + +// softWrapLine wraps long lines +func softWrapLine(src, linebreak string, n int) string { + if int(float64(n)*softWrapLineToleranceFactor) > len(src) { + return src + } + + line := 1 // line number + for i := 0; i < len(src); { + s := src[i] + // 32 ASCII number for space + if i >= n*line && i%n >= 0 && s == 32 { + src = fmt.Sprintf("%s%s%s", src[:i], linebreak, src[i+1:]) + i += len(linebreak) + line++ + continue + } + i++ + } + return src +} + +func addChanges(body []byte, changes []string) string { + lines := strings.Split(string(body), "\n") + + i := 0 + found := false + for ; i < len(lines); i++ { + if strings.HasPrefix(lines[i], "-") { + found = true + continue + } + + if found && lines[i] == "" { + break + } + } + + items := make([]string, 0) + items = append(items, lines[:i]...) + for _, s := range changes { + items = append(items, softWrapLine("- "+s, "\n ", lineWidth)) + } + + items = append(items, lines[i:]...) + return strings.Join(items, "\n") +} + +// updateChangelog updates changelog with CRD changes. +// To do so, it reads into memory files before they changed. +// Then compares with updates and finds the changes. +func updateChangelog(operatorPath, crdCharts string) (func() error, error) { + crdDir := path.Join(crdCharts, crdDestinationDir) + wasFiles, err := readFiles(crdDir) + if err != nil { + return nil, err + } + + return func() error { + hasFiles, err := readFiles(crdDir) + if err != nil { + return err + } + + // Finds changes per Kind + changes := make([]string, 0) + for _, k := range sortedKeys(hasFiles) { + kindChanges, err := genChangelog(wasFiles[k], hasFiles[k]) + if err != nil { + return err + } + changes = append(changes, kindChanges...) + } + + // Reads changelogFile + changelogPath := path.Join(operatorPath, changelogFile) + changelogBody, err := os.ReadFile(changelogPath) + if err != nil { + return err + } + + // Writes changes to changelogFile + changelogUpdated := addChanges(changelogBody, changes) + return os.WriteFile(changelogPath, []byte(changelogUpdated), 0644) + }, nil +} + +func readFiles(p string) (map[string][]byte, error) { + files, err := os.ReadDir(p) + if err != nil { + return nil, err + } + + result := make(map[string][]byte) + for _, file := range files { + if file.IsDir() { + continue + } + b, err := os.ReadFile(path.Join(p, file.Name())) + if err != nil { + return nil, err + } + result[file.Name()] = b + } + + return result, nil +} + +// sortedKeys returns map's keys sorted to have predictable output +func sortedKeys[K constraints.Ordered, V any](m map[K]V) []K { + keys := make([]K, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +} + +// mergedKeys returns merged keys of multiple maps +func mergedKeys[K constraints.Ordered, V any](maps ...map[K]V) []K { + unique := make(map[K]bool) + for _, m := range maps { + for k := range m { + unique[k] = true + } + } + return sortedKeys[K](unique) +} diff --git a/generators/charts/changelog_test.go b/generators/charts/changelog_test.go new file mode 100644 index 000000000..3bd180537 --- /dev/null +++ b/generators/charts/changelog_test.go @@ -0,0 +1,84 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSoftWrapLine(t *testing.T) { + src := "Add `Kafka.userConfig.kafka.sasl_oauthbearer_expected_audience`: The (optional) comma-delimited setting for the broker to use to verify that the JWT was issued for one of the expected audiences." + expect := "Add `Kafka.userConfig.kafka.sasl_oauthbearer_expected_audience`: The (optional) comma-delimited setting\n for the broker to use to verify that the JWT was issued for one of the expected audiences." + assert.Equal(t, expect, softWrapLine(src, "\n ", 100)) +} + +const testOldCRD = ` +spec: + names: + kind: Kafka + versions: + - schema: + openAPIV3Schema: + properties: + spec: + properties: + cloudName: + description: Cloud the service runs in. + maxLength: 120 + type: string + enum: [foo, bar] + minimum: 1.0 + maximum: 2.0 + karapace: + description: Switch the service to use Karapace for schema registry and REST proxy + type: boolean + topic_name: + description: The durable single partition topic + format: ^[1-9]*(GiB|G)* + maxLength: 111 + type: string +` + +const testNewCRD = ` +spec: + names: + kind: Kafka + versions: + - schema: + openAPIV3Schema: + properties: + spec: + properties: + cloudName: + description: Cloud the service runs in. + maxLength: 256 + type: string + enum: [foo, bar, baz] + minimum: 3.0 + maximum: 4.0 + disk_space: + description: The disk space of the service. + type: string + topic_name: + description: The durable single partition topic + format: ^[1-9][0-9]*(GiB|G)* + maxLength: 249 + minLength: 1 + minimum: 1 + maximum: 1000000 + enum: [foo, bar, baz] +` + +func TestGenChangelog(t *testing.T) { + changes, err := genChangelog([]byte(testOldCRD), []byte(testNewCRD)) + require.NoError(t, err) + + expect := []string{"Add `Kafka` field `disk_space`, type `string`: The disk space of the service", + "Change `Kafka` field `cloudName`: enum ~`[bar, foo]`~ → `[bar, baz, foo]`, maxLength ~`120`~ → `256`, maximum ~`2`~ → `4`, minimum ~`1`~ → `3`", + "Change `Kafka` field `topic_name`: enum `[bar, baz, foo]`, format ~`^[1-9]*(GiB|G)*`~ → `^[1-9][0-9]*(GiB|G)*`, maxLength ~`111`~ → `249`, maximum `1000000`, minLength `1`, minimum `1`", + "Remove `Kafka` field `karapace`, type `boolean`: Switch the service to use Karapace for schema registry and REST proxy", + } + + assert.Equal(t, expect, changes) +} diff --git a/generators/charts/crds.go b/generators/charts/crds.go index 1cbcd7f48..7b20806d0 100644 --- a/generators/charts/crds.go +++ b/generators/charts/crds.go @@ -11,12 +11,16 @@ import ( ) // allCRDYaml contains all crds -const allCRDYaml = "aiven.io_crd-all.gen.yaml" +const ( + allCRDYaml = "aiven.io_crd-all.gen.yaml" + crdSourceDir = "config/crd/bases/" + crdDestinationDir = "templates" +) // copyCRDs copies CRDs, like MySQL, Postgres, etc func copyCRDs(operatorPath, crdCharts string) error { - srcCRDs := path.Join(operatorPath, "config/crd/bases/") - dstCRDs := path.Join(crdCharts, "templates") + srcCRDs := path.Join(operatorPath, crdSourceDir) + dstCRDs := path.Join(crdCharts, crdDestinationDir) err := cp.Copy(srcCRDs, dstCRDs) if err != nil { return err diff --git a/generators/charts/main.go b/generators/charts/main.go index 1d1fbd225..fb1e9ce61 100644 --- a/generators/charts/main.go +++ b/generators/charts/main.go @@ -51,10 +51,17 @@ func generate(version, operatorPath, operatorCharts, crdCharts string) error { return err } + // Changelog is generated from old CRDs. + // Reads them first, and then compares with updated files. + commitChangelog, err := updateChangelog(operatorPath, crdCharts) + if err != nil { + return err + } + err = copyCRDs(operatorPath, crdCharts) if err != nil { return err } - return nil + return commitChangelog() } diff --git a/generators/charts/version.go b/generators/charts/version.go index 5bd8f7149..7701a7ae4 100644 --- a/generators/charts/version.go +++ b/generators/charts/version.go @@ -32,7 +32,7 @@ func updateVersion(version, operatorPath string, charts ...string) error { return err } } - return updateChangelog(version, path.Join(operatorPath, "CHANGELOG.md")) + return updateChangelogVersion(version, path.Join(operatorPath, changelogFile)) } // updateChartVersion replaces version in Chart.yaml @@ -67,9 +67,9 @@ const ( headerVersionDateFormat = "2006-01-02" ) -// updateChangelog updates CHANGELOG.md, sets version header. +// updateChangelogVersion updates CHANGELOG.md, sets version header. // If version already exists, updates release date -func updateChangelog(version, filePath string) error { +func updateChangelogVersion(version, filePath string) error { f, err := os.ReadFile(filePath) if err != nil { return err