From ce61c9f14f5117452b2f9d2037f406498f8a6aea Mon Sep 17 00:00:00 2001 From: Murad Biashimov Date: Wed, 16 Oct 2024 13:03:28 +0200 Subject: [PATCH] feat: add changelog generator --- Makefile | 8 +- changelog/differ.go | 191 +++++++++++++++++++++++++++++++++++++++ changelog/differ_test.go | 99 ++++++++++++++++++++ changelog/main.go | 162 +++++++++++++++++++++++++++++++++ changelog/text.go | 87 ++++++++++++++++++ 5 files changed, 545 insertions(+), 2 deletions(-) create mode 100644 changelog/differ.go create mode 100644 changelog/differ_test.go create mode 100644 changelog/main.go create mode 100644 changelog/text.go diff --git a/Makefile b/Makefile index fec7b3dcd..4741990d1 100644 --- a/Makefile +++ b/Makefile @@ -174,10 +174,14 @@ sweep-check: generate: gen-go docs - +OLD_SCHEMA ?= .oldSchema.json +CHANGELOG := PROVIDER_AIVEN_ENABLE_BETA=1 go run ./changelog/... gen-go: - go generate ./... + $(CHANGELOG) --save > $(OLD_SCHEMA) + go generate ./...; $(MAKE) fmt-imports + $(CHANGELOG) --diff < $(OLD_SCHEMA) + rm $(OLD_SCHEMA) docs: $(TFPLUGINDOCS) diff --git a/changelog/differ.go b/changelog/differ.go new file mode 100644 index 000000000..58092b14c --- /dev/null +++ b/changelog/differ.go @@ -0,0 +1,191 @@ +package main + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + + "github.com/google/go-cmp/cmp" + "github.com/samber/lo" +) + +func diffItems(resourceType ResourceType, was, have *Item) (*Diff, error) { + // Added or removed + if was == nil || have == nil { + action := ChangeTypeAdd + if have == nil { + action = ChangeTypeRemove + have = was + } + + return &Diff{ + Type: action, + ResourceType: resourceType, + Description: removeEnum(have.Description), + Item: have, + }, nil + } + + // Equal items + if cmp.Equal(was, have) { + return nil, nil + } + + // Compare all the fields + wasMap, err := toMap(was) + if err != nil { + return nil, err + } + + haveMap, err := toMap(have) + if err != nil { + return nil, err + } + + entries := make([]string, 0) + for k, wasValue := range wasMap { + haveValue := haveMap[k] + if cmp.Equal(wasValue, haveValue) { + continue + } + + var entry string + switch k { + case "deprecated": + entry = "remove deprecation" + if have.Deprecated != "" { + entry = fmt.Sprintf("deprecate: %s", have.Deprecated) + } + case "beta": + entry = "marked as beta" + if !haveValue.(bool) { + entry = "no longer beta" + } + default: + // The rest of the fields will have diff-like entry + entry = fmt.Sprintf("%s ~~`%s`~~ -> `%s`", k, strValue(wasValue), strValue(haveValue)) + + // Fixes formatting issues + entry = strings.ReplaceAll(entry, "``", "`") + } + + entries = append(entries, entry) + } + + if len(entries) == 0 { + return nil, nil + } + + return &Diff{ + Type: ChangeTypeChange, + ResourceType: resourceType, + Description: strings.Join(entries, ", "), + Item: have, + }, nil +} + +func diffItemMaps(was, have ItemMap) ([]string, error) { + result := make([]string, 0) + kinds := []ResourceType{ResourceKind, DataSourceKind} + for _, kind := range kinds { + wasItems := was[kind] + haveItems := have[kind] + keys := append(lo.Keys(wasItems), lo.Keys(haveItems)...) + slices.Sort(keys) + + var skipPrefix string + seen := make(map[string]bool) + for _, k := range keys { + if seen[k] { + continue + } + + // Skips duplicate keys + seen[k] = true + + // When a resource added or removed, it skips all its fields until the next resource + if skipPrefix != "" && strings.HasPrefix(k, skipPrefix) { + continue + } + + skipPrefix = "" + wasVal, wasOk := wasItems[k] + haveVal, haveOk := haveItems[k] + if wasOk != haveOk { + // Resource added or removed, must skip all its fields + skipPrefix = k + } + + change, err := diffItems(kind, wasVal, haveVal) + if err != nil { + return nil, fmt.Errorf("failed to compare %s %s: %w", kind, k, err) + } + + if change != nil { + result = append(result, change.String()) + } + } + } + return result, nil +} + +type DiffType string + +const ( + ChangeTypeAdd DiffType = "Add" + ChangeTypeRemove DiffType = "Remove" + ChangeTypeChange DiffType = "Change" +) + +type Diff struct { + Type DiffType + ResourceType ResourceType + Description string + Item *Item +} + +func (c *Diff) String() string { + // resource name + field name + path := strings.SplitN(c.Item.Path, ".", 2) + + // e.g.: "Add resource `aiven_project`" + msg := fmt.Sprintf("%s %s `%s`", c.Type, c.ResourceType, path[0]) + + // e.g.: "field `project`" + if len(path) > 1 { + msg = fmt.Sprintf("%s field `%s`", msg, path[1]) + } + + // Adds beta if needed + if hasBeta(c.Description) { + msg = fmt.Sprintf("%s _(beta)_", msg) + } + + // Adds description + const maxSize = 120 + + msg += ": " + msg += shorten(maxSize-len(msg), c.Description) + return msg +} + +func toMap(item *Item) (map[string]any, error) { + b, err := json.Marshal(item) + if err != nil { + return nil, err + } + + m := make(map[string]any) + err = json.Unmarshal(b, &m) + if err != nil { + return nil, err + } + + m["enum"] = findEnums(item.Description) + m["beta"] = hasBeta(item.Description) + m["type"] = strValueType(item.Type) + m["elemType"] = strValueType(item.ElemType) + delete(m, "description") // Not needed to compare descriptions + return m, err +} diff --git a/changelog/differ_test.go b/changelog/differ_test.go new file mode 100644 index 000000000..cbc518132 --- /dev/null +++ b/changelog/differ_test.go @@ -0,0 +1,99 @@ +package main + +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/stretchr/testify/assert" +) + +func TestCompare(t *testing.T) { + tests := []struct { + name string + expect string + kind ResourceType + old, new *Item + }{ + { + name: "change enums", + expect: "Change resource `foo` field `bar`: enum ~~`bar`, `baz`~~ -> `foo`, `baz`", + kind: ResourceKind, + old: &Item{ + Type: schema.TypeString, + Path: "foo.bar", + Description: "Foo. The possible values are `bar`, `baz`.", + }, + new: &Item{ + Type: schema.TypeString, + Path: "foo.bar", + Description: "Foo. The possible values are `foo`, `baz`.", + }, + }, + { + name: "add resource field", + expect: "Add resource `foo` field `bar`: Foo", + kind: ResourceKind, + new: &Item{ + Type: schema.TypeString, + Path: "foo.bar", + Description: "Foo", + }, + }, + { + name: "remove resource field", + expect: "Remove resource `foo` field `bar`: Foo", + kind: ResourceKind, + old: &Item{ + Type: schema.TypeString, + Path: "foo.bar", + Description: "Foo", + }, + }, + { + name: "remove beta from the field", + expect: "Change resource `foo` field `bar`: no longer beta", + kind: ResourceKind, + old: &Item{ + Type: schema.TypeString, + Path: "foo.bar", + Description: "PROVIDER_AIVEN_ENABLE_BETA", + }, + new: &Item{ + Type: schema.TypeString, + Path: "foo.bar", + Description: "Foo", + }, + }, + { + name: "add beta resource", + expect: "Add resource `foo` _(beta)_: does stuff, PROVIDER_AIVEN_ENABLE_BETA", + kind: ResourceKind, + new: &Item{ + Type: schema.TypeString, + Path: "foo", + Description: "does stuff, PROVIDER_AIVEN_ENABLE_BETA", + }, + }, + { + name: "change type", + expect: "Change resource `foo` field `bar`: type ~~`list`~~ -> `set`", + kind: ResourceKind, + old: &Item{ + Type: schema.TypeList, + Path: "foo.bar", + }, + new: &Item{ + Type: schema.TypeSet, + Path: "foo.bar", + }, + }, + } + + for _, opt := range tests { + t.Run(opt.name, func(t *testing.T) { + got, err := diffItems(opt.kind, opt.old, opt.new) + assert.NoError(t, err) + assert.Equal(t, opt.expect, got.String()) + }) + } +} diff --git a/changelog/main.go b/changelog/main.go new file mode 100644 index 000000000..f73cc0cc7 --- /dev/null +++ b/changelog/main.go @@ -0,0 +1,162 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "os" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + + "github.com/aiven/terraform-provider-aiven/internal/plugin/util" + "github.com/aiven/terraform-provider-aiven/internal/sdkprovider/provider" +) + +func main() { + err := exec() + if err != nil { + log.Fatal(err) + } +} + +func exec() error { + if !util.IsBeta() { + return fmt.Errorf("please enable beta mode, i.e. set %s=1", util.AivenEnableBeta) + } + + save := flag.Bool("save", false, "output current schema") + diff := flag.Bool("diff", false, "compare current schema with imported schema") + changelog := flag.String("changelog", "", "write changes to file") + flag.Parse() + + if *save == *diff { + return fmt.Errorf("either --save or --diff must be set") + } + + // Loads the current Provider schema + p, err := provider.Provider("dev") + if err != nil { + return err + } + + newMap, err := fromProvider(p) + if err != nil { + return err + } + + if *save { + return json.NewEncoder(os.Stdout).Encode(&newMap) + } + + // Outputs diff with the current Provider schema + if *diff { + b, err := io.ReadAll(os.Stdin) + if err != nil { + return err + } + + var oldMap ItemMap + err = json.Unmarshal(b, &oldMap) + if err != nil { + return err + } + + entries, err := diffItemMaps(oldMap, newMap) + if err != nil { + return err + } + + // todo: write to file + if *changelog == "" { + for _, l := range entries { + fmt.Printf("- %s\n", l) + } + } + } + + return nil +} + +func fromProvider(p *schema.Provider) (ItemMap, error) { + // Item names might clash between resources and data sources + // Splits into separate maps + sourceMaps := map[ResourceType]map[string]*schema.Resource{ + ResourceKind: p.ResourcesMap, + DataSourceKind: p.DataSourcesMap, + } + + items := make(ItemMap) + for kind, m := range sourceMaps { + items[kind] = make(map[string]*Item) + for name, r := range m { + res := &Item{ + Name: name, + Path: name, + Description: r.Description, + Type: schema.TypeList, + } + for k, v := range r.Schema { + walked := walkSchema(k, v, res) + for i := range walked { + item := walked[i] + items[kind][item.Path] = item + } + } + } + } + return items, nil +} + +type ResourceType string + +const ( + ResourceKind ResourceType = "resource" + DataSourceKind ResourceType = "datasource" +) + +type ItemMap map[ResourceType]map[string]*Item + +type Item struct { + Name string `json:"name"` // TF field name + Path string `json:"path"` // full path to the item, e.g. `aiven_project.project` + + // TF schema fields + Description string `json:"description"` + ForceNew bool `json:"forceNew"` + Optional bool `json:"optional"` + Sensitive bool `json:"sensitive"` + MaxItems int `json:"maxItems"` + Deprecated string `json:"deprecated"` + Type schema.ValueType `json:"type"` + ElemType schema.ValueType `json:"elemType"` +} + +func walkSchema(name string, this *schema.Schema, parent *Item) []*Item { + item := &Item{ + Name: name, + Path: fmt.Sprintf("%s.%s", parent.Path, name), + ForceNew: this.ForceNew, + Optional: this.Optional, + Sensitive: this.Sensitive, + MaxItems: this.MaxItems, + Description: this.Description, + Deprecated: this.Deprecated, + Type: this.Type, + } + + items := []*Item{item} + + // Properties + switch elem := this.Elem.(type) { + case *schema.Schema: + item.ElemType = elem.Type + case *schema.Resource: + for k, child := range elem.Schema { + items = append(items, walkSchema(k, child, item)...) + } + } + + return items +} diff --git a/changelog/text.go b/changelog/text.go new file mode 100644 index 000000000..a6466b6e5 --- /dev/null +++ b/changelog/text.go @@ -0,0 +1,87 @@ +package main + +import ( + "fmt" + "regexp" + "strings" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + + "github.com/aiven/terraform-provider-aiven/internal/plugin/util" + "github.com/aiven/terraform-provider-aiven/internal/schemautil/userconfig" +) + +func hasBeta(description string) bool { + return strings.Contains(description, util.AivenEnableBeta) +} + +var reEnum = regexp.MustCompile("(?i)enum: `.+`\\.?\\s*") + +// removeEnum removes enum values from the description to keep it brief +func removeEnum(text string) string { + return reEnum.ReplaceAllString(text, "") +} + +var reCode = regexp.MustCompile("`[^`]+`") + +func findEnums(description string) []string { + parts := strings.Split(description, userconfig.PossibleValuesPrefix) + if len(parts) != 2 { + return nil + } + + return reCode.FindAllString(parts[1], -1) +} + +// strValue formats Go value into humanreadable string +func strValue(src any) string { + switch v := src.(type) { + case string: + return v + case []string: + return strings.Join(v, ", ") + default: + return fmt.Sprintf("%v", v) + } +} + +// strValueType returns the string representation of the schema.ValueType +func strValueType(t schema.ValueType) string { + switch t { + case schema.TypeBool: + return "bool" + case schema.TypeString: + return "string" + case schema.TypeInt: + return "int" + case schema.TypeFloat: + return "float" + case schema.TypeList: + return "list" + case schema.TypeMap: + return "map" + case schema.TypeSet: + return "set" + default: + return "unknown" + } +} + +// shorten shortens the text to the given size. +func shorten(size int, text string) string { + if size < 1 || len(text) <= size { + return text + } + + const sep = ". " + brief := "" + chunks := strings.Split(text, sep) + for i := 0; len(brief) <= size && i < len(chunks); i++ { + if i > 0 { + brief += sep + } + brief += chunks[i] + } + + return brief +}