From a40410dd5a0bcb09706cba307ed20ae1e7b514d7 Mon Sep 17 00:00:00 2001 From: Marco Molteni Date: Mon, 29 Jan 2024 20:03:31 +0100 Subject: [PATCH] add "remove" command Part of PCI-3617 --- CHANGELOG.md | 1 + README.md | 25 +++++++++++++++- cmdremove.go | 54 ++++++++++++++++++++++++++++++++++ cmdremove_test.go | 29 ++++++++++++++++++ go.mod | 10 ++++++- go.sum | 26 ++++++++++------ main.go | 22 +++++++++++--- terravalet_script_test.go | 29 ++++++++++++++++++ testdata/remove/01_plan.txt | 31 +++++++++++++++++++ testdata/remove/01_up.sh | 18 ++++++++++++ testdata/script/cmd-remove.txt | 50 +++++++++++++++++++++++++++++++ 11 files changed, 280 insertions(+), 15 deletions(-) create mode 100644 cmdremove.go create mode 100644 cmdremove_test.go create mode 100644 terravalet_script_test.go create mode 100644 testdata/remove/01_plan.txt create mode 100644 testdata/remove/01_up.sh create mode 100644 testdata/script/cmd-remove.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index e843de0..8bf72e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 #### New +- New command `remove` to remove resources from the Terraform state. See the README for details. - The script generated by `terravalet import` now prints each command before executing it. This helps to understand which command is being executed. ### Changes diff --git a/README.md b/README.md index dd21b49..7949f9a 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ A tool to help with advanced, low-level [Terraform](https://www.terraform.io/) o - Rename resources within the same Terraform state, with optional fuzzy match. - Move resources from one Terraform state to another. - Import existing resources into Terraform state. +- Remove existing resources from Terraform state. **DISCLAIMER Manipulating Terraform state is inherently dangerous. It is your responsibility to be careful and ensure you UNDERSTAND what you are doing**. @@ -57,11 +58,12 @@ After the creation of Terravalet, Terraform introduced the `moved` block, which ## Usage -There are three modes of operation: +Terravalet supports multiple operations: - [Rename resources](#rename-resources-within-the-same-state) within the same Terraform state, with optional fuzzy match. - [Move resources](#-move-resources-from-one-state-to-another) from one Terraform state to another. - [Import existing resources](#-import-existing-resources) into Terraform state. +- [Remove existing resources](#removing-existing-resources) from Terraform state. They will be explained in the following sections. @@ -399,6 +401,27 @@ NON ignorable errors: 1. Provider specific argument ID is wrong. +# Removing existing resources + +Although `terraform state rm` allows to remove _individual_ resources, but when a real-world resource is composed of multiple terraform resources, using `terraform state rm` becomes tedious and error-prone. Even worse when multiple high-level resources are removed together. + +Thus, `terravalet remove` parses a plan file and creates all the `state rm` commands for you. + +1. Remove the resources in the Terraform configuration files. +2. Generate the plan file: + ``` + $ terraform -chdir= plan -no-color > remove-plan.txt + ``` +3. Run `terravalet remove`. As usual, this is a safe operation, since it will only generate a script file: + ``` + $ terravalet remove --up=remove.sh --plan=remove-plan.txt + ``` +4. Carefully examine the generated script file `remove.sh`! +5. Execute the scrip. + ``` + $ sh ./remove.sh + ``` + # Making a release ## Setup diff --git a/cmdremove.go b/cmdremove.go new file mode 100644 index 0000000..7d62fb8 --- /dev/null +++ b/cmdremove.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "io" + "os" + "strings" +) + +func doRemove(planPath string, upPath string) error { + planFile, err := os.Open(planPath) + if err != nil { + return fmt.Errorf("remove: opening the plan file: %s", err) + } + defer planFile.Close() + + upFile, err := os.Create(upPath) + if err != nil { + return fmt.Errorf("remove: creating the up file: %s", err) + } + defer upFile.Close() + + toCreate, toDestroy, err := parse(planFile) + if err != nil { + return fmt.Errorf("remove: parsing plan: %s", err) + } + if toCreate.Size() > 0 { + return fmt.Errorf("remove: plan contains resources to create: %v", + sorted(toCreate.List())) + } + + var bld strings.Builder + generateRemoveScript(&bld, sorted(toDestroy.List())) + _, err = upFile.WriteString(bld.String()) + if err != nil { + return fmt.Errorf("remove: writing script file: %s", err) + } + + return nil +} + +func generateRemoveScript(wr io.Writer, addresses []string) { + fmt.Fprintf(wr, `#! /bin/sh +# DO NOT EDIT. Generated by https://github.com/pix4D/terravalet +# This script will remove %d items. + +set -e + +`, len(addresses)) + for _, addr := range addresses { + fmt.Fprintf(wr, "terraform state rm '%s'\n", addr) + } + fmt.Fprintln(wr) +} diff --git a/cmdremove_test.go b/cmdremove_test.go new file mode 100644 index 0000000..0db0be7 --- /dev/null +++ b/cmdremove_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "strings" + "testing" + + "github.com/go-quicktest/qt" +) + +func TestGenerateRemoveScript(t *testing.T) { + addresses := []string{ + `module.a.b.c["foo"]`, + `module.a.b.d["foo.AN-"]`, + } + var bld strings.Builder + want := `#! /bin/sh +# DO NOT EDIT. Generated by https://github.com/pix4D/terravalet +# This script will remove 2 items. + +set -e + +terraform state rm 'module.a.b.c["foo"]' +terraform state rm 'module.a.b.d["foo.AN-"]' + +` + + generateRemoveScript(&bld, addresses) + qt.Assert(t, qt.Equals(bld.String(), want)) +} diff --git a/go.mod b/go.mod index c70874e..022d79e 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,16 @@ go 1.21 require ( github.com/alexflint/go-arg v1.4.3 github.com/dexyk/stringosim v0.0.0-20170922105913-9d0b3e91a842 + github.com/go-quicktest/qt v1.101.0 github.com/google/go-cmp v0.6.0 + github.com/rogpeppe/go-internal v1.11.0 github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e ) -require github.com/alexflint/go-scalar v1.2.0 // indirect +require ( + github.com/alexflint/go-scalar v1.2.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect + golang.org/x/tools v0.1.12 // indirect +) diff --git a/go.sum b/go.sum index 03b80f0..587a5d1 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,9 @@ -github.com/alexflint/go-arg v1.4.2 h1:lDWZAXxpAnZUq4qwb86p/3rIJJ2Li81EoMbTMujhVa0= -github.com/alexflint/go-arg v1.4.2/go.mod h1:9iRbDxne7LcR/GSvEr7ma++GLpdIU1zrghf2y2768kM= github.com/alexflint/go-arg v1.4.3 h1:9rwwEBpMXfKQKceuZfYcwuc/7YY7tWJbFsgG5cAU/uo= github.com/alexflint/go-arg v1.4.3/go.mod h1:3PZ/wp/8HuqRZMUUgu7I+e1qcpUbvmS258mRXkFH4IA= -github.com/alexflint/go-scalar v1.0.0/go.mod h1:GpHzbCOZXEKMEcygYQ5n/aa4Aq84zbxjy3MxYW0gjYw= -github.com/alexflint/go-scalar v1.1.0 h1:aaAouLLzI9TChcPXotr6gUhq+Scr8rl0P9P4PnltbhM= github.com/alexflint/go-scalar v1.1.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+WQBRw= github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -14,19 +11,30 @@ github.com/dexyk/stringosim v0.0.0-20170922105913-9d0b3e91a842 h1:FWXGhOthNyZKdK github.com/dexyk/stringosim v0.0.0-20170922105913-9d0b3e91a842/go.mod h1:PfVoEMbmPGFArz22/wIefW9CzuQhdnE+C9ikEzJvb9Q= github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA= github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e h1:7q6NSFZDeGfvvtIRwBrU/aegEYJYmvev0cHAwo17zZQ= github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 44380a1..42fabd0 100644 --- a/main.go +++ b/main.go @@ -16,21 +16,27 @@ var ( ) func main() { + os.Exit(Main()) +} + +func Main() int { if err := run(); err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) + return 1 } + return 0 } -type args struct { +type Args struct { Rename *RenameCmd `arg:"subcommand:rename" help:"rename resources in the same root environment"` MoveAfter *MoveAfterCmd `arg:"subcommand:move-after" help:"move resources from one root environment to AFTER another"` MoveBefore *MoveBeforeCmd `arg:"subcommand:move-before" help:"move resources from one root environment to BEFORE another"` Import *ImportCmd `arg:"subcommand:import" help:"import resources generated out-of-band of Terraform"` + Remove *RemoveCmd `arg:"subcommand:remove" help:"remove resources"` Version *struct{} `arg:"subcommand:version" help:"show version"` } -func (args) Description() string { +func (Args) Description() string { return "terravalet - helps with advanced Terraform operations\n" } @@ -64,8 +70,13 @@ type ImportCmd struct { SrcPlanPath string `arg:"--src-plan,required" help:"path to the SRC terraform plan in JSON format"` } +type RemoveCmd struct { + Up string `arg:"required" help:"path of the up script to generate (NNN_TITLE.up.sh)"` + Plan string `arg:"required" help:"path to to the output of 'terraform plan -no-color'"` +} + func run() error { - var args args + var args Args parser := arg.MustParse(&args) if parser.Subcommand() == nil { @@ -86,6 +97,9 @@ func run() error { case args.Import != nil: cmd := args.Import return doImport(cmd.Up, cmd.Down, cmd.SrcPlanPath, cmd.ResourceDefs) + case args.Remove != nil: + cmd := args.Remove + return doRemove(cmd.Plan, cmd.Up) case args.Version != nil: fmt.Println("terravalet", fullVersion) return nil diff --git a/terravalet_script_test.go b/terravalet_script_test.go new file mode 100644 index 0000000..81c88f6 --- /dev/null +++ b/terravalet_script_test.go @@ -0,0 +1,29 @@ +// This file runs tests using the 'testscript' package. +// To understand, see: +// - https://github.com/rogpeppe/go-internal +// - https://bitfieldconsulting.com/golang/test-scripts + +package main + +import ( + "os" + "testing" + + "github.com/rogpeppe/go-internal/testscript" +) + +func TestMain(m *testing.M) { + // The commands map holds the set of command names, each with an associated + // run function which should return the code to pass to os.Exit. + // When [testscript.Run] is called, these commands are installed as regular + // commands in the shell path, so can be invoked with "exec". + os.Exit(testscript.RunMain(m, map[string]func() int{ + "terravalet": Main, + })) +} + +func TestScript(t *testing.T) { + testscript.Run(t, testscript.Params{ + Dir: "testdata/script", + }) +} diff --git a/testdata/remove/01_plan.txt b/testdata/remove/01_plan.txt new file mode 100644 index 0000000..207f7eb --- /dev/null +++ b/testdata/remove/01_plan.txt @@ -0,0 +1,31 @@ +Terraform used the selected providers to generate the following execution +plan. Resource actions are indicated with the following symbols: + - destroy + +Terraform will perform the following actions: + + # module.github.github_branch_default.default["foo-c"] will be destroyed + # (because key ["foo-c"] is not in for_each map) + - resource "github_branch_default" "default" { + + # module.github.github_repository.repos["foo-c"] will be destroyed + # (because key ["foo-c"] is not in for_each map) + - resource "github_repository" "repos" { + + # module.github.github_repository_autolink_reference.repo_autolinks["foo-c.AN-"] will be destroyed + # (because key ["foo-c.AN-"] is not in for_each map) + - resource "github_repository_autolink_reference" "repo_autolinks" { + + # module.github.github_repository_autolink_reference.repo_autolinks["foo-c.CV-"] will be destroyed + # (because key ["foo-c.CV-"] is not in for_each map) + - resource "github_repository_autolink_reference" "repo_autolinks" { + + # module.github.github_repository_autolink_reference.repo_autolinks["foo-c.OPF-"] will be destroyed + # (because key ["foo-c.OPF-"] is not in for_each map) + - resource "github_repository_autolink_reference" "repo_autolinks" { + + # module.github.github_repository_collaborators.repo_collaborators["foo-c"] will be destroyed + # (because key ["foo-c"] is not in for_each map) + - resource "github_repository_collaborators" "repo_collaborators" { + +Plan: 0 to add, 0 to change, 6 to destroy. diff --git a/testdata/remove/01_up.sh b/testdata/remove/01_up.sh new file mode 100644 index 0000000..c3b81da --- /dev/null +++ b/testdata/remove/01_up.sh @@ -0,0 +1,18 @@ +#! /bin/sh +# DO NOT EDIT. Generated by https://github.com/pix4D/terravalet +# +# This script will detach 6 items. + +set -e + +terraform state rm 'module.github.github_branch_default.default["crane-camera-ccqc"]' + +terraform state rm 'module.github.github_repository.repos["crane-camera-ccqc"]' + +terraform state rm 'module.github.github_repository_autolink_reference.repo_autolinks["crane-camera-ccqc.AN-"]' + +terraform state rm 'module.github.github_repository_autolink_reference.repo_autolinks["crane-camera-ccqc.CV-"]' + +terraform state rm 'module.github.github_repository_autolink_reference.repo_autolinks["crane-camera-ccqc.OPF-"]' + +terraform state rm 'module.github.github_repository_collaborators.repo_collaborators["crane-camera-ccqc"]' diff --git a/testdata/script/cmd-remove.txt b/testdata/script/cmd-remove.txt new file mode 100644 index 0000000..2176d28 --- /dev/null +++ b/testdata/script/cmd-remove.txt @@ -0,0 +1,50 @@ +exec terravalet remove --up=foo_up.sh --plan=detach.plan.txt +! stderr . +cmp foo_up.sh foo_up.sh.want + +-- foo_up.sh.want -- +#! /bin/sh +# DO NOT EDIT. Generated by https://github.com/pix4D/terravalet +# This script will remove 6 items. + +set -e + +terraform state rm 'module.github.github_branch_default.default["foo"]' +terraform state rm 'module.github.github_repository.repos["foo"]' +terraform state rm 'module.github.github_repository_autolink_reference.repo_autolinks["foo.AN-"]' +terraform state rm 'module.github.github_repository_autolink_reference.repo_autolinks["foo.CV-"]' +terraform state rm 'module.github.github_repository_autolink_reference.repo_autolinks["foo.OPF-"]' +terraform state rm 'module.github.github_repository_collaborators.repo_collaborators["foo"]' + +-- detach.plan.txt -- +Terraform used the selected providers to generate the following execution +plan. Resource actions are indicated with the following symbols: + - destroy + +Terraform will perform the following actions: + + # module.github.github_branch_default.default["foo"] will be destroyed + # (because key ["foo"] is not in for_each map) + - resource "github_branch_default" "default" { + + # module.github.github_repository.repos["foo"] will be destroyed + # (because key ["foo"] is not in for_each map) + - resource "github_repository" "repos" { + + # module.github.github_repository_autolink_reference.repo_autolinks["foo.AN-"] will be destroyed + # (because key ["foo.AN-"] is not in for_each map) + - resource "github_repository_autolink_reference" "repo_autolinks" { + + # module.github.github_repository_autolink_reference.repo_autolinks["foo.CV-"] will be destroyed + # (because key ["foo.CV-"] is not in for_each map) + - resource "github_repository_autolink_reference" "repo_autolinks" { + + # module.github.github_repository_autolink_reference.repo_autolinks["foo.OPF-"] will be destroyed + # (because key ["foo.OPF-"] is not in for_each map) + - resource "github_repository_autolink_reference" "repo_autolinks" { + + # module.github.github_repository_collaborators.repo_collaborators["foo"] will be destroyed + # (because key ["foo"] is not in for_each map) + - resource "github_repository_collaborators" "repo_collaborators" { + +Plan: 0 to add, 0 to change, 6 to destroy.