Merge pull request #60 from Pix4D/cmd-remove

add "remove" command

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 #### New
 
+- New command `remove` to remove resources from the Terraform state. See the README for details.
 - The script generated by `terravalet import` now prints each command before executing it. This helps to understand which command is being executed.
 
 ### Changes

README.md

@@ -5,6 +5,7 @@ A tool to help with advanced, low-level [Terraform](https://www.terraform.io/) o
 - Rename resources within the same Terraform state, with optional fuzzy match.
 - Move resources from one Terraform state to another.
 - Import existing resources into Terraform state.
+- Remove existing resources from Terraform state.
 
 **DISCLAIMER Manipulating Terraform state is inherently dangerous. It is your responsibility to be careful and ensure you UNDERSTAND what you are doing**.
 
@@ -57,11 +58,12 @@ After the creation of Terravalet, Terraform introduced the `moved` block, which
 
 ## Usage
 
-There are three modes of operation:
+Terravalet supports multiple operations:
 
 - [Rename resources](#rename-resources-within-the-same-state) within the same Terraform state, with optional fuzzy match.
 - [Move resources](#-move-resources-from-one-state-to-another) from one Terraform state to another.
 - [Import existing resources](#-import-existing-resources) into Terraform state.
+- [Remove existing resources](#removing-existing-resources) from Terraform state.
 
 They will be explained in the following sections.
 
@@ -399,6 +401,27 @@ NON ignorable errors:
 
 1. Provider specific argument ID is wrong.
 
+# Removing existing resources
+
+Although `terraform state rm` allows to remove _individual_ resources, but when a real-world resource is composed of multiple terraform resources, using `terraform state rm` becomes tedious and error-prone. Even worse when multiple high-level resources are removed together.
+
+Thus, `terravalet remove` parses a plan file and creates all the `state rm` commands for you.
+
+1. Remove the resources in the Terraform configuration files.
+2. Generate the plan file:
+   ```
+   $ terraform -chdir=<the tf root> plan -no-color > remove-plan.txt
+   ```
+3. Run `terravalet remove`. As usual, this is a safe operation, since it will only generate a script file:
+   ```
+   $ terravalet remove --up=remove.sh --plan=remove-plan.txt
+   ```
+4. Carefully examine the generated script file `remove.sh`!
+5. Execute the scrip.
+   ```
+   $ sh ./remove.sh
+   ```
+
 # Making a release
 
 ## Setup

cmdremove.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+func doRemove(planPath string, upPath string) error {
+	planFile, err := os.Open(planPath)
+	if err != nil {
+		return fmt.Errorf("remove: opening the plan file: %s", err)
+	}
+	defer planFile.Close()
+
+	upFile, err := os.Create(upPath)
+	if err != nil {
+		return fmt.Errorf("remove: creating the up file: %s", err)
+	}
+	defer upFile.Close()
+
+	toCreate, toDestroy, err := parse(planFile)
+	if err != nil {
+		return fmt.Errorf("remove: parsing plan: %s", err)
+	}
+	if toCreate.Size() > 0 {
+		return fmt.Errorf("remove: plan contains resources to create: %v",
+			sorted(toCreate.List()))
+	}
+
+	var bld strings.Builder
+	generateRemoveScript(&bld, sorted(toDestroy.List()))
+	_, err = upFile.WriteString(bld.String())
+	if err != nil {
+		return fmt.Errorf("remove: writing script file: %s", err)
+	}
+
+	return nil
+}
+
+func generateRemoveScript(wr io.Writer, addresses []string) {
+	fmt.Fprintf(wr, `#! /bin/sh
+# DO NOT EDIT. Generated by https://github.com/pix4D/terravalet
+# This script will remove %d items.
+
+set -e
+
+`, len(addresses))
+	for _, addr := range addresses {
+		fmt.Fprintf(wr, "terraform state rm '%s'\n", addr)
+	}
+	fmt.Fprintln(wr)
+}

cmdremove_test.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/go-quicktest/qt"
+)
+
+func TestGenerateRemoveScript(t *testing.T) {
+	addresses := []string{
+		`module.a.b.c["foo"]`,
+		`module.a.b.d["foo.AN-"]`,
+	}
+	var bld strings.Builder
+	want := `#! /bin/sh
+# DO NOT EDIT. Generated by https://github.com/pix4D/terravalet
+# This script will remove 2 items.
+
+set -e
+
+terraform state rm 'module.a.b.c["foo"]'
+terraform state rm 'module.a.b.d["foo.AN-"]'
+
+`
+
+	generateRemoveScript(&bld, addresses)
+	qt.Assert(t, qt.Equals(bld.String(), want))
+}

go.mod

@@ -5,8 +5,16 @@ go 1.21
 require (
 	github.com/alexflint/go-arg v1.4.3
 	github.com/dexyk/stringosim v0.0.0-20170922105913-9d0b3e91a842
+	github.com/go-quicktest/qt v1.101.0
 	github.com/google/go-cmp v0.6.0
+	github.com/rogpeppe/go-internal v1.11.0
 	github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e
 )
 
-require github.com/alexflint/go-scalar v1.2.0 // indirect
+require (
+	github.com/alexflint/go-scalar v1.2.0 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
+	golang.org/x/tools v0.1.12 // indirect
+)

go.sum

@@ -1,32 +1,40 @@
-github.com/alexflint/go-arg v1.4.2 h1:lDWZAXxpAnZUq4qwb86p/3rIJJ2Li81EoMbTMujhVa0=
-github.com/alexflint/go-arg v1.4.2/go.mod h1:9iRbDxne7LcR/GSvEr7ma++GLpdIU1zrghf2y2768kM=
 github.com/alexflint/go-arg v1.4.3 h1:9rwwEBpMXfKQKceuZfYcwuc/7YY7tWJbFsgG5cAU/uo=
 github.com/alexflint/go-arg v1.4.3/go.mod h1:3PZ/wp/8HuqRZMUUgu7I+e1qcpUbvmS258mRXkFH4IA=
-github.com/alexflint/go-scalar v1.0.0/go.mod h1:GpHzbCOZXEKMEcygYQ5n/aa4Aq84zbxjy3MxYW0gjYw=
-github.com/alexflint/go-scalar v1.1.0 h1:aaAouLLzI9TChcPXotr6gUhq+Scr8rl0P9P4PnltbhM=
 github.com/alexflint/go-scalar v1.1.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o=
 github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+WQBRw=
 github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dexyk/stringosim v0.0.0-20170922105913-9d0b3e91a842 h1:FWXGhOthNyZKdK0YVyDrkg5dCXOfKvexcRG37U1v6AQ=
 github.com/dexyk/stringosim v0.0.0-20170922105913-9d0b3e91a842/go.mod h1:PfVoEMbmPGFArz22/wIefW9CzuQhdnE+C9ikEzJvb9Q=
 github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
 github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
-github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e h1:7q6NSFZDeGfvvtIRwBrU/aegEYJYmvev0cHAwo17zZQ=
 github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -16,21 +16,27 @@ var (
 )
 
 func main() {
+	os.Exit(Main())
+}
+
+func Main() int {
 	if err := run(); err != nil {
 		fmt.Fprintf(os.Stderr, "error: %v\n", err)
-		os.Exit(1)
+		return 1
 	}
+	return 0
 }
 
-type args struct {
+type Args struct {
 	Rename     *RenameCmd     `arg:"subcommand:rename" help:"rename resources in the same root environment"`
 	MoveAfter  *MoveAfterCmd  `arg:"subcommand:move-after" help:"move resources from one root environment to AFTER another"`
 	MoveBefore *MoveBeforeCmd `arg:"subcommand:move-before" help:"move resources from one root environment to BEFORE another"`
 	Import     *ImportCmd     `arg:"subcommand:import" help:"import resources generated out-of-band of Terraform"`
+	Remove     *RemoveCmd     `arg:"subcommand:remove" help:"remove resources"`
 	Version    *struct{}      `arg:"subcommand:version" help:"show version"`
 }
 
-func (args) Description() string {
+func (Args) Description() string {
 	return "terravalet - helps with advanced Terraform operations\n"
 }
 
@@ -64,8 +70,13 @@ type ImportCmd struct {
 	SrcPlanPath  string `arg:"--src-plan,required" help:"path to the SRC terraform plan in JSON format"`
 }
 
+type RemoveCmd struct {
+	Up   string `arg:"required" help:"path of the up script to generate (NNN_TITLE.up.sh)"`
+	Plan string `arg:"required" help:"path to to the output of 'terraform plan -no-color'"`
+}
+
 func run() error {
-	var args args
+	var args Args
 
 	parser := arg.MustParse(&args)
 	if parser.Subcommand() == nil {
@@ -86,6 +97,9 @@ func run() error {
 	case args.Import != nil:
 		cmd := args.Import
 		return doImport(cmd.Up, cmd.Down, cmd.SrcPlanPath, cmd.ResourceDefs)
+	case args.Remove != nil:
+		cmd := args.Remove
+		return doRemove(cmd.Plan, cmd.Up)
 	case args.Version != nil:
 		fmt.Println("terravalet", fullVersion)
 		return nil

terravalet_script_test.go

@@ -0,0 +1,29 @@
+// This file runs tests using the 'testscript' package.
+// To understand, see:
+// - https://github.com/rogpeppe/go-internal
+// - https://bitfieldconsulting.com/golang/test-scripts
+
+package main
+
+import (
+	"os"
+	"testing"
+
+	"github.com/rogpeppe/go-internal/testscript"
+)
+
+func TestMain(m *testing.M) {
+	// The commands map holds the set of command names, each with an associated
+	// run function which should return the code to pass to os.Exit.
+	// When [testscript.Run] is called, these commands are installed as regular
+	// commands in the shell path, so can be invoked with "exec".
+	os.Exit(testscript.RunMain(m, map[string]func() int{
+		"terravalet": Main,
+	}))
+}
+
+func TestScript(t *testing.T) {
+	testscript.Run(t, testscript.Params{
+		Dir: "testdata/script",
+	})
+}

testdata/remove/01_plan.txt

@@ -0,0 +1,31 @@
+Terraform used the selected providers to generate the following execution
+plan. Resource actions are indicated with the following symbols:
+  - destroy
+
+Terraform will perform the following actions:
+
+  # module.github.github_branch_default.default["foo-c"] will be destroyed
+  # (because key ["foo-c"] is not in for_each map)
+  - resource "github_branch_default" "default" {
+
+  # module.github.github_repository.repos["foo-c"] will be destroyed
+  # (because key ["foo-c"] is not in for_each map)
+  - resource "github_repository" "repos" {
+
+  # module.github.github_repository_autolink_reference.repo_autolinks["foo-c.AN-"] will be destroyed
+  # (because key ["foo-c.AN-"] is not in for_each map)
+  - resource "github_repository_autolink_reference" "repo_autolinks" {
+
+  # module.github.github_repository_autolink_reference.repo_autolinks["foo-c.CV-"] will be destroyed
+  # (because key ["foo-c.CV-"] is not in for_each map)
+  - resource "github_repository_autolink_reference" "repo_autolinks" {
+
+  # module.github.github_repository_autolink_reference.repo_autolinks["foo-c.OPF-"] will be destroyed
+  # (because key ["foo-c.OPF-"] is not in for_each map)
+  - resource "github_repository_autolink_reference" "repo_autolinks" {
+
+  # module.github.github_repository_collaborators.repo_collaborators["foo-c"] will be destroyed
+  # (because key ["foo-c"] is not in for_each map)
+  - resource "github_repository_collaborators" "repo_collaborators" {
+
+Plan: 0 to add, 0 to change, 6 to destroy.

testdata/remove/01_up.sh

@@ -0,0 +1,18 @@
+#! /bin/sh
+# DO NOT EDIT. Generated by https://github.com/pix4D/terravalet
+#
+# This script will detach 6 items.
+
+set -e
+
+terraform state rm 'module.github.github_branch_default.default["crane-camera-ccqc"]'
+
+terraform state rm 'module.github.github_repository.repos["crane-camera-ccqc"]'
+
+terraform state rm 'module.github.github_repository_autolink_reference.repo_autolinks["crane-camera-ccqc.AN-"]'
+
+terraform state rm 'module.github.github_repository_autolink_reference.repo_autolinks["crane-camera-ccqc.CV-"]'
+
+terraform state rm 'module.github.github_repository_autolink_reference.repo_autolinks["crane-camera-ccqc.OPF-"]'
+
+terraform state rm 'module.github.github_repository_collaborators.repo_collaborators["crane-camera-ccqc"]'

testdata/script/cmd-remove.txt

@@ -0,0 +1,50 @@
+exec terravalet remove --up=foo_up.sh --plan=detach.plan.txt
+! stderr .
+cmp foo_up.sh foo_up.sh.want
+
+-- foo_up.sh.want --
+#! /bin/sh
+# DO NOT EDIT. Generated by https://github.com/pix4D/terravalet
+# This script will remove 6 items.
+
+set -e
+
+terraform state rm 'module.github.github_branch_default.default["foo"]'
+terraform state rm 'module.github.github_repository.repos["foo"]'
+terraform state rm 'module.github.github_repository_autolink_reference.repo_autolinks["foo.AN-"]'
+terraform state rm 'module.github.github_repository_autolink_reference.repo_autolinks["foo.CV-"]'
+terraform state rm 'module.github.github_repository_autolink_reference.repo_autolinks["foo.OPF-"]'
+terraform state rm 'module.github.github_repository_collaborators.repo_collaborators["foo"]'
+
+-- detach.plan.txt --
+Terraform used the selected providers to generate the following execution
+plan. Resource actions are indicated with the following symbols:
+  - destroy
+
+Terraform will perform the following actions:
+
+  # module.github.github_branch_default.default["foo"] will be destroyed
+  # (because key ["foo"] is not in for_each map)
+  - resource "github_branch_default" "default" {
+
+  # module.github.github_repository.repos["foo"] will be destroyed
+  # (because key ["foo"] is not in for_each map)
+  - resource "github_repository" "repos" {
+
+  # module.github.github_repository_autolink_reference.repo_autolinks["foo.AN-"] will be destroyed
+  # (because key ["foo.AN-"] is not in for_each map)
+  - resource "github_repository_autolink_reference" "repo_autolinks" {
+
+  # module.github.github_repository_autolink_reference.repo_autolinks["foo.CV-"] will be destroyed
+  # (because key ["foo.CV-"] is not in for_each map)
+  - resource "github_repository_autolink_reference" "repo_autolinks" {
+
+  # module.github.github_repository_autolink_reference.repo_autolinks["foo.OPF-"] will be destroyed
+  # (because key ["foo.OPF-"] is not in for_each map)
+  - resource "github_repository_autolink_reference" "repo_autolinks" {
+
+  # module.github.github_repository_collaborators.repo_collaborators["foo"] will be destroyed
+  # (because key ["foo"] is not in for_each map)
+  - resource "github_repository_collaborators" "repo_collaborators" {
+
+Plan: 0 to add, 0 to change, 6 to destroy.