Skip to content

Commit

Permalink
Add terraform import script (#130)
Browse files Browse the repository at this point in the history
  • Loading branch information
ichung08 authored Aug 21, 2024
1 parent 556d27e commit 532f085
Show file tree
Hide file tree
Showing 16 changed files with 4,342 additions and 32 deletions.
65 changes: 65 additions & 0 deletions .github/workflows/build-release-import-script.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
name: Build and Release Import Script

on:
push:
tags:
- 'import-v*'

permissions:
contents: write

jobs:
build:
name: Build
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
with:
go-version-file: 'go.mod'
cache: true
- run: go mod download
- run: go build -v -o import ./import/import_script.go

lint:
name: Lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
with:
go-version-file: 'go.mod'
cache: true
- run: go fmt ./import/import_script.go
- run: go vet ./import/import_script.go
- run: make validate-fmt

goreleaser:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
with:
fetch-depth: 0
- uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
with:
go-version-file: 'go.mod'
cache: true
- name: Import GPG key
uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4 # v6.1.0
id: import_gpg
with:
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
passphrase: ${{ secrets.PASSPHRASE }}
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@5742e2a039330cbb23ebf35f046f814d4c6ff811 # v5.1.0
with:
args: release --clean --config .goreleaser.import.yml # Use the import-specific config
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
- name: Upload Terraform import script artifact
uses: actions/upload-artifact@v3
with:
name: import
path: dist/import*
3 changes: 2 additions & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ jobs:
cache: true
- run: go mod download
- run: go build -v .
- run: make test
- name: Run unit tests
run: make test

lint:
name: Lint
Expand Down
55 changes: 55 additions & 0 deletions .github/workflows/import-script-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
name: Import Script Tests

on:
pull_request:
push:
branches:
- main

# Testing only needs permissions to read the repository contents.
permissions:
contents: read

jobs:
# Ensure project builds before running testing matrix
build:
name: Build
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
with:
go-version-file: 'go.mod'
cache: true
- run: go mod download
- name: Build import script
run: go build -o import_script ./import/import_script.go

test-import-script:
name: Import Script Tests
needs: build
timeout-minutes: 20
strategy:
fail-fast: true
matrix:
terraform:
- latest
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
with:
go-version-file: 'go.mod'
cache: true
- uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
with:
terraform_version: ${{ matrix.terraform }}
terraform_wrapper: false
- run: go mod download
- name: Run import script tests
env:
SKIP_IMPORT_SCRIPT_TEST: 1
HOSTED_ORGANIZATION_API_TOKEN: ${{ secrets.DEV_HOSTED_ORGANIZATION_API_TOKEN }}
HOSTED_ORGANIZATION_ID: clx42kkcm01fo01o06agtmshg
run: make test-import-script
4 changes: 2 additions & 2 deletions .github/workflows/testacc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ jobs:
terraform:
- latest
provider_version:
- v0.3.0
- v0.3.1
- v0.2.1
- v0.1.0-alpha
- main
Expand Down Expand Up @@ -162,7 +162,7 @@ jobs:
terraform:
- latest
provider_version:
- v0.3.0
- v0.3.1
- v0.2.1
- v0.1.0-alpha
- main
Expand Down
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,15 @@ website/node_modules
*.test
*.iml
*.tfvars
import.tf
generated.tf
import/import_script

website/vendor
test_results
vendor/
.terraformrc
.terraform.lock.hcl

# Test exclusions
!command/test-fixtures/**/*.tfstate
Expand Down
48 changes: 48 additions & 0 deletions .goreleaser.import.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
project_name: terraform-import-script

builds:
- main: ./import/import.go
binary: terraform-import-script
goos:
- linux
- windows
- darwin
goarch:
- amd64
- arm64

archives:
- format: tar.gz
name_template: >-
{{ .ProjectName }}_
{{- title .Os }}_
{{- if eq .Arch "amd64" }}x86_64
{{- else if eq .Arch "386" }}i386
{{- else }}{{ .Arch }}{{ end }}
format_overrides:
- goos: windows
format: zip

changelog:
sort: asc
filters:
exclude:
- '^docs:'
- '^test:'

signs:
- cmd: gpg
args:
- "--output"
- "${signature}"
- "--detach-sign"
- "${artifact}"
artifacts: checksum

# Customize the release name to differentiate it from your main project releases
release:
name_template: "Terraform Import Script {{.Tag}}"
header: |
# Terraform Import Script Release {{.Tag}}
This release contains the Terraform import script, a standalone tool for importing resources into Terraform.
33 changes: 32 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ $(ENVTEST_ASSETS_DIR):
mkdir -p $(ENVTEST_ASSETS_DIR)
OAPI_CODEGEN ?= $(ENVTEST_ASSETS_DIR)/oapi-codegen

MOCKERY_VERSION := 2.42.0

# Run acceptance tests
.PHONY: testacc
testacc:
Expand All @@ -18,14 +20,43 @@ testacc:
.PHONY: test
test:
go vet ./...
TF_ACC="" go test ./... -v $(TESTARGS)
TF_ACC="" SKIP_IMPORT_SCRIPT_TEST="" SKIP_IMPORT_SCRIPT_TEST_DEV="" go test ./... -v $(TESTARGS)

# Run script tests
.PHONY: test-import-script
test-import-script:
go test ./import/... -v $(TESTARGS)

.PHONY: fmt
fmt:
gofmt -w ./
goimports -w ./
[ -z "$$CIRCLECI" ] || git diff --exit-code --color=always # In CI: exit if anything changed

.PHONY: mock
mock: ensure-mockery generate-mocks

.PHONY: ensure-mockery
ensure-mockery:
@if ! mockery --version | grep -q $(MOCKERY_VERSION); then \
echo "Installing Mockery $(MOCKERY_VERSION)"; \
go install $(MOCKERY_PACKAGE); \
fi

.PHONY: generate-mocks
generate-mocks:
@echo "Generating mocks..."
@rm -rf mocks
@mockery --name=ClientWithResponsesInterface \
--dir=./internal/clients/iam \
--output=./internal/mocks/iam \
--outpkg=mocks_iam
@mockery --name=ClientWithResponsesInterface \
--dir=./internal/clients/platform \
--output=./internal/mocks/platform \
--outpkg=mocks_platform
@echo "Mocks generated successfully."

.PHONY: validate-fmt
validate-fmt:
@output=$$(gofmt -l ./); \
Expand Down
47 changes: 47 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -203,5 +203,52 @@ You will also need to set all the environment variables described in `internal/p

The acceptance tests will run against the Astronomer API and create/read/update/delete real resources.

## Importing Existing Resources
The Astro Terraform Import Script is a tool designed to help you import existing Astro resources into your Terraform configuration.
Currently, this script automates the process of generating Terraform import blocks and resource configurations for the following resources: workspaces, deployments, clusters, hybrid cluster workspace authorizations, API tokens, teams, team roles, and user roles.

To use the import script, download the `import_script` executable file from [releases](https://github.com/astronomer/terraform-provider-astro/releases) and run it with the following command:

```
./import [options]
```

### Options

- `-resources`: Comma-separated list of resources to import. Accepted values are workspace, deployment, cluster, api_token, team, team_roles, user_roles.
- `-token`: API token to authenticate with the Astro platform. If not provided, the script will attempt to use the `ASTRO_API_TOKEN` environment variable.
- `-host`: API host to connect to. Default is https://api.astronomer.io. Use "dev" for https://api.astronomer-dev.io or "stage" for https://api.astronomer-stage.io.
- `-organizationId`: Organization ID to import resources from.
- `-runTerraformInit`: Run `terraform init` after generating the import configuration.
- `-help`: Display help information.

### Examples

1. Import workspaces and deployments:
```
./import -resources workspace,deployment -token your_api_token -organizationId your_org_id
```

2. Import all supported resources and run Terraform init:
```
./import -resources workspace,deployment,cluster,api_token,team,team_roles,user_roles -token your_api_token -organizationId your_org_id -runTerraformInit
```

3. Use a different API host (e.g., dev environment):
```
./import -resources workspace -token your_api_token -organizationId your_org_id -host dev
```

### Output

The script will generate two main files:

1. `import.tf`: Contains the Terraform import blocks for the specified resources.
2. `generated.tf`: Contains the Terraform resource configurations for the imported resources.

### Notes

- Ensure you have the necessary permissions in your Astro organization to access the resources you're attempting to import.
- The generated Terraform configurations may require some manual adjustment to match your specific requirements or to resolve any conflicts.
- Always review the generated files before applying them to your Terraform state.

23 changes: 15 additions & 8 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,11 @@ require (
github.com/hashicorp/terraform-plugin-testing v1.8.0
github.com/lucsky/cuid v1.2.1
github.com/oapi-codegen/runtime v1.1.1
github.com/onsi/ginkgo/v2 v2.20.0
github.com/onsi/gomega v1.34.1
github.com/samber/lo v1.39.0
github.com/stretchr/testify v1.9.0
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
)

require (
Expand All @@ -33,8 +36,11 @@ require (
github.com/cloudflare/circl v1.3.7 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/hashicorp/cli v1.1.6 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
Expand Down Expand Up @@ -71,24 +77,25 @@ require (
github.com/rivo/uniseg v0.2.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/yuin/goldmark v1.7.1 // indirect
github.com/yuin/goldmark-meta v1.1.0 // indirect
github.com/zclconf/go-cty v1.14.4 // indirect
go.abhg.dev/goldmark/frontmatter v0.2.0 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/tools v0.17.0 // indirect
golang.org/x/crypto v0.26.0 // indirect
golang.org/x/mod v0.20.0 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.23.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/tools v0.24.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/grpc v1.63.2 // indirect
google.golang.org/protobuf v1.34.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
Loading

0 comments on commit 532f085

Please sign in to comment.