Skip to content

Commit

Permalink
🐛 Fix GKE cloud test
Browse files Browse the repository at this point in the history
`GitHubSecurityLab/actions-permissions/monitor` added a MITM proxy to intercept calls.
This proxy signed the GKE API cert which lead to errors:
```
Unable to connect to the server: tls: failed to verify certificate: x509: certificate signed by unknown authority
```

Signed-off-by: Christian Zunker <[email protected]>
  • Loading branch information
czunker committed Nov 9, 2023
1 parent 6756963 commit 2e6e2dc
Show file tree
Hide file tree
Showing 6 changed files with 111 additions and 102 deletions.
2 changes: 2 additions & 0 deletions .github/terraform/gke/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ resource "google_container_cluster" "cluster" {

remove_default_node_pool = true
initial_node_count = 1
deletion_protection = false
}

resource "google_container_node_pool" "node_pool" {
name = "mondoo-operator-pool-${random_string.suffix.result}"
location = "us-central1-a"
project = var.project_id
cluster = google_container_cluster.cluster.id
node_count = 1

Expand Down
2 changes: 1 addition & 1 deletion .github/terraform/gke/outputs.tf
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ resource "local_file" "kubeconfig" {
depends_on = [google_container_cluster.cluster]
content = module.gke_auth.kubeconfig_raw
filename = "kubeconfig"
}
}
185 changes: 88 additions & 97 deletions .github/workflows/cloud-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,102 +38,100 @@ env:
CNSPEC_IMAGE_TAG: ${{ github.event.inputs.cnspecImageTag || 'edge-latest-rootless' }}

jobs:
aks-integration-test:
runs-on: ubuntu-latest
name: AKS integration tests

env:
ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/aks/kubeconfig') }}
# aks-integration-test:
# runs-on: ubuntu-latest
# name: AKS integration tests

# env:
# ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
# ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
# ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
# ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
# KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/aks/kubeconfig') }}

strategy:
fail-fast: false
matrix:
k8s-version: ["1.25", "1.26", "1.27"]
# strategy:
# fail-fast: false
# matrix:
# k8s-version: ["1.25", "1.26", "1.27"]

steps:
- uses: GitHubSecurityLab/actions-permissions/monitor@v1
with:
config: ${{ vars.PERMISSIONS_CONFIG }}
- uses: actions/checkout@v4
with:
fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile
# steps:
# - uses: actions/checkout@v4
# with:
# fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile

- name: Import environment variables from file
run: cat ".github/env" >> $GITHUB_ENV
# - name: Import environment variables from file
# run: cat ".github/env" >> $GITHUB_ENV

- name: Setup Terraform
uses: hashicorp/setup-terraform@v3
# - name: Setup Terraform
# uses: hashicorp/setup-terraform@v3

- name: Terraform init
run: terraform init
working-directory: .github/terraform/aks
# - name: Terraform init
# run: terraform init
# working-directory: .github/terraform/aks

- name: Terraform plan
run: terraform plan -out aks-${{ matrix.k8s-version }}.json
env:
TF_VAR_k8s_version: ${{ matrix.k8s-version }}
working-directory: .github/terraform/aks

- name: Terraform apply
run: terraform apply -auto-approve aks-${{ matrix.k8s-version }}.json
env:
TF_VAR_k8s_version: ${{ matrix.k8s-version }}
working-directory: .github/terraform/aks

- uses: actions/setup-go@v4
with:
go-version: "${{ env.golang-version }}"
cache: true

- name: Get operator version
run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV

- name: Wait a bit for the cluster to become more stable
run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s

- name: Run integration tests
env:
MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }}
MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342
MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query
run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=aks make test/integration/ci

- name: Clean up AKS terraform
run: terraform destroy -auto-approve
if: success() || failure()
working-directory: .github/terraform/aks
# - name: Terraform plan
# run: terraform plan -out aks-${{ matrix.k8s-version }}.json
# env:
# TF_VAR_k8s_version: ${{ matrix.k8s-version }}
# working-directory: .github/terraform/aks

# - name: Terraform apply
# run: terraform apply -auto-approve aks-${{ matrix.k8s-version }}.json
# env:
# TF_VAR_k8s_version: ${{ matrix.k8s-version }}
# working-directory: .github/terraform/aks

# - uses: actions/setup-go@v4
# with:
# go-version: "${{ env.golang-version }}"
# cache: true

# - name: Get operator version
# run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV

# - name: Wait a bit for the cluster to become more stable
# run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s

# - name: Run integration tests
# env:
# MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }}
# MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342
# MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query
# run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=aks make test/integration/ci

# - name: Clean up AKS terraform
# run: terraform destroy -auto-approve
# if: success() || failure()
# working-directory: .github/terraform/aks

- run: mv integration-tests.xml integration-tests-aks-${{ matrix.k8s-version }}.xml
if: success() || failure()

- name: Upload cloud test results
uses: actions/upload-artifact@v3 # upload test results
if: success() || failure() # run this step even if previous step failed
with: # upload a combined archive with unit and integration test results
name: cloud-test-results
path: |
integration-tests-aks-${{ matrix.k8s-version }}.xml
.github/terraform/aks/aks-${{ matrix.k8s-version }}.json
- name: Upload test logs artifact
uses: actions/upload-artifact@v3
if: failure()
with:
name: test-logs-aks-${{ matrix.k8s-version }}
path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/
# - run: mv integration-tests.xml integration-tests-aks-${{ matrix.k8s-version }}.xml
# if: success() || failure()

# - name: Upload cloud test results
# uses: actions/upload-artifact@v3 # upload test results
# if: success() || failure() # run this step even if previous step failed
# with: # upload a combined archive with unit and integration test results
# name: cloud-test-results
# path: |
# integration-tests-aks-${{ matrix.k8s-version }}.xml
# .github/terraform/aks/aks-${{ matrix.k8s-version }}.json

# - name: Upload test logs artifact
# uses: actions/upload-artifact@v3
# if: failure()
# with:
# name: test-logs-aks-${{ matrix.k8s-version }}
# path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/

eks-integration-test:
runs-on: ubuntu-latest
name: EKS integration tests

strategy:
fail-fast: false
max-parallel: 3 # Otherwise we will hit AWS VPC quota limits
matrix:
k8s-version: ["1.24", "1.25", "1.26", "1.27", "1.28"]
k8s-version: ["1.25"] #, "1.25", "1.26", "1.27", "1.28"]

env:
TF_VAR_test_name: ${{ github.event.inputs.mondooOperatorImageTag }}
Expand All @@ -142,9 +140,6 @@ jobs:
AWS_REGION: us-east-2

steps:
- uses: GitHubSecurityLab/actions-permissions/monitor@v1
with:
config: ${{ vars.PERMISSIONS_CONFIG }}
- uses: actions/checkout@v4
with:
fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile
Expand Down Expand Up @@ -217,16 +212,13 @@ jobs:
strategy:
fail-fast: false
matrix:
k8s-version: ["1.25", "1.26", "1.27"]
k8s-version: ["1.25"] #, "1.26", "1.27"]

env:
GOOGLE_APPLICATION_CREDENTIALS: ${{ format('{0}/{1}', github.workspace, 'gcp_sa.json') }}
KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/gke/kubeconfig') }}

steps:
- uses: GitHubSecurityLab/actions-permissions/monitor@v1
with:
config: ${{ vars.PERMISSIONS_CONFIG }}
- uses: actions/checkout@v4
with:
fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile
Expand Down Expand Up @@ -260,11 +252,14 @@ jobs:
go-version: "${{ env.golang-version }}"
cache: true

- name: Manually install dep
run: go get github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/[email protected]

- name: Get operator version
run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV

- name: Wait a bit for the cluster to become more stable
run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s
run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=300s

- name: Run integration tests
env:
Expand Down Expand Up @@ -302,12 +297,10 @@ jobs:
test-report:
name: Report test results
runs-on: ubuntu-latest
needs: [eks-integration-test,aks-integration-test,gke-integration-test]
#needs: [eks-integration-test,aks-integration-test,gke-integration-test]
needs: [gke-integration-test]
if: always()
steps:
- uses: GitHubSecurityLab/actions-permissions/monitor@v1
with:
config: ${{ vars.PERMISSIONS_CONFIG }}
- uses: actions/checkout@v4
with:
fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile
Expand All @@ -327,13 +320,11 @@ jobs:
discord-notification:
runs-on: ubuntu-latest
name: Send Discord notification
needs: [eks-integration-test,aks-integration-test,gke-integration-test]
#needs: [eks-integration-test,aks-integration-test,gke-integration-test]
needs: [gke-integration-test]
# Run only if the previous job has failed and only if it's running against the main branch
if: ${{ always() && contains(join(needs.*.result, ','), 'fail') && github.ref_name == 'main' }}
steps:
- uses: GitHubSecurityLab/actions-permissions/monitor@v1
with:
config: ${{ vars.PERMISSIONS_CONFIG }}
- uses: sarisia/actions-status-discord@v1
with:
webhook: ${{ secrets.DISCORD_WEBHOOK }}
Expand Down
7 changes: 7 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust
$(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./pkg/webhooks/..."

generate: controller-gen gomockgen prep/tools ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
echo "Running generate"
$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
go generate ./controllers/... ./pkg/...

Expand Down Expand Up @@ -222,6 +223,7 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in

.PHONY: generate-manifests
generate-manifests: manifests kustomize ## Generates manifests and pipes into a yaml file
echo "Running generate-manifests"
cp config/manager/kustomization.yaml config/manager/kustomization.yaml.before_kustomize
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
$(KUSTOMIZE) build config/default > mondoo-operator-manifests.yaml
Expand Down Expand Up @@ -268,7 +270,9 @@ endif
.PHONY: controller-gen
controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
$(CONTROLLER_GEN): $(LOCALBIN)
echo "Installing controller-gen"
test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
go mod tidy

.PHONY: envtest
envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
Expand All @@ -278,6 +282,7 @@ $(ENVTEST): $(LOCALBIN)
GOTESTSUM = $(LOCALBIN)/gotestsum
gotestsum: $(GOTESTSUM) ## Download gotestsum locally if necessary.
$(GOTESTSUM): $(LOCALBIN)
echo "Installing gotestsum"
test -s $(LOCALBIN)/gotestsum || GOBIN=$(LOCALBIN) go install gotest.tools/gotestsum@latest

GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
Expand Down Expand Up @@ -402,10 +407,12 @@ test/spell-check:
# An alternative (especially for local development) is to soft-link a local copy of the repo
# yourself. We don't pin submodules at this time, but we may want to check if they are up to date here.
prep/tools: prep/tools/ranger
echo "Running prep/tools"
command -v protoc-gen-go || go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
command -v protoc-gen-rangerrpc-swagger || go install go.mondoo.com/ranger-rpc/protoc-gen-rangerrpc-swagger@latest

prep/tools/ranger:
echo "prep/tools/ranger"
go install go.mondoo.com/ranger-rpc/protoc-gen-rangerrpc@latest

prep/ci/protoc:
Expand Down
9 changes: 5 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ require (
github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect
github.com/GaijinEntertainment/go-exhaustruct/v3 v3.1.0 // indirect
github.com/GoogleCloudPlatform/berglas v1.0.3 // indirect
github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.9.2 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/OpenPeeDeeP/depguard/v2 v2.1.0 // indirect
Expand Down Expand Up @@ -118,9 +119,9 @@ require (
github.com/go-critic/go-critic v0.9.0 // indirect
github.com/go-jose/go-jose/v3 v3.0.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonpointer v0.20.0 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-openapi/swag v0.22.4 // indirect
github.com/go-toolsmith/astcast v1.1.0 // indirect
github.com/go-toolsmith/astcopy v1.1.0 // indirect
github.com/go-toolsmith/astequal v1.1.0 // indirect
Expand Down Expand Up @@ -208,7 +209,7 @@ require (
github.com/mbilski/exhaustivestruct v1.2.0 // indirect
github.com/mgechev/revive v1.3.4 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moricho/tparallel v0.3.1 // indirect
github.com/mtibben/percent v0.2.1 // indirect
Expand Down Expand Up @@ -326,7 +327,7 @@ require (
github.com/google/go-containerregistry v0.16.1
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.4.0
github.com/imdario/mergo v0.3.12 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
Expand Down
Loading

0 comments on commit 2e6e2dc

Please sign in to comment.