diff --git a/.github/pre-req.sh b/.github/pre-req.sh index bec30214..4692815e 100755 --- a/.github/pre-req.sh +++ b/.github/pre-req.sh @@ -10,7 +10,9 @@ sudo apt-get update sudo apt-get install apt-transport-https ca-certificates sudo apt-get install cf-cli #CF Login -cf login --skip-ssl-validation -a $API_ENDPOINT -u $API_USER -p $API_PASSWORD -o system -s system +API_PASSWORD_DECRYPTED=$(echo "$API_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") +cf login --skip-ssl-validation -a "$API_ENDPOINT" -u "$API_USER" -p "$API_PASSWORD_DECRYPTED" + #Create splunk-ci org and space if [ "`cf o | grep "splunk-ci-org"`" == "splunk-ci-org" ]; then echo "splunk-ci-org org already exists" @@ -21,4 +23,13 @@ else cf target -o splunk-ci-org cf create-space splunk-ci-space cf target -o "splunk-ci-org" -s "splunk-ci-space" +fi + +gem install cf-uaac +uaac target "$API_UAA_ENDPOINT" --skip-ssl-validation +API_CLIENT_PASSWORD_DECRYPTED=$(echo "$API_CLIENT_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") +uaac token client get "$API_USER" -s "$API_CLIENT_PASSWORD_DECRYPTED" + +if [ $(uaac client get "$CLIENT_ID" | grep -woc "$CLIENT_ID") -eq 0 ]; then + uaac client add "$CLIENT_ID" --name splunk-firehose --secret "$CLIENT_SECRET" --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only fi \ No newline at end of file diff --git a/.github/update_manifest.sh b/.github/update_manifest.sh index 7d3baaa0..f144062f 100755 --- a/.github/update_manifest.sh +++ b/.github/update_manifest.sh @@ -2,10 +2,12 @@ set -e #Set below params in github env variable settings # API_ENDPOINT, API_USER, API_PASSWORD, SPLUNK_TOKEN, SPLUNK_HOST, SPLUNK_INDEX, SPLUNK_METRIC_INDEX +API_PASSWORD_DEC=$(echo "$API_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") +echo "$API_PASSWORD_DEC" #Update manifest for deployment sed -i 's@API_ENDPOINT:.*@'"API_ENDPOINT: $API_ENDPOINT"'@' scripts/ci_nozzle_manifest.yml sed -i 's@API_USER:.*@'"API_USER: $API_USER"'@' scripts/ci_nozzle_manifest.yml -sed -i 's@API_PASSWORD:.*@'"API_PASSWORD: $API_PASSWORD"'@' scripts/ci_nozzle_manifest.yml +sed -i 's@API_PASSWORD:.*@'"API_PASSWORD: $API_PASSWORD_DEC"'@' scripts/ci_nozzle_manifest.yml sed -i 's@CLIENT_ID:.*@'"CLIENT_ID: $CLIENT_ID"'@' scripts/ci_nozzle_manifest.yml sed -i 's@CLIENT_SECRET:.*@'"CLIENT_SECRET: $CLIENT_SECRET"'@' scripts/ci_nozzle_manifest.yml sed -i 's@SPLUNK_HOST:.*@'"SPLUNK_HOST: $SPLUNK_HOST"'@' scripts/ci_nozzle_manifest.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e7597637..6fe753af 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,9 +4,13 @@ on: branches: [ "develop" ] pull_request: branches: [ "develop" ] - + workflow_dispatch: +env: + GO_VERSION: '1.18.1' + RUBY_VERSION: '3.3' + jobs: workflow_approval: name: Approve workflow @@ -44,7 +48,7 @@ jobs: fossa test --debug env: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} - + semgrep: runs-on: ubuntu-latest needs: @@ -59,10 +63,74 @@ jobs: with: publishToken: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} + create-env: + needs: + - workflow_approval + runs-on: ubuntu-latest + timeout-minutes: 60 + outputs: + API_ENDPOINT: ${{ steps.get-credentials.outputs.API_ENDPOINT }} + API_UAA_ENDPOINT: ${{ steps.get-credentials.outputs.API_UAA_ENDPOINT }} + API_PASSWORD: ${{ steps.get-credentials.outputs.API_PASSWORD }} + API_CLIENT_PASSWORD: ${{ steps.get-credentials.outputs.API_CLIENT_PASSWORD }} + ENV_ID: ${{ steps.get-credentials.outputs.ENV_ID }} + steps: + - name: Install Selfservice + run: | + wget https://github.com/cf-platform-eng/selfservice/releases/download/0.2.9/selfservice + chmod +x selfservice + ls -latr + + - name: Get authorization token + run: | + echo "$(./selfservice auth $API_TOKEN | cut -c 8-)" >> "$GITHUB_ENV" + echo "::add-mask::$AUTH_TOKEN" + env: + API_TOKEN: ${{ secrets.API_TOKEN }} + + - name: Claim environment and wait + run: | + echo ENV_ID="$(./selfservice claimAndWait isv_ci_tas_srt_4_0 | jq -r '.id')" >> "$GITHUB_ENV" + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Set up Brew + uses: Homebrew/actions/setup-homebrew@master + + - name: Install hammer and cf + run: | + brew tap pivotal/hammer https://github.com/pivotal/hammer + brew install hammer + + brew tap pivotal-cf/om https://github.com/pivotal-cf/om + brew install om + brew install cloudfoundry/tap/cf-cli + + - name: Credentials + id: get-credentials + run: | + ./selfservice get $ENV_ID | jq -r '.credentials' > env.json + echo API_ENDPOINT="https://api.$(jq -r '.sys_domain' ./env.json)" >> "$GITHUB_OUTPUT" + echo API_UAA_ENDPOINT="https://uaa.$(jq -r '.sys_domain' ./env.json)" >> "$GITHUB_OUTPUT" + API_PASSWORD="$(hammer -t ./env.json om credentials -- -p cf -t json -c .uaa.admin_credentials | jq -r '.password')" + API_PASSWORD_ENCRYPTED="$(echo $API_PASSWORD | openssl aes-256-cbc -a -pbkdf2 -salt -pass pass:$ENCRYPT_KEY)" + echo API_PASSWORD=$API_PASSWORD_ENCRYPTED >> "$GITHUB_OUTPUT" + API_CLIENT_PASSWORD="$(hammer -t ./env.json om credentials -- -p cf -t json -c .uaa.admin_client_credentials | jq -r '.password')" + API_CLIENT_PASSWORD_ENCRYPTED="$(echo $API_CLIENT_PASSWORD | openssl aes-256-cbc -a -pbkdf2 -salt -pass pass:$ENCRYPT_KEY)" + echo API_CLIENT_PASSWORD=$API_CLIENT_PASSWORD_ENCRYPTED >> "$GITHUB_OUTPUT" + echo ENV_ID=$ENV_ID >> "$GITHUB_OUTPUT" + env: + ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} + + build: runs-on: ubuntu-latest needs: - - workflow_approval + - create-env + steps: - name: Checkout uses: actions/checkout@v3 @@ -71,7 +139,7 @@ jobs: repository: ${{github.event.pull_request.head.repo.full_name}} - uses: actions/setup-go@v3 with: - go-version: '1.17.0' + go-version: ${{ env.GO_VERSION }} - run: go version # Install Dependencies @@ -79,7 +147,7 @@ jobs: run: ls -la - name: Install Dependencies run: | - go mod vendor + go mod vendor # Builder - name: Builder @@ -96,10 +164,11 @@ jobs: path: splunk-firehose-nozzle deploy_nozzle: - - env: - API_ENDPOINT: ${{ secrets.API_ENDPOINT }} - API_PASSWORD: ${{ secrets.API_PASSWORD }} + env: + API_ENDPOINT: ${{ needs.create-env.outputs.API_ENDPOINT }} + API_PASSWORD: ${{ needs.create-env.outputs.API_PASSWORD }} + API_CLIENT_PASSWORD: ${{ needs.create-env.outputs.API_CLIENT_PASSWORD }} + API_UAA_ENDPOINT: ${{ needs.create-env.outputs.API_UAA_ENDPOINT }} API_USER: ${{ secrets.API_USER }} CLIENT_ID: ${{ secrets.CLIENT_ID }} CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} @@ -108,20 +177,26 @@ jobs: SPLUNK_HOST: ${{ secrets.SPLUNK_HOST }} SPLUNK_METRIC_INDEX: ${{ secrets.SPLUNK_METRIC_INDEX }} - needs: build + needs: + - build + - create-env runs-on: ubuntu-latest steps: - - name: Checkout uses: actions/checkout@v3 with: ref: ${{github.event.pull_request.head.sha}} repository: ${{github.event.pull_request.head.repo.full_name}} + - uses: actions/setup-go@v3 with: - go-version: '1.17.0' + go-version: ${{ env.GO_VERSION }} - run: go version + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.RUBY_VERSION }} + - run: ruby -v - uses: actions/download-artifact@v3 with: @@ -130,21 +205,29 @@ jobs: # Install dependencies - name: Install dependencies run: | - go mod vendor + go mod vendor # Deploy nozzle - name: Deploy nozzle run: | - .github/update_manifest.sh - .github/pre-req.sh - cf push -f scripts/ci_nozzle_manifest.yml -u process --random-route + .github/update_manifest.sh + .github/pre-req.sh + cf push -f scripts/ci_nozzle_manifest.yml -u process --random-route + env: + ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} + # Nozzle Log + - name: Nozzle Log + run: | + cf logs splunk-firehose-nozzle & + tile-builder: - - env: - API_ENDPOINT: ${{ secrets.API_ENDPOINT }} - API_PASSWORD: ${{ secrets.API_PASSWORD }} + env: + API_ENDPOINT: ${{ needs.create-env.outputs.API_ENDPOINT }} + API_PASSWORD: ${{ needs.create-env.outputs.API_PASSWORD }} + API_CLIENT_PASSWORD: ${{ needs.create-env.outputs.API_CLIENT_PASSWORD }} + API_UAA_ENDPOINT: ${{ needs.create-env.outputs.API_UAA_ENDPOINT }} API_USER: ${{ secrets.API_USER }} CLIENT_ID: ${{ secrets.CLIENT_ID }} CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} @@ -153,37 +236,32 @@ jobs: SPLUNK_HOST: ${{ secrets.SPLUNK_HOST }} SPLUNK_METRIC_INDEX: ${{ secrets.SPLUNK_METRIC_INDEX }} - - needs: deploy_nozzle + needs: + - deploy_nozzle + - create-env runs-on: ubuntu-latest steps: - - name: Checkout uses: actions/checkout@v3 with: ref: ${{github.event.pull_request.head.sha}} repository: ${{github.event.pull_request.head.repo.full_name}} + - uses: actions/setup-go@v3 with: - go-version: '1.17.0' + go-version: ${{ env.GO_VERSION }} - run: go version - - uses: actions/download-artifact@v3 with: name: splunk-firehose-nozzle - + # Tile builder - name: Tile builder run: | - .github/tile-builder.sh - - # Nozzle Log - - name: Nozzle Log - run: | - cf logs splunk-firehose-nozzle & - + .github/tile-builder.sh + - name: Get tile name run: | echo "tile_name=$(ls tile/product | grep ".pivotal")" >> "$GITHUB_ENV" @@ -191,18 +269,21 @@ jobs: - name: Upload tile uses: actions/upload-artifact@v2 with: - name: ${{ env.tile_name }} - path: tile/product/${{ env.tile_name }} + name: ${{ env.tile_name }} + path: tile/product/${{ env.tile_name }} # Skip test for now! execute_tests: - - needs: tile-builder + needs: + - tile-builder + - create-env runs-on: ubuntu-latest - env: - API_ENDPOINT: ${{ secrets.API_ENDPOINT }} - API_PASSWORD: ${{ secrets.API_PASSWORD }} + env: + API_ENDPOINT: ${{ needs.create-env.outputs.API_ENDPOINT }} + API_PASSWORD: ${{ needs.create-env.outputs.API_PASSWORD }} + API_CLIENT_PASSWORD: ${{ needs.create-env.outputs.API_CLIENT_PASSWORD }} + API_UAA_ENDPOINT: ${{ needs.create-env.outputs.API_UAA_ENDPOINT }} API_USER: ${{ secrets.API_USER }} CLIENT_ID: ${{ secrets.CLIENT_ID }} CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} @@ -215,58 +296,93 @@ jobs: SPLUNK_HOST: ${{ secrets.SPLUNK_HOST }} steps: - - name: Checkout uses: actions/checkout@v3 with: ref: ${{github.event.pull_request.head.sha}} repository: ${{github.event.pull_request.head.repo.full_name}} + - uses: actions/setup-go@v3 with: - go-version: '1.17.0' + go-version: ${{ env.GO_VERSION }} - run: go version + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.RUBY_VERSION }} + - run: ruby -v - uses: actions/download-artifact@v3 with: name: splunk-firehose-nozzle - # Install dependencies - name: Install dependencies run: | go mod vendor chmod +x splunk-firehose-nozzle - + # Deploy data-gen - name: Deploy data-gen run: | .github/pre-req.sh cf push -f scripts/data_gen_manifest.yml -u process -p tools/data_gen --random-route sleep 10 - + cf apps + env: + ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} # Nozzle Log - name: Nozzle Log run: | - cf logs splunk-firehose-nozzle & + cf logs splunk-firehose-nozzle & # Prepare test environment - name: Prepare test environment run: | .github/pre-functional-test.sh - # Executing tests - name: Executing tests - run: | - .github/functional-test.sh - + run: | + .github/functional-test.sh + # Teardown - name: Teardown if: always() run: | - echo "Teardown deployment env" - cf delete splunk-firehose-nozzle -f - cf delete data_gen -f - cf delete-org splunk-ci-org -f + API_PASSWORD_DEC=$(echo "$API_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") + cf login --skip-ssl-validation -a "$API_ENDPOINT" -u "$API_USER" -p "$API_PASSWORD_DEC" + echo "Teardown deployment env" + cf target -o "splunk-ci-org" -s "splunk-ci-space" + cf delete splunk-firehose-nozzle -f + cf delete data_gen -f + cf delete-org splunk-ci-org -f + env: + ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} + + + teardown-env: + runs-on: ubuntu-latest + needs: + - create-env + - execute_tests + if: always() + env: + ENV_ID: ${{ needs.create-env.outputs.ENV_ID }} + steps: + - name: install-selfservice + run: | + wget https://github.com/cf-platform-eng/selfservice/releases/download/0.2.9/selfservice + chmod +x selfservice + ls -latr + + - name: auth + run: | + echo "$(./selfservice auth $API_TOKEN | cut -c 8-)" >> "$GITHUB_ENV" + env: + API_TOKEN: ${{ secrets.API_TOKEN }} + + - name: release environment + run: | + ./selfservice release $ENV_ID \ No newline at end of file