diff --git a/.github/workflows/docker-build-api-executors-tag.yaml b/.github/workflows/docker-build-api-executors-tag.yaml index 9a578b3faf0..c51fc43f865 100644 --- a/.github/workflows/docker-build-api-executors-tag.yaml +++ b/.github/workflows/docker-build-api-executors-tag.yaml @@ -27,15 +27,10 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v1 - - name: Go Cache - uses: actions/cache@v2 + - name: Setup Golang with Cache + uses: magnetikonline/action-golang-cache@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + go-version: "1.21" - name: Login to DockerHub uses: docker/login-action@v1 @@ -191,6 +186,98 @@ jobs: DOCKER_BUILDX_CACHE_FROM: "type=gha" DOCKER_BUILDX_CACHE_TO: "type=gha,mode=max" + executor_jmeterd: + runs-on: ubuntu-latest + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - uses: sigstore/cosign-installer@v3.0.5 + - uses: anchore/sbom-action/download-syft@v0.14.2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Go Cache + uses: actions/cache@v2 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push README to Dockerhub + uses: christian-korneck/update-container-description-action@v1 + env: + DOCKER_USER: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKER_PASS: ${{ secrets.DOCKERHUB_TOKEN }} + with: + destination_container_repo: "kubeshop/testkube-jmeterd-executor" + provider: dockerhub + short_description: 'Testkube jmeterd executor' + readme_file: "./contrib/executor/jmeterd/README.md" + + - name: Release + uses: goreleaser/goreleaser-action@v4 + with: + distribution: goreleaser + version: latest + args: release -f goreleaser_files/.goreleaser-docker-build-executor-jmeterd.yml + env: + GITHUB_TOKEN: ${{ secrets.CI_BOT_TOKEN }} + DOCKER_BUILDX_BUILDER: "${{ steps.buildx.outputs.name }}" + DOCKER_BUILDX_CACHE_FROM: "type=gha" + DOCKER_BUILDX_CACHE_TO: "type=gha,mode=max" + + jmeterd_slave: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Docker Cache + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: ./contrib/executor/jmeterd/build/slaves/Dockerfile + push: true + tags: kubeshop/testkube-jmeterd-slave:${{ github.event.release.tag_name }},kubeshop/testkube-jmeterd-slave:latest + platforms: linux/amd64,linux/arm64 + executor_maven: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/docker-build-develop.yaml b/.github/workflows/docker-build-develop.yaml index 44c71b93f5a..e087c683e60 100644 --- a/.github/workflows/docker-build-develop.yaml +++ b/.github/workflows/docker-build-develop.yaml @@ -22,15 +22,10 @@ jobs: id: buildx uses: docker/setup-buildx-action@v1 - - name: Go Cache - uses: actions/cache@v2 + - name: Setup Golang with Cache + uses: magnetikonline/action-golang-cache@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + go-version: "1.21" - name: Login to DockerHub uses: docker/login-action@v1 @@ -170,6 +165,98 @@ jobs: run: | docker push kubeshop/testkube-jmeter-executor:${{ steps.github_sha.outputs.sha_short }} + executor_jmeterd: + runs-on: ubuntu-latest + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Go Cache + uses: actions/cache@v2 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - id: commit + uses: prompt/actions-commit-hash@v3 + + - name: Release + uses: goreleaser/goreleaser-action@v4 + with: + distribution: goreleaser + version: latest + args: release -f goreleaser_files/.goreleaser-docker-build-executor-jmeterd-commit-only.yml --snapshot + env: + GITHUB_TOKEN: ${{ secrets.CI_BOT_TOKEN }} + DOCKER_BUILDX_BUILDER: "${{ steps.buildx.outputs.name }}" + DOCKER_BUILDX_CACHE_FROM: "type=gha" + DOCKER_BUILDX_CACHE_TO: "type=gha,mode=max" + + - name: Push multi-arch image to the Registry + run: | + docker push kubeshop/testkube-jmeterd-executor:${{ steps.commit.outputs.short }}-amd64 + docker push kubeshop/testkube-jmeterd-executor:${{ steps.commit.outputs.short }}-arm64v8 + + docker manifest create kubeshop/testkube-jmeterd-executor:${{ steps.commit.outputs.short }} --amend kubeshop/testkube-jmeterd-executor:${{ steps.commit.outputs.short }}-amd64 --amend kubeshop/testkube-jmeterd-executor:${{ steps.commit.outputs.short }}-arm64v8 + docker manifest push -p kubeshop/testkube-jmeterd-executor:${{ steps.commit.outputs.short }} + + jmeterd_slave: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Docker Cache + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - id: commit + uses: prompt/actions-commit-hash@v3 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: ./contrib/executor/jmeterd/build/slaves/Dockerfile + push: true + tags: kubeshop/testkube-jmeterd-slave:${{ steps.commit.outputs.short }},kubeshop/testkube-jmeterd-slave:latest + platforms: linux/amd64,linux/arm64 + executor_maven: runs-on: ubuntu-latest steps: @@ -367,7 +454,7 @@ jobs: docker push kubeshop/testkube-playwright-executor:${{ steps.github_sha.outputs.sha_short }} workflow_dispatch: - needs: [api, single_executor, executor_jmeter, executor_maven, executor_gradle, executor_cypress, executor_playwright] + needs: [api, single_executor, executor_jmeter, executor_jmeterd, jmeterd_slave, executor_maven, executor_gradle, executor_cypress, executor_playwright] runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/docker-build-release.yaml b/.github/workflows/docker-build-release.yaml index a05316c3a3a..32e70ae96e8 100644 --- a/.github/workflows/docker-build-release.yaml +++ b/.github/workflows/docker-build-release.yaml @@ -23,15 +23,10 @@ jobs: id: buildx uses: docker/setup-buildx-action@v1 - - name: Go Cache - uses: actions/cache@v2 + - name: Setup Golang with Cache + uses: magnetikonline/action-golang-cache@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + go-version: "1.21" - name: Login to DockerHub uses: docker/login-action@v1 @@ -170,7 +165,59 @@ jobs: run: | docker push kubeshop/testkube-jmeter-executor:${{ steps.github_sha.outputs.sha_short }} - executor_maven: + executor_jmeterd: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Go Cache + uses: actions/cache@v2 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - id: commit + uses: prompt/actions-commit-hash@v3 + + - name: Release + uses: goreleaser/goreleaser-action@v4 + with: + distribution: goreleaser + version: latest + args: release -f goreleaser_files/.goreleaser-docker-build-executor-jmeterd-commit-only.yml --snapshot + env: + GITHUB_TOKEN: ${{ secrets.CI_BOT_TOKEN }} + DOCKER_BUILDX_BUILDER: "${{ steps.buildx.outputs.name }}" + DOCKER_BUILDX_CACHE_FROM: "type=gha" + DOCKER_BUILDX_CACHE_TO: "type=gha,mode=max" + + - name: Push Docker images + run: | + docker push kubeshop/testkube-jmeterd-executor:${{ steps.commit.outputs.short }}-amd64 + docker push kubeshop/testkube-jmeterd-executor:${{ steps.commit.outputs.short }}-arm64v8 + + docker manifest create kubeshop/testkube-jmeterd-executor:${{ steps.commit.outputs.short }} --amend kubeshop/testkube-jmeterd-executor:${{ steps.commit.outputs.short }}-amd64 --amend kubeshop/testkube-jmeterd-executor:${{ steps.commit.outputs.short }}-arm64v8 + docker manifest push -p kubeshop/testkube-jmeterd-executor:${{ steps.commit.outputs.short }} + + jmeterd_slave: runs-on: ubuntu-latest steps: - name: Checkout @@ -183,6 +230,45 @@ jobs: id: buildx uses: docker/setup-buildx-action@v1 + - name: Docker Cache + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - id: commit + uses: prompt/actions-commit-hash@v3 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: ./contrib/executor/jmeterd/build/slaves/Dockerfile + push: true + tags: kubeshop/testkube-jmeterd-slave:${{ steps.commit.outputs.short }} + platforms: linux/amd64,linux/arm64 + + executor_maven: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Go Cache uses: actions/cache@v2 with: @@ -367,7 +453,7 @@ jobs: docker push kubeshop/testkube-playwright-executor:${{ steps.github_sha.outputs.sha_short }} workflow_dispatch: - needs: [ api, single_executor, executor_jmeter, executor_maven, executor_gradle, executor_cypress, executor_playwright ] + needs: [ api, single_executor, executor_jmeter, executor_jmeterd, jmeterd_slave, executor_maven, executor_gradle, executor_cypress, executor_playwright ] runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000000..ec9953b5f24 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,72 @@ +# This workflow uses actions that are not certified by GitHub. They are provided +# by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: '35 11 * * 1' + push: + branches: [ "develop", "main" ] + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + # Uncomment the permissions below if installing in a private repository. + # contents: read + # actions: read + + steps: + - name: "Checkout code" + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 # v2.1.2 + with: + results_file: results.sarif + results_format: sarif + # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: + # - you want to enable the Branch-Protection check on a *public* repository, or + # - you are installing Scorecard on a *private* repository + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. + # repo_token: ${{ secrets.SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories: + # - `publish_results` will always be set to `false`, regardless + # of the value entered here. + publish_results: true + + # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF + # format to the repository Actions tab. + - name: "Upload artifact" + uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard. + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@17573ee1cc1b9d061760f3a006fc4aac4f944fd5 # v2.2.4 + with: + sarif_file: results.sarif diff --git a/Makefile b/Makefile index 55cbfde44f8..0f074967193 100644 --- a/Makefile +++ b/Makefile @@ -29,19 +29,28 @@ endef use-env-file: $(call setup_env) -run-api: use-env-file +.PHONY: refresh-config +refresh-config: + wget "https://raw.githubusercontent.com/kubeshop/helm-charts/develop/charts/testkube-api/executors.json" -O config/executors.json & + wget "https://raw.githubusercontent.com/kubeshop/helm-charts/develop/charts/testkube-api/job-container-template.yml" -O config/job-container-template.yml & + wget "https://raw.githubusercontent.com/kubeshop/helm-charts/develop/charts/testkube-api/job-scraper-template.yml" -O config/job-scraper-template.yml & + wget "https://raw.githubusercontent.com/kubeshop/helm-charts/develop/charts/testkube-api/job-template.yml" -O config/job-template.yml & + wget "https://raw.githubusercontent.com/kubeshop/helm-charts/develop/charts/testkube-api/pvc-container-template.yml" -O config/pvc-container-template.yml & + wget "https://raw.githubusercontent.com/kubeshop/helm-charts/develop/charts/testkube-api/slack-config.json" -O config/slack-config.json & + wget "https://raw.githubusercontent.com/kubeshop/helm-charts/develop/charts/testkube-api/slack-template.json" -O config/slack-template.json + +run-api: use-env-file refresh-config TESTKUBE_DASHBOARD_URI=$(DASHBOARD_URI) APISERVER_CONFIG=testkube-api-server-config-testkube TESTKUBE_ANALYTICS_ENABLED=$(TESTKUBE_ANALYTICS_ENABLED) TESTKUBE_NAMESPACE=$(NAMESPACE) SCRAPPERENABLED=true STORAGE_SSL=true DEBUG=$(DEBUG) APISERVER_PORT=8088 go run -ldflags='$(LD_FLAGS)' cmd/api-server/main.go -run-api-race-detector: use-env-file +run-api-race-detector: use-env-file refresh-config TESTKUBE_DASHBOARD_URI=$(DASHBOARD_URI) APISERVER_CONFIG=testkube-api-server-config-testkube TESTKUBE_NAMESPACE=$(NAMESPACE) DEBUG=1 APISERVER_PORT=8088 go run -race -ldflags='$(LD_FLAGS)' cmd/api-server/main.go -run-api-telepresence: use-env-file +run-api-telepresence: use-env-file refresh-config TESTKUBE_DASHBOARD_URI=$(DASHBOARD_URI) APISERVER_CONFIG=testkube-api-server-config-testkube TESTKUBE_NAMESPACE=$(NAMESPACE) DEBUG=1 API_MONGO_DSN=mongodb://testkube-mongodb:27017 APISERVER_PORT=8088 go run cmd/api-server/main.go run-mongo-dev: docker run --name mongodb -p 27017:27017 --rm mongo - build: build-api-server build-testkube-bin build-api-server: @@ -80,12 +89,6 @@ docker-build-api: docker-build-cli: env SLACK_BOT_CLIENT_ID=** SLACK_BOT_CLIENT_SECRET=** ANALYTICS_TRACKING_ID=** ANALYTICS_API_KEY=** SEGMENTIO_KEY=** CLOUD_SEGMENTIO_KEY=** DOCKER_BUILDX_CACHE_FROM=type=registry,ref=docker.io/kubeshop/testkube-cli:latest ALPINE_IMAGE=alpine:3.18.0 goreleaser release -f .builds-linux.goreleaser.yml --rm-dist --snapshot - -#make docker-build-cli SLACK_BOT_CLIENT_ID=** SLACK_BOT_CLIENT_SECRET=** ANALYTICS_TRACKING_ID=** ANALYTICS_API_KEY=** SEGMENTIO_KEY=** CLOUD_SEGMENTIO_KEY=** DOCKER_BUILDX_CACHE_FROM=type=registry,ref=docker.io/kubeshop/testkube-cli:latest ALPINE_IMAGE=alpine:3.18.0 -docker-build-cli: - goreleaser release -f .builds-linux.goreleaser.yml --rm-dist --snapshot - - #make docker-build-executor EXECUTOR=zap GITHUB_TOKEN=*** DOCKER_BUILDX_CACHE_FROM=type=registry,ref=docker.io/kubeshop/testkube-zap-executor:latest #add ALPINE_IMAGE=alpine:3.18.0 env var for building of curl and scraper executor docker-build-executor: @@ -160,7 +163,6 @@ create-examples: execute-testkube-cli-test-suite: test/run.sh - test-reload-sanity-test: kubectl delete secrets sanity-secrets -ntestkube kubectl delete test sanity -ntestkube || true @@ -180,7 +182,6 @@ test-api-port-forwarded: test-api-on-cluster: kubectl testkube run test sanity -f -p api_uri=http://testkube-api-server:8088 -p test_api_uri=http://testkube-api-server:8088 -p test_type=postman/collection -p test_name=fill-me -p execution_name=fill-me - cover: @go test -failfast -count=1 -v -tags test -coverprofile=./testCoverage.txt ./... && go tool cover -html=./testCoverage.txt -o testCoverage.html && rm ./testCoverage.txt open testCoverage.html @@ -202,6 +203,7 @@ version-bump-dev: commands-reference: go run cmd/kubectl-testkube/main.go generate doc +.PHONY: docs docs: commands-reference prerelease: @@ -230,7 +232,6 @@ video: ffmpeg -y -r 30 -f image2pipe -vcodec ppm -i stream.out -b 65536K movie.mp4 - port-forward-minio: kubectl port-forward svc/testkube-minio-service-testkube 9090:9090 -ntestkube diff --git a/api/v1/testkube.yaml b/api/v1/testkube.yaml index 425fcad903d..c1049a648fe 100644 --- a/api/v1/testkube.yaml +++ b/api/v1/testkube.yaml @@ -1634,6 +1634,7 @@ paths: - api parameters: - $ref: "#/components/parameters/ID" + - $ref: "#/components/parameters/SkipDeleteExecutions" summary: "Delete test" description: "Deletes a test" operationId: deleteTest @@ -2500,6 +2501,243 @@ paths: items: $ref: "#/components/schemas/Problem" + /templates: + get: + tags: + - templates + - api + summary: "List templates" + description: "List templates available in cluster" + operationId: listTemplates + parameters: + - $ref: "#/components/parameters/Selector" + responses: + 200: + description: "successful operation" + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Template" + text/yaml: + schema: + type: string + 400: + description: "problem with input for CRD generation" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + 502: + description: "problem with read information from kubernetes cluster" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + post: + tags: + - template + - api + summary: "Create new template" + description: "Create new template based on variables passed in request" + operationId: createTemplate + requestBody: + description: template request body data + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/TemplateCreateRequest" + text/yaml: + schema: + type: string + responses: + 200: + description: "successful operation" + content: + text/yaml: + schema: + type: string + 201: + description: "successful operation" + content: + application/json: + schema: + $ref: "#/components/schemas/Template" + 400: + description: "problem with template definition - probably some bad input occurs (invalid JSON body or similar)" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + 502: + description: "problem with communicating with kubernetes cluster" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + delete: + tags: + - template + - api + summary: "Delete templates" + description: "Deletes labeled templates" + operationId: deleteTemplates + parameters: + - $ref: "#/components/parameters/Selector" + responses: + 204: + description: "no content" + 502: + description: "problem with read information from kubernetes cluster" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + + /templates/{id}: + delete: + parameters: + - $ref: "#/components/parameters/ID" + tags: + - api + - template + summary: "Delete template" + description: "Deletes template by its name" + operationId: deleteTemplate + responses: + 204: + description: template deleted successfuly + 404: + description: "template not found" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + 502: + description: "problem with communicating with kubernetes cluster" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + + get: + parameters: + - $ref: "#/components/parameters/ID" + tags: + - api + - template + summary: "Get template details" + description: "Returns template" + operationId: getTemplate + responses: + 200: + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/Template" + text/yaml: + schema: + type: string + 400: + description: "problem with input for CRD generation" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + 404: + description: "template not found" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + 500: + description: "problem with getting template data" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + 502: + description: "problem with communicating with kubernetes cluster" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + patch: + parameters: + - $ref: "#/components/parameters/ID" + tags: + - template + - api + summary: "Update new template" + description: "Update new template based on variables passed in request" + operationId: updateTemplate + requestBody: + description: template request body data + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/TemplateUpdateRequest" + text/yaml: + schema: + type: string + responses: + 200: + description: "successful operation" + content: + application/json: + schema: + $ref: "#/components/schemas/Template" + 400: + description: "problem with template definition - probably some bad input occurs (invalid JSON body or similar)" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + 404: + description: "template not found" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + 502: + description: "problem with communicating with kubernetes cluster" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + /config: patch: tags: @@ -2951,6 +3189,32 @@ paths: items: $ref: "#/components/schemas/Problem" + /secrets: + get: + tags: + - secrets + - api + summary: "List secrets" + description: "List secrets available in cluster" + operationId: listSecrets + responses: + 200: + description: "successful operation" + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Secret" + 502: + description: "problem with communicating with kubernetes cluster or git server" + content: + application/problem+json: + schema: + type: array + items: + $ref: "#/components/schemas/Problem" + components: schemas: ExecutionsMetrics: @@ -3171,6 +3435,9 @@ components: $ref: "#/components/schemas/TestSuiteExecutionRequest" status: $ref: "#/components/schemas/TestSuiteStatus" + readOnly: + type: boolean + description: if test suite is offline and cannot be executed TestSuiteV2: type: object @@ -3598,6 +3865,10 @@ components: type: string description: test namespace example: "testkube" + description: + type: string + description: test description + example: "this test is used for that purpose" type: type: string description: test type @@ -3625,6 +3896,10 @@ components: type: string description: schedule to run test example: "* * * * *" + readOnly: + type: boolean + description: if test is offline and cannot be executed + uploads: type: array items: @@ -3957,7 +4232,10 @@ components: postRunScript: type: string description: script to run after test execution - example: "sleep 30" + example: "sleep 30" + executePostRunScriptBeforeScraping: + type: boolean + description: execute post run script before scraping (prebuilt executor only) runningContext: $ref: "#/components/schemas/RunningContext" description: running context for the test execution @@ -4206,6 +4484,10 @@ components: type: string description: helm chart version example: "1.4.14" + dashboardUri: + type: string + description: dashboard uri + example: "http://localhost:8080" Repository: description: repository representation for tests in git repositories @@ -4310,6 +4592,13 @@ components: items: type: string description: artifact directories for scraping + storageBucket: + type: string + description: artifact bucket storage + example: test1-artifacts + omitFolderPerExecution: + type: boolean + description: don't use a separate folder for execution artifacts ArtifactUpdateRequest: description: artifact request update body @@ -4459,9 +4748,15 @@ components: jobTemplate: type: string description: job template extensions + jobTemplateReference: + type: string + description: name of the template resource cronJobTemplate: type: string description: cron job template extensions + cronJobTemplateReference: + type: string + description: name of the template resource contentRequest: $ref: "#/components/schemas/TestContentRequest" description: adjusting parameters for test content @@ -4473,9 +4768,21 @@ components: type: string description: script to run after test execution example: "sleep 30" + executePostRunScriptBeforeScraping: + type: boolean + description: execute post run script before scraping (prebuilt executor only) scraperTemplate: type: string description: scraper template extensions + scraperTemplateReference: + type: string + description: name of the template resource + pvcTemplate: + type: string + description: pvc template extensions + pvcTemplateReference: + type: string + description: name of the template resource envConfigMaps: type: array description: "config map references" @@ -4561,9 +4868,30 @@ components: runningContext: $ref: "#/components/schemas/RunningContext" description: running context for the test suite execution + jobTemplate: + type: string + description: job template extensions + jobTemplateReference: + type: string + description: name of the template resource cronJobTemplate: type: string description: cron job template extensions + cronJobTemplateReference: + type: string + description: name of the template resource + scraperTemplate: + type: string + description: scraper template extensions + scraperTemplateReference: + type: string + description: name of the template resource + pvcTemplate: + type: string + description: pvc template extensions + pvcTemplateReference: + type: string + description: name of the template resource concurrencyLevel: type: integer format: int32 @@ -4690,6 +5018,8 @@ components: image: description: Image for kube-job type: string + slaves: + $ref: "#/components/schemas/SlavesMeta" imagePullSecrets: type: array description: "container image pull secrets" @@ -4728,6 +5058,9 @@ components: jobTemplate: description: Job template to launch executor type: string + jobTemplateReference: + type: string + description: name of the template resource labels: type: object description: "executor labels" @@ -4812,6 +5145,17 @@ components: allOf: - $ref: "#/components/schemas/ExecutorMeta" + SlavesMeta: + description: Slave data for executing tests in distributed environment + type: object + properties: + image: + description: slave image + type: string + example: kubeshop/ex-slaves-image:latest + required: + - image + RunningContext: description: running context for test or test suite execution type: object @@ -4860,9 +5204,12 @@ components: payloadTemplate: type: string description: golang based template for notification payload + payloadTemplateReference: + type: string + description: name of the template resource headers: type: object - description: "webhook headers" + description: "webhook headers (golang template supported)" additionalProperties: type: string example: @@ -5089,6 +5436,8 @@ components: $ref: "#/components/schemas/TestTriggerExecutions" testSelector: $ref: "#/components/schemas/TestTriggerSelector" + concurrencyPolicy: + $ref: "#/components/schemas/TestTriggerConcurrencyPolicies" LocalObjectReference: description: Reference to Kubernetes object @@ -5256,6 +5605,14 @@ components: example: Content-Type: "application/xml" + TestTriggerConcurrencyPolicies: + description: supported concurrency policies for test triggers + type: string + enum: + - allow + - forbid + - replace + TestTriggerKeyMap: type: object required: @@ -5263,6 +5620,7 @@ components: - actions - executions - events + - concurrencyPolicies properties: resources: type: array @@ -5284,13 +5642,13 @@ components: type: array items: type: string - description: list of supported values for resources + description: list of supported values for actions example: ["run"] executions: type: array items: type: string - description: list of supported values for resources + description: list of supported values for executions example: ["test", "testsuite"] events: type: object @@ -5310,6 +5668,12 @@ components: type: string description: list of supported values for conditions example: ["Available", "Progressing"] + concurrencyPolicies: + type: array + items: + type: string + description: list of supported values for concurrency policies + example: ["allow", "forbid", "replace"] TestSourceBatchRequest: description: Test source batch request @@ -5345,6 +5709,78 @@ components: description: deleted test sources example: ["name7", "name8", "name9"] + Template: + description: Golang based template + type: object + required: + - name + - type + - body + properties: + name: + type: string + description: template name for reference + example: "webhook-template" + namespace: + type: string + description: template namespace + example: "testkube" + type: + $ref: "#/components/schemas/TemplateType" + body: + type: string + description: template body to use + example: "{\"id\": \"{{ .Id }}\"}" + labels: + type: object + description: "template labels" + additionalProperties: + type: string + example: + env: "prod" + app: "backend" + + TemplateType: + description: template type by purpose + type: string + enum: + - job + - container + - cronjob + - scraper + - pvc + - webhook + + TemplateCreateRequest: + description: template create request body + type: object + allOf: + - $ref: "#/components/schemas/Template" + + TemplateUpdateRequest: + description: template update request body + type: object + nullable: true + allOf: + - $ref: "#/components/schemas/Template" + + Secret: + description: Secret with keys + type: object + required: + - name + properties: + name: + type: string + description: secret name + example: "git-secret" + keys: + type: array + description: secret keys + items: + type: string + example: ["key1", "key2", "key3"] + # # Errors # @@ -5536,6 +5972,14 @@ components: type: string description: mask to filter files required: false + SkipDeleteExecutions: + in: query + name: skipDeleteExecutions + schema: + type: boolean + default: false + description: dont delete executions + required: false requestBodies: UploadsBody: description: "Upload files request body data" diff --git a/cmd/api-server/main.go b/cmd/api-server/main.go index b18a0ec142c..a07dd9448fb 100644 --- a/cmd/api-server/main.go +++ b/cmd/api-server/main.go @@ -6,14 +6,14 @@ import ( "encoding/json" "flag" "fmt" - "io" "net" "os" "os/signal" - "path/filepath" "strings" "syscall" + executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" + "google.golang.org/grpc" cloudartifacts "github.com/kubeshop/testkube/pkg/cloud/data/artifact" @@ -31,6 +31,7 @@ import ( "github.com/kubeshop/testkube/internal/common" "github.com/kubeshop/testkube/internal/config" + parser "github.com/kubeshop/testkube/internal/template" "github.com/kubeshop/testkube/pkg/version" "github.com/kubeshop/testkube/pkg/cloud" @@ -43,6 +44,7 @@ import ( "github.com/pkg/errors" + "github.com/kubeshop/testkube/internal/app/api/debug" "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/pkg/agent" kubeexecutor "github.com/kubeshop/testkube/pkg/executor" @@ -57,16 +59,16 @@ import ( "github.com/kubeshop/testkube/pkg/k8sclient" "github.com/kubeshop/testkube/pkg/triggers" - kubeclient "github.com/kubeshop/testkube-operator/client" - executorsclientv1 "github.com/kubeshop/testkube-operator/client/executors/v1" - scriptsclient "github.com/kubeshop/testkube-operator/client/scripts/v2" - testexecutionsclientv1 "github.com/kubeshop/testkube-operator/client/testexecutions/v1" - testsclientv1 "github.com/kubeshop/testkube-operator/client/tests" - testsclientv3 "github.com/kubeshop/testkube-operator/client/tests/v3" - testsourcesclientv1 "github.com/kubeshop/testkube-operator/client/testsources/v1" - testsuiteexecutionsclientv1 "github.com/kubeshop/testkube-operator/client/testsuiteexecutions/v1" - testsuitesclientv2 "github.com/kubeshop/testkube-operator/client/testsuites/v2" - testsuitesclientv3 "github.com/kubeshop/testkube-operator/client/testsuites/v3" + kubeclient "github.com/kubeshop/testkube-operator/pkg/client" + scriptsclient "github.com/kubeshop/testkube-operator/pkg/client/scripts/v2" + templatesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1" + testexecutionsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testexecutions/v1" + testsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/tests" + testsclientv3 "github.com/kubeshop/testkube-operator/pkg/client/tests/v3" + testsourcesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testsources/v1" + testsuiteexecutionsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testsuiteexecutions/v1" + testsuitesclientv2 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v2" + testsuitesclientv3 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v3" apiv1 "github.com/kubeshop/testkube/internal/app/api/v1" "github.com/kubeshop/testkube/internal/migrations" "github.com/kubeshop/testkube/pkg/configmap" @@ -153,6 +155,15 @@ func main() { grpcClient = cloud.NewTestKubeCloudAPIClient(grpcConn) } + if cfg.EnableDebugServer { + debugSrv := debug.NewDebugServer(cfg.DebugListenAddr) + + g.Go(func() error { + log.DefaultLogger.Infof("starting debug pprof server") + return debugSrv.ListenAndServe() + }) + } + // k8s scriptsClient := scriptsclient.NewClient(kubeClient, cfg.TestkubeNamespace) testsClientV1 := testsclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) @@ -164,6 +175,7 @@ func main() { testsourcesClient := testsourcesclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) testExecutionsClient := testexecutionsclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) testsuiteExecutionsClient := testsuiteexecutionsclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) + templatesClient := templatesclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) clientset, err := k8sclient.ConnectToK8s() if err != nil { @@ -315,7 +327,7 @@ func main() { ui.ExitOnError("Sync default executors", err) } - jobTemplate, err := parseJobTemplate(cfg) + jobTemplate, err := parser.ParseJobTemplate(cfg) if err != nil { ui.ExitOnError("Creating job templates", err) } @@ -332,9 +344,11 @@ func main() { testsClientV3, clientset, testExecutionsClient, + templatesClient, cfg.TestkubeRegistry, cfg.TestkubePodStartTimeout, clusterId, + cfg.TestkubeDashboardURI, ) if err != nil { ui.ExitOnError("Creating executor client", err) @@ -357,9 +371,11 @@ func main() { executorsClient, testsClientV3, testExecutionsClient, + templatesClient, cfg.TestkubeRegistry, cfg.TestkubePodStartTimeout, clusterId, + cfg.TestkubeDashboardURI, ) if err != nil { ui.ExitOnError("Creating container executor", err) @@ -381,6 +397,8 @@ func main() { configMapConfig, configMapClient, testsuiteExecutionsClient, + eventBus, + cfg.TestkubeDashboardURI, ) slackLoader, err := newSlackLoader(cfg, envs) @@ -412,9 +430,12 @@ func main() { storageClient, cfg.GraphqlPort, artifactStorage, + templatesClient, cfg.CDEventsTarget, cfg.TestkubeDashboardURI, cfg.TestkubeHelmchartVersion, + mode, + eventBus, ) if mode == common.ModeAgent { @@ -460,6 +481,9 @@ func main() { log.DefaultLogger, configMapConfig, executorsClient, + executor, + eventBus, + metrics, triggers.WithHostnameIdentifier(), triggers.WithTestkubeNamespace(cfg.TestkubeNamespace), triggers.WithWatcherNamespaces(cfg.TestkubeWatcherNamespaces), @@ -512,22 +536,8 @@ func main() { } } -func parseJobTemplate(cfg *config.Config) (template string, err error) { - template, err = loadConfigFromStringOrFile( - cfg.TestkubeTemplateJob, - cfg.TestkubeConfigDir, - "job-template.yml", - "job template", - ) - if err != nil { - return "", err - } - - return template, nil -} - func parseContainerTemplates(cfg *config.Config) (t kubeexecutor.Templates, err error) { - t.Job, err = loadConfigFromStringOrFile( + t.Job, err = parser.LoadConfigFromStringOrFile( cfg.TestkubeContainerTemplateJob, cfg.TestkubeConfigDir, "job-container-template.yml", @@ -537,7 +547,7 @@ func parseContainerTemplates(cfg *config.Config) (t kubeexecutor.Templates, err return t, err } - t.Scraper, err = loadConfigFromStringOrFile( + t.Scraper, err = parser.LoadConfigFromStringOrFile( cfg.TestkubeContainerTemplateScraper, cfg.TestkubeConfigDir, "job-scraper-template.yml", @@ -547,7 +557,7 @@ func parseContainerTemplates(cfg *config.Config) (t kubeexecutor.Templates, err return t, err } - t.PVC, err = loadConfigFromStringOrFile( + t.PVC, err = parser.LoadConfigFromStringOrFile( cfg.TestkubeContainerTemplatePVC, cfg.TestkubeConfigDir, "pvc-container-template.yml", @@ -561,7 +571,7 @@ func parseContainerTemplates(cfg *config.Config) (t kubeexecutor.Templates, err } func parseDefaultExecutors(cfg *config.Config) (executors []testkube.ExecutorDetails, err error) { - rawExecutors, err := loadConfigFromStringOrFile( + rawExecutors, err := parser.LoadConfigFromStringOrFile( cfg.TestkubeDefaultExecutors, cfg.TestkubeConfigDir, "executors.json", @@ -579,7 +589,7 @@ func parseDefaultExecutors(cfg *config.Config) (executors []testkube.ExecutorDet } func newSlackLoader(cfg *config.Config, envs map[string]string) (*slack.SlackLoader, error) { - slackTemplate, err := loadConfigFromStringOrFile( + slackTemplate, err := parser.LoadConfigFromStringOrFile( cfg.SlackTemplate, cfg.TestkubeConfigDir, "slack-template.json", @@ -589,41 +599,13 @@ func newSlackLoader(cfg *config.Config, envs map[string]string) (*slack.SlackLoa return nil, err } - slackConfig, err := loadConfigFromStringOrFile(cfg.SlackConfig, cfg.TestkubeConfigDir, "slack-config.json", "slack config") + slackConfig, err := parser.LoadConfigFromStringOrFile(cfg.SlackConfig, cfg.TestkubeConfigDir, "slack-config.json", "slack config") if err != nil { return nil, err } - return slack.NewSlackLoader(slackTemplate, slackConfig, cfg.TestkubeClusterName, testkube.AllEventTypes, envs), nil -} - -func loadConfigFromStringOrFile(inputString, configDir, filename, configType string) (raw string, err error) { - var data []byte - - if inputString != "" { - if isBase64Encoded(inputString) { - data, err = base64.StdEncoding.DecodeString(inputString) - if err != nil { - return "", errors.Wrapf(err, "error decoding %s from base64", configType) - } - raw = string(data) - log.DefaultLogger.Infof("parsed %s from base64 env var", configType) - } else { - raw = inputString - log.DefaultLogger.Infof("parsed %s from plain env var", configType) - } - } else if f, err := os.Open(filepath.Join(configDir, filename)); err == nil { - data, err = io.ReadAll(f) - if err != nil { - return "", errors.Wrapf(err, "error reading file %s from config dir %s", filename, configDir) - } - raw = string(data) - log.DefaultLogger.Infof("loaded %s from file %s", configType, filepath.Join(configDir, filename)) - } else { - log.DefaultLogger.Infof("no %s config found", configType) - } - - return raw, nil + return slack.NewSlackLoader(slackTemplate, slackConfig, cfg.TestkubeClusterName, cfg.TestkubeDashboardURI, + testkube.AllEventTypes, envs), nil } func isBase64Encoded(base64Val string) bool { diff --git a/cmd/kubectl-testkube/commands/cloud/init.go b/cmd/kubectl-testkube/commands/cloud/init.go index 9a32476c7f7..bef55faaffd 100644 --- a/cmd/kubectl-testkube/commands/cloud/init.go +++ b/cmd/kubectl-testkube/commands/cloud/init.go @@ -18,7 +18,7 @@ func NewInitCmd() *cobra.Command { cmd := &cobra.Command{ Use: "init", - Short: "Install Helm chart registry in current kubectl context and update dependencies", + Short: "Install Testkube Cloud Agent and connect to Testkube Cloud environment", Aliases: []string{"install"}, Run: func(cmd *cobra.Command, args []string) { ui.Info("WELCOME TO") @@ -76,9 +76,12 @@ func NewInitCmd() *cobra.Command { cmd.Flags().StringVar(&options.Chart, "chart", "kubeshop/testkube", "chart name (usually you don't need to change it)") cmd.Flags().StringVar(&options.Name, "name", "testkube", "installation name (usually you don't need to change it)") - cmd.Flags().StringVar(&options.Namespace, "namespace", "testkube", "namespace where to install") cmd.Flags().StringVar(&options.Values, "values", "", "path to Helm values file") + cmd.Flags().BoolVar(&options.MultiNamespace, "multi-namespace", false, "multi namespace mode") + cmd.Flags().BoolVar(&options.NoOperator, "no-operator", false, "should operator be installed (for more instances in multi namespace mode it should be set to true)") + cmd.Flags().StringVar(&options.Namespace, "namespace", "testkube", "namespace where to install") + cmd.Flags().StringVar(&options.CloudAgentToken, "agent-token", "", "Testkube Cloud agent key") cmd.Flags().StringVar(&options.CloudOrgId, "org-id", "", "Testkube Cloud organization id") cmd.Flags().StringVar(&options.CloudEnvId, "env-id", "", "Testkube Cloud environment id") diff --git a/cmd/kubectl-testkube/commands/common/cloudcontext.go b/cmd/kubectl-testkube/commands/common/cloudcontext.go index fb989c4cb16..178a2813616 100644 --- a/cmd/kubectl-testkube/commands/common/cloudcontext.go +++ b/cmd/kubectl-testkube/commands/common/cloudcontext.go @@ -75,7 +75,7 @@ func UiContextHeader(cmd *cobra.Command, cfg config.Data) { header += ui.DarkGray("Env: ") + ui.White(envName) } else { header += ui.DarkGray("Context: ") + ui.White(cfg.ContextType) + ui.DarkGray(" ("+Version+")") + separator - header += ui.DarkGray("Namespace: ") + ui.White(cfg.Namespace) + header += ui.DarkGray("Namespace: ") + ui.White(cmd.Flag("namespace").Value.String()) } fmt.Println(header) diff --git a/cmd/kubectl-testkube/commands/common/helper.go b/cmd/kubectl-testkube/commands/common/helper.go index 249f8495ff4..e9b347eb835 100644 --- a/cmd/kubectl-testkube/commands/common/helper.go +++ b/cmd/kubectl-testkube/commands/common/helper.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/pkg/errors" "github.com/skratchdot/open-golang/open" "github.com/spf13/cobra" @@ -30,7 +31,9 @@ type HelmOptions struct { CloudOrgId, CloudEnvId string CloudUris CloudUris // For debug - DryRun bool + DryRun bool + MultiNamespace bool + NoOperator bool } const ( @@ -99,6 +102,8 @@ func HelmUpgradeOrInstallTestkubeCloud(options HelmOptions, cfg config.Data, isM args = append(args, "--set", fmt.Sprintf("testkube-api.cloud.orgId=%s", options.CloudOrgId)) } + args = append(args, "--set", fmt.Sprintf("testkube-api.multinamespace.enabled=%t", options.MultiNamespace)) + args = append(args, "--set", fmt.Sprintf("testkube-operator.enabled=%t", !options.NoOperator)) args = append(args, "--set", fmt.Sprintf("testkube-dashboard.enabled=%t", !options.NoDashboard)) args = append(args, "--set", fmt.Sprintf("mongodb.enabled=%t", !options.NoMongo)) args = append(args, "--set", fmt.Sprintf("testkube-api.minio.enabled=%t", !options.NoMinio)) @@ -143,6 +148,8 @@ func HelmUpgradeOrInstalTestkube(options HelmOptions) error { ui.ExitOnError("updating helm repositories", err) args = []string{"upgrade", "--install", "--create-namespace", "--namespace", options.Namespace} + args = append(args, "--set", fmt.Sprintf("testkube-api.multinamespace.enabled=%t", options.MultiNamespace)) + args = append(args, "--set", fmt.Sprintf("testkube-operator.enabled=%t", !options.NoOperator)) args = append(args, "--set", fmt.Sprintf("testkube-dashboard.enabled=%t", !options.NoDashboard)) args = append(args, "--set", fmt.Sprintf("mongodb.enabled=%t", !options.NoMongo)) args = append(args, "--set", fmt.Sprintf("testkube-api.minio.enabled=%t", !options.NoMinio)) @@ -214,7 +221,7 @@ func PopulateLoginDataToContext(orgID, envID, token, refreshToken string, option cfg, err := PopulateOrgAndEnvNames(cfg, orgID, envID, options.CloudRootDomain) if err != nil { - return err + return errors.Wrap(err, "error populating org and env names") } return config.Save(cfg) @@ -344,13 +351,13 @@ func PopulateOrgAndEnvNames(cfg config.Data, orgId, envId, rootDomain string) (c orgClient := cloudclient.NewOrganizationsClient(rootDomain, cfg.CloudContext.ApiKey) org, err := orgClient.Get(cfg.CloudContext.OrganizationId) if err != nil { - return cfg, err + return cfg, errors.Wrap(err, "error getting organization") } envsClient := cloudclient.NewEnvironmentsClient(rootDomain, cfg.CloudContext.ApiKey, cfg.CloudContext.OrganizationId) env, err := envsClient.Get(cfg.CloudContext.EnvironmentId) if err != nil { - return cfg, err + return cfg, errors.Wrap(err, "error getting environment") } cfg.CloudContext.OrganizationName = org.Name diff --git a/cmd/kubectl-testkube/commands/common/render/common.go b/cmd/kubectl-testkube/commands/common/render/common.go index ec809bba66c..ad957f27881 100644 --- a/cmd/kubectl-testkube/commands/common/render/common.go +++ b/cmd/kubectl-testkube/commands/common/render/common.go @@ -2,14 +2,16 @@ package render import ( "encoding/json" + "fmt" "io" "os" - "text/template" "gopkg.in/yaml.v2" + "github.com/kubeshop/testkube/pkg/api/v1/client" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/ui" + "github.com/kubeshop/testkube/pkg/utils" ) type OutputType string @@ -21,7 +23,7 @@ const ( OutputPretty OutputType = "pretty" ) -type CliObjRenderer func(ui *ui.UI, obj interface{}) error +type CliObjRenderer func(client client.Client, ui *ui.UI, obj interface{}) error func RenderJSON(obj interface{}, w io.Writer) error { return json.NewEncoder(w).Encode(obj) @@ -32,7 +34,7 @@ func RenderYaml(obj interface{}, w io.Writer) error { } func RenderGoTemplate(item interface{}, w io.Writer, tpl string) error { - tmpl, err := template.New("result").Parse(tpl) + tmpl, err := utils.NewTemplate("result").Parse(tpl) if err != nil { return err } @@ -41,7 +43,7 @@ func RenderGoTemplate(item interface{}, w io.Writer, tpl string) error { } func RenderGoTemplateList(list []interface{}, w io.Writer, tpl string) error { - tmpl, err := template.New("result").Parse(tpl) + tmpl, err := utils.NewTemplate("result").Parse(tpl) if err != nil { return err } @@ -63,7 +65,7 @@ func RenderPrettyList(obj ui.TableData, w io.Writer) error { return nil } -func RenderExecutionResult(execution *testkube.Execution, logsOnly bool) { +func RenderExecutionResult(client client.Client, execution *testkube.Execution, logsOnly bool) { result := execution.ExecutionResult if result == nil { @@ -84,6 +86,11 @@ func RenderExecutionResult(execution *testkube.Execution, logsOnly bool) { if !logsOnly { duration := execution.EndTime.Sub(execution.StartTime) ui.Success("Test execution completed with success in " + duration.String()) + + info, err := client.GetServerInfo() + ui.ExitOnError("getting server info", err) + + PrintExecutionURIs(execution, info.DashboardUri) } case result.IsAborted(): @@ -99,6 +106,11 @@ func RenderExecutionResult(execution *testkube.Execution, logsOnly bool) { ui.UseStderr() ui.Warn("Test execution failed:\n") ui.Errf(result.ErrorMessage) + + info, err := client.GetServerInfo() + ui.ExitOnError("getting server info", err) + + PrintExecutionURIs(execution, info.DashboardUri) } ui.Info(result.Output) @@ -118,3 +130,24 @@ func RenderExecutionResult(execution *testkube.Execution, logsOnly bool) { } } + +func PrintExecutionURIs(execution *testkube.Execution, dashboardURI string) { + ui.NL() + ui.Link("Test URI:", fmt.Sprintf("%s/tests/%s", dashboardURI, execution.TestName)) + ui.Link("Test Execution URI:", fmt.Sprintf("%s/tests/%s/executions/%s", dashboardURI, + execution.TestName, execution.Id)) + ui.NL() +} + +func PrintTestSuiteExecutionURIs(execution *testkube.TestSuiteExecution, dashboardURI string) { + ui.NL() + testSuiteName := "" + if execution.TestSuite != nil { + testSuiteName = execution.TestSuite.Name + } + + ui.Link("Test Suite URI:", fmt.Sprintf("%s/test-suites/%s", dashboardURI, testSuiteName)) + ui.Link("Test Suite Execution URI:", fmt.Sprintf("%s/test-suites/%s/executions/%s", dashboardURI, + testSuiteName, execution.Id)) + ui.NL() +} diff --git a/cmd/kubectl-testkube/commands/common/render/obj.go b/cmd/kubectl-testkube/commands/common/render/obj.go index cbb5cf56e94..f1a23fc7d0f 100644 --- a/cmd/kubectl-testkube/commands/common/render/obj.go +++ b/cmd/kubectl-testkube/commands/common/render/obj.go @@ -6,6 +6,7 @@ import ( "github.com/spf13/cobra" + "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common" "github.com/kubeshop/testkube/pkg/ui" ) @@ -15,7 +16,12 @@ func Obj(cmd *cobra.Command, obj interface{}, w io.Writer, renderer ...CliObjRen switch outputType { case OutputPretty: if len(renderer) > 0 { // if custom renderer is set render using custom pretty renderer - return renderer[0](ui.NewUI(ui.Verbose, w), obj) + client, _, err := common.GetClient(cmd) + if err != nil { + return err + } + + return renderer[0](client, ui.NewUI(ui.Verbose, w), obj) } return RenderYaml(obj, w) // fallback to yaml case OutputYAML: diff --git a/cmd/kubectl-testkube/commands/create.go b/cmd/kubectl-testkube/commands/create.go index 4e5e344afa0..46de9b05cc9 100644 --- a/cmd/kubectl-testkube/commands/create.go +++ b/cmd/kubectl-testkube/commands/create.go @@ -6,6 +6,7 @@ import ( "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common/validator" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/executors" + "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/templates" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/tests" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/testsources" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/testsuites" @@ -41,6 +42,7 @@ func NewCreateCmd() *cobra.Command { cmd.AddCommand(webhooks.NewCreateWebhookCmd()) cmd.AddCommand(executors.NewCreateExecutorCmd()) cmd.AddCommand(testsources.NewCreateTestSourceCmd()) + cmd.AddCommand(templates.NewCreateTemplateCmd()) cmd.PersistentFlags().BoolVar(&crdOnly, "crd-only", false, "generate only crd") diff --git a/cmd/kubectl-testkube/commands/delete.go b/cmd/kubectl-testkube/commands/delete.go index cb6385dec90..8f8cfa87727 100644 --- a/cmd/kubectl-testkube/commands/delete.go +++ b/cmd/kubectl-testkube/commands/delete.go @@ -6,6 +6,7 @@ import ( "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common/validator" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/executors" + "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/templates" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/tests" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/testsources" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/testsuites" @@ -40,6 +41,7 @@ func NewDeleteCmd() *cobra.Command { cmd.AddCommand(webhooks.NewDeleteWebhookCmd()) cmd.AddCommand(executors.NewDeleteExecutorCmd()) cmd.AddCommand(testsources.NewDeleteTestSourceCmd()) + cmd.AddCommand(templates.NewDeleteTemplateCmd()) return cmd } diff --git a/cmd/kubectl-testkube/commands/executors/common.go b/cmd/kubectl-testkube/commands/executors/common.go index 37d084097fd..7cdf14b2d10 100644 --- a/cmd/kubectl-testkube/commands/executors/common.go +++ b/cmd/kubectl-testkube/commands/executors/common.go @@ -32,6 +32,7 @@ func NewUpsertExecutorOptionsFromFlags(cmd *cobra.Command) (options apiClient.Up return options, err } + jobTemplateReference := cmd.Flag("job-template-reference").Value.String() jobTemplate := cmd.Flag("job-template").Value.String() jobTemplateContent := "" if jobTemplate != "" { @@ -83,19 +84,20 @@ func NewUpsertExecutorOptionsFromFlags(cmd *cobra.Command) (options apiClient.Up } options = apiClient.UpsertExecutorOptions{ - Name: name, - Types: types, - ExecutorType: executorType, - Image: image, - ImagePullSecrets: imageSecrets, - Command: command, - Args: executorArgs, - Uri: uri, - ContentTypes: contentTypes, - JobTemplate: jobTemplateContent, - Features: features, - Labels: labels, - Meta: meta, + Name: name, + Types: types, + ExecutorType: executorType, + Image: image, + ImagePullSecrets: imageSecrets, + Command: command, + Args: executorArgs, + Uri: uri, + ContentTypes: contentTypes, + JobTemplate: jobTemplateContent, + JobTemplateReference: jobTemplateReference, + Features: features, + Labels: labels, + Meta: meta, } return options, nil @@ -123,6 +125,10 @@ func NewUpdateExecutorOptionsFromFlags(cmd *cobra.Command) (options apiClient.Up "image", &options.Image, }, + { + "job-template-reference", + &options.JobTemplateReference, + }, } for _, field := range fields { diff --git a/cmd/kubectl-testkube/commands/executors/create.go b/cmd/kubectl-testkube/commands/executors/create.go index 1dbf04372cd..de1eee972f8 100644 --- a/cmd/kubectl-testkube/commands/executors/create.go +++ b/cmd/kubectl-testkube/commands/executors/create.go @@ -1,6 +1,7 @@ package executors import ( + "fmt" "strconv" "github.com/spf13/cobra" @@ -14,9 +15,10 @@ import ( func NewCreateExecutorCmd() *cobra.Command { var ( - types, command, executorArgs, imagePullSecretNames, features, contentTypes []string - name, executorType, image, uri, jobTemplate, iconURI, docsURI string - labels, tooltips map[string]string + types, command, executorArgs, imagePullSecretNames, features, contentTypes []string + name, executorType, image, uri, jobTemplate, iconURI, docsURI, jobTemplateReference string + labels, tooltips map[string]string + update bool ) cmd := &cobra.Command{ @@ -40,7 +42,25 @@ func NewCreateExecutorCmd() *cobra.Command { executor, _ := client.GetExecutor(name) if name == executor.Name { - ui.Failf("Executor with name '%s' already exists in namespace %s", name, namespace) + if cmd.Flag("update").Changed { + if !update { + ui.Failf("Executor with name '%s' already exists in namespace %s, ", executor.Name, namespace) + } + } else { + ok := ui.Confirm(fmt.Sprintf("Executor with name '%s' already exists in namespace %s, ", executor.Name, namespace) + + "do you want to overwrite it?") + if !ok { + ui.Failf("Executor creation was aborted") + } + } + + options, err := NewUpdateExecutorOptionsFromFlags(cmd) + ui.ExitOnError("getting executor options", err) + + _, err = client.UpdateExecutor(options) + ui.ExitOnError("updating executor "+name+" in namespace "+namespace, err) + + ui.SuccessAndExit("Executor updated", name) } } @@ -72,12 +92,14 @@ func NewCreateExecutorCmd() *cobra.Command { cmd.Flags().StringArrayVar(&command, "command", []string{}, "command passed to image in executor") cmd.Flags().StringArrayVar(&executorArgs, "args", []string{}, "args passed to image in executor") cmd.Flags().StringVarP(&jobTemplate, "job-template", "j", "", "if executor needs to be launched using custom job specification, then a path to template file should be provided") + cmd.Flags().StringVarP(&jobTemplateReference, "job-template-reference", "", "", "reference to job template for using with executor") cmd.Flags().StringToStringVarP(&labels, "label", "l", nil, "label key value pair: --label key1=value1") cmd.Flags().StringArrayVar(&features, "feature", []string{}, "feature provided by executor") cmd.Flags().StringVarP(&iconURI, "icon-uri", "", "", "URI to executor icon") cmd.Flags().StringVarP(&docsURI, "docs-uri", "", "", "URI to executor docs") cmd.Flags().StringArrayVar(&contentTypes, "content-type", []string{}, "list of supported content types for executor") cmd.Flags().StringToStringVarP(&tooltips, "tooltip", "", nil, "tooltip key value pair: --tooltip key1=value1") + cmd.Flags().BoolVar(&update, "update", false, "update, if executor already exists") return cmd } diff --git a/cmd/kubectl-testkube/commands/executors/get.go b/cmd/kubectl-testkube/commands/executors/get.go index 47f3f2fd9a0..033561ded0d 100644 --- a/cmd/kubectl-testkube/commands/executors/get.go +++ b/cmd/kubectl-testkube/commands/executors/get.go @@ -86,6 +86,7 @@ func mapExecutorDetailsToCreateExecutorOptions(namespace string, executor *testk options.Args = executor.Executor.Args options.Uri = executor.Executor.Uri options.Labels = executor.Executor.Labels + options.JobTemplateReference = executor.Executor.JobTemplateReference if executor.Executor.JobTemplate != "" { options.JobTemplate = fmt.Sprintf("%q", executor.Executor.JobTemplate) } diff --git a/cmd/kubectl-testkube/commands/executors/update.go b/cmd/kubectl-testkube/commands/executors/update.go index bc8f08ab1c2..f87c1700791 100644 --- a/cmd/kubectl-testkube/commands/executors/update.go +++ b/cmd/kubectl-testkube/commands/executors/update.go @@ -9,9 +9,9 @@ import ( func UpdateExecutorCmd() *cobra.Command { var ( - types, command, executorArgs, imagePullSecretNames, features, contentTypes []string - name, executorType, image, uri, jobTemplate, iconURI, docsURI string - labels, tooltips map[string]string + types, command, executorArgs, imagePullSecretNames, features, contentTypes []string + name, executorType, image, uri, jobTemplate, iconURI, docsURI, jobTemplateReference string + labels, tooltips map[string]string ) cmd := &cobra.Command{ @@ -52,6 +52,7 @@ func UpdateExecutorCmd() *cobra.Command { cmd.Flags().StringArrayVar(&command, "command", []string{}, "command passed to image in executor") cmd.Flags().StringArrayVar(&executorArgs, "args", []string{}, "args passed to image in executor") cmd.Flags().StringVarP(&jobTemplate, "job-template", "j", "", "if executor needs to be launched using custom job specification, then a path to template file should be provided") + cmd.Flags().StringVarP(&jobTemplateReference, "job-template-reference", "", "", "reference to job template for using with executor") cmd.Flags().StringToStringVarP(&labels, "label", "l", nil, "label key value pair: --label key1=value1") cmd.Flags().StringArrayVar(&features, "feature", []string{}, "feature provided by executor") cmd.Flags().StringVarP(&iconURI, "icon-uri", "", "", "URI to executor icon") diff --git a/cmd/kubectl-testkube/commands/get.go b/cmd/kubectl-testkube/commands/get.go index c7cfb17dfad..b0355e79623 100644 --- a/cmd/kubectl-testkube/commands/get.go +++ b/cmd/kubectl-testkube/commands/get.go @@ -8,6 +8,7 @@ import ( "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common/validator" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/context" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/executors" + "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/templates" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/tests" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/testsources" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/testsuites" @@ -46,6 +47,7 @@ func NewGetCmd() *cobra.Command { cmd.AddCommand(testsuites.NewTestSuiteExecutionCmd()) cmd.AddCommand(testsources.NewGetTestSourceCmd()) cmd.AddCommand(context.NewGetContextCmd()) + cmd.AddCommand(templates.NewGetTemplateCmd()) cmd.PersistentFlags().StringP("output", "o", "pretty", "output type can be one of json|yaml|pretty|go-template") cmd.PersistentFlags().StringP("go-template", "", "{{.}}", "go template to render") diff --git a/cmd/kubectl-testkube/commands/github/issue.go b/cmd/kubectl-testkube/commands/github/issue.go index 33b49c91af0..2e2864ebb4c 100644 --- a/cmd/kubectl-testkube/commands/github/issue.go +++ b/cmd/kubectl-testkube/commands/github/issue.go @@ -4,13 +4,13 @@ import ( "bytes" "errors" "fmt" - "html/template" prShared "github.com/cli/cli/v2/pkg/cmd/pr/shared" "github.com/skratchdot/open-golang/open" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/ui" + "github.com/kubeshop/testkube/pkg/utils" ) const ( @@ -76,7 +76,7 @@ func buildTicket(d testkube.DebugInfo) (string, string, error) { if d.ClientVersion == "" || d.ClusterVersion == "" { return "", "", errors.New("client version and cluster version must be populated to create debug message") } - t, err := template.New("debug").Parse(Template) + t, err := utils.NewTemplate("debug").Parse(Template) if err != nil { return "", "", fmt.Errorf("cannot create template: %w", err) } diff --git a/cmd/kubectl-testkube/commands/root.go b/cmd/kubectl-testkube/commands/root.go index c0b27a0cd0d..b871110c1e0 100644 --- a/cmd/kubectl-testkube/commands/root.go +++ b/cmd/kubectl-testkube/commands/root.go @@ -87,7 +87,9 @@ var RootCmd = &cobra.Command{ } serverCfg, err := client.GetConfig() - ui.WarnOnError("getting config", err) + if ui.Verbose && err != nil { + ui.Err(err) + } if clientCfg.TelemetryEnabled != serverCfg.EnableTelemetry && err == nil { if serverCfg.EnableTelemetry { diff --git a/cmd/kubectl-testkube/commands/templates/common.go b/cmd/kubectl-testkube/commands/templates/common.go new file mode 100644 index 00000000000..e2ca1f283c2 --- /dev/null +++ b/cmd/kubectl-testkube/commands/templates/common.go @@ -0,0 +1,104 @@ +package templates + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + apiv1 "github.com/kubeshop/testkube/pkg/api/v1/client" + "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/ui" +) + +// NewCreateTemplateOptionsFromFlags creates create template options from command flags +func NewCreateTemplateOptionsFromFlags(cmd *cobra.Command) (options apiv1.CreateTemplateOptions, err error) { + name := cmd.Flag("name").Value.String() + namespace := cmd.Flag("namespace").Value.String() + if err != nil { + return options, err + } + + templateType := testkube.TemplateType(cmd.Flag("template-type").Value.String()) + + if templateType != testkube.JOB_TemplateType && templateType != testkube.CRONJOB_TemplateType && + templateType != testkube.SCRAPER_TemplateType && templateType != testkube.PVC_TemplateType && + templateType != testkube.WEBHOOK_TemplateType { + ui.Failf("invalid template type: %s. use one of job|container|cronjob|scraper|pvc|webhook", templateType) + } + + body := cmd.Flag("body").Value.String() + bodyContent := "" + if body != "" { + b, err := os.ReadFile(body) + ui.ExitOnError("reading template body", err) + bodyContent = string(b) + } + + labels, err := cmd.Flags().GetStringToString("label") + if err != nil { + return options, err + } + + options = apiv1.CreateTemplateOptions{ + Name: name, + Namespace: namespace, + Type_: &templateType, + Labels: labels, + Body: bodyContent, + } + + return options, nil +} + +// NewUpdateTemplateOptionsFromFlags creates update template options from command flags +func NewUpdateTemplateOptionsFromFlags(cmd *cobra.Command) (options apiv1.UpdateTemplateOptions, err error) { + var fields = []struct { + name string + destination **string + }{ + { + "name", + &options.Name, + }, + } + + for _, field := range fields { + if cmd.Flag(field.name).Changed { + value := cmd.Flag(field.name).Value.String() + *field.destination = &value + } + } + + if cmd.Flag("template-type").Changed { + templateType := testkube.TemplateType(cmd.Flag("template-type").Value.String()) + if templateType != testkube.JOB_TemplateType && templateType != testkube.CRONJOB_TemplateType && + templateType != testkube.SCRAPER_TemplateType && templateType != testkube.PVC_TemplateType && + templateType != testkube.WEBHOOK_TemplateType { + ui.Failf("invalid template type: %s. use one of job|container|cronjob|scraper|pvc|webhook", templateType) + } + options.Type_ = &templateType + } + + if cmd.Flag("body").Changed { + body := cmd.Flag("body").Value.String() + b, err := os.ReadFile(body) + if err != nil { + return options, fmt.Errorf("reading template body %w", err) + } + + value := string(b) + options.Body = &value + } + + if cmd.Flag("label").Changed { + labels, err := cmd.Flags().GetStringToString("label") + if err != nil { + return options, err + } + + options.Labels = &labels + } + + return options, nil +} diff --git a/cmd/kubectl-testkube/commands/templates/create.go b/cmd/kubectl-testkube/commands/templates/create.go new file mode 100644 index 00000000000..8d03a7d5b55 --- /dev/null +++ b/cmd/kubectl-testkube/commands/templates/create.go @@ -0,0 +1,95 @@ +package templates + +import ( + "fmt" + "strconv" + + "github.com/spf13/cobra" + + "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common" + apiv1 "github.com/kubeshop/testkube/pkg/api/v1/client" + "github.com/kubeshop/testkube/pkg/crd" + "github.com/kubeshop/testkube/pkg/ui" +) + +func NewCreateTemplateCmd() *cobra.Command { + var ( + name string + templateType string + labels map[string]string + body string + update bool + ) + + cmd := &cobra.Command{ + Use: "template", + Aliases: []string{"tp"}, + Short: "Create a new Template.", + Long: `Create a new Template Custom Resource.`, + Run: func(cmd *cobra.Command, args []string) { + crdOnly, err := strconv.ParseBool(cmd.Flag("crd-only").Value.String()) + ui.ExitOnError("parsing flag value", err) + + if name == "" { + ui.Failf("pass valid name (in '--name' flag)") + } + + namespace := cmd.Flag("namespace").Value.String() + var client apiv1.Client + if !crdOnly { + client, namespace, err = common.GetClient(cmd) + ui.ExitOnError("getting client", err) + + template, _ := client.GetTemplate(name) + if name == template.Name { + if cmd.Flag("update").Changed { + if !update { + ui.Failf("Template with name '%s' already exists in namespace %s, ", template.Name, namespace) + } + } else { + ok := ui.Confirm(fmt.Sprintf("Template with name '%s' already exists in namespace %s, ", template.Name, namespace) + + "do you want to overwrite it?") + if !ok { + ui.Failf("Template creation was aborted") + } + } + + options, err := NewUpdateTemplateOptionsFromFlags(cmd) + ui.ExitOnError("getting template options", err) + + _, err = client.UpdateTemplate(options) + ui.ExitOnError("updating template "+name+" in namespace "+namespace, err) + + ui.SuccessAndExit("Template updated", name) + } + } + + options, err := NewCreateTemplateOptionsFromFlags(cmd) + ui.ExitOnError("getting template options", err) + + if !crdOnly { + _, err := client.CreateTemplate(options) + ui.ExitOnError("creating template "+name+" in namespace "+namespace, err) + + ui.Success("Template created", name) + } else { + if options.Body != "" { + options.Body = fmt.Sprintf("%q", options.Body) + } + + data, err := crd.ExecuteTemplate(crd.TemplateTemplate, options) + ui.ExitOnError("executing crd template", err) + + ui.Info(data) + } + }, + } + + cmd.Flags().StringVarP(&name, "name", "n", "", "unique template name - mandatory") + cmd.Flags().StringVarP(&templateType, "template-type", "", "", "template type one of job|container|cronjob|scraper|pvc|webhook") + cmd.Flags().StringToStringVarP(&labels, "label", "l", nil, "label key value pair: --label key1=value1") + cmd.Flags().StringVarP(&body, "body", "", "", "a path to template file to use as template body") + cmd.Flags().BoolVar(&update, "update", false, "update, if template already exists") + + return cmd +} diff --git a/cmd/kubectl-testkube/commands/templates/delete.go b/cmd/kubectl-testkube/commands/templates/delete.go new file mode 100644 index 00000000000..f6caef2981c --- /dev/null +++ b/cmd/kubectl-testkube/commands/templates/delete.go @@ -0,0 +1,48 @@ +package templates + +import ( + "strings" + + "github.com/spf13/cobra" + + "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common" + "github.com/kubeshop/testkube/pkg/ui" +) + +func NewDeleteTemplateCmd() *cobra.Command { + var name string + var selectors []string + + cmd := &cobra.Command{ + + Use: "template ", + Aliases: []string{"tp"}, + Short: "Delete a template.", + Long: `Delete a template and pass the template name to be deleted.`, + Run: func(cmd *cobra.Command, args []string) { + client, _, err := common.GetClient(cmd) + ui.ExitOnError("getting client", err) + + if len(args) > 0 { + name = args[0] + err := client.DeleteTemplate(name) + ui.ExitOnError("deleting template: "+name, err) + ui.SuccessAndExit("Succesfully deleted template", name) + } + + if len(selectors) != 0 { + selector := strings.Join(selectors, ",") + err := client.DeleteTemplates(selector) + ui.ExitOnError("deleting templates by labels: "+selector, err) + ui.SuccessAndExit("Succesfully deleted templates by labels", selector) + } + + ui.Failf("Pass Template name or labels to delete by labels") + }, + } + + cmd.Flags().StringVarP(&name, "name", "n", "", "unique template name, you can also pass it as first argument") + cmd.Flags().StringSliceVarP(&selectors, "label", "l", nil, "label key value pair: --label key1=value1") + + return cmd +} diff --git a/cmd/kubectl-testkube/commands/templates/get.go b/cmd/kubectl-testkube/commands/templates/get.go new file mode 100644 index 00000000000..cd5e3c60155 --- /dev/null +++ b/cmd/kubectl-testkube/commands/templates/get.go @@ -0,0 +1,74 @@ +package templates + +import ( + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + + "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common" + "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common/render" + "github.com/kubeshop/testkube/pkg/crd" + "github.com/kubeshop/testkube/pkg/ui" +) + +func NewGetTemplateCmd() *cobra.Command { + var name string + var selectors []string + var crdOnly bool + + cmd := &cobra.Command{ + Use: "template ", + Aliases: []string{"templates", "tp"}, + Short: "Get template details.", + Long: `Get template allows you to change the output format. To get single details, pass the template name as the first argument.`, + Run: func(cmd *cobra.Command, args []string) { + client, _, err := common.GetClient(cmd) + ui.ExitOnError("getting client", err) + + firstEntry := true + if len(args) > 0 { + name := args[0] + template, err := client.GetTemplate(name) + ui.ExitOnError("getting template: "+name, err) + + if crdOnly { + if template.Body != "" { + template.Body = fmt.Sprintf("%q", template.Body) + } + + common.UIPrintCRD(crd.TemplateTemplate, template, &firstEntry) + return + } + + err = render.Obj(cmd, template, os.Stdout) + ui.ExitOnError("rendering obj", err) + } else { + templates, err := client.ListTemplates(strings.Join(selectors, ",")) + ui.ExitOnError("getting templates", err) + + if crdOnly { + for _, template := range templates { + if template.Body != "" { + template.Body = fmt.Sprintf("%q", template.Body) + } + + common.UIPrintCRD(crd.TemplateTemplate, template, &firstEntry) + } + + return + } + + err = render.List(cmd, templates, os.Stdout) + ui.ExitOnError("rendering list", err) + } + }, + } + + cmd.Flags().StringVarP(&name, "name", "n", "", "unique template name, you can also pass it as argument") + cmd.Flags().StringSliceVarP(&selectors, "label", "l", nil, "label key value pair: --label key1=value1") + cmd.Flags().BoolVar(&crdOnly, "crd-only", false, "show only test crd") + + return cmd +} diff --git a/cmd/kubectl-testkube/commands/templates/update.go b/cmd/kubectl-testkube/commands/templates/update.go new file mode 100644 index 00000000000..571f89e8749 --- /dev/null +++ b/cmd/kubectl-testkube/commands/templates/update.go @@ -0,0 +1,52 @@ +package templates + +import ( + "github.com/spf13/cobra" + + "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common" + "github.com/kubeshop/testkube/pkg/ui" +) + +func UpdateTemplateCmd() *cobra.Command { + var ( + name string + templateType string + labels map[string]string + body string + ) + + cmd := &cobra.Command{ + Use: "template", + Aliases: []string{"templates", "tp"}, + Short: "Update Template", + Long: `Update Template Custom Resource.`, + Run: func(cmd *cobra.Command, args []string) { + if name == "" { + ui.Failf("pass valid name (in '--name' flag)") + } + + client, namespace, err := common.GetClient(cmd) + ui.ExitOnError("getting client", err) + + template, _ := client.GetTemplate(name) + if name != template.Name { + ui.Failf("Template with name '%s' not exists in namespace %s", name, namespace) + } + + options, err := NewUpdateTemplateOptionsFromFlags(cmd) + ui.ExitOnError("getting template options", err) + + _, err = client.UpdateTemplate(options) + ui.ExitOnError("updating template "+name+" in namespace "+namespace, err) + + ui.Success("Template updated", name) + }, + } + + cmd.Flags().StringVarP(&name, "name", "n", "", "unique template name - mandatory") + cmd.Flags().StringVarP(&templateType, "template-type", "", "", "template type one of job|container|cronjob|scraper|pvc|webhook") + cmd.Flags().StringToStringVarP(&labels, "label", "l", nil, "label key value pair: --label key1=value1") + cmd.Flags().StringVarP(&body, "body", "", "", "a path to template file to use as template body") + + return cmd +} diff --git a/cmd/kubectl-testkube/commands/tests/common.go b/cmd/kubectl-testkube/commands/tests/common.go index 0e79c1dd640..a4d17d8ebbb 100644 --- a/cmd/kubectl-testkube/commands/tests/common.go +++ b/cmd/kubectl-testkube/commands/tests/common.go @@ -228,11 +228,20 @@ func newArtifactRequestFromFlags(cmd *cobra.Command) (request *testkube.Artifact return nil, err } - if artifactStorageClassName != "" || artifactVolumeMountPath != "" || len(dirs) != 0 { + artifactStorageBucket := cmd.Flag("artifact-storage-bucket").Value.String() + artifactOmitFolderPerExecution, err := cmd.Flags().GetBool("artifact-omit-folder-per-execution") + if err != nil { + return nil, err + } + + if artifactStorageClassName != "" || artifactVolumeMountPath != "" || len(dirs) != 0 || + artifactStorageBucket != "" || artifactOmitFolderPerExecution { request = &testkube.ArtifactRequest{ - StorageClassName: artifactStorageClassName, - VolumeMountPath: artifactVolumeMountPath, - Dirs: dirs, + StorageClassName: artifactStorageClassName, + VolumeMountPath: artifactVolumeMountPath, + Dirs: dirs, + StorageBucket: artifactStorageBucket, + OmitFolderPerExecution: artifactOmitFolderPerExecution, } } @@ -380,89 +389,82 @@ func newExecutionRequestFromFlags(cmd *cobra.Command) (request *testkube.Executi imageSecrets = append(imageSecrets, testkube.LocalObjectReference{Name: secretName}) } - jobTemplateContent := "" - jobTemplate := cmd.Flag("job-template").Value.String() - if jobTemplate != "" { - b, err := os.ReadFile(jobTemplate) - if err != nil { - return nil, err - } - - jobTemplateContent = string(b) + jobTemplateReference := cmd.Flag("job-template-reference").Value.String() + cronJobTemplateReference := cmd.Flag("cronjob-template-reference").Value.String() + scraperTemplateReference := cmd.Flag("scraper-template-reference").Value.String() + pvcTemplateReference := cmd.Flag("pvc-template-reference").Value.String() + executePostRunScriptBeforeScraping, err := cmd.Flags().GetBool("execute-postrun-script-before-scraping") + if err != nil { + return nil, err } - cronJobTemplateContent := "" - cronJobTemplate := cmd.Flag("cronjob-template").Value.String() - if cronJobTemplate != "" { - b, err := os.ReadFile(cronJobTemplate) - if err != nil { - return nil, err - } - - cronJobTemplateContent = string(b) + request = &testkube.ExecutionRequest{ + Name: executionName, + Variables: variables, + Image: image, + Command: command, + Args: executorArgs, + ArgsMode: mode, + ImagePullSecrets: imageSecrets, + Envs: envs, + SecretEnvs: secretEnvs, + HttpProxy: httpProxy, + HttpsProxy: httpsProxy, + ActiveDeadlineSeconds: timeout, + JobTemplateReference: jobTemplateReference, + CronJobTemplateReference: cronJobTemplateReference, + ScraperTemplateReference: scraperTemplateReference, + PvcTemplateReference: pvcTemplateReference, + NegativeTest: negativeTest, + ExecutePostRunScriptBeforeScraping: executePostRunScriptBeforeScraping, } - preRunScriptContent := "" - preRunScript := cmd.Flag("prerun-script").Value.String() - if preRunScript != "" { - b, err := os.ReadFile(preRunScript) - if err != nil { - return nil, err - } - - preRunScriptContent = string(b) + var fields = []struct { + source string + destination *string + }{ + { + cmd.Flag("job-template").Value.String(), + &request.JobTemplate, + }, + { + cmd.Flag("cronjob-template").Value.String(), + &request.CronJobTemplate, + }, + { + cmd.Flag("prerun-script").Value.String(), + &request.PreRunScript, + }, + { + cmd.Flag("postrun-script").Value.String(), + &request.PostRunScript, + }, + { + cmd.Flag("scraper-template").Value.String(), + &request.ScraperTemplate, + }, + { + cmd.Flag("pvc-template").Value.String(), + &request.PvcTemplate, + }, } - postRunScriptContent := "" - postRunScript := cmd.Flag("postrun-script").Value.String() - if postRunScript != "" { - b, err := os.ReadFile(postRunScript) - if err != nil { - return nil, err - } - - postRunScriptContent = string(b) - } + for _, field := range fields { + if field.source != "" { + b, err := os.ReadFile(field.source) + if err != nil { + return nil, err + } - scraperTemplateContent := "" - scraperTemplate := cmd.Flag("scraper-template").Value.String() - if scraperTemplate != "" { - b, err := os.ReadFile(scraperTemplate) - if err != nil { - return nil, err + *field.destination = string(b) } - - scraperTemplateContent = string(b) } - envConfigMaps, envSecrets, err := newEnvReferencesFromFlags(cmd) + request.EnvConfigMaps, request.EnvSecrets, err = newEnvReferencesFromFlags(cmd) if err != nil { return nil, err } - request = &testkube.ExecutionRequest{ - Name: executionName, - Variables: variables, - Image: image, - Command: command, - Args: executorArgs, - ArgsMode: mode, - ImagePullSecrets: imageSecrets, - Envs: envs, - SecretEnvs: secretEnvs, - HttpProxy: httpProxy, - HttpsProxy: httpsProxy, - ActiveDeadlineSeconds: timeout, - JobTemplate: jobTemplateContent, - CronJobTemplate: cronJobTemplateContent, - PreRunScript: preRunScriptContent, - PostRunScript: postRunScriptContent, - ScraperTemplate: scraperTemplateContent, - NegativeTest: negativeTest, - EnvConfigMaps: envConfigMaps, - EnvSecrets: envSecrets, - } - request.ArtifactRequest, err = newArtifactRequestFromFlags(cmd) if err != nil { return nil, err @@ -482,6 +484,7 @@ func NewUpsertTestOptionsFromFlags(cmd *cobra.Command) (options apiclientv1.Upse file := cmd.Flag("file").Value.String() executorType := cmd.Flag("type").Value.String() namespace := cmd.Flag("namespace").Value.String() + description := cmd.Flag("description").Value.String() labels, err := cmd.Flags().GetStringToString("label") if err != nil { return options, err @@ -502,14 +505,15 @@ func NewUpsertTestOptionsFromFlags(cmd *cobra.Command) (options apiclientv1.Upse sourceName = cmd.Flag("source").Value.String() } options = apiclientv1.UpsertTestOptions{ - Name: name, - Type_: executorType, - Content: content, - Source: sourceName, - Namespace: namespace, - Schedule: schedule, - Uploads: copyFiles, - Labels: labels, + Name: name, + Description: description, + Type_: executorType, + Content: content, + Source: sourceName, + Namespace: namespace, + Schedule: schedule, + Uploads: copyFiles, + Labels: labels, } options.ExecutionRequest, err = newExecutionRequestFromFlags(cmd) @@ -641,6 +645,10 @@ func NewUpdateTestOptionsFromFlags(cmd *cobra.Command) (options apiclientv1.Upda "source", &options.Source, }, + { + "description", + &options.Description, + }, } for _, field := range fields { @@ -783,6 +791,22 @@ func newExecutionUpdateRequestFromFlags(cmd *cobra.Command) (request *testkube.E "args-mode", &request.ArgsMode, }, + { + "job-template-reference", + &request.JobTemplateReference, + }, + { + "cronjob-template-reference", + &request.CronJobTemplateReference, + }, + { + "scraper-template-reference", + &request.ScraperTemplateReference, + }, + { + "pvc-template-reference", + &request.PvcTemplateReference, + }, } var nonEmpty bool @@ -904,84 +928,52 @@ func newExecutionUpdateRequestFromFlags(cmd *cobra.Command) (request *testkube.E nonEmpty = true } - if cmd.Flag("job-template").Changed { - jobTemplateContent := "" - jobTemplate := cmd.Flag("job-template").Value.String() - if jobTemplate != "" { - b, err := os.ReadFile(jobTemplate) - if err != nil { - return nil, err - } - - jobTemplateContent = string(b) - } - - request.JobTemplate = &jobTemplateContent - nonEmpty = true - } - - if cmd.Flag("cronjob-template").Changed { - cronJobTemplateContent := "" - cronJobTemplate := cmd.Flag("cronjob-template").Value.String() - if cronJobTemplate != "" { - b, err := os.ReadFile(cronJobTemplate) - if err != nil { - return nil, err - } - - cronJobTemplateContent = string(b) - } - - request.CronJobTemplate = &cronJobTemplateContent - nonEmpty = true - } - - if cmd.Flag("prerun-script").Changed { - preRunScriptContent := "" - preRunScript := cmd.Flag("prerun-script").Value.String() - if preRunScript != "" { - b, err := os.ReadFile(preRunScript) - if err != nil { - return nil, err - } - - preRunScriptContent = string(b) - } - - request.PreRunScript = &preRunScriptContent - nonEmpty = true + var values = []struct { + source string + destination **string + }{ + { + "job-template", + &request.JobTemplate, + }, + { + "cronjob-template", + &request.CronJobTemplate, + }, + { + "prerun-script", + &request.PreRunScript, + }, + { + "postrun-script", + &request.PostRunScript, + }, + { + "scraper-template", + &request.ScraperTemplate, + }, + { + "pvc-template", + &request.PvcTemplate, + }, } - if cmd.Flag("postrun-script").Changed { - postRunScriptContent := "" - postRunScript := cmd.Flag("postrun-script").Value.String() - if postRunScript != "" { - b, err := os.ReadFile(postRunScript) - if err != nil { - return nil, err - } - - postRunScriptContent = string(b) - } - - request.PostRunScript = &postRunScriptContent - nonEmpty = true - } + for _, value := range values { + if cmd.Flag(value.source).Changed { + data := "" + name := cmd.Flag(value.source).Value.String() + if name != "" { + b, err := os.ReadFile(name) + if err != nil { + return nil, err + } - if cmd.Flag("scraper-template").Changed { - scraperTemplateContent := "" - scraperTemplate := cmd.Flag("scraper-template").Value.String() - if scraperTemplate != "" { - b, err := os.ReadFile(scraperTemplate) - if err != nil { - return nil, err + data = string(b) } - scraperTemplateContent = string(b) + *value.destination = &data + nonEmpty = true } - - request.ScraperTemplate = &scraperTemplateContent - nonEmpty = true } if cmd.Flag("mount-configmap").Changed || cmd.Flag("variable-configmap").Changed { @@ -1002,6 +994,15 @@ func newExecutionUpdateRequestFromFlags(cmd *cobra.Command) (request *testkube.E nonEmpty = true } + if cmd.Flag("execute-postrun-script-before-scraping").Changed { + executePostRunScriptBeforeScraping, err := cmd.Flags().GetBool("execute-postrun-script-before-scraping") + if err != nil { + return nil, err + } + request.ExecutePostRunScriptBeforeScraping = &executePostRunScriptBeforeScraping + nonEmpty = true + } + artifactRequest, err := newArtifactUpdateRequestFromFlags(cmd) if err != nil { return nil, err @@ -1037,6 +1038,10 @@ func newArtifactUpdateRequestFromFlags(cmd *cobra.Command) (request *testkube.Ar "artifact-volume-mount-path", &request.VolumeMountPath, }, + { + "artifact-storage-bucket", + &request.StorageBucket, + }, } var nonEmpty bool @@ -1058,6 +1063,16 @@ func newArtifactUpdateRequestFromFlags(cmd *cobra.Command) (request *testkube.Ar nonEmpty = true } + if cmd.Flag("artifact-omit-folder-per-execution").Changed { + value, err := cmd.Flags().GetBool("artifact-omit-folder-per-execution") + if err != nil { + return nil, err + } + + request.OmitFolderPerExecution = &value + nonEmpty = true + } + if nonEmpty { return request, nil } diff --git a/cmd/kubectl-testkube/commands/tests/create.go b/cmd/kubectl-testkube/commands/tests/create.go index 513c85dd64d..343f46eac5c 100644 --- a/cmd/kubectl-testkube/commands/tests/create.go +++ b/cmd/kubectl-testkube/commands/tests/create.go @@ -17,38 +17,47 @@ import ( // CreateCommonFlags are common flags for creating all test types type CreateCommonFlags struct { - ExecutorType string - Labels map[string]string - Variables map[string]string - SecretVariables map[string]string - Schedule string - ExecutorArgs []string - ArgsMode string - ExecutionName string - VariablesFile string - Envs map[string]string - SecretEnvs map[string]string - HttpProxy, HttpsProxy string - SecretVariableReferences map[string]string - CopyFiles []string - Image string - Command []string - ImagePullSecretNames []string - Timeout int64 - ArtifactStorageClassName string - ArtifactVolumeMountPath string - ArtifactDirs []string - JobTemplate string - CronJobTemplate string - PreRunScript string - PostRunScript string - ScraperTemplate string - NegativeTest bool - MountConfigMaps map[string]string - VariableConfigMaps []string - MountSecrets map[string]string - VariableSecrets []string - UploadTimeout string + ExecutorType string + Labels map[string]string + Variables map[string]string + SecretVariables map[string]string + Schedule string + ExecutorArgs []string + ArgsMode string + ExecutionName string + VariablesFile string + Envs map[string]string + SecretEnvs map[string]string + HttpProxy, HttpsProxy string + SecretVariableReferences map[string]string + CopyFiles []string + Image string + Command []string + ImagePullSecretNames []string + Timeout int64 + ArtifactStorageClassName string + ArtifactVolumeMountPath string + ArtifactDirs []string + JobTemplate string + JobTemplateReference string + CronJobTemplate string + CronJobTemplateReference string + PreRunScript string + PostRunScript string + ExecutePostRunScriptBeforeScraping bool + ScraperTemplate string + ScraperTemplateReference string + PvcTemplate string + PvcTemplateReference string + NegativeTest bool + MountConfigMaps map[string]string + VariableConfigMaps []string + MountSecrets map[string]string + VariableSecrets []string + UploadTimeout string + ArtifactStorageBucket string + ArtifactOmitFolderPerExecution bool + Description string } // NewCreateTestsCmd is a command tp create new Test Custom Resource @@ -72,6 +81,7 @@ func NewCreateTestsCmd() *cobra.Command { gitAuthType string sourceName string flags CreateCommonFlags + update bool ) cmd := &cobra.Command{ @@ -96,7 +106,25 @@ func NewCreateTestsCmd() *cobra.Command { test, _ := client.GetTest(testName) if testName == test.Name { - ui.Failf("Test with name '%s' already exists in namespace %s", testName, namespace) + if cmd.Flag("update").Changed { + if !update { + ui.Failf("Test with name '%s' already exists in namespace %s, ", testName, namespace) + } + } else { + ok := ui.Confirm(fmt.Sprintf("Test with name '%s' already exists in namespace %s, ", testName, namespace) + + "do you want to overwrite it?") + if !ok { + ui.Failf("Test creation was aborted") + } + } + + options, err := NewUpdateTestOptionsFromFlags(cmd) + ui.ExitOnError("getting test options", err) + + test, err = client.UpdateTest(options) + ui.ExitOnError("updating test "+testName+" in namespace "+namespace, err) + + ui.SuccessAndExit("Test updated", namespace, "/", testName) } } @@ -172,6 +200,7 @@ func NewCreateTestsCmd() *cobra.Command { cmd.Flags().StringVarP(&gitCertificateSecret, "git-certificate-secret", "", "", "if git repository is private we can use certificate as an auth parameter stored in a kubernetes secret name") cmd.Flags().StringVarP(&gitAuthType, "git-auth-type", "", "basic", "auth type for git requests one of basic|header") cmd.Flags().StringVarP(&sourceName, "source", "", "", "source name - will be used together with content parameters") + cmd.Flags().BoolVar(&update, "update", false, "update, if test already exists") cmd.Flags().MarkDeprecated("env", "env is deprecated use variable instead") cmd.Flags().MarkDeprecated("secret-env", "secret-env is deprecated use secret-variable instead") @@ -207,16 +236,25 @@ func AddCreateFlags(cmd *cobra.Command, flags *CreateCommonFlags) { cmd.Flags().StringVar(&flags.ArtifactVolumeMountPath, "artifact-volume-mount-path", "", "artifact volume mount path for container executor") cmd.Flags().StringArrayVarP(&flags.ArtifactDirs, "artifact-dir", "", []string{}, "artifact dirs for scraping") cmd.Flags().StringVar(&flags.JobTemplate, "job-template", "", "job template file path for extensions to job template") + cmd.Flags().StringVar(&flags.JobTemplateReference, "job-template-reference", "", "reference to job template to use for the test") cmd.Flags().StringVar(&flags.CronJobTemplate, "cronjob-template", "", "cron job template file path for extensions to cron job template") + cmd.Flags().StringVar(&flags.CronJobTemplateReference, "cronjob-template-reference", "", "reference to cron job template to use for the test") cmd.Flags().StringVarP(&flags.PreRunScript, "prerun-script", "", "", "path to script to be run before test execution") cmd.Flags().StringVarP(&flags.PostRunScript, "postrun-script", "", "", "path to script to be run after test execution") + cmd.Flags().BoolVarP(&flags.ExecutePostRunScriptBeforeScraping, "execute-postrun-script-before-scraping", "", false, "whether to execute postrun scipt before scraping or not (prebuilt executor only)") cmd.Flags().StringVar(&flags.ScraperTemplate, "scraper-template", "", "scraper template file path for extensions to scraper template") + cmd.Flags().StringVar(&flags.ScraperTemplateReference, "scraper-template-reference", "", "reference to scraper template to use for the test") + cmd.Flags().StringVar(&flags.PvcTemplate, "pvc-template", "", "pvc template file path for extensions to pvc template") + cmd.Flags().StringVar(&flags.PvcTemplateReference, "pvc-template-reference", "", "reference to pvc template to use for the test") cmd.Flags().BoolVar(&flags.NegativeTest, "negative-test", false, "negative test, if enabled, makes failure an expected and correct test result. If the test fails the result will be set to success, and vice versa") cmd.Flags().StringToStringVarP(&flags.MountConfigMaps, "mount-configmap", "", map[string]string{}, "config map value pair for mounting it to executor pod: --mount-configmap configmap_name=configmap_mountpath") cmd.Flags().StringArrayVar(&flags.VariableConfigMaps, "variable-configmap", []string{}, "config map name used to map all keys to basis variables") cmd.Flags().StringToStringVarP(&flags.MountSecrets, "mount-secret", "", map[string]string{}, "secret value pair for mounting it to executor pod: --mount-secret secret_name=secret_mountpath") cmd.Flags().StringArrayVar(&flags.VariableSecrets, "variable-secret", []string{}, "secret name used to map all keys to secret variables") cmd.Flags().StringVar(&flags.UploadTimeout, "upload-timeout", "", "timeout to use when uploading files, example: 30s") + cmd.Flags().StringVar(&flags.ArtifactStorageBucket, "artifact-storage-bucket", "", "artifact storage class name for container executor") + cmd.Flags().BoolVarP(&flags.ArtifactOmitFolderPerExecution, "artifact-omit-folder-per-execution", "", false, "don't store artifacts in execution folder") + cmd.Flags().StringVarP(&flags.Description, "description", "", "", "test description") } func validateExecutorTypeAndContent(executorType, contentType string, executors testkube.ExecutorsDetails) error { diff --git a/cmd/kubectl-testkube/commands/tests/executions.go b/cmd/kubectl-testkube/commands/tests/executions.go index aec8fb9f80c..3e0b9719a51 100644 --- a/cmd/kubectl-testkube/commands/tests/executions.go +++ b/cmd/kubectl-testkube/commands/tests/executions.go @@ -35,7 +35,7 @@ func NewGetExecutionCmd() *cobra.Command { ui.ExitOnError("getting test execution: "+executionID, err) if logsOnly { - render.RenderExecutionResult(&execution, logsOnly) + render.RenderExecutionResult(client, &execution, logsOnly) } else { err = render.Obj(cmd, execution, os.Stdout, renderer.ExecutionRenderer) ui.ExitOnError("rendering execution", err) diff --git a/cmd/kubectl-testkube/commands/tests/renderer/execution_obj.go b/cmd/kubectl-testkube/commands/tests/renderer/execution_obj.go index be3320eae4a..5b48a046372 100644 --- a/cmd/kubectl-testkube/commands/tests/renderer/execution_obj.go +++ b/cmd/kubectl-testkube/commands/tests/renderer/execution_obj.go @@ -5,11 +5,12 @@ import ( "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common/render" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/renderer" + "github.com/kubeshop/testkube/pkg/api/v1/client" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/ui" ) -func ExecutionRenderer(ui *ui.UI, obj interface{}) error { +func ExecutionRenderer(client client.Client, ui *ui.UI, obj interface{}) error { execution, ok := obj.(testkube.Execution) if !ok { return fmt.Errorf("can't render execution, expecrted obj to be testkube.Execution but got '%T'", obj) @@ -56,7 +57,7 @@ func ExecutionRenderer(ui *ui.UI, obj interface{}) error { ui.Warn(" Auth type: ", execution.Content.Repository.AuthType) } - render.RenderExecutionResult(&execution, false) + render.RenderExecutionResult(client, &execution, false) ui.NL() diff --git a/cmd/kubectl-testkube/commands/tests/renderer/test_obj.go b/cmd/kubectl-testkube/commands/tests/renderer/test_obj.go index 46e99f2dc0d..bbb78e1f036 100644 --- a/cmd/kubectl-testkube/commands/tests/renderer/test_obj.go +++ b/cmd/kubectl-testkube/commands/tests/renderer/test_obj.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/renderer" + "github.com/kubeshop/testkube/pkg/api/v1/client" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/ui" ) @@ -14,7 +15,7 @@ type mountParams struct { path string } -func TestRenderer(ui *ui.UI, obj interface{}) error { +func TestRenderer(client client.Client, ui *ui.UI, obj interface{}) error { test, ok := obj.(testkube.Test) if !ok { return fmt.Errorf("can't use '%T' as testkube.Test in RenderObj for test", obj) @@ -23,6 +24,10 @@ func TestRenderer(ui *ui.UI, obj interface{}) error { ui.Warn("Name: ", test.Name) ui.Warn("Namespace:", test.Namespace) ui.Warn("Created: ", test.Created.String()) + if test.Description != "" { + ui.NL() + ui.Warn("Description: ", test.Description) + } if len(test.Labels) > 0 { ui.NL() ui.Warn("Labels: ", testkube.MapToString(test.Labels)) @@ -101,6 +106,10 @@ func TestRenderer(ui *ui.UI, obj interface{}) error { ui.Warn(" Args mode: ", test.ExecutionRequest.ArgsMode) } + if test.ExecutionRequest.ArgsMode != "" { + ui.Warn(" Args mode: ", test.ExecutionRequest.ArgsMode) + } + if len(test.ExecutionRequest.Envs) > 0 { ui.NL() ui.Warn("(deprecated) Envs: ", testkube.MapToString(test.ExecutionRequest.Envs)) @@ -125,14 +134,28 @@ func TestRenderer(ui *ui.UI, obj interface{}) error { } if test.ExecutionRequest.ArtifactRequest != nil { - ui.Warn(" Artifact request: ") - ui.Warn(" Storage class name: ", test.ExecutionRequest.ArtifactRequest.StorageClassName) - ui.Warn(" Volume mount path: ", test.ExecutionRequest.ArtifactRequest.VolumeMountPath) - ui.Warn(" Dirs: ", strings.Join(test.ExecutionRequest.ArtifactRequest.Dirs, ",")) + ui.Warn(" Artifact request: ") + ui.Warn(" Storage class name: ", test.ExecutionRequest.ArtifactRequest.StorageClassName) + ui.Warn(" Volume mount path: ", test.ExecutionRequest.ArtifactRequest.VolumeMountPath) + ui.Warn(" Dirs: ", strings.Join(test.ExecutionRequest.ArtifactRequest.Dirs, ",")) + ui.Warn(" Storage bucket: ", test.ExecutionRequest.ArtifactRequest.StorageBucket) + ui.Warn(" Omit folder per execution: ", fmt.Sprint(test.ExecutionRequest.ArtifactRequest.OmitFolderPerExecution)) } if test.ExecutionRequest.JobTemplate != "" { - ui.Warn(" Job template: ", "\n", test.ExecutionRequest.JobTemplate) + ui.Warn(" Job template: ", "\n", test.ExecutionRequest.JobTemplate) + } + + if test.ExecutionRequest.JobTemplateReference != "" { + ui.Warn(" Job template reference: ", test.ExecutionRequest.JobTemplateReference) + } + + if test.ExecutionRequest.CronJobTemplate != "" { + ui.Warn(" Cron job template: ", "\n", test.ExecutionRequest.CronJobTemplate) + } + + if test.ExecutionRequest.CronJobTemplateReference != "" { + ui.Warn(" Cron job template reference: ", test.ExecutionRequest.CronJobTemplateReference) } if test.ExecutionRequest.CronJobTemplate != "" { @@ -140,15 +163,28 @@ func TestRenderer(ui *ui.UI, obj interface{}) error { } if test.ExecutionRequest.PreRunScript != "" { - ui.Warn(" Pre run script: ", "\n", test.ExecutionRequest.PreRunScript) + ui.Warn(" Pre run script: ", "\n", test.ExecutionRequest.PreRunScript) } if test.ExecutionRequest.PostRunScript != "" { - ui.Warn(" Post run script: ", "\n", test.ExecutionRequest.PostRunScript) + ui.Warn(" Post run script: ", "\n", test.ExecutionRequest.PostRunScript) } + ui.Warn(" Execute postrun script before scraping: ", fmt.Sprint(test.ExecutionRequest.ExecutePostRunScriptBeforeScraping)) if test.ExecutionRequest.ScraperTemplate != "" { - ui.Warn(" Scraper template: ", "\n", test.ExecutionRequest.ScraperTemplate) + ui.Warn(" Scraper template: ", "\n", test.ExecutionRequest.ScraperTemplate) + } + + if test.ExecutionRequest.ScraperTemplateReference != "" { + ui.Warn(" Scraper template reference: ", test.ExecutionRequest.ScraperTemplateReference) + } + + if test.ExecutionRequest.PvcTemplate != "" { + ui.Warn(" PVC template: ", "\n", test.ExecutionRequest.PvcTemplate) + } + + if test.ExecutionRequest.PvcTemplateReference != "" { + ui.Warn(" PVC template reference: ", test.ExecutionRequest.PvcTemplateReference) } var mountConfigMaps, mountSecrets []mountParams diff --git a/cmd/kubectl-testkube/commands/tests/run.go b/cmd/kubectl-testkube/commands/tests/run.go index 2d6323b364b..e019d501b17 100644 --- a/cmd/kubectl-testkube/commands/tests/run.go +++ b/cmd/kubectl-testkube/commands/tests/run.go @@ -20,46 +20,53 @@ const WatchInterval = 2 * time.Second func NewRunTestCmd() *cobra.Command { var ( - name string - image string - iterations int - watchEnabled bool - binaryArgs []string - variables map[string]string - secretVariables map[string]string - variablesFile string - downloadArtifactsEnabled bool - downloadDir string - envs map[string]string - secretEnvs map[string]string - selectors []string - concurrencyLevel int - httpProxy, httpsProxy string - executionLabels map[string]string - secretVariableReferences map[string]string - copyFiles []string - artifactStorageClassName string - artifactVolumeMountPath string - artifactDirs []string - jobTemplate string - gitBranch string - gitCommit string - gitPath string - gitWorkingDir string - preRunScript string - postRunScript string - scraperTemplate string - negativeTest bool - mountConfigMaps map[string]string - variableConfigMaps []string - mountSecrets map[string]string - variableSecrets []string - uploadTimeout string - format string - masks []string - runningContext string - command []string - argsMode string + name string + image string + iterations int + watchEnabled bool + binaryArgs []string + variables map[string]string + secretVariables map[string]string + variablesFile string + downloadArtifactsEnabled bool + downloadDir string + envs map[string]string + secretEnvs map[string]string + selectors []string + concurrencyLevel int + httpProxy, httpsProxy string + executionLabels map[string]string + secretVariableReferences map[string]string + copyFiles []string + artifactStorageClassName string + artifactVolumeMountPath string + artifactDirs []string + jobTemplate string + jobTemplateReference string + gitBranch string + gitCommit string + gitPath string + gitWorkingDir string + preRunScript string + postRunScript string + executePostRunScriptBeforeScraping bool + scraperTemplate string + scraperTemplateReference string + pvcTemplate string + pvcTemplateReference string + negativeTest bool + mountConfigMaps map[string]string + variableConfigMaps []string + mountSecrets map[string]string + variableSecrets []string + uploadTimeout string + format string + masks []string + runningContext string + command []string + argsMode string + artifactStorageBucket string + artifactOmitFolderPerExecution bool ) cmd := &cobra.Command{ @@ -80,43 +87,11 @@ func NewRunTestCmd() *cobra.Command { envConfigMaps, envSecrets, err := newEnvReferencesFromFlags(cmd) ui.WarnOnError("getting env config maps and secrets", err) - jobTemplateContent := "" - if jobTemplate != "" { - b, err := os.ReadFile(jobTemplate) - ui.ExitOnError("reading job template", err) - jobTemplateContent = string(b) - } - - preRunScriptContent := "" - if preRunScript != "" { - b, err := os.ReadFile(preRunScript) - ui.ExitOnError("reading pre run script", err) - preRunScriptContent = string(b) - } - - postRunScriptContent := "" - if postRunScript != "" { - b, err := os.ReadFile(postRunScript) - ui.ExitOnError("reading post run script", err) - postRunScriptContent = string(b) - } - - scraperTemplateContent := "" - if scraperTemplate != "" { - b, err := os.ReadFile(scraperTemplate) - ui.ExitOnError("reading scraper template", err) - scraperTemplateContent = string(b) - } - mode := "" if cmd.Flag("args-mode").Changed { mode = argsMode } - var executions []testkube.Execution - client, namespace, err := common.GetClient(cmd) - ui.ExitOnError("getting client", err) - options := apiv1.ExecuteTestOptions{ ExecutionVariables: variables, ExecutionLabels: executionLabels, @@ -128,10 +103,9 @@ func NewRunTestCmd() *cobra.Command { HTTPSProxy: httpsProxy, Envs: envs, Image: image, - JobTemplate: jobTemplateContent, - PreRunScriptContent: preRunScriptContent, - PostRunScriptContent: postRunScriptContent, - ScraperTemplate: scraperTemplateContent, + JobTemplateReference: jobTemplateReference, + ScraperTemplateReference: scraperTemplateReference, + PvcTemplateReference: pvcTemplateReference, IsNegativeTestChangedOnRun: false, EnvConfigMaps: envConfigMaps, EnvSecrets: envSecrets, @@ -139,13 +113,61 @@ func NewRunTestCmd() *cobra.Command { Type_: string(testkube.RunningContextTypeUserCLI), Context: runningContext, }, + ExecutePostRunScriptBeforeScraping: executePostRunScriptBeforeScraping, + } + + var fields = []struct { + source string + title string + destination *string + }{ + { + jobTemplate, + "job template", + &options.JobTemplate, + }, + { + preRunScript, + "pre run script", + &options.PreRunScriptContent, + }, + { + postRunScript, + "post run script", + &options.PostRunScriptContent, + }, + { + scraperTemplate, + "scraper template", + &options.ScraperTemplate, + }, + { + pvcTemplate, + "pvc template", + &options.PvcTemplate, + }, } - if artifactStorageClassName != "" || artifactVolumeMountPath != "" || len(artifactDirs) != 0 { + for _, field := range fields { + if field.source != "" { + b, err := os.ReadFile(field.source) + ui.ExitOnError("reading "+field.title, err) + *field.destination = string(b) + } + } + + var executions []testkube.Execution + client, namespace, err := common.GetClient(cmd) + ui.ExitOnError("getting client", err) + + if artifactStorageClassName != "" || artifactVolumeMountPath != "" || len(artifactDirs) != 0 || + artifactStorageBucket != "" || artifactOmitFolderPerExecution { options.ArtifactRequest = &testkube.ArtifactRequest{ - StorageClassName: artifactStorageClassName, - VolumeMountPath: artifactVolumeMountPath, - Dirs: artifactDirs, + StorageClassName: artifactStorageClassName, + VolumeMountPath: artifactVolumeMountPath, + Dirs: artifactDirs, + StorageBucket: artifactStorageBucket, + OmitFolderPerExecution: artifactOmitFolderPerExecution, } } @@ -236,7 +258,7 @@ func NewRunTestCmd() *cobra.Command { ui.ExitOnError("getting recent execution data id:"+execution.Id, err) } - render.RenderExecutionResult(&execution, false) + render.RenderExecutionResult(client, &execution, false) if execution.Id != "" { if downloadArtifactsEnabled { @@ -280,13 +302,18 @@ func NewRunTestCmd() *cobra.Command { cmd.Flags().StringVar(&artifactVolumeMountPath, "artifact-volume-mount-path", "", "artifact volume mount path for container executor") cmd.Flags().StringArrayVarP(&artifactDirs, "artifact-dir", "", []string{}, "artifact dirs for scraping") cmd.Flags().StringVar(&jobTemplate, "job-template", "", "job template file path for extensions to job template") + cmd.Flags().StringVar(&jobTemplateReference, "job-template-reference", "", "reference to job template to use for the test") cmd.Flags().StringVarP(&gitBranch, "git-branch", "", "", "if uri is git repository we can set additional branch parameter") cmd.Flags().StringVarP(&gitCommit, "git-commit", "", "", "if uri is git repository we can use commit id (sha) parameter") cmd.Flags().StringVarP(&gitPath, "git-path", "", "", "if repository is big we need to define additional path to directory/file to checkout partially") cmd.Flags().StringVarP(&gitWorkingDir, "git-working-dir", "", "", "if repository contains multiple directories with tests (like monorepo) and one starting directory we can set working directory parameter") cmd.Flags().StringVarP(&preRunScript, "prerun-script", "", "", "path to script to be run before test execution") cmd.Flags().StringVarP(&postRunScript, "postrun-script", "", "", "path to script to be run after test execution") + cmd.Flags().BoolVarP(&executePostRunScriptBeforeScraping, "execute-postrun-script-before-scraping", "", false, "whether to execute postrun scipt before scraping or not (prebuilt executor only)") cmd.Flags().StringVar(&scraperTemplate, "scraper-template", "", "scraper template file path for extensions to scraper template") + cmd.Flags().StringVar(&scraperTemplateReference, "scraper-template-reference", "", "reference to scraper template to use for the test") + cmd.Flags().StringVar(&pvcTemplate, "pvc-template", "", "pvc template file path for extensions to pvc template") + cmd.Flags().StringVar(&pvcTemplateReference, "pvc-template-reference", "", "reference to pvc template to use for the test") cmd.Flags().BoolVar(&negativeTest, "negative-test", false, "negative test, if enabled, makes failure an expected and correct test result. If the test fails the result will be set to success, and vice versa") cmd.Flags().StringToStringVarP(&mountConfigMaps, "mount-configmap", "", map[string]string{}, "config map value pair for mounting it to executor pod: --mount-configmap configmap_name=configmap_mountpath") cmd.Flags().StringArrayVar(&variableConfigMaps, "variable-configmap", []string{}, "config map name used to map all keys to basis variables") @@ -298,6 +325,8 @@ func NewRunTestCmd() *cobra.Command { cmd.Flags().StringVar(&format, "format", "folder", "data format for storing files, one of folder|archive") cmd.Flags().StringArrayVarP(&masks, "mask", "", []string{}, "regexp to filter downloaded files, single or comma separated, like report/.* or .*\\.json,.*\\.js$") cmd.Flags().StringVar(&runningContext, "context", "", "running context description for test execution") + cmd.Flags().StringVar(&artifactStorageBucket, "artifact-storage-bucket", "", "artifact storage class name for container executor") + cmd.Flags().BoolVarP(&artifactOmitFolderPerExecution, "artifact-omit-folder-per-execution", "", false, "don't store artifacts in execution folder") return cmd } diff --git a/cmd/kubectl-testkube/commands/tests/update.go b/cmd/kubectl-testkube/commands/tests/update.go index f9628d1207c..890b1b604cb 100644 --- a/cmd/kubectl-testkube/commands/tests/update.go +++ b/cmd/kubectl-testkube/commands/tests/update.go @@ -10,53 +10,22 @@ import ( func NewUpdateTestsCmd() *cobra.Command { var ( - testName string - testContentType string - file string - executorType string - uri string - gitUri string - gitBranch string - gitCommit string - gitPath string - gitUsername string - gitToken string - sourceName string - labels map[string]string - variables map[string]string - secretVariables map[string]string - schedule string - executorArgs []string - argsMode string - executionName string - variablesFile string - envs map[string]string - secretEnvs map[string]string - httpProxy, httpsProxy string - gitUsernameSecret map[string]string - gitTokenSecret map[string]string - secretVariableReferences map[string]string - copyFiles []string - image string - command []string - imagePullSecretNames []string - timeout int64 - gitWorkingDir string - gitCertificateSecret string - gitAuthType string - artifactStorageClassName string - artifactVolumeMountPath string - artifactDirs []string - jobTemplate string - cronJobTemplate string - preRunScript string - postRunScript string - scraperTemplate string - negativeTest bool - mountConfigMaps map[string]string - variableConfigMaps []string - mountSecrets map[string]string - variableSecrets []string + testName string + testContentType string + file string + uri string + gitUri string + gitBranch string + gitCommit string + gitPath string + gitUsername string + gitToken string + sourceName string + gitUsernameSecret map[string]string + gitTokenSecret map[string]string + gitWorkingDir string + gitCertificateSecret string + gitAuthType string ) cmd := &cobra.Command{ @@ -91,55 +60,23 @@ func NewUpdateTestsCmd() *cobra.Command { cmd.Flags().StringVarP(&testName, "name", "n", "", "unique test name - mandatory") cmd.Flags().StringVarP(&file, "file", "f", "", "test file - will try to read content from stdin if not specified") cmd.Flags().StringVarP(&testContentType, "test-content-type", "", "", "content type of test one of string|file-uri|git") - - cmd.Flags().StringVarP(&executorType, "type", "t", "", "test type (defaults to postman-collection)") - cmd.Flags().StringVarP(&uri, "uri", "", "", "URI of resource - will be loaded by http GET") cmd.Flags().StringVarP(&gitUri, "git-uri", "", "", "Git repository uri") cmd.Flags().StringVarP(&gitBranch, "git-branch", "", "", "if uri is git repository we can set additional branch parameter") cmd.Flags().StringVarP(&gitCommit, "git-commit", "", "", "if uri is git repository we can use commit id (sha) parameter") cmd.Flags().StringVarP(&gitPath, "git-path", "", "", "if repository is big we need to define additional path to directory/file to checkout partially") + cmd.Flags().StringVarP(&gitWorkingDir, "git-working-dir", "", "", "if repository contains multiple directories with tests (like monorepo) and one starting directory we can set working directory parameter") cmd.Flags().StringVarP(&gitUsername, "git-username", "", "", "if git repository is private we can use username as an auth parameter") cmd.Flags().StringVarP(&gitToken, "git-token", "", "", "if git repository is private we can use token as an auth parameter") - cmd.Flags().StringVarP(&sourceName, "source", "", "", "source name - will be used together with content parameters") - cmd.Flags().StringToStringVarP(&labels, "label", "l", nil, "label key value pair: --label key1=value1") - cmd.Flags().StringToStringVarP(&variables, "variable", "v", nil, "variable key value pair: -v key1=value1") - cmd.Flags().StringToStringVarP(&secretVariables, "secret-variable", "s", nil, "secret variable key value pair: -s key1=value1") - cmd.Flags().StringVarP(&schedule, "schedule", "", "", "test schedule in a cron job form: * * * * *") - cmd.Flags().StringArrayVarP(&command, "command", "", []string{}, "command passed to image in executor") - cmd.Flags().StringArrayVarP(&executorArgs, "executor-args", "", []string{}, "executor binary additional arguments") - cmd.Flags().StringVarP(&argsMode, "args-mode", "", "append", "usage mode for arguments. one of append|override") - cmd.Flags().StringVarP(&executionName, "execution-name", "", "", "execution name, if empty will be autogenerated") - cmd.Flags().StringVarP(&variablesFile, "variables-file", "", "", "variables file path, e.g. postman env file - will be passed to executor if supported") - cmd.Flags().StringToStringVarP(&envs, "env", "", map[string]string{}, "envs in a form of name1=val1 passed to executor") - cmd.Flags().StringToStringVarP(&secretEnvs, "secret-env", "", map[string]string{}, "secret envs in a form of secret_key1=secret_name1 passed to executor") - cmd.Flags().StringVar(&httpProxy, "http-proxy", "", "http proxy for executor containers") - cmd.Flags().StringVar(&httpsProxy, "https-proxy", "", "https proxy for executor containers") cmd.Flags().StringToStringVarP(&gitUsernameSecret, "git-username-secret", "", map[string]string{}, "git username secret in a form of secret_name1=secret_key1 for private repository") cmd.Flags().StringToStringVarP(&gitTokenSecret, "git-token-secret", "", map[string]string{}, "git token secret in a form of secret_name1=secret_key1 for private repository") - cmd.Flags().StringToStringVarP(&secretVariableReferences, "secret-variable-reference", "", nil, "secret variable references in a form name1=secret_name1=secret_key1") - cmd.Flags().StringArrayVarP(©Files, "copy-files", "", []string{}, "file path mappings from host to pod of form source:destination") - cmd.Flags().StringVarP(&image, "image", "i", "", "image for container executor") - cmd.Flags().StringArrayVar(&imagePullSecretNames, "image-pull-secrets", []string{}, "secret name used to pull the image in container executor") - cmd.Flags().Int64Var(&timeout, "timeout", 0, "duration in seconds for test to timeout. 0 disables timeout.") - cmd.Flags().StringVarP(&gitWorkingDir, "git-working-dir", "", "", "if repository contains multiple directories with tests (like monorepo) and one starting directory we can set working directory parameter") cmd.Flags().StringVarP(&gitCertificateSecret, "git-certificate-secret", "", "", "if git repository is private we can use certificate as an auth parameter stored in a kubernetes secret name") cmd.Flags().StringVarP(&gitAuthType, "git-auth-type", "", "basic", "auth type for git requests one of basic|header") - cmd.Flags().StringVar(&artifactStorageClassName, "artifact-storage-class-name", "", "artifact storage class name for container executor") - cmd.Flags().StringVar(&artifactVolumeMountPath, "artifact-volume-mount-path", "", "artifact volume mount path for container executor") - cmd.Flags().StringArrayVarP(&artifactDirs, "artifact-dir", "", []string{}, "artifact dirs for scraping") - cmd.Flags().StringVar(&jobTemplate, "job-template", "", "job template file path for extensions to job template") - cmd.Flags().StringVar(&cronJobTemplate, "cronjob-template", "", "cron job template file path for extensions to cron job template") - cmd.Flags().StringVarP(&preRunScript, "prerun-script", "", "", "path to script to be run before test execution") - cmd.Flags().StringVarP(&postRunScript, "postrun-script", "", "", "path to script to be run after test execution") - cmd.Flags().StringVar(&scraperTemplate, "scraper-template", "", "scraper template file path for extensions to scraper template") - cmd.Flags().BoolVar(&negativeTest, "negative-test", false, "negative test, if enabled, makes failure an expected and correct test result. If the test fails the result will be set to success, and vice versa") - cmd.Flags().StringToStringVarP(&mountConfigMaps, "mount-configmap", "", map[string]string{}, "config map value pair for mounting it to executor pod: --mount-configmap configmap_name=configmap_mountpath") - cmd.Flags().StringArrayVar(&variableConfigMaps, "variable-configmap", []string{}, "config map name used to map all keys to basis variables") - cmd.Flags().StringToStringVarP(&mountSecrets, "mount-secret", "", map[string]string{}, "secret value pair for mounting it to executor pod: --mount-secret secret_name=secret_mountpath") - cmd.Flags().StringArrayVar(&variableSecrets, "variable-secret", []string{}, "secret name used to map all keys to secret variables") + cmd.Flags().StringVarP(&sourceName, "source", "", "", "source name - will be used together with content parameters") cmd.Flags().MarkDeprecated("env", "env is deprecated use variable instead") cmd.Flags().MarkDeprecated("secret-env", "secret-env is deprecated use secret-variable instead") + AddCreateFlags(cmd, &CreateCommonFlags{}) + return cmd } diff --git a/cmd/kubectl-testkube/commands/testsources/create.go b/cmd/kubectl-testkube/commands/testsources/create.go index c451fddd83a..eedeed52afa 100644 --- a/cmd/kubectl-testkube/commands/testsources/create.go +++ b/cmd/kubectl-testkube/commands/testsources/create.go @@ -29,6 +29,7 @@ func NewCreateTestSourceCmd() *cobra.Command { gitTokenSecret map[string]string gitCertificateSecret string gitAuthType string + update bool ) cmd := &cobra.Command{ @@ -52,7 +53,25 @@ func NewCreateTestSourceCmd() *cobra.Command { testsource, _ := client.GetTestSource(name) if name == testsource.Name { - ui.Failf("TestSource with name '%s' already exists in namespace %s", name, namespace) + if cmd.Flag("update").Changed { + if !update { + ui.Failf("TestSource with name '%s' already exists in namespace %s, ", testsource.Name, namespace) + } + } else { + ok := ui.Confirm(fmt.Sprintf("TestSource with name '%s' already exists in namespace %s, ", testsource.Name, namespace) + + "do you want to overwrite it?") + if !ok { + ui.Failf("TestSource creation was aborted") + } + } + + options, err := NewUpdateTestSourceOptionsFromFlags(cmd) + ui.ExitOnError("getting test source options", err) + + _, err = client.UpdateTestSource(options) + ui.ExitOnError("updating test source "+name+" in namespace "+namespace, err) + + ui.SuccessAndExit("TestSource updated", name) } } @@ -84,7 +103,7 @@ func NewCreateTestSourceCmd() *cobra.Command { cmd.Flags().StringToStringVarP(&labels, "label", "l", nil, "label key value pair: --label key1=value1") cmd.Flags().StringVarP(&sourceType, "source-type", "", "", "source type of test one of string|file-uri|git") cmd.Flags().StringVarP(&file, "file", "f", "", "source file - will be read from stdin if not specified") - cmd.Flags().StringVarP(&uri, "uri", "u", "", "URI which should be called when given event occurs") + cmd.Flags().StringVarP(&uri, "uri", "u", "", "URI which should be called to get test content") cmd.Flags().StringVarP(&gitUri, "git-uri", "", "", "Git repository uri") cmd.Flags().StringVarP(&gitBranch, "git-branch", "", "", "if uri is git repository we can set additional branch parameter") cmd.Flags().StringVarP(&gitCommit, "git-commit", "", "", "if uri is git repository we can use commit id (sha) parameter") @@ -96,6 +115,7 @@ func NewCreateTestSourceCmd() *cobra.Command { cmd.Flags().StringVarP(&gitCertificateSecret, "git-certificate-secret", "", "", "if git repository is private we can use certificate as an auth parameter stored in a kubernetes secret name") cmd.Flags().StringVarP(&gitWorkingDir, "git-working-dir", "", "", "if repository contains multiple directories with tests (like monorepo) and one starting directory we can set working directory parameter") cmd.Flags().StringVarP(&gitAuthType, "git-auth-type", "", "basic", "auth type for git requests one of basic|header") + cmd.Flags().BoolVar(&update, "update", false, "update, if test source already exists") return cmd } diff --git a/cmd/kubectl-testkube/commands/testsources/renderer/testsource_obj.go b/cmd/kubectl-testkube/commands/testsources/renderer/testsource_obj.go index a3062143f37..b4dfcb905af 100644 --- a/cmd/kubectl-testkube/commands/testsources/renderer/testsource_obj.go +++ b/cmd/kubectl-testkube/commands/testsources/renderer/testsource_obj.go @@ -3,11 +3,12 @@ package renderer import ( "fmt" + "github.com/kubeshop/testkube/pkg/api/v1/client" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/ui" ) -func TestSourceRenderer(ui *ui.UI, obj interface{}) error { +func TestSourceRenderer(client client.Client, ui *ui.UI, obj interface{}) error { testSource, ok := obj.(testkube.TestSource) if !ok { return fmt.Errorf("can't use '%T' as testkube.TestSource in RenderObj for test source", obj) diff --git a/cmd/kubectl-testkube/commands/testsources/update.go b/cmd/kubectl-testkube/commands/testsources/update.go index 4d460ad067a..c2a81be3c6e 100644 --- a/cmd/kubectl-testkube/commands/testsources/update.go +++ b/cmd/kubectl-testkube/commands/testsources/update.go @@ -58,7 +58,7 @@ func UpdateTestSourceCmd() *cobra.Command { cmd.Flags().StringToStringVarP(&labels, "label", "l", nil, "label key value pair: --label key1=value1") cmd.Flags().StringVarP(&sourceType, "source-type", "", "", "source type of test one of string|file-uri|git") cmd.Flags().StringVarP(&file, "file", "f", "", "source file - will be read from stdin if not specified") - cmd.Flags().StringVarP(&uri, "uri", "u", "", "URI which should be called when given event occurs") + cmd.Flags().StringVarP(&uri, "uri", "u", "", "URI which should be called to get test content") cmd.Flags().StringVarP(&gitUri, "git-uri", "", "", "Git repository uri") cmd.Flags().StringVarP(&gitBranch, "git-branch", "", "", "if uri is git repository we can set additional branch parameter") cmd.Flags().StringVarP(&gitCommit, "git-commit", "", "", "if uri is git repository we can use commit id (sha) parameter") @@ -67,8 +67,8 @@ func UpdateTestSourceCmd() *cobra.Command { cmd.Flags().StringVarP(&gitToken, "git-token", "", "", "if git repository is private we can use token as an auth parameter") cmd.Flags().StringToStringVarP(&gitUsernameSecret, "git-username-secret", "", map[string]string{}, "git username secret in a form of secret_name1=secret_key1 for private repository") cmd.Flags().StringToStringVarP(&gitTokenSecret, "git-token-secret", "", map[string]string{}, "git token secret in a form of secret_name1=secret_key1 for private repository") - cmd.Flags().StringVarP(&gitWorkingDir, "git-working-dir", "", "", "if repository contains multiple directories with tests (like monorepo) and one starting directory we can set working directory parameter") cmd.Flags().StringVarP(&gitCertificateSecret, "git-certificate-secret", "", "", "if git repository is private we can use certificate as an auth parameter stored in a kubernetes secret name") + cmd.Flags().StringVarP(&gitWorkingDir, "git-working-dir", "", "", "if repository contains multiple directories with tests (like monorepo) and one starting directory we can set working directory parameter") cmd.Flags().StringVarP(&gitAuthType, "git-auth-type", "", "basic", "auth type for git requests one of basic|header") return cmd diff --git a/cmd/kubectl-testkube/commands/testsuites/common.go b/cmd/kubectl-testkube/commands/testsuites/common.go index 7403c5a8616..2203f8439c4 100644 --- a/cmd/kubectl-testkube/commands/testsuites/common.go +++ b/cmd/kubectl-testkube/commands/testsuites/common.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common" + "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common/render" apiclientv1 "github.com/kubeshop/testkube/pkg/api/v1/client" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/ui" @@ -37,7 +38,7 @@ func printExecution(execution testkube.TestSuiteExecution, startTime time.Time) ui.NL() } -func uiPrintExecutionStatus(execution testkube.TestSuiteExecution) { +func uiPrintExecutionStatus(client apiclientv1.Client, execution testkube.TestSuiteExecution) { if execution.Status == nil { return } @@ -52,9 +53,19 @@ func uiPrintExecutionStatus(execution testkube.TestSuiteExecution) { case execution.IsPassed(): ui.Success("Test Suite execution completed with sucess in " + execution.Duration) + info, err := client.GetServerInfo() + ui.ExitOnError("getting server info", err) + + render.PrintTestSuiteExecutionURIs(&execution, info.DashboardUri) + case execution.IsFailed(): ui.UseStderr() ui.Errf("Test Suite execution failed") + + info, err := client.GetServerInfo() + ui.ExitOnError("getting server info", err) + + render.PrintTestSuiteExecutionURIs(&execution, info.DashboardUri) os.Exit(1) } @@ -149,25 +160,55 @@ func NewTestSuiteUpsertOptionsFromFlags(cmd *cobra.Command) (options apiclientv1 return options, fmt.Errorf("validating schedule %w", err) } - cronJobTemplateContent := "" - cronJobTemplate := cmd.Flag("cronjob-template").Value.String() - if cronJobTemplate != "" { - b, err := os.ReadFile(cronJobTemplate) - if err != nil { - return options, err - } - - cronJobTemplateContent = string(b) - } + jobTemplateReference := cmd.Flag("job-template-reference").Value.String() + cronJobTemplateReference := cmd.Flag("cronjob-template-reference").Value.String() + scraperTemplateReference := cmd.Flag("scraper-template-reference").Value.String() + pvcTemplateReference := cmd.Flag("pvc-template-reference").Value.String() options.Schedule = schedule options.ExecutionRequest = &testkube.TestSuiteExecutionRequest{ - Variables: variables, - Name: cmd.Flag("execution-name").Value.String(), - HttpProxy: cmd.Flag("http-proxy").Value.String(), - HttpsProxy: cmd.Flag("https-proxy").Value.String(), - Timeout: timeout, - CronJobTemplate: cronJobTemplateContent, + Variables: variables, + Name: cmd.Flag("execution-name").Value.String(), + HttpProxy: cmd.Flag("http-proxy").Value.String(), + HttpsProxy: cmd.Flag("https-proxy").Value.String(), + Timeout: timeout, + JobTemplateReference: jobTemplateReference, + CronJobTemplateReference: cronJobTemplateReference, + ScraperTemplateReference: scraperTemplateReference, + PvcTemplateReference: pvcTemplateReference, + } + + var fields = []struct { + source string + destination *string + }{ + { + cmd.Flag("job-template").Value.String(), + &options.ExecutionRequest.JobTemplate, + }, + { + cmd.Flag("cronjob-template").Value.String(), + &options.ExecutionRequest.CronJobTemplate, + }, + { + cmd.Flag("scraper-template").Value.String(), + &options.ExecutionRequest.ScraperTemplate, + }, + { + cmd.Flag("pvc-template").Value.String(), + &options.ExecutionRequest.PvcTemplate, + }, + } + + for _, field := range fields { + if field.source != "" { + b, err := os.ReadFile(field.source) + if err != nil { + return options, err + } + + *field.destination = string(b) + } } return options, nil @@ -268,20 +309,44 @@ func NewTestSuiteUpdateOptionsFromFlags(cmd *cobra.Command) (options apiclientv1 nonEmpty = true } - if cmd.Flag("cronjob-template").Changed { - cronJobTemplateContent := "" - cronJobTemplate := cmd.Flag("cronjob-template").Value.String() - if cronJobTemplate != "" { - b, err := os.ReadFile(cronJobTemplate) - if err != nil { - return options, err + var values = []struct { + source string + destination **string + }{ + { + "job-template", + &executionRequest.JobTemplate, + }, + { + "cronjob-template", + &executionRequest.CronJobTemplate, + }, + { + "scraper-template", + &executionRequest.ScraperTemplate, + }, + { + "pvc-template", + &executionRequest.PvcTemplate, + }, + } + + for _, value := range values { + if cmd.Flag(value.source).Changed { + data := "" + name := cmd.Flag(value.source).Value.String() + if name != "" { + b, err := os.ReadFile(name) + if err != nil { + return options, err + } + + data = string(b) } - cronJobTemplateContent = string(b) + *value.destination = &data + nonEmpty = true } - - executionRequest.CronJobTemplate = &cronJobTemplateContent - nonEmpty = true } var executionFields = []struct { @@ -300,6 +365,22 @@ func NewTestSuiteUpdateOptionsFromFlags(cmd *cobra.Command) (options apiclientv1 "https-proxy", &executionRequest.HttpsProxy, }, + { + "job-template-reference", + &executionRequest.JobTemplateReference, + }, + { + "cronjob-template-reference", + &executionRequest.CronJobTemplateReference, + }, + { + "scraper-template-reference", + &executionRequest.ScraperTemplateReference, + }, + { + "pvc-template-reference", + &executionRequest.PvcTemplateReference, + }, } for _, field := range executionFields { diff --git a/cmd/kubectl-testkube/commands/testsuites/create.go b/cmd/kubectl-testkube/commands/testsuites/create.go index 018ff38a907..8f45f09bfc0 100644 --- a/cmd/kubectl-testkube/commands/testsuites/create.go +++ b/cmd/kubectl-testkube/commands/testsuites/create.go @@ -1,6 +1,7 @@ package testsuites import ( + "fmt" "strconv" "github.com/robfig/cron" @@ -26,7 +27,15 @@ func NewCreateTestSuitesCmd() *cobra.Command { httpProxy, httpsProxy string secretVariableReferences map[string]string timeout int32 + jobTemplate string cronJobTemplate string + scraperTemplate string + pvcTemplate string + jobTemplateReference string + cronJobTemplateReference string + scraperTemplateReference string + pvcTemplateReference string + update bool ) cmd := &cobra.Command{ @@ -49,9 +58,28 @@ func NewCreateTestSuitesCmd() *cobra.Command { client, namespace, err := common.GetClient(cmd) ui.ExitOnError("getting client", err) - test, _ := client.GetTestSuite(options.Name) - if options.Name == test.Name { - ui.Failf("TestSuite with name '%s' already exists in namespace %s", options.Name, namespace) + testSuite, _ := client.GetTestSuite(options.Name) + + if options.Name == testSuite.Name { + if cmd.Flag("update").Changed { + if !update { + ui.Failf("TestSuite with name '%s' already exists in namespace %s, ", testSuite.Name, namespace) + } + } else { + ok := ui.Confirm(fmt.Sprintf("TestSuite with name '%s' already exists in namespace %s, ", testSuite.Name, namespace) + + "do you want to overwrite it?") + if !ok { + ui.Failf("TestSuite creation was aborted") + } + } + + options, err := NewTestSuiteUpdateOptionsFromFlags(cmd) + ui.ExitOnError("getting test suite options", err) + + testSuite, err = client.UpdateTestSuite(options) + ui.ExitOnError("updating TestSuite "+testSuite.Name+" in namespace "+namespace, err) + + ui.SuccessAndExit("TestSuite updated", testSuite.Name) } _, err = client.CreateTestSuite(apiClient.UpsertTestSuiteOptions(options)) @@ -79,7 +107,15 @@ func NewCreateTestSuitesCmd() *cobra.Command { cmd.Flags().StringVar(&httpsProxy, "https-proxy", "", "https proxy for executor containers") cmd.Flags().StringToStringVarP(&secretVariableReferences, "secret-variable-reference", "", nil, "secret variable references in a form name1=secret_name1=secret_key1") cmd.Flags().Int32Var(&timeout, "timeout", 0, "duration in seconds for test suite to timeout. 0 disables timeout.") + cmd.Flags().StringVar(&jobTemplate, "job-template", "", "job template file path for extensions to job template") cmd.Flags().StringVar(&cronJobTemplate, "cronjob-template", "", "cron job template file path for extensions to cron job template") + cmd.Flags().StringVar(&scraperTemplate, "scraper-template", "", "scraper template file path for extensions to scraper template") + cmd.Flags().StringVar(&pvcTemplate, "pvc-template", "", "pvc template file path for extensions to pvc template") + cmd.Flags().StringVar(&jobTemplateReference, "job-template-reference", "", "reference to job template to use for the test") + cmd.Flags().StringVar(&cronJobTemplateReference, "cronjob-template-reference", "", "reference to cron job template to use for the test") + cmd.Flags().StringVar(&scraperTemplateReference, "scraper-template-reference", "", "reference to scraper template to use for the test") + cmd.Flags().StringVar(&pvcTemplateReference, "pvc-template-reference", "", "reference to pvc template to use for the test") + cmd.Flags().BoolVar(&update, "update", false, "update, if test suite already exists") return cmd } diff --git a/cmd/kubectl-testkube/commands/testsuites/renderer/execution_obj.go b/cmd/kubectl-testkube/commands/testsuites/renderer/execution_obj.go index 965b0b732eb..24e910010d6 100644 --- a/cmd/kubectl-testkube/commands/testsuites/renderer/execution_obj.go +++ b/cmd/kubectl-testkube/commands/testsuites/renderer/execution_obj.go @@ -4,11 +4,13 @@ import ( "fmt" "os" + "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common/render" + "github.com/kubeshop/testkube/pkg/api/v1/client" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/ui" ) -func TestSuiteExecutionRenderer(ui *ui.UI, obj interface{}) error { +func TestSuiteExecutionRenderer(client client.Client, ui *ui.UI, obj interface{}) error { execution, ok := obj.(testkube.TestSuiteExecution) if !ok { return fmt.Errorf("can't render execution, expecrted obj to be testkube.Execution but got '%T'", obj) @@ -26,6 +28,12 @@ func TestSuiteExecutionRenderer(ui *ui.UI, obj interface{}) error { ui.Warn("Type: ", execution.RunningContext.Type_) ui.Warn("Context:", execution.RunningContext.Context) } + + info, err := client.GetServerInfo() + ui.ExitOnError("getting server info", err) + + render.PrintTestSuiteExecutionURIs(&execution, info.DashboardUri) + ui.Table(execution, os.Stdout) ui.NL() diff --git a/cmd/kubectl-testkube/commands/testsuites/renderer/testsuite_obj.go b/cmd/kubectl-testkube/commands/testsuites/renderer/testsuite_obj.go index 399481f2db1..f9c62ff7d69 100644 --- a/cmd/kubectl-testkube/commands/testsuites/renderer/testsuite_obj.go +++ b/cmd/kubectl-testkube/commands/testsuites/renderer/testsuite_obj.go @@ -5,11 +5,12 @@ import ( "strings" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/renderer" + "github.com/kubeshop/testkube/pkg/api/v1/client" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/ui" ) -func TestSuiteRenderer(ui *ui.UI, obj interface{}) error { +func TestSuiteRenderer(client client.Client, ui *ui.UI, obj interface{}) error { ts, ok := obj.(testkube.TestSuite) if !ok { return fmt.Errorf("can't use '%T' as testkube.TestSuite in RenderObj for test suite", obj) @@ -17,6 +18,10 @@ func TestSuiteRenderer(ui *ui.UI, obj interface{}) error { ui.Warn("Name: ", ts.Name) ui.Warn("Namespace:", ts.Namespace) + if ts.Description != "" { + ui.NL() + ui.Warn("Description: ", ts.Description) + } if len(ts.Labels) > 0 { ui.NL() ui.Warn("Labels: ", testkube.MapToString(ts.Labels)) @@ -29,7 +34,7 @@ func TestSuiteRenderer(ui *ui.UI, obj interface{}) error { if ts.ExecutionRequest != nil { ui.Warn("Execution request: ") if ts.ExecutionRequest.Name != "" { - ui.Warn(" Name: ", ts.ExecutionRequest.Name) + ui.Warn(" Name: ", ts.ExecutionRequest.Name) } if len(ts.ExecutionRequest.Variables) > 0 { @@ -37,15 +42,43 @@ func TestSuiteRenderer(ui *ui.UI, obj interface{}) error { } if ts.ExecutionRequest.HttpProxy != "" { - ui.Warn(" Http proxy: ", ts.ExecutionRequest.HttpProxy) + ui.Warn(" Http proxy: ", ts.ExecutionRequest.HttpProxy) } if ts.ExecutionRequest.HttpsProxy != "" { - ui.Warn(" Https proxy: ", ts.ExecutionRequest.HttpsProxy) + ui.Warn(" Https proxy: ", ts.ExecutionRequest.HttpsProxy) + } + + if ts.ExecutionRequest.JobTemplate != "" { + ui.Warn(" Job template: ", "\n", ts.ExecutionRequest.JobTemplate) + } + + if ts.ExecutionRequest.JobTemplateReference != "" { + ui.Warn(" Job template reference: ", ts.ExecutionRequest.JobTemplateReference) } if ts.ExecutionRequest.CronJobTemplate != "" { - ui.Warn(" Cron job template: ", ts.ExecutionRequest.CronJobTemplate) + ui.Warn(" Cron job template: ", "\n", ts.ExecutionRequest.CronJobTemplate) + } + + if ts.ExecutionRequest.CronJobTemplateReference != "" { + ui.Warn(" Cron job template reference: ", ts.ExecutionRequest.CronJobTemplateReference) + } + + if ts.ExecutionRequest.ScraperTemplate != "" { + ui.Warn(" Scraper template: ", "\n", ts.ExecutionRequest.ScraperTemplate) + } + + if ts.ExecutionRequest.ScraperTemplateReference != "" { + ui.Warn(" Scraper template reference: ", ts.ExecutionRequest.ScraperTemplateReference) + } + + if ts.ExecutionRequest.PvcTemplate != "" { + ui.Warn(" PVC template: ", "\n", ts.ExecutionRequest.PvcTemplate) + } + + if ts.ExecutionRequest.PvcTemplateReference != "" { + ui.Warn(" PVC template reference: ", ts.ExecutionRequest.PvcTemplateReference) } } diff --git a/cmd/kubectl-testkube/commands/testsuites/run.go b/cmd/kubectl-testkube/commands/testsuites/run.go index 76fdd1c92e4..7324b79824e 100644 --- a/cmd/kubectl-testkube/commands/testsuites/run.go +++ b/cmd/kubectl-testkube/commands/testsuites/run.go @@ -2,6 +2,7 @@ package testsuites import ( "fmt" + "os" "strings" "time" @@ -31,6 +32,12 @@ func NewRunTestSuiteCmd() *cobra.Command { gitPath string gitWorkingDir string runningContext string + jobTemplate string + scraperTemplate string + pvcTemplate string + jobTemplateReference string + scraperTemplateReference string + pvcTemplateReference string ) cmd := &cobra.Command{ @@ -46,20 +53,53 @@ func NewRunTestSuiteCmd() *cobra.Command { var executions []testkube.TestSuiteExecution - variables, err := common.CreateVariables(cmd, false) - ui.WarnOnError("getting variables", err) options := apiv1.ExecuteTestSuiteOptions{ - ExecutionVariables: variables, - HTTPProxy: httpProxy, - HTTPSProxy: httpsProxy, - ExecutionLabels: executionLabels, + HTTPProxy: httpProxy, + HTTPSProxy: httpsProxy, + ExecutionLabels: executionLabels, RunningContext: &testkube.RunningContext{ Type_: string(testkube.RunningContextTypeUserCLI), Context: runningContext, }, - ConcurrencyLevel: int32(concurrencyLevel), + ConcurrencyLevel: int32(concurrencyLevel), + JobTemplateReference: jobTemplateReference, + ScraperTemplateReference: scraperTemplateReference, + PvcTemplateReference: pvcTemplateReference, + } + + var fields = []struct { + source string + title string + destination *string + }{ + { + jobTemplate, + "job template", + &options.JobTemplate, + }, + { + scraperTemplate, + "scraper template", + &options.ScraperTemplate, + }, + { + pvcTemplate, + "pvc template", + &options.PvcTemplate, + }, } + for _, field := range fields { + if field.source != "" { + b, err := os.ReadFile(field.source) + ui.ExitOnError("reading "+field.title, err) + *field.destination = string(b) + } + } + + options.ExecutionVariables, err = common.CreateVariables(cmd, false) + ui.WarnOnError("getting variables", err) + if gitBranch != "" || gitCommit != "" || gitPath != "" || gitWorkingDir != "" { options.ContentRequest = &testkube.TestContentRequest{ Repository: &testkube.RepositoryParameters{ @@ -108,7 +148,7 @@ func NewRunTestSuiteCmd() *cobra.Command { printExecution(execution, startTime) ui.ExitOnError("getting recent execution data id:"+execution.Id, err) - uiPrintExecutionStatus(execution) + uiPrintExecutionStatus(client, execution) uiShellTestSuiteGetCommandBlock(execution.Id) if execution.Id != "" { @@ -139,6 +179,12 @@ func NewRunTestSuiteCmd() *cobra.Command { cmd.Flags().StringVarP(&gitPath, "git-path", "", "", "if repository is big we need to define additional path to directory/file to checkout partially") cmd.Flags().StringVarP(&gitWorkingDir, "git-working-dir", "", "", "if repository contains multiple directories with tests (like monorepo) and one starting directory we can set working directory parameter") cmd.Flags().StringVar(&runningContext, "context", "", "running context description for test suite execution") + cmd.Flags().StringVar(&jobTemplate, "job-template", "", "job template file path for extensions to job template") + cmd.Flags().StringVar(&scraperTemplate, "scraper-template", "", "scraper template file path for extensions to scraper template") + cmd.Flags().StringVar(&pvcTemplate, "pvc-template", "", "pvc template file path for extensions to pvc template") + cmd.Flags().StringVar(&jobTemplateReference, "job-template-reference", "", "reference to job template to use for the test") + cmd.Flags().StringVar(&scraperTemplateReference, "scraper-template-reference", "", "reference to scraper template to use for the test") + cmd.Flags().StringVar(&pvcTemplateReference, "pvc-template-reference", "", "reference to pvc template to use for the test") return cmd } diff --git a/cmd/kubectl-testkube/commands/testsuites/update.go b/cmd/kubectl-testkube/commands/testsuites/update.go index 7e02ce52af9..abd0772f831 100644 --- a/cmd/kubectl-testkube/commands/testsuites/update.go +++ b/cmd/kubectl-testkube/commands/testsuites/update.go @@ -20,7 +20,14 @@ func UpdateTestSuitesCmd() *cobra.Command { httpProxy, httpsProxy string secretVariableReferences map[string]string timeout int32 + jobTemplate string cronJobTemplate string + scraperTemplate string + pvcTemplate string + jobTemplateReference string + cronJobTemplateReference string + scraperTemplateReference string + pvcTemplateReference string ) cmd := &cobra.Command{ @@ -62,10 +69,17 @@ func UpdateTestSuitesCmd() *cobra.Command { cmd.Flags().StringVarP(&schedule, "schedule", "", "", "test suite schedule in a cron job form: * * * * *") cmd.Flags().StringVarP(&executionName, "execution-name", "", "", "execution name, if empty will be autogenerated") cmd.Flags().StringVar(&httpProxy, "http-proxy", "", "http proxy for executor containers") - cmd.Flags().StringToStringVarP(&secretVariableReferences, "secret-variable-reference", "", nil, "secret variable references in a form name1=secret_name1=secret_key1") cmd.Flags().StringVar(&httpsProxy, "https-proxy", "", "https proxy for executor containers") + cmd.Flags().StringToStringVarP(&secretVariableReferences, "secret-variable-reference", "", nil, "secret variable references in a form name1=secret_name1=secret_key1") cmd.Flags().Int32Var(&timeout, "timeout", 0, "duration in seconds for test suite to timeout. 0 disables timeout.") + cmd.Flags().StringVar(&jobTemplate, "job-template", "", "job template file path for extensions to job template") cmd.Flags().StringVar(&cronJobTemplate, "cronjob-template", "", "cron job template file path for extensions to cron job template") + cmd.Flags().StringVar(&scraperTemplate, "scraper-template", "", "scraper template file path for extensions to scraper template") + cmd.Flags().StringVar(&pvcTemplate, "pvc-template", "", "pvc template file path for extensions to pvc template") + cmd.Flags().StringVar(&jobTemplateReference, "job-template-reference", "", "reference to job template to use for the test") + cmd.Flags().StringVar(&cronJobTemplateReference, "cronjob-template-reference", "", "reference to cron job template to use for the test") + cmd.Flags().StringVar(&scraperTemplateReference, "scraper-template-reference", "", "reference to scraper template to use for the test") + cmd.Flags().StringVar(&pvcTemplateReference, "pvc-template-reference", "", "reference to pvc template to use for the test") return cmd } diff --git a/cmd/kubectl-testkube/commands/testsuites/watch.go b/cmd/kubectl-testkube/commands/testsuites/watch.go index e93d73ce058..0f1a141164e 100644 --- a/cmd/kubectl-testkube/commands/testsuites/watch.go +++ b/cmd/kubectl-testkube/commands/testsuites/watch.go @@ -36,7 +36,7 @@ func NewWatchTestSuiteExecutionCmd() *cobra.Command { printExecution(execution, startTime) ui.ExitOnError("getting recent execution data id:"+execution.Id, err) - uiPrintExecutionStatus(execution) + uiPrintExecutionStatus(client, execution) uiShellTestSuiteGetCommandBlock(execution.Id) }, } diff --git a/cmd/kubectl-testkube/commands/update.go b/cmd/kubectl-testkube/commands/update.go index 157a5eb0f38..cd1b77ff3d7 100644 --- a/cmd/kubectl-testkube/commands/update.go +++ b/cmd/kubectl-testkube/commands/update.go @@ -6,6 +6,7 @@ import ( "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common/validator" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/executors" + "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/templates" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/tests" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/testsources" "github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/testsuites" @@ -37,6 +38,7 @@ func NewUpdateCmd() *cobra.Command { cmd.AddCommand(testsources.UpdateTestSourceCmd()) cmd.AddCommand(executors.UpdateExecutorCmd()) cmd.AddCommand(webhooks.UpdateWebhookCmd()) + cmd.AddCommand(templates.UpdateTemplateCmd()) return cmd } diff --git a/cmd/kubectl-testkube/commands/webhooks/common.go b/cmd/kubectl-testkube/commands/webhooks/common.go index fbff807dd63..c818cc62035 100644 --- a/cmd/kubectl-testkube/commands/webhooks/common.go +++ b/cmd/kubectl-testkube/commands/webhooks/common.go @@ -42,16 +42,18 @@ func NewCreateWebhookOptionsFromFlags(cmd *cobra.Command) (options apiv1.CreateW return options, err } + payloadTemplateReference := cmd.Flag("payload-template-reference").Value.String() options = apiv1.CreateWebhookOptions{ - Name: name, - Namespace: namespace, - Events: webhooksmapper.MapStringArrayToCRDEvents(events), - Uri: uri, - Selector: selector, - Labels: labels, - PayloadObjectField: payloadObjectField, - PayloadTemplate: payloadTemplateContent, - Headers: headers, + Name: name, + Namespace: namespace, + Events: webhooksmapper.MapStringArrayToCRDEvents(events), + Uri: uri, + Selector: selector, + Labels: labels, + PayloadObjectField: payloadObjectField, + PayloadTemplate: payloadTemplateContent, + Headers: headers, + PayloadTemplateReference: payloadTemplateReference, } return options, nil @@ -79,6 +81,10 @@ func NewUpdateWebhookOptionsFromFlags(cmd *cobra.Command) (options apiv1.UpdateW "payload-field", &options.PayloadObjectField, }, + { + "payload-template-reference", + &options.PayloadTemplateReference, + }, } for _, field := range fields { diff --git a/cmd/kubectl-testkube/commands/webhooks/create.go b/cmd/kubectl-testkube/commands/webhooks/create.go index b615e33eab5..046cd2c75a1 100644 --- a/cmd/kubectl-testkube/commands/webhooks/create.go +++ b/cmd/kubectl-testkube/commands/webhooks/create.go @@ -14,13 +14,15 @@ import ( func NewCreateWebhookCmd() *cobra.Command { var ( - events []string - name, uri string - selector string - labels map[string]string - payloadObjectField string - payloadTemplate string - headers map[string]string + events []string + name, uri string + selector string + labels map[string]string + payloadObjectField string + payloadTemplate string + headers map[string]string + payloadTemplateReference string + update bool ) cmd := &cobra.Command{ @@ -44,7 +46,25 @@ func NewCreateWebhookCmd() *cobra.Command { webhook, _ := client.GetWebhook(name) if name == webhook.Name { - ui.Failf("Webhook with name '%s' already exists in namespace %s", name, namespace) + if cmd.Flag("update").Changed { + if !update { + ui.Failf("Webhook with name '%s' already exists in namespace %s, ", webhook.Name, namespace) + } + } else { + ok := ui.Confirm(fmt.Sprintf("Webhook with name '%s' already exists in namespace %s, ", webhook.Name, namespace) + + "do you want to overwrite it?") + if !ok { + ui.Failf("Webhook creation was aborted") + } + } + + options, err := NewUpdateWebhookOptionsFromFlags(cmd) + ui.ExitOnError("getting webhook options", err) + + _, err = client.UpdateWebhook(options) + ui.ExitOnError("updating webhook "+name+" in namespace "+namespace, err) + + ui.SuccessAndExit("Webhook updated", name) } } @@ -71,12 +91,14 @@ func NewCreateWebhookCmd() *cobra.Command { cmd.Flags().StringVarP(&name, "name", "n", "", "unique webhook name - mandatory") cmd.Flags().StringArrayVarP(&events, "events", "e", []string{}, "event types handled by webhook e.g. start-test|end-test") - cmd.Flags().StringVarP(&uri, "uri", "u", "", "URI which should be called when given event occurs") + cmd.Flags().StringVarP(&uri, "uri", "u", "", "URI which should be called when given event occurs (golang template supported)") cmd.Flags().StringVarP(&selector, "selector", "", "", "expression to select tests and test suites for webhook events: --selector app=backend") cmd.Flags().StringToStringVarP(&labels, "label", "l", nil, "label key value pair: --label key1=value1") cmd.Flags().StringVarP(&payloadObjectField, "payload-field", "", "", "field to use for notification object payload") cmd.Flags().StringVarP(&payloadTemplate, "payload-template", "", "", "if webhook needs to send a custom notification, then a path to template file should be provided") - cmd.Flags().StringToStringVarP(&headers, "header", "", nil, "webhook header value pair: --header Content-Type=application/xml") + cmd.Flags().StringToStringVarP(&headers, "header", "", nil, "webhook header value pair (golang template supported): --header Content-Type=application/xml") + cmd.Flags().StringVar(&payloadTemplateReference, "payload-template-reference", "", "reference to payload template to use for the webhook") + cmd.Flags().BoolVar(&update, "update", false, "update, if webhook already exists") return cmd } diff --git a/cmd/kubectl-testkube/commands/webhooks/update.go b/cmd/kubectl-testkube/commands/webhooks/update.go index 81d2a469a33..8f0c524d026 100644 --- a/cmd/kubectl-testkube/commands/webhooks/update.go +++ b/cmd/kubectl-testkube/commands/webhooks/update.go @@ -9,13 +9,14 @@ import ( func UpdateWebhookCmd() *cobra.Command { var ( - events []string - name, uri string - selector string - labels map[string]string - payloadObjectField string - payloadTemplate string - headers map[string]string + events []string + name, uri string + selector string + labels map[string]string + payloadObjectField string + payloadTemplate string + headers map[string]string + payloadTemplateReference string ) cmd := &cobra.Command{ @@ -48,12 +49,13 @@ func UpdateWebhookCmd() *cobra.Command { cmd.Flags().StringVarP(&name, "name", "n", "", "unique webhook name - mandatory") cmd.Flags().StringArrayVarP(&events, "events", "e", []string{}, "event types handled by webhook e.g. start-test|end-test") - cmd.Flags().StringVarP(&uri, "uri", "u", "", "URI which should be called when given event occurs") + cmd.Flags().StringVarP(&uri, "uri", "u", "", "URI which should be called when given event occurs (golang template supported)") cmd.Flags().StringVarP(&selector, "selector", "", "", "expression to select tests and test suites for webhook events: --selector app=backend") cmd.Flags().StringToStringVarP(&labels, "label", "l", nil, "label key value pair: --label key1=value1") cmd.Flags().StringVarP(&payloadObjectField, "payload-field", "", "", "field to use for notification object payload") cmd.Flags().StringVarP(&payloadTemplate, "payload-template", "", "", "if webhook needs to send a custom notification, then a path to template file should be provided") - cmd.Flags().StringToStringVarP(&headers, "header", "", nil, "webhook header value pair: --header Content-Type=application/xml") + cmd.Flags().StringToStringVarP(&headers, "header", "", nil, "webhook header value pair (golang template supported): --header Content-Type=application/xml") + cmd.Flags().StringVar(&payloadTemplateReference, "payload-template-reference", "", "reference to payload template to use for the webhook") return cmd } diff --git a/cmd/kubectl-testkube/config/config_test.go b/cmd/kubectl-testkube/config/config_test.go index c75069dfebb..52988e3cdf2 100644 --- a/cmd/kubectl-testkube/config/config_test.go +++ b/cmd/kubectl-testkube/config/config_test.go @@ -1,7 +1,6 @@ package config import ( - "io/ioutil" "os" "testing" @@ -10,7 +9,7 @@ import ( func TestSave(t *testing.T) { // override default directory - dir, err := ioutil.TempDir("", "test-config-save") + dir, err := os.MkdirTemp("", "test-config-save") assert.NoError(t, err) defaultDirectory = dir diff --git a/config/executors.json b/config/executors.json index 33fbcfa1440..2b660a40094 100644 --- a/config/executors.json +++ b/config/executors.json @@ -1,301 +1,532 @@ [ - { - "name": "playwright-executor", - "executor": { - "executorType": "job", - "image": "kubeshop/testkube-playwright-executor:1.10.39", - "types": [ - "playwright/test" - ], - "contentTypes": [ - "git-dir", - "git" - ], - "features": [ - "artifacts" - ], - "meta": { - "iconURI": "playwright", - "docsURI": "https://docs.testkube.io/test-types/executor-playwright" - } + { + "name": "tracetest-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-tracetest-executor:224f02d", + "command": [ + "tracetest" + ], + "args": [ + "test", + "run", + "--server-url", + "", + "--definition", + "", + "--wait-for-result", + "--output", + "pretty" + ], + "types": [ + "tracetest/test" + ], + "contentTypes": [ + "string", + "file-uri", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "tracetest", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-tracetest" + } + } + }, + { + "name": "zap-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-zap-executor:224f02d", + "command": [ + "" + ], + "args": [ + "" + ], + "types": [ + "zap/api", + "zap/baseline", + "zap/full" + ], + "contentTypes": [ + "string", + "file-uri", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "zap", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-zap" + } + } + }, + { + "name": "playwright-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-playwright-executor:224f02d", + "command": [ + "" + ], + "args": [ + "", + "playwright", + "test" + ], + "types": [ + "playwright/test" + ], + "contentTypes": [ + "git-dir", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "playwright", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-playwright" + } + } + }, + { + "name": "jmeter-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-jmeter-executor:224f02d", + "command": [ + "" + ], + "args": [ + "-n", + "-j", + "", + "-t", + "", + "-l", + "", + "-e", + "-o", + "", + "" + ], + "types": [ + "jmeter/test" + ], + "contentTypes": [ + "string", + "file-uri", + "git-file", + "git-dir", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "jmeter", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-jmeter" + } + } + }, + { + "name": "jmeterd-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-jmeterd-executor:224f02d", + "command": [ + "" + ], + "slaves": { + "image": "kubeshop/testkube-jmeterd-slave:224f02d" + }, + "args": [ + "-n", + "-j", + "", + "-t", + "", + "-l", + "", + "-e", + "-o", + "", + "" + ], + "types": [ + "jmeterd/test" + ], + "contentTypes": [ + "string", + "file-uri", + "git-file", + "git-dir", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "jmeter", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-jmeter" + } + } + }, + { + "name": "ginkgo-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-ginkgo-executor:224f02d", + "command": [ + "ginkgo" + ], + "args": [ + "-r", + "-p", + "--randomize-all", + "--randomize-suites", + "--keep-going", + "--trace", + "--junit-report", + "", + "", + "" + ], + "types": [ + "ginkgo/test" + ], + "contentTypes": [ + "string", + "file-uri", + "git-file", + "git-dir", + "git" + ], + "features": [ + "artifacts", + "junit-report" + ], + "meta": { + "iconURI": "ginkgo", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-ginkgo" + } + } + }, + { + "name": "maven-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-maven-executor:224f02d", + "command": [ + "mvn" + ], + "args": [ + "--settings", + "", + "", + "-Duser.home", + "" + ], + "types": [ + "maven/project", + "maven/test", + "maven/integration-test" + ], + "contentTypes": [ + "git-dir", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "maven", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-maven" + } + } + }, + { + "name": "gradle-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-gradle-executor:224f02d", + "command": [ + "gradle" + ], + "args": [ + "--no-daemon", + "", + "-p", + "" + ], + "types": [ + "gradle/project", + "gradle/test", + "gradle/integrationTest" + ], + "contentTypes": [ + "git-dir", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "gradle", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-gradle" + } + } + }, + { + "name": "kubepug-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-kubepug-executor:224f02d", + "command": [ + "kubepug" + ], + "args": [ + "--format=json", + "--input-file", + "" + ], + "types": [ + "kubepug/yaml", + "kubepug/json" + ], + "contentTypes": [ + "string", + "file-uri", + "git-file", + "git-dir", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "kubepug", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-kubepug" + } + } + }, + { + "name": "soapui-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-soapui-executor:224f02d", + "command": [ + "/bin/sh", + "/usr/local/SmartBear/EntryPoint.sh" + ], + "args": [ + "" + ], + "types": [ + "soapui/xml" + ], + "contentTypes": [ + "string", + "file-uri", + "git-file", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "soapui", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-soapui" + } + } + }, + { + "name": "k6-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-k6-executor:224f02d", + "command": [ + "k6" + ], + "args": [ + "", + "", + "" + ], + "types": [ + "k6/script" + ], + "contentTypes": [ + "string", + "file-uri", + "git-file", + "git-dir", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "k6", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-k6" + } + } + }, + { + "name": "cypress-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-cypress-executor:224f02d", + "command": [ + "./node_modules/cypress/bin/cypress" + ], + "args": [ + "run", + "--reporter", + "junit", + "--reporter-options", + "mochaFile=,toConsole=false", + "--project", + "", + "--env", + "" + ], + "types": [ + "cypress/project" + ], + "contentTypes": [ + "git-dir", + "git" + ], + "features": [ + "artifacts", + "junit-report" + ], + "meta": { + "iconURI": "cypress", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-cypress" + } + } + }, + { + "name": "curl-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-curl-executor:224f02d", + "command": [ + "curl" + ], + "args": [ + "-is" + ], + "types": [ + "curl/test" + ], + "contentTypes": [ + "string", + "file-uri", + "git-file", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "curl", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-curl" + } + } + }, + { + "name": "postman-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-postman-executor:224f02d", + "command": [ + "newman" + ], + "args": [ + "run", + "", + "-e", + "", + "--reporters", + "cli,json", + "--reporter-json-export", + "" + ], + "types": [ + "postman/collection" + ], + "contentTypes": [ + "string", + "file-uri", + "git-file", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "postman", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-postman" + } + } + }, + { + "name": "artillery-executor", + "executor": { + "executorType": "job", + "image": "kubeshop/testkube-artillery-executor:224f02d", + "command": [ + "artillery" + ], + "args": [ + "run", + "", + "--dotenv", + "", + "-o", + "" + ], + "types": [ + "artillery/test" + ], + "contentTypes": [ + "string", + "file-uri", + "git-file", + "git-dir", + "git" + ], + "features": [ + "artifacts" + ], + "meta": { + "iconURI": "artillery", + "docsURI": "https://kubeshop.github.io/testkube/test-types/executor-artillery" + } + } + }, + { + "name": "scraper-executor", + "executor": { + "executorType": "scraper", + "image": "kubeshop/testkube-scraper-executor:224f02d", + "types": [] + } + }, + { + "name": "init-executor", + "executor": { + "executorType": "init", + "image": "kubeshop/testkube-init-executor:224f02d", + "types": [] + } } - }, - { - "name": "jmeter-executor", - "executor": { - "executorType": "job", - "image": "kubeshop/testkube-jmeter-executor:1.10.39", - "types": [ - "jmeter/test" - ], - "contentTypes": [ - "string", - "file-uri", - "git-file", - "git" - ], - "features": [ - "artifacts" - ], - "meta": { - "iconURI": "jmeter", - "docsURI": "https://docs.testkube.io/test-types/executor-jmeter" - } - } - }, - { - "name": "ginkgo-executor", - "executor": { - "executorType": "job", - "image": "kubeshop/testkube-ginkgo-executor:1.10.39", - "types": [ - "ginkgo/test" - ], - "contentTypes": [ - "string", - "file-uri", - "git-file", - "git-dir", - "git" - ], - "features": [ - "artifacts", - "junit-report" - ], - "meta": { - "iconURI": "ginkgo", - "docsURI": "https://docs.testkube.io/test-types/executor-ginkgo" - } - } - }, - { - "name": "maven-executor", - "executor": { - "executorType": "job", - "image": "kubeshop/testkube-maven-executor:1.10.39", - "types": [ - "maven/project", - "maven/test", - "maven/integration-test" - ], - "contentTypes": [ - "git-dir", - "git" - ], - "features": [ - - ], - "meta": { - "iconURI": "maven", - "docsURI": "https://docs.testkube.io/test-types/executor-maven" - } - } - }, - { - "name": "gradle-executor", - "executor": { - "executorType": "job", - "image": "kubeshop/testkube-gradle-executor:1.10.39", - "types": [ - "gradle/project", - "gradle/test", - "gradle/integrationTest" - ], - "contentTypes": [ - "git-dir", - "git" - ], - "features": [ - - ], - "meta": { - "iconURI": "gradle", - "docsURI": "https://docs.testkube.io/test-types/executor-gradle" - } - } - }, - { - "name": "kubepug-executor", - "executor": { - "executorType": "job", - "image": "kubeshop/testkube-kubepug-executor:1.10.39", - "types": [ - "kubepug/yaml", - "kubepug/json" - ], - "contentTypes": [ - "string", - "file-uri", - "git-file", - "git-dir", - "git" - ], - "features": [ - - ], - "meta": { - "iconURI": "kubepug", - "docsURI": "https://docs.testkube.io/test-types/executor-kubepug" - } - } - }, - { - "name": "soapui-executor", - "executor": { - "executorType": "job", - "image": "kubeshop/testkube-soapui-executor:1.10.39", - "types": [ - "soapui/xml" - ], - "contentTypes": [ - "string", - "file-uri", - "git-file", - "git" - ], - "features": [ - "artifacts" - ], - "meta": { - "iconURI": "soapui", - "docsURI": "https://docs.testkube.io/test-types/executor-soapui" - } - } - }, - { - "name": "k6-executor", - "executor": { - "executorType": "job", - "image": "kubeshop/testkube-k6-executor:1.10.39", - "types": [ - "k6/script" - ], - "contentTypes": [ - "string", - "file-uri", - "git-file", - "git-dir", - "git" - ], - "features": [ - - ], - "meta": { - "iconURI": "k6", - "docsURI": "https://docs.testkube.io/test-types/executor-k6" - } - } - }, - { - "name": "cypress-executor", - "executor": { - "executorType": "job", - "image": "kubeshop/testkube-cypress-executor:1.10.39", - "types": [ - "cypress/project" - ], - "contentTypes": [ - "git-dir", - "git" - ], - "features": [ - "artifacts", - "junit-report" - ], - "meta": { - "iconURI": "cypress", - "docsURI": "https://docs.testkube.io/test-types/executor-cypress" - } - } - }, - { - "name": "curl-executor", - "executor": { - "executorType": "job", - "image": "kubeshop/testkube-curl-executor:1.10.39", - "types": [ - "curl/test" - ], - "contentTypes": [ - "string", - "file-uri", - "git-file", - "git" - ], - "features": [ - - ], - "meta": { - "iconURI": "curl", - "docsURI": "https://docs.testkube.io/test-types/executor-curl" - } - } - }, - { - "name": "postman-executor", - "executor": { - "executorType": "job", - "image": "kubeshop/testkube-postman-executor:1.10.39", - "types": [ - "postman/collection" - ], - "contentTypes": [ - "string", - "file-uri", - "git-file", - "git" - ], - "features": [ - - ], - "meta": { - "iconURI": "postman", - "docsURI": "https://docs.testkube.io/test-types/executor-postman" - } - } - }, - { - "name": "artillery-executor", - "executor": { - "executorType": "job", - "image": "kubeshop/testkube-artillery-executor:1.10.39", - "types": [ - "artillery/test" - ], - "contentTypes": [ - "string", - "file-uri", - "git-file", - "git-dir", - "git" - ], - "features": [ - - ], - "meta": { - "iconURI": "artillery", - "docsURI": "https://docs.testkube.io/test-types/executor-artillery" - } - } - }, - { - "name": "scraper-executor", - "executor": { - "executorType": "scraper", - "image": "kubeshop/testkube-scraper-executor:1.10.39", - "types": [ - - ] - } - }, - { - "name": "init-executor", - "executor": { - "executorType": "init", - "image": "kubeshop/testkube-init-executor:1.10.39", - "types": [ - - ] - } - } -] \ No newline at end of file +] diff --git a/config/job-container-template.yml b/config/job-container-template.yml index 8e565aa9e87..1ecb62530fe 100644 --- a/config/job-container-template.yml +++ b/config/job-container-template.yml @@ -11,110 +11,110 @@ spec: spec: {{- if ne .InitImage "" }} initContainers: - - name: {{ .Name }}-init - {{- if .Registry }} - image: {{ .Registry }}/{{ .InitImage }} - {{- else }} - image: {{ .InitImage }} - {{- end }} - imagePullPolicy: IfNotPresent - command: - - "/bin/runner" - - '{{ .Jsn }}' - volumeMounts: - - name: data-volume - mountPath: /data - {{- if .CertificateSecret }} - - name: {{ .CertificateSecret }} - mountPath: /etc/certs - {{- end }} - {{- if .ArtifactRequest }} - {{- if .ArtifactRequest.VolumeMountPath }} - - name: artifact-volume - mountPath: {{ .ArtifactRequest.VolumeMountPath }} - {{- end }} - {{- end }} - {{- range $configmap := .EnvConfigMaps }} - {{- if and $configmap.Mount $configmap.Reference }} - - name: {{ $configmap.Reference.Name }} - mountPath: {{ $configmap.MountPath }} - {{- end }} - {{- end }} - {{- range $secret := .EnvSecrets }} - {{- if and $secret.Mount $secret.Reference }} - - name: {{ $secret.Reference.Name }} - mountPath: {{ $secret.MountPath }} - {{- end }} - {{- end }} + - name: {{ .Name }}-init + {{- if .Registry }} + image: {{ .Registry }}/{{ .InitImage }} + {{- else }} + image: {{ .InitImage }} + {{- end }} + imagePullPolicy: IfNotPresent + command: + - "/bin/runner" + - '{{ .Jsn }}' + volumeMounts: + - name: data-volume + mountPath: /data + {{- if .CertificateSecret }} + - name: {{ .CertificateSecret }} + mountPath: /etc/certs + {{- end }} + {{- if .ArtifactRequest }} + {{- if .ArtifactRequest.VolumeMountPath }} + - name: artifact-volume + mountPath: {{ .ArtifactRequest.VolumeMountPath }} + {{- end }} + {{- end }} + {{- range $configmap := .EnvConfigMaps }} + {{- if and $configmap.Mount $configmap.Reference }} + - name: {{ $configmap.Reference.Name }} + mountPath: {{ $configmap.MountPath }} + {{- end }} + {{- end }} + {{- range $secret := .EnvSecrets }} + {{- if and $secret.Mount $secret.Reference }} + - name: {{ $secret.Reference.Name }} + mountPath: {{ $secret.MountPath }} + {{- end }} + {{- end }} {{- end }} containers: - - name: "{{ .Name }}" - {{- if .Registry }} - image: {{ .Registry }}/{{ .Image }} - {{- else }} - image: {{ .Image }} - {{- end }} - imagePullPolicy: IfNotPresent - {{- if gt (len .Command) 0 }} - command: - {{- range $cmd := .Command }} - - {{ $cmd -}} - {{- end }} - {{- end -}} - {{- if gt (len .Args) 0 }} - args: - {{- range $arg := .Args }} - - {{ $arg -}} - {{- end }} - {{- end }} - {{- if .WorkingDir }} - workingDir: {{ .WorkingDir }} - {{- end }} - volumeMounts: - - name: data-volume - mountPath: /data - {{- if .CertificateSecret }} - - name: {{ .CertificateSecret }} - mountPath: /etc/certs - {{- end }} - {{- if .ArtifactRequest }} - {{- if .ArtifactRequest.VolumeMountPath }} - - name: artifact-volume - mountPath: {{ .ArtifactRequest.VolumeMountPath }} - {{- end }} - {{- end }} - {{- range $configmap := .EnvConfigMaps }} - {{- if and $configmap.Mount $configmap.Reference }} - - name: {{ $configmap.Reference.Name }} - mountPath: {{ $configmap.MountPath }} - {{- end }} - {{- end }} - {{- range $secret := .EnvSecrets }} - {{- if and $secret.Mount $secret.Reference }} - - name: {{ $secret.Reference.Name }} - mountPath: {{ $secret.MountPath }} - {{- end }} - {{- end }} - volumes: + - name: "{{ .Name }}" + {{- if .Registry }} + image: {{ .Registry }}/{{ .Image }} + {{- else }} + image: {{ .Image }} + {{- end }} + imagePullPolicy: IfNotPresent + {{- if gt (len .Command) 0 }} + command: + {{- range $cmd := .Command }} + - {{ $cmd -}} + {{- end }} + {{- end -}} + {{- if gt (len .Args) 0 }} + args: + {{- range $arg := .Args }} + - {{ $arg -}} + {{- end }} + {{- end }} + {{- if .WorkingDir }} + workingDir: {{ .WorkingDir }} + {{- end }} + volumeMounts: - name: data-volume - emptyDir: {} + mountPath: /data + {{- if .CertificateSecret }} + - name: {{ .CertificateSecret }} + mountPath: /etc/certs + {{- end }} + {{- if .ArtifactRequest }} + {{- if .ArtifactRequest.VolumeMountPath }} + - name: artifact-volume + mountPath: {{ .ArtifactRequest.VolumeMountPath }} + {{- end }} + {{- end }} + {{- range $configmap := .EnvConfigMaps }} + {{- if and $configmap.Mount $configmap.Reference }} + - name: {{ $configmap.Reference.Name }} + mountPath: {{ $configmap.MountPath }} + {{- end }} + {{- end }} + {{- range $secret := .EnvSecrets }} + {{- if and $secret.Mount $secret.Reference }} + - name: {{ $secret.Reference.Name }} + mountPath: {{ $secret.MountPath }} + {{- end }} + {{- end }} + volumes: + - name: data-volume + emptyDir: {} {{- if .CertificateSecret }} - name: {{ .CertificateSecret }} secret: secretName: {{ .CertificateSecret }} {{- end }} {{- if .ArtifactRequest }} - {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} + {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} - name: artifact-volume persistentVolumeClaim: claimName: {{ .Name }}-pvc - {{- end }} + {{- end }} {{- end }} {{- range $configmap := .EnvConfigMaps }} {{- if and $configmap.Mount $configmap.Reference }} - name: {{ $configmap.Reference.Name }} - configmap: - name: {{ $configmap.Reference.Name }} + configmap: + name: {{ $configmap.Reference.Name }} {{- end }} {{- end }} {{- range $secret := .EnvSecrets }} diff --git a/config/job-scraper-template.yml b/config/job-scraper-template.yml index 89c84c7a620..fb5b58ec0dd 100644 --- a/config/job-scraper-template.yml +++ b/config/job-scraper-template.yml @@ -10,23 +10,23 @@ spec: template: spec: containers: - - name: {{ .Name }}-scraper - {{- if .Registry }} - image: {{ .Registry }}/{{ .ScraperImage }} - {{- else }} - image: {{ .ScraperImage }} - {{- end }} - imagePullPolicy: IfNotPresent - command: - - "/bin/runner" - - '{{ .Jsn }}' - {{- if .ArtifactRequest }} - {{- if .ArtifactRequest.VolumeMountPath }} - volumeMounts: - - name: artifact-volume - mountPath: {{ .ArtifactRequest.VolumeMountPath }} - {{- end }} + - name: {{ .Name }}-scraper + {{- if .Registry }} + image: {{ .Registry }}/{{ .ScraperImage }} + {{- else }} + image: {{ .ScraperImage }} + {{- end }} + imagePullPolicy: IfNotPresent + command: + - "/bin/runner" + - '{{ .Jsn }}' + {{- if .ArtifactRequest }} + {{- if .ArtifactRequest.VolumeMountPath }} + volumeMounts: + - name: artifact-volume + mountPath: {{ .ArtifactRequest.VolumeMountPath }} {{- end }} + {{- end }} {{- if .ArtifactRequest }} {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} volumes: diff --git a/config/job-template.yml b/config/job-template.yml index 29116f7b076..cfb74c0f5b4 100644 --- a/config/job-template.yml +++ b/config/job-template.yml @@ -10,68 +10,68 @@ spec: template: spec: initContainers: - - name: {{ .Name }}-init - {{- if .Registry }} - image: {{ .Registry }}/{{ .InitImage }} - {{- else }} - image: {{ .InitImage }} - {{- end }} - imagePullPolicy: IfNotPresent - command: - - "/bin/runner" - - '{{ .Jsn }}' - volumeMounts: - - name: data-volume - mountPath: /data - {{- if .CertificateSecret }} - - name: {{ .CertificateSecret }} - mountPath: /etc/certs - {{- end }} - {{- range $configmap := .EnvConfigMaps }} - {{- if and $configmap.Mount $configmap.Reference }} - - name: {{ $configmap.Reference.Name }} - mountPath: {{ $configmap.MountPath }} - {{- end }} - {{- end }} - {{- range $secret := .EnvSecrets }} - {{- if and $secret.Mount $secret.Reference }} - - name: {{ $secret.Reference.Name }} - mountPath: {{ $secret.MountPath }} - {{- end }} - {{- end }} + - name: {{ .Name }}-init + {{- if .Registry }} + image: {{ .Registry }}/{{ .InitImage }} + {{- else }} + image: {{ .InitImage }} + {{- end }} + imagePullPolicy: IfNotPresent + command: + - "/bin/runner" + - '{{ .Jsn }}' + volumeMounts: + - name: data-volume + mountPath: /data + {{- if .CertificateSecret }} + - name: {{ .CertificateSecret }} + mountPath: /etc/certs + {{- end }} + {{- range $configmap := .EnvConfigMaps }} + {{- if and $configmap.Mount $configmap.Reference }} + - name: {{ $configmap.Reference.Name }} + mountPath: {{ $configmap.MountPath }} + {{- end }} + {{- end }} + {{- range $secret := .EnvSecrets }} + {{- if and $secret.Mount $secret.Reference }} + - name: {{ $secret.Reference.Name }} + mountPath: {{ $secret.MountPath }} + {{- end }} + {{- end }} containers: - - name: "{{ .Name }}" - {{- if .Registry }} - image: {{ .Registry }}/{{ .Image }} - {{- else }} - image: {{ .Image }} - {{- end }} - imagePullPolicy: IfNotPresent - command: - - "/bin/runner" - - '{{ .Jsn }}' - volumeMounts: - - name: data-volume - mountPath: /data - {{- if .CertificateSecret }} - - name: {{ .CertificateSecret }} - mountPath: /etc/certs - {{- end }} - {{- range $configmap := .EnvConfigMaps }} - {{- if and $configmap.Mount $configmap.Reference }} - - name: {{ $configmap.Reference.Name }} - mountPath: {{ $configmap.MountPath }} - {{- end }} - {{- end }} - {{- range $secret := .EnvSecrets }} - {{- if and $secret.Mount $secret.Reference }} - - name: {{ $secret.Reference.Name }} - mountPath: {{ $secret.MountPath }} - {{- end }} - {{- end }} - volumes: + - name: "{{ .Name }}" + {{- if .Registry }} + image: {{ .Registry }}/{{ .Image }} + {{- else }} + image: {{ .Image }} + {{- end }} + imagePullPolicy: IfNotPresent + command: + - "/bin/runner" + - '{{ .Jsn }}' + volumeMounts: - name: data-volume - emptyDir: {} + mountPath: /data + {{- if .CertificateSecret }} + - name: {{ .CertificateSecret }} + mountPath: /etc/certs + {{- end }} + {{- range $configmap := .EnvConfigMaps }} + {{- if and $configmap.Mount $configmap.Reference }} + - name: {{ $configmap.Reference.Name }} + mountPath: {{ $configmap.MountPath }} + {{- end }} + {{- end }} + {{- range $secret := .EnvSecrets }} + {{- if and $secret.Mount $secret.Reference }} + - name: {{ $secret.Reference.Name }} + mountPath: {{ $secret.MountPath }} + {{- end }} + {{- end }} + volumes: + - name: data-volume + emptyDir: {} {{- if .CertificateSecret }} - name: {{ .CertificateSecret }} secret: diff --git a/config/pvc-container-template.yml b/config/pvc-container-template.yml index ea4d32a28ee..7b1a0a214d7 100644 --- a/config/pvc-container-template.yml +++ b/config/pvc-container-template.yml @@ -9,4 +9,4 @@ spec: - ReadWriteOnce resources: requests: - storage: 1Gi \ No newline at end of file + storage: 1Gi diff --git a/config/slack-config.json b/config/slack-config.json index 055910f4885..c9336c080d7 100644 --- a/config/slack-config.json +++ b/config/slack-config.json @@ -1,20 +1,20 @@ [ - { - "ChannelID": "", - "selector": {}, - "testName": [], - "testSuiteName": [], - "events": [ - "start-test", - "end-test-success", - "end-test-failed", - "end-test-aborted", - "end-test-timeout", - "start-testsuite", - "end-testsuite-success", - "end-testsuite-failed", - "end-testsuite-aborted", - "end-testsuite-timeout" - ] - } -] \ No newline at end of file + { + "ChannelID": "", + "selector": {}, + "testName": [], + "testSuiteName": [], + "events": [ + "start-test", + "end-test-success", + "end-test-failed", + "end-test-aborted", + "end-test-timeout", + "start-testsuite", + "end-testsuite-success", + "end-testsuite-failed", + "end-testsuite-aborted", + "end-testsuite-timeout" + ] + } + ] \ No newline at end of file diff --git a/config/slack-template.json b/config/slack-template.json index dfac6cdc679..b0db6c27025 100644 --- a/config/slack-template.json +++ b/config/slack-template.json @@ -1,143 +1,183 @@ { - "blocks": [ - { - "type": "section", - "text": { - "type": "plain_text", - "emoji": true, - "text": "Execution {{ .ExecutionName }} of {{ .TestName }} status {{ .Status }}" - } - }, - { - "type": "context", - "elements": [ - { - "type": "image", - "image_url": "{{ if eq .Status "failed" }}https://raw.githubusercontent.com/kubeshop/testkube/d3380bc4bf4534ef1fb88cdce5d346dca8898986/assets/imageFailed.png{{ else if eq .Status "passed" }}https://raw.githubusercontent.com/kubeshop/testkube/d3380bc4bf4534ef1fb88cdce5d346dca8898986/assets/imagePassed.png{{ else }}https://raw.githubusercontent.com/kubeshop/testkube/d3380bc4bf4534ef1fb88cdce5d346dca8898986/assets/imagePending.png{{ end }}", - "alt_text": "notifications warning icon" - } - {{ if (gt .TotalSteps 0 )}} - , - { - "type": "mrkdwn", - "text": "* {{ .FailedSteps }}/{{ .TotalSteps }} STEPS FAILED*" - } - {{ end }} - ] - }, - { - "type": "divider" - }, - { - "type": "section", - "fields": [ - { - "type": "mrkdwn", - "text": "*Test Name*" - }, - { - "type": "mrkdwn", - "text": "*Type*" - }, - { - "type": "plain_text", - "text": "{{ .TestName }}", - "emoji": true - }, - { - "type": "plain_text", - "text": "{{ .TestType }}", - "emoji": true - } - ] - }, - {{ if .Namespace}} - { - "type": "section", - "fields": [ - { - "type": "mrkdwn", - "text": "*Namespace*" - }, - { - "type": "mrkdwn", - "text": "*Labels*" - }, - { - "type": "plain_text", - "text": "{{ .Namespace }} ", - "emoji": true - }, - { - "type": "plain_text", - "text": "{{ .Labels }} ", - "emoji": true - } - ] - }, - {{ end }} - { - "type": "section", - "fields": [ - { - "type": "mrkdwn", - "text": "*Start Time*" - }, - { - "type": "mrkdwn", - "text": "*End Time*" - }, - { - "type": "plain_text", - "text": "{{ .StartTime }}", - "emoji": true - }, - { - "type": "plain_text", - "text": "{{ .EndTime }}", - "emoji": true - } - ] - }, - {{ if .Duration }} - { - "type": "section", - "fields": [ - { - "type": "mrkdwn", - "text": "*Duration*" - }, - { - "type": "mrkdwn", - "text": " " - }, - { - "type": "plain_text", - "text": "{{ .Duration }}", - "emoji": true - } - ] - }, - {{ end }} - { - "type": "divider" - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "*Test Execution Results*" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "`kubectl testkube get execution {{ .ExecutionName }} `\n" - } - }, - { - "type": "divider" - } - ] + "blocks": [ + { + "type": "section", + "text": { + "type": "plain_text", + "emoji": true, + "text": "Execution {{ .ExecutionName }} of {{ .TestName }} status {{ .Status }}" + } + }, + { + "type": "context", + "elements": [ + { + "type": "image", + "image_url": "{{ if eq .Status "failed" }}https://raw.githubusercontent.com/kubeshop/testkube/d3380bc4bf4534ef1fb88cdce5d346dca8898986/assets/imageFailed.png{{ else if eq .Status "passed" }}https://raw.githubusercontent.com/kubeshop/testkube/d3380bc4bf4534ef1fb88cdce5d346dca8898986/assets/imagePassed.png{{ else }}https://raw.githubusercontent.com/kubeshop/testkube/d3380bc4bf4534ef1fb88cdce5d346dca8898986/assets/imagePending.png{{ end }}", + "alt_text": "notifications warning icon" + } + {{ if (gt .TotalSteps 0 )}} + , + { + "type": "mrkdwn", + "text": "* {{ .FailedSteps }}/{{ .TotalSteps }} STEPS FAILED*" + } + {{ end }} + ] + }, + { + "type": "divider" + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Test Name*" + }, + { + "type": "mrkdwn", + "text": "*Type*" + }, + { + "type": "plain_text", + "text": "{{ .TestName }}", + "emoji": true + }, + { + "type": "plain_text", + "text": "{{ .TestType }}", + "emoji": true + } + ] + }, + {{ if .Namespace}} + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Namespace*" + }, + { + "type": "mrkdwn", + "text": "*Labels*" + }, + { + "type": "plain_text", + "text": "{{ .Namespace }} ", + "emoji": true + }, + { + "type": "plain_text", + "text": "{{ .Labels }} ", + "emoji": true + } + ] + }, + {{ end }} + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Start Time*" + }, + { + "type": "mrkdwn", + "text": "*End Time*" + }, + { + "type": "plain_text", + "text": "{{ .StartTime }}", + "emoji": true + }, + { + "type": "plain_text", + "text": "{{ .EndTime }}", + "emoji": true + } + ] + }, + {{ if .Duration }} + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Duration*" + }, + { + "type": "mrkdwn", + "text": " " + }, + { + "type": "plain_text", + "text": "{{ .Duration }}", + "emoji": true + } + ] + }, + {{ end }} + { + "type": "divider" + }, + {{ if .ClusterName }} + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Cluster Name: {{ .ClusterName }}" + } + }, + {{ end }} + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*Test Execution CLI*" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "`kubectl testkube get execution {{ .ExecutionName }} `\n" + } + }, + {{ if eq .TestType "Test Suite" }} + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Test Suite URI: {{ .DashboardURI }}/test-suites/{{ .TestName }}" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Test Suite Execution URI: {{ .DashboardURI }}/test-suites/{{ .TestName }}/executions/{{ .ExecutionID }}" + } + }, + {{ else }} + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Test URI: {{ .DashboardURI }}/tests/{{ .TestName }}" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Test Execution URI: {{ .DashboardURI }}/tests/{{ .TestName }}/executions/{{ .ExecutionID }}" + } + }, + {{ end }} + { + "type": "divider" + } + ] } diff --git a/contrib/docker/jmeter/Makefile b/contrib/docker/jmeter/Makefile new file mode 100644 index 00000000000..db7d0c99833 --- /dev/null +++ b/contrib/docker/jmeter/Makefile @@ -0,0 +1,19 @@ +# Variables +DOCKER_REPOSITORY = kubeshop +DOCKER_IMAGE_NAME = jmeter +DOCKER_TAG = 5.5 + +# Build the Docker image +.PHONY: build +build: + @echo "Building Docker image..." + @docker buildx build --platform linux/amd64,linux/arm64 -f jmeter5.5.ubi8.8.Dockerfile -t $(DOCKER_REPOSITORY)/$(DOCKER_IMAGE_NAME):$(DOCKER_TAG) . + +.PHONY: push +push: build + @echo "Pushing Docker image..." + @docker buildx build --push --platform linux/amd64,linux/arm64 -f jmeter5.5.ubi8.8.Dockerfile -t $(DOCKER_REPOSITORY)/$(DOCKER_IMAGE_NAME):$(DOCKER_TAG) . + +test: build + @echo "Testing Docker image..." + @docker run --rm -it $(DOCKER_REPOSITORY)/$(DOCKER_IMAGE_NAME):$(DOCKER_TAG) --version \ No newline at end of file diff --git a/contrib/docker/jmeter/README.md b/contrib/docker/jmeter/README.md new file mode 100644 index 00000000000..c1c386061ef --- /dev/null +++ b/contrib/docker/jmeter/README.md @@ -0,0 +1,25 @@ +# JMeter + +This repository contains Dockerfiles for JMeter builds which are used by the Testkube JMeter Executor. + +Currently supported builds: +* JMeter 5.5 with OpenJDK 17 built on RHEL UBI 8.8 (minimal) + +## Development + +Use the following `make` targets to build and push the images: + +To build the JMeter Docker image use: +```bash +make build +``` + +To do a quick test run of the JMeter Docker image use: +```bash +make test +``` + +To push the JMeter Docker image to the registry use: +```bash +make push +``` \ No newline at end of file diff --git a/contrib/docker/jmeter/jmeter5.5.ubi8.8.Dockerfile b/contrib/docker/jmeter/jmeter5.5.ubi8.8.Dockerfile new file mode 100644 index 00000000000..a84d1ebf2ac --- /dev/null +++ b/contrib/docker/jmeter/jmeter5.5.ubi8.8.Dockerfile @@ -0,0 +1,33 @@ +# Use Red Hat's Universal Base Image 8 +FROM redhat/ubi8-minimal:8.8 + +ENV JAVA_VERSION=17 +ENV JMETER_VERSION=5.5 + +# Labels and authorship +LABEL org.opencontainers.image.title="JMeter" \ + org.opencontainers.image.description="Red Hat UBI with Java $JAVA_VERSION and JMeter $JMETER_VERSION" \ + org.opencontainers.image.version="$JMETER_VERSION" \ + org.opencontainers.image.maintainer="support@testkube.io" \ + org.opencontainers.image.vendor="testkube" \ + org.opencontainers.image.url="https://cloud.testkube.io" \ + org.opencontainers.image.source="https://github.com/kubeshop/testkube/tree/develop/contrib/docker/jmeter" + +# Update the system and install required libraries +RUN microdnf update -y && \ + microdnf install curl unzip java-$JAVA_VERSION-openjdk tar && \ + microdnf clean all + +# Install JMeter +RUN curl -L https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz | tar xz -C /opt/ && \ + mv /opt/apache-jmeter-$JMETER_VERSION /opt/jmeter + +# Set JMeter Home and add JMeter bin directory to the PATH +ENV JMETER_HOME /opt/jmeter +ENV PATH $JMETER_HOME/bin:$PATH + +# Expose the required JMeter ports +EXPOSE 60000 + +# Command to run JMeter tests +ENTRYPOINT [ "jmeter" ] diff --git a/contrib/executor/artillery/pkg/runner/artillery.go b/contrib/executor/artillery/pkg/runner/artillery.go index a97f911b17e..1502eb543c2 100644 --- a/contrib/executor/artillery/pkg/runner/artillery.go +++ b/contrib/executor/artillery/pkg/runner/artillery.go @@ -12,6 +12,7 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/content" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" @@ -122,6 +123,14 @@ func (r *ArtilleryRunner) Run(ctx context.Context, execution testkube.Execution) result = MapTestSummaryToResults(artilleryResult) output.PrintLog(fmt.Sprintf("%s Mapped test summary to Execution Results...", ui.IconCheckMark)) + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + output.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + output.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + if r.Params.ScrapperEnabled { directories := []string{ testReportFile, diff --git a/contrib/executor/curl/pkg/runner/runner.go b/contrib/executor/curl/pkg/runner/runner.go index 1aa5d2e6eb7..ef91f46243f 100644 --- a/contrib/executor/curl/pkg/runner/runner.go +++ b/contrib/executor/curl/pkg/runner/runner.go @@ -3,7 +3,9 @@ package runner import ( "context" "encoding/json" + "fmt" "os" + "path/filepath" "regexp" "strconv" "strings" @@ -14,8 +16,10 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" contentPkg "github.com/kubeshop/testkube/pkg/executor/content" "github.com/kubeshop/testkube/pkg/executor/env" + "github.com/kubeshop/testkube/pkg/executor/output" outputPkg "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" "github.com/kubeshop/testkube/pkg/executor/scraper" @@ -69,7 +73,25 @@ func (r *CurlRunner) Run(ctx context.Context, execution testkube.Execution) (res } if fileInfo.IsDir() { - return result, testkube.ErrTestContentTypeNotFile + scriptName := execution.Args[len(execution.Args)-1] + if workingDir != "" { + path = filepath.Join(r.Params.DataDir, "repo") + if execution.Content != nil && execution.Content.Repository != nil { + scriptName = filepath.Join(execution.Content.Repository.Path, scriptName) + } + } + + execution.Args = execution.Args[:len(execution.Args)-1] + output.PrintLogf("%s It is a directory test - trying to find file from the last executor argument %s in directory %s", ui.IconWorld, scriptName, path) + + // sanity checking for test script + scriptFile := filepath.Join(path, workingDir, scriptName) + fileInfo, errFile := os.Stat(scriptFile) + if errors.Is(errFile, os.ErrNotExist) || fileInfo.IsDir() { + output.PrintLogf("%s Could not find file %s in the directory, error: %s", ui.IconCross, scriptName, errFile) + return *result.Err(errors.Errorf("could not find file %s in the directory: %v", scriptName, errFile)), nil + } + path = scriptFile } content, err := os.ReadFile(path) @@ -124,6 +146,14 @@ func (r *CurlRunner) Run(ctx context.Context, execution testkube.Execution) (res return *result.Err(err), nil } + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + outputPkg.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + outputPkg.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + // scrape artifacts first even if there are errors above if r.Params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { outputPkg.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) diff --git a/contrib/executor/curl/pkg/runner/template.go b/contrib/executor/curl/pkg/runner/template.go index 23bd62dc7d1..6e274cada54 100644 --- a/contrib/executor/curl/pkg/runner/template.go +++ b/contrib/executor/curl/pkg/runner/template.go @@ -2,7 +2,8 @@ package runner import ( "strings" - "text/template" + + "github.com/kubeshop/testkube/pkg/utils" ) // ResolveTemplates fills the string array with the values if they are templated @@ -21,7 +22,7 @@ func ResolveTemplates(stringsToResolve []string, params map[string]string) error // ResolveTemplate fills a string with the values if they are templated func ResolveTemplate(stringToResolve string, params map[string]string) (string, error) { - ut, err := template.New("cmd").Parse(stringToResolve) + ut, err := utils.NewTemplate("cmd").Parse(stringToResolve) if err != nil { return "", err diff --git a/contrib/executor/cypress/pkg/runner/cypress.go b/contrib/executor/cypress/pkg/runner/cypress.go index 82faa1533bb..b98c0d6344f 100644 --- a/contrib/executor/cypress/pkg/runner/cypress.go +++ b/contrib/executor/cypress/pkg/runner/cypress.go @@ -13,6 +13,7 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/content" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" @@ -109,7 +110,8 @@ func (r *CypressRunner) Run(ctx context.Context, execution testkube.Execution) ( envVars = append(envVars, fmt.Sprintf("%s=%s", value.Name, value.Value)) } - junitReportPath := filepath.Join(projectPath, "results/junit.xml") + junitReportDir := filepath.Join(projectPath, "results") + junitReportPath := filepath.Join(projectPath, "results/junit-[hash].xml") var project string if workingDir != "" { @@ -142,7 +144,7 @@ func (r *CypressRunner) Run(ctx context.Context, execution testkube.Execution) ( output.PrintLogf("%s Test run command %s %s", ui.IconRocket, command, strings.Join(args, " ")) out, err = executor.Run(runPath, command, envManager, args...) out = envManager.ObfuscateSecrets(out) - suites, serr := junit.IngestFile(junitReportPath) + suites, serr := junit.IngestDir(junitReportDir) result = MapJunitToExecutionResults(out, suites) output.PrintLogf("%s Mapped Junit to Execution Results...", ui.IconCheckMark) @@ -157,10 +159,18 @@ func (r *CypressRunner) Run(ctx context.Context, execution testkube.Execution) ( } } + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + output.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + output.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + // scrape artifacts first even if there are errors above if r.Params.ScrapperEnabled { directories := []string{ - junitReportPath, + junitReportDir, filepath.Join(projectPath, "cypress/videos"), filepath.Join(projectPath, "cypress/screenshots"), } diff --git a/contrib/executor/ginkgo/pkg/runner/runner.go b/contrib/executor/ginkgo/pkg/runner/runner.go index 59380fe4748..4b4f17e85ce 100644 --- a/contrib/executor/ginkgo/pkg/runner/runner.go +++ b/contrib/executor/ginkgo/pkg/runner/runner.go @@ -13,6 +13,7 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/content" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" @@ -153,8 +154,15 @@ func (r *GinkgoRunner) Run(ctx context.Context, execution testkube.Execution) (r output.PrintLogf("%s Mapped Junit to Execution Results...", ui.IconCheckMark) } - // scrape artifacts first even if there are errors above + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + output.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + if err = agent.RunScript(execution.PostRunScript); err != nil { + output.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + + // scrape artifacts first even if there are errors above if r.Params.ScrapperEnabled { directories := []string{ reportsPath, diff --git a/contrib/executor/gradle/pkg/runner/runner.go b/contrib/executor/gradle/pkg/runner/runner.go index 83f0fdb859f..b171fc035b2 100644 --- a/contrib/executor/gradle/pkg/runner/runner.go +++ b/contrib/executor/gradle/pkg/runner/runner.go @@ -14,6 +14,7 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" @@ -159,6 +160,14 @@ func (r *GradleRunner) Run(ctx context.Context, execution testkube.Execution) (r } } + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + output.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + output.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + // scrape artifacts first even if there are errors above if r.params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { output.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) diff --git a/contrib/executor/init/pkg/runner/runner.go b/contrib/executor/init/pkg/runner/runner.go index a7f4c96c79d..776292de2e9 100755 --- a/contrib/executor/init/pkg/runner/runner.go +++ b/contrib/executor/init/pkg/runner/runner.go @@ -136,7 +136,7 @@ func (r *InitRunner) Run(ctx context.Context, execution testkube.Execution) (res output.PrintLogf("%s Could not chmod for data dir: %s", ui.IconCross, err.Error()) } - if execution.ArtifactRequest != nil { + if execution.ArtifactRequest != nil && execution.ArtifactRequest.StorageClassName != "" { mountPath := filepath.Join(r.Params.DataDir, "artifacts") if execution.ArtifactRequest.VolumeMountPath != "" { mountPath = execution.ArtifactRequest.VolumeMountPath diff --git a/contrib/executor/jmeter/pkg/runner/runner.go b/contrib/executor/jmeter/pkg/runner/runner.go index c10f3b017e1..d3b1e0db775 100644 --- a/contrib/executor/jmeter/pkg/runner/runner.go +++ b/contrib/executor/jmeter/pkg/runner/runner.go @@ -13,6 +13,7 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/content" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" @@ -196,6 +197,14 @@ func (r *JMeterRunner) Run(ctx context.Context, execution testkube.Execution) (r output.PrintLogf("%s Mapped JMeter results to Execution Results...", ui.IconCheckMark) + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + output.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + output.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + // scrape artifacts first even if there are errors above if r.Params.ScrapperEnabled { directories := []string{ diff --git a/contrib/executor/jmeterd/.dockerignore b/contrib/executor/jmeterd/.dockerignore new file mode 100644 index 00000000000..e181b4bbb6b --- /dev/null +++ b/contrib/executor/jmeterd/.dockerignore @@ -0,0 +1,10 @@ +.git +.gitignore +.golangci.yml +CODE_OF_CONDUCT.md +CONTRIBUTING.md +LICENSE +Makefile +README.md +temp +data \ No newline at end of file diff --git a/contrib/executor/jmeterd/.env.sample b/contrib/executor/jmeterd/.env.sample new file mode 100644 index 00000000000..9b3e4d4d166 --- /dev/null +++ b/contrib/executor/jmeterd/.env.sample @@ -0,0 +1,6 @@ +# used if storage backend is behind HTTPS, should be set to false for local development +RUNNER_SSL=false +# used to enable/disable scrapper, should be set to false for local development +RUNNER_SCRAPPERENABLED=false +# path to the data/ directory where JMeter will run and store results +RUNNER_DATADIR=./data diff --git a/contrib/executor/jmeterd/.gitignore b/contrib/executor/jmeterd/.gitignore new file mode 100644 index 00000000000..a67117cf392 --- /dev/null +++ b/contrib/executor/jmeterd/.gitignore @@ -0,0 +1,35 @@ +### Go template +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +vendor/ + +### JetBrains +.idea +*.iml + +# Helm +Chart.lock + +.DS_Store +bin/ +site +dist/ +data/ +temp/ + +.vscode + +### Environment +.env \ No newline at end of file diff --git a/contrib/executor/jmeterd/.golangci.yml b/contrib/executor/jmeterd/.golangci.yml new file mode 100644 index 00000000000..a0ae72e0c50 --- /dev/null +++ b/contrib/executor/jmeterd/.golangci.yml @@ -0,0 +1,29 @@ +run: + timeout: 5m + +linters: + disable-all: true + enable: + - errcheck + - goimports + - govet + - staticcheck + - revive + - unused + - errname + - errorlint + - gocyclo + - gofmt + - goimports + - misspell + - predeclared + +linters-settings: + govet: + check-shadowing: true + lll: + line-length: 150 + misspell: + locale: US + goimports: + local-prefixes: github.com/kubeshop/testkube,github.com/kubeshop/testkube-executor-jmeter diff --git a/contrib/executor/jmeterd/CODE_OF_CONDUCT.md b/contrib/executor/jmeterd/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..fa481824840 --- /dev/null +++ b/contrib/executor/jmeterd/CODE_OF_CONDUCT.md @@ -0,0 +1,30 @@ +# testkube (by Kubeshop) Community Code of Conduct + +Testkube follows the CNCF Code of Conduct. The text of the CNCF CoC is replicated below. If you notice that this is out of date, please file an issue. + +If you notice a violation of the Code of Conduct at an event or meeting, in Slack, or in another communication mechanism, reach out to the Testkube Code of Conduct Committee. You can reach us by email at contact@kubeshop.io Your anonymity will be protected. + +# CNCF Community Code of Conduct v1.0 + +## Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities. + +We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, or nationality. + +## Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery +- Personal attacks +- Trolling or insulting/derogatory comments +- Public or private harassment +- Publishing other's private information, such as physical or electronic addresses, without explicit permission +- Other unethical or unprofessional conduct. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. By adopting this Code of Conduct, project maintainers commit themselves to fairly and consistently applying these principles to every aspect of managing this project. Project maintainers who do not follow or enforce the Code of Conduct may be permanently removed from the project team. + +This code of conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior in Testkube may be reported by contacting the Testkube Code of Conduct Committee via contact@kubeshop.io. + +This Code of Conduct is adapted from the Contributor Covenant (https://contributor-covenant.org), version 1.2.0, available at https://contributor-covenant.org/version/1/2/0/ diff --git a/contrib/executor/jmeterd/CONTRIBUTING.md b/contrib/executor/jmeterd/CONTRIBUTING.md new file mode 100644 index 00000000000..8588d097621 --- /dev/null +++ b/contrib/executor/jmeterd/CONTRIBUTING.md @@ -0,0 +1,44 @@ +# Contribution to Testkube + +Thanks for reaching out contribution 🎉 + +If you're new in Open-source community there is nice guide how to start contributing to projects: +https://github.com/firstcontributions/first-contributions + +## Code of Conduct + +This project and everyone participating in it is governed by the Testkube [code of conduct](CODE_OF_CONDUCT.md) + +## Have questions or idea? + +We're using github discussions for managing pre-development ideas and clarifications, feel free to add one at our [Q&A discussion page](https://github.com/kubeshop/testkube/discussions/categories/q-a) + +New ideas should be placed in [Ideas discussion page](https://github.com/kubeshop/testkube/discussions/categories/ideas) + + + +## General guidance for contributing to Testkube project + +You're very welcome to help in testkube development, there is a lot of incoming work to do :). + +We're trying hard to limit technical debt from the beginning so we defined simple rules into Testkube repo to help with it. + +### For golang based components + +- Always use gofmt (there is only one true way of doing code formatting ;) ). +- Follow golang good practices (proverbs) in your code. +- Tests are your friend (we will target 80% CC in our code). +- Use clean names, don't brake basic design patterns and rules. + +### For infrastructure / Kubernetes based components + +- We're using helm charts to build and share Testkube +- Comment non-obvious decisions +- Use current Helm/Kubernetes versions + + +## How can I help? + +- By fixing [one of many Issues](https://github.com/kubeshop/testkube/issues) - simply fork our repo and create new Pull Request with new code changes. +- By helping to reach out valid results [from discussions](https://github.com/kubeshop/testkube/discussions) + diff --git a/contrib/executor/jmeterd/LICENSE b/contrib/executor/jmeterd/LICENSE new file mode 100644 index 00000000000..4c691a5bad3 --- /dev/null +++ b/contrib/executor/jmeterd/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Kubeshop + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/contrib/executor/jmeterd/Makefile b/contrib/executor/jmeterd/Makefile new file mode 100644 index 00000000000..7ff7eab147f --- /dev/null +++ b/contrib/executor/jmeterd/Makefile @@ -0,0 +1,64 @@ +REPOSITORY ?= kubeshop +NAME ?= testkube-jmeterd-executor +SLAVES_NAME ?= testkube-jmeterd-slave +LOCAL_TAG ?= 999.0.0 +BIN_DIR ?= $(HOME)/bin + +.PHONY: build +build: + go build -o $(BIN_DIR)/$(NAME) cmd/agent/main.go + +.PHONY: build-local-linux +build-local-linux: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/runner cmd/agent/main.go + +.PHONY: run +run: + EXECUTOR_PORT=8082 go run cmd/agent/main.go ${run_args} + +.PHONY: docker-build +docker-build: + docker build -t $(REPOSITORY)/$(NAME) -f build/agent/Dockerfile . + +.PHONY: docker-build-local +docker-build-local: build-local-linux + docker build -t $(REPOSITORY)/$(NAME):$(LOCAL_TAG) -f build/agent/local.Dockerfile . + +docker-build-slaves: + docker build -t $(REPOSITORY)/$(SLAVES_NAME):$(LOCAL_TAG) -f build/slaves/Dockerfile . + +.PHONY: kind-load-local +kind-load-local: build-local-linux + kind load docker-image kubeshop/testkube-jmeterd-executor:999.0.0 + +.PHONY: test +test: + go test ./... -cover + +.PHONY: integration-test +integration-test: + INTEGRATION=y gotestsum --format short-verbose -- -run _Integration -cover ./... + +.PHONY: cover +cover: + @go test -failfast -count=1 -v -tags test -coverprofile=./testCoverage.txt ./... && go tool cover -html=./testCoverage.txt -o testCoverage.html && rm ./testCoverage.txt + open testCoverage.html + +.PHONY: version-bump +version-bump: version-bump-patch + +.PHONY: version-bump-patch +version-bump-patch: + go run cmd/tools/main.go bump -k patch + +.PHONY: version-bump-minor +version-bump-minor: + go run cmd/tools/main.go bump -k minor + +.PHONY: version-bump-major +version-bump-major: + go run cmd/tools/main.go bump -k major + +.PHONY: version-bump-dev +version-bump-dev: + go run cmd/tools/main.go bump --dev diff --git a/contrib/executor/jmeterd/README.md b/contrib/executor/jmeterd/README.md new file mode 100644 index 00000000000..386f96863fc --- /dev/null +++ b/contrib/executor/jmeterd/README.md @@ -0,0 +1,161 @@ +![Testkube Logo](https://raw.githubusercontent.com/kubeshop/testkube/main/assets/testkube-color-gray.png) + +[![Go Report Card](https://goreportcard.com/badge/github.com/kubeshop/testkube-executor-jmeter)](https://goreportcard.com/report/github.com/kubeshop/testkube-executor-jmeter) +[![Go Reference](https://pkg.go.dev/badge/github.com/kubeshop/testkube-executor-jmeter.svg)](https://pkg.go.dev/github.com/kubeshop/testkube-executor-jmeter) +[![License](https://img.shields.io/github/license/kubeshop/testkube-executor-jmeter)]() + +# Distributed JMeter Executor +An extension of JMeter Executor which can run the JMeter Tests in distributed mode by creating slave pods and distributing the test among them. + +## What is an Executor? + +Executor is nothing more than a program wrapped into Docker container which gets JSON (testube.Execution) OpenAPI based document as an input and returns a stream of JSON output lines (testkube.ExecutorOutput), +where each output line is simply wrapped in this JSON, similar to the structured logging idea. + +## Features +This executor is an extension of JMeter executor and has all the features of JMeter executor. In addition to that, it has the following features: +* Can run JMeter tests in distributed mode by creating slave pods and distributing the test among them. +* Supports defining plugins for a test in a git repo by placing plugins in a directory named `plugins` in the test folder. +* Supports overriding JMeter `user.properties` file by placing a custom `user.properties` file in the test folder. + +## Usage + +### Supported Environment Variables + +1. **MASTER_OVERRIDE_JVM_ARGS / SLAVES_OVERRIDE_JVM_ARGS**: Used to override default memory options for JMeter master/slaves. Example: `MASTER_OVERRIDE_JVM_ARGS=-Xmn256m -Xms512m -Xmx512m`. + +2. **SLAVES_COUNT**: Specifies the number of slave pods required for Distributed JMeter tests. Example: `SLAVES_COUNT=3`. Default value of `SLAVES_COUNT` is 1. + +3. **MASTER_ADDITIONAL_JVM_ARGS / SLAVES_ADDITIONAL_JMETER_ARGS**: Allows exporting additional JVM arguments for slaves/master. Example: `MASTER_ADDITIONAL_JVM_ARGS=-Xmx1024m -Xms512m -XX:MaxMetaspaceSize=256m`. + +4. **SLAVES_ADDITIONAL_JMETER_ARGS**: Provides additional JVM arguments for JMeter server / slaves. Example: `SLAVES_ADDITIONAL_JMETER_ARGS=jmeter-server -Jserver.rmi.ssl.disable=true -Dserver_port=60000`. + +### Guide + +Below guide will provide you the details about how to run a Jmeter test in distributed environment. + +1. File option: + + When you provide a test (.jmx) file to `Distributed JMeter ( JMeter in distributed mode )`, + the executor of `Distributed JMeter` will spawn number of slaves pods specified by user through `SLAVES_COUNT` environment variable as described above and run the test on all the slave pods. + +2. Git Option: + Using Git flow of the executor we can have use advanced features of `Distributed JMeter` executor which is not possible with JMeter executor: + + - Additional files required by a particular test like a CSV or JSON file can be provided through git repo. + There is an example of using a CSV file by test (.jmx) file in the `example` folder of `Distributed JMeter`. + - Dynamic plugins required for a test by keeping the plugins inside the test folder in a directory named `plugins` in the git repo. + - Overriding the JMeter `user.properties` can be provided by using custom `user.properties` file in the git repo. + + +For using the Git option and to avail all the above features, user should have the following directory structure in the git repo: + + ``` + github.com/`/`/--- + + |-test1/--- + |- testfile1.jmx + |- userdata.csv ( or any other additional file ) + |- user.properties + |- plugins/--- + |- plugin-manager.jar + |- + + |-test2/--- + |- testfile2.jmx + |- userdata.json ( or any other additional file ) + |- user.properties + |- plugins/--- + |- plugin-manager.jar + |- + + ``` + +For additional info, see the [GitFlow Example test for Distributed JMeter](./examples/gitflow/README.md). + +## Local development + +### Prerequisites + +Make sure the following tools are installed on your machine and available in your PATH: +* [JMeter](https://jmeter.apache.org/download_jmeter.cgi) - pure Java application designed to load test functional behavior and measure performance + +### Setup +1. Create a directory called `data/` where JMeter will run and store results (best practice is to create it in the project root because it is git-ignored) +2. Create a JMeter XML project file and save it as a file named `test-content` in the newly created `data/` directory +3. Create an execution JSON file and save it as a file named `execution.json` based on the template below (best practice is to save it in the `temp/` folder in the project root because it is git-ignored) + ```json + { + "id": "jmeterd-test", + "args": [], + "variables": {}, + "content": { + "type": "string" + } + } + ``` +4. You need to provide the `RUNNER_SCRAPPERENABLED`, `RUNNER_SSL` and `RUNNER_DATADIR` environment variables and run the Executor using the `make run run_args="-f|--file "` make command where `-f|--file ` argument is the path to the `execution.json` file you created in step 3. + ```bash + RUNNER_SCRAPPERENABLED=false RUNNER_SSL=false RUNNER_DATADIR="./data" make run run_args="-f temp/execution.json" + ``` + +#### Execution JSON + +Execution JSON stores information required for an Executor to run the configured tests. + +Breakdown of the Execution JSON: +```json +{ + "args": ["-n", "-t", "test.jmx"], + "variables": { + "example": { + "type": "basic", + "name": "example", + "value": "some-value" + } + }, + "content": { + "type": "string" + } +} +``` +* **args** - array of strings which will be passed to JMeter as arguments + * example: `["-n", "-t", "test.jmx"]` +* **variables** - map of variables which will be passed to JMeter as arguments + * example: `{"example": {"type": "basic", "name": "example", "value": "some-value"}}` +* **content.type** - used to specify that JMeter XML is provided as a text file + +#### Environment Variables +```bash +RUNNER_SSL=false # used if storage backend is behind HTTPS, should be set to false for local development +RUNNER_SCRAPPERENABLED=false # used to enable/disable scrapper, should be set to false for local development +RUNNER_DATADIR= # path to the data/ directory where JMeter will run and store results +``` + +## Testing in Kubernetes + +### Prerequisites +* Kubernetes cluster with Testkube installed (best practice is to install it in the `testkube` namespace) + +### Guide + +After validating locally that the Executor changes work as expected, next step is to test whether Testkube can successfully schedule a Test using the new Executor image. + +NOTE: The following commands assume that Testkube is installed in the `testkube` namespace, if you have it installed in a different namespace, please adjust the `--namespace` flag accordingly. + +The following steps need to be executed in order for Testkube to use the new Executor image: +1. Build the new Executor image using the `make docker-build-local` command. By default, the image will be tagged as `kubeshop/testkube-executor-jmeter:999.0.0` unless a `LOCAL_TAG` environment variable is provided before the command. +2. Now you need to make the image accessible in Kubernetes, there are a couple of approaches: + * *kind* - `kind load docker-image --name ` (e.g. `kind load docker-image testkube-executor-jmeter:999.0.0 --name testkube-k8s-cluster`) + * *minikube* - `minikube image load --profile ` (e.g. `minikube image load testkube-executor-jmeter:999.0.0 --profile k8s-cluster-test`) + * *Docker Desktop* - just by building the image locally, it becomes accessible in the Docker Desktop Kubernetes cluster + * *other* - you can push the image to a registry and then Testkube will pull it in Kubernetes (assuming it has credentials for it if needed) +3. Edit the Job Template and change the `imagePullPolicy` to `IfNotPresent` + * Edit the ConfigMap `testkube-api-server` either by running `kubectl edit configmap testkube-api-server --namespace testkube` or by using a tool like Monokle + * Find the `job-template.yml` key and change the `imagePullPolicy` field in the `containers` section to `IfNotPresent` +4. Edit the Executors configuration and change the base image to use the newly created image: + * Edit the ConfigMap `testkube-api-server` either by running `kubectl edit configmap testkube-api-server --namespace testkube` or by using a tool like Monokle + * Find the `executors.json` key and change the `executor.image` field to use the newly created image for the JMeter Executor (`name` field is `jmeter-executor`) +5. Restart the API Server by running `kubectl rollout restart deployment testkube-api-server --namespace testkube` + +Testkube should now use the new image for the Executor and you can schedule a Test with your preferred method. diff --git a/contrib/executor/jmeterd/build/agent/Dockerfile b/contrib/executor/jmeterd/build/agent/Dockerfile new file mode 100644 index 00000000000..27259e16863 --- /dev/null +++ b/contrib/executor/jmeterd/build/agent/Dockerfile @@ -0,0 +1,11 @@ +# syntax=docker/dockerfile:1 +FROM kubeshop/jmeter:5.5 +COPY jmeterd /bin/runner + +RUN microdnf update -y && microdnf install -y ca-certificates git && microdnf clean all +ENV ENTRYPOINT_CMD="/executor_entrypoint.sh" +WORKDIR /root/ +COPY ./contrib/executor/jmeterd/scripts/entrypoint.sh /executor_entrypoint.sh +COPY ./contrib/executor/jmeterd/scripts/jmeter-master.sh /executor_entrypoint_master.sh + +ENTRYPOINT ["/bin/runner"] diff --git a/contrib/executor/jmeterd/build/agent/local.Dockerfile b/contrib/executor/jmeterd/build/agent/local.Dockerfile new file mode 100644 index 00000000000..aeefa13e996 --- /dev/null +++ b/contrib/executor/jmeterd/build/agent/local.Dockerfile @@ -0,0 +1,16 @@ +# syntax=docker/dockerfile:1 + +FROM kubeshop/jmeter:5.5 + +RUN microdnf update -y && microdnf install -y ca-certificates git && microdnf clean all + +WORKDIR /root/ + +ENV ENTRYPOINT_CMD="/executor_entrypoint.sh" + +COPY dist/runner /bin/runner +COPY scripts/entrypoint.sh /executor_entrypoint.sh +COPY scripts/jmeter-master.sh /executor_entrypoint_master.sh + +ENTRYPOINT ["/bin/runner"] + diff --git a/contrib/executor/jmeterd/build/slaves/Dockerfile b/contrib/executor/jmeterd/build/slaves/Dockerfile new file mode 100644 index 00000000000..6014293199d --- /dev/null +++ b/contrib/executor/jmeterd/build/slaves/Dockerfile @@ -0,0 +1,9 @@ +FROM kubeshop/jmeter:5.5 + + +EXPOSE 1099 60001 +ENV SSL_DISABLED true + +COPY ./contrib/executor/jmeterd/scripts/jmeter-slaves.sh /jmeter_slaves_entrypoint.sh +RUN chmod +x /jmeter_slaves_entrypoint.sh +ENTRYPOINT /jmeter_slaves_entrypoint.sh \ No newline at end of file diff --git a/contrib/executor/jmeterd/cmd/agent/main.go b/contrib/executor/jmeterd/cmd/agent/main.go new file mode 100644 index 00000000000..cb61a04966f --- /dev/null +++ b/contrib/executor/jmeterd/cmd/agent/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "context" + "os" + + "github.com/pkg/errors" + + "github.com/kubeshop/testkube/contrib/executor/jmeterd/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/executor/agent" + "github.com/kubeshop/testkube/pkg/executor/output" +) + +func main() { + ctx := context.Background() + params, err := envs.LoadTestkubeVariables() + if err != nil { + output.PrintError(os.Stderr, errors.Errorf("could not initialize Distributed JMeter Executor environment variables: %v", err)) + os.Exit(1) + } + r, err := runner.NewRunner(ctx, params) + if err != nil { + output.PrintError(os.Stderr, errors.Wrap(err, "error instantiating Distributed JMeter Executor")) + os.Exit(1) + } + agent.Run(ctx, r, os.Args) +} diff --git a/contrib/executor/jmeterd/cmd/tools/main.go b/contrib/executor/jmeterd/cmd/tools/main.go new file mode 100644 index 00000000000..39a186a395d --- /dev/null +++ b/contrib/executor/jmeterd/cmd/tools/main.go @@ -0,0 +1,22 @@ +package main + +import "github.com/kubeshop/testkube/cmd/tools/commands" + +var ( + commit string + version string + builtBy string + date string +) + +func init() { + // pass data from goreleaser to commands package + commands.Version = version + commands.BuiltBy = builtBy + commands.Commit = commit + commands.Date = date +} + +func main() { + commands.Execute() +} diff --git a/contrib/executor/jmeterd/examples/executor.yaml b/contrib/executor/jmeterd/examples/executor.yaml new file mode 100644 index 00000000000..176bda2d9e7 --- /dev/null +++ b/contrib/executor/jmeterd/examples/executor.yaml @@ -0,0 +1,11 @@ +apiVersion: executor.testkube.io/v1 +kind: Executor +metadata: + name: jmeterd-executor + namespace: testkube +spec: + features: + - artifacts + image: kubeshop/testkube-jmeterd-executor:dev-008 + types: + - jmeterd/test diff --git a/contrib/executor/jmeterd/examples/gitflow/README.md b/contrib/executor/jmeterd/examples/gitflow/README.md new file mode 100644 index 00000000000..233d63df6f3 --- /dev/null +++ b/contrib/executor/jmeterd/examples/gitflow/README.md @@ -0,0 +1,74 @@ +# GitFlow Example test for Distributed JMeter + +This test is an example of how to run a distributed JMeter test using a git repo as a source and how to use the advanced features of the executor. + +## Test Breakdown + +### Plugins +All the plugins required by the test are kept in the `plugins` directory of the test folder in the git repo. + +### Additional Files +* **CSV**: The test references a CSV file named `Credentials.csv` located in the `data/` directory relative to the project home directory (`${PROJECT_HOME}`). + This CSV should contain columns `USERNAME` and `PASSWORD`. + +### Environment Variables +* **DATA_CONFIG**: Used to determine the directory of the CSV data file. It defaults to `${PROJECT_HOME}` if not provided. + +### Properties +* **JMETER_UC1_NBUSERS**: Number of users for the test. Defaults to `2` if not provided. +* **JMETER_UC1_RAMPUP**: Ramp-up period for the test in seconds. Defaults to `2` if not provided. +* **JMETER_URI_PATH**: The URI path to test against. Defaults to `/pricing` if not provided. + +## Steps to execute this Test + +### Testkube Dashboard +1. Open the Testkube Dashboard and create a new test. +2. Type a test name (i.e. `jmeterd-example`) and select `jmeterd/test` as test type. +3. Select `Git` as the source type and fill the following details: + * Git Repository URI: https://github.com/kubeshop/testkube + * Branch: develop + * Path: contrib/executor/jmeterd/examples/gitflow +4. Click **Create** to create the test. +5. Select **Settings** tab and then open the **Variables & Secrets** tab from the left menu. +6. Add a new variable called **SLAVES_COUNT** and set it to the number of slave pods you want to spawn for the test. +7. Add another variable called **DATA_CONFIG** and set it to `/data/repo/contrib/executor/jmeterd/examples/gitflow` +8. Click on **Save** underneath **the Variables & Secrets** section. +9. In the Arguments section, set the following arguments: `-GJMETER_UC1_NBUSERS=5 jmeter-properties-external.jmx` +10. Click on **Save** underneath the **Arguments** section. +11. Run the test + +### CRD + +You can also apply the following Test CRD to create the example test: +```yaml +apiVersion: tests.testkube.io/v3 +kind: Test +metadata: + name: jmeterd-example + namespace: testkube + labels: + executor: jmeterd-executor + test-type: jmeterd-test +spec: + type: jmeterd/test + content: + type: git + repository: + type: git + uri: https://github.com/kubeshop/testkube + branch: develop + path: contrib/executor/jmeterd/examples/gitflow + executionRequest: + variables: + DATA_CONFIG: + name: DATA_CONFIG + value: "/data/repo/contrib/executor/jmeterd/examples/gitflow" + type: basic + SLAVES_COUNT: + name: SLAVES_COUNT + value: "2" + type: basic + args: + - "-GJMETER_UC1_NBUSERS=5" + - "jmeter-properties-external.jmx" +``` \ No newline at end of file diff --git a/contrib/executor/jmeterd/examples/gitflow/csvdata/Credentials.csv b/contrib/executor/jmeterd/examples/gitflow/csvdata/Credentials.csv new file mode 100644 index 00000000000..c106f4c47e5 --- /dev/null +++ b/contrib/executor/jmeterd/examples/gitflow/csvdata/Credentials.csv @@ -0,0 +1 @@ +USER_01,PASSWORD_01 USER_02,PASSWORD_02 USER_03,PASSWORD_03 USER_04,PASSWORD_04 USER_05,PASSWORD_05 \ No newline at end of file diff --git a/contrib/executor/jmeterd/examples/gitflow/jmeter-properties-external.jmx b/contrib/executor/jmeterd/examples/gitflow/jmeter-properties-external.jmx new file mode 100644 index 00000000000..443b1536fa9 --- /dev/null +++ b/contrib/executor/jmeterd/examples/gitflow/jmeter-properties-external.jmx @@ -0,0 +1,203 @@ + + + + + Kubeshop site simple perf test + false + true + false + + + + PATH + /pricing + = + + + + + + + + + + UC1_NBUSERS + ${__property(JMETER_UC1_NBUSERS,,2)} + = + + + UC1_RAMPUP + ${__property(JMETER_UC1_RAMPUP,,2)} + = + + + URI_PATH + ${__property(JMETER_URI_PATH,,/pricing)} + = + + + PROJECT_HOME + ${__BeanShell(import org.apache.jmeter.services.FileServer; FileServer.getFileServer().getBaseDir();)} + = + + + + + + , + + ${__env(DATA_CONFIG,,${__eval(${PROJECT_HOME})})}/csvdata/Credentials.csv + false + false + true + shareMode.group + false + USERNAME,PASSWORD + + + + continue + + false + 1 + + ${UC1_NBUSERS} + ${UC1_RAMPUP} + false + + + true + + + + false + false + + + + String uc1_nbusers = vars.get("UC1_NBUSERS"); +String uc1_rampup = vars.get("UC1_RAMPUP"); +String uri_path = vars.get("URI_PATH"); +String username = vars.get("USERNAME"); +String password = vars.get("PASSWORD"); +log.info("================================="); +log.info("UC1_NBUSERS: " + uc1_nbusers); +log.info("UC1_RAMPUP: " + uc1_rampup); +log.info("URI_PATH: " + uri_path); +log.info("USERNAME: " + username); +log.info("PASSWORD: " + password); +log.info("================================="); + + + false + + + + 6 + true + false + + + + + + + testkube.io + + https + + ${PATH} + GET + true + false + true + false + + + + + + + + SOME_NONExisting_String + + + Assertion.response_data + false + 20 + + + + + + + + testkube.io + + https + + ${PATH} + GET + true + false + true + false + + + + + + + + SOME_NONExisting_String + + + Assertion.response_data + false + 20 + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 2 + true + true + true + true + true + true + + + + + + + + + diff --git a/contrib/executor/jmeterd/examples/gitflow/plugins/JMeterPlugins-Extras.jar b/contrib/executor/jmeterd/examples/gitflow/plugins/JMeterPlugins-Extras.jar new file mode 100644 index 00000000000..7b62b70643d Binary files /dev/null and b/contrib/executor/jmeterd/examples/gitflow/plugins/JMeterPlugins-Extras.jar differ diff --git a/contrib/executor/jmeterd/examples/gitflow/plugins/jmeter-parallel-0.11.jar b/contrib/executor/jmeterd/examples/gitflow/plugins/jmeter-parallel-0.11.jar new file mode 100644 index 00000000000..cd79edec962 Binary files /dev/null and b/contrib/executor/jmeterd/examples/gitflow/plugins/jmeter-parallel-0.11.jar differ diff --git a/contrib/executor/jmeterd/examples/gitflow/plugins/jmeter-plugins-dummy-0.4.jar b/contrib/executor/jmeterd/examples/gitflow/plugins/jmeter-plugins-dummy-0.4.jar new file mode 100644 index 00000000000..2bb188708c6 Binary files /dev/null and b/contrib/executor/jmeterd/examples/gitflow/plugins/jmeter-plugins-dummy-0.4.jar differ diff --git a/contrib/executor/jmeterd/examples/gitflow/plugins/jmeter-plugins-functions-2.2.jar b/contrib/executor/jmeterd/examples/gitflow/plugins/jmeter-plugins-functions-2.2.jar new file mode 100644 index 00000000000..99866f754aa Binary files /dev/null and b/contrib/executor/jmeterd/examples/gitflow/plugins/jmeter-plugins-functions-2.2.jar differ diff --git a/contrib/executor/jmeterd/examples/gitflow/plugins/jmeter-plugins-manager-1.9.jar b/contrib/executor/jmeterd/examples/gitflow/plugins/jmeter-plugins-manager-1.9.jar new file mode 100644 index 00000000000..eac78dcefda Binary files /dev/null and b/contrib/executor/jmeterd/examples/gitflow/plugins/jmeter-plugins-manager-1.9.jar differ diff --git a/contrib/executor/jmeterd/examples/gitflow/user.properties b/contrib/executor/jmeterd/examples/gitflow/user.properties new file mode 100644 index 00000000000..89d54a422f8 --- /dev/null +++ b/contrib/executor/jmeterd/examples/gitflow/user.properties @@ -0,0 +1,174 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Sample user.properties file + +#--------------------------------------------------------------------------- +# Classpath configuration +#--------------------------------------------------------------------------- +# +# List of paths (separated by ;) to search for additional JMeter plugin classes, +# for example new GUI elements and samplers. +# A path item can either be a jar file or a directory. +# Any jar file in such a directory will be automatically included, +# jar files in sub directories are ignored. +# The given value is in addition to any jars found in the lib/ext directory. +# Do not use this for utility or plugin dependency jars. +#search_paths=/app1/lib;/app2/lib + +# List of paths that JMeter will search for utility and plugin dependency classes. +# Use your platform path separator (java.io.File.pathSeparatorChar in Java) to separate multiple paths. +# A path item can either be a jar file or a directory. +# Any jar file in such a directory will be automatically included, +# jar files in sub directories are ignored. +# The given value is in addition to any jars found in the lib directory. +# All entries will be added to the class path of the system class loader +# and also to the path of the JMeter internal loader. +# Paths with spaces may cause problems for the JVM +#Example for windows (; separator) +#user.classpath=../classes;../lib;../app1/jar1.jar;../app2/jar2.jar +#Example for linux (:separator) +#user.classpath=../classes:../lib:../app1/jar1.jar:../app2/jar2.jar + +# List of paths (separated by ;) that JMeter will search for utility +# and plugin dependency classes. +# A path item can either be a jar file or a directory. +# Any jar file in such a directory will be automatically included, +# jar files in sub directories are ignored. +# The given value is in addition to any jars found in the lib directory +# or given by the user.classpath property. +# All entries will be added to the path of the JMeter internal loader only. +# For plugin dependencies using plugin_dependency_paths should be preferred over +# user.classpath. +#plugin_dependency_paths=../dependencies/lib;../app1/jar1.jar;../app2/jar2.jar + +#--------------------------------------------------------------------------- +# Reporting configuration +#--------------------------------------------------------------------------- + +# Configure this property to change the report title +#jmeter.reportgenerator.report_title=Apache JMeter Dashboard + +# Used to generate a report based on a date range if needed +# Default date format (from SimpleDateFormat Java API and Locale.ENGLISH) +#jmeter.reportgenerator.date_format=yyyyMMddHHmmss +# Date range start date using date_format property +#jmeter.reportgenerator.start_date= +# Date range end date using date_format property +#jmeter.reportgenerator.end_date= + +# Change this parameter if you want to change the granularity of over time graphs. +# Set to 60000 ms by default +#jmeter.reportgenerator.overall_granularity=60000 + +# Sets the size of the sliding window used by percentile evaluation. +# Caution : higher value provides a better accuracy but needs more memory. +#jmeter.reportgenerator.statistic_window = 20000 + +# Change this parameter if you want to change the granularity of Response time distribution +# Set to 100 ms by default +#jmeter.reportgenerator.graph.responseTimeDistribution.property.set_granularity=100 + +# Change this parameter if you want to keep only some samples. +# Regular Expression which Indicates which samples to keep for graphs and statistics generation. +# Empty value means no filtering +#jmeter.reportgenerator.sample_filter= + +# Change this parameter if you want to override the APDEX satisfaction threshold. +# Set to 500 ms by default +#jmeter.reportgenerator.apdex_satisfied_threshold=500 + +# Change this parameter if you want to override the APDEX tolerance threshold. +# Set to 1500 ms by default +#jmeter.reportgenerator.apdex_tolerated_threshold=1500 + +# Indicates which graph series are filtered (regular expression) +# In the below example we filter on Search and Order samples +# Note that the end of the pattern should always include (-success|-failure)?$ +# TransactionsPerSecondGraphConsumer suffixes transactions with "-success" or "-failure" depending +# on the result +#jmeter.reportgenerator.exporter.html.series_filter=^(Search|Order)(-success|-failure)?$ + +# Indicates whether only controller samples are displayed on graphs that support it. +#jmeter.reportgenerator.exporter.html.show_controllers_only=false + +# This property is used by menu item "Export transactions for report" +# It is used to select which transactions by default will be exported +#jmeter.reportgenerator.exported_transactions_pattern=[a-zA-Z0-9_\\-{}\\$\\.]*[-_][0-9]* + + +## Custom graph definition +#jmeter.reportgenerator.graph.custom_mm_hit.classname=org.apache.jmeter.report.processor.graph.impl.CustomGraphConsumer +#jmeter.reportgenerator.graph.custom_mm_hit.title=Graph Title +#jmeter.reportgenerator.graph.custom_mm_hit.property.set_Y_Axis=Response Time (ms) +#jmeter.reportgenerator.graph.custom_mm_hit.property.set_X_Axis=Over Time +#jmeter.reportgenerator.graph.custom_mm_hit.property.set_granularity=${jmeter.reportgenerator.overall_granularity} +#jmeter.reportgenerator.graph.custom_mm_hit.property.setSampleVariableName=VarName +#jmeter.reportgenerator.graph.custom_mm_hit.property.setContentMessage=Message for graph point label + +######################################################################## +################## DISTRIBUTED TESTING CONFIGURATION ################## +######################################################################## +# Type of keystore : JKS +# +#server.rmi.ssl.keystore.type=JKS +# +# Keystore file that contains private key +# +#server.rmi.ssl.keystore.file=rmi_keystore.jks +# +# Password of keystore +# +#server.rmi.ssl.keystore.password=changeit +# +# Key alias +# +#server.rmi.ssl.keystore.alias=rmi +# +# Type of truststore : JKS +# +#server.rmi.ssl.truststore.type=JKS +# +# Keystore file that contains certificate +# +#server.rmi.ssl.truststore.file=rmi_keystore.jks +# +# Password of Trust store +# +#server.rmi.ssl.truststore.password=changeit +# +# Set this if you don't want to use SSL for RMI +# +#server.rmi.ssl.disable=false + +jmeter.reportgenerator.overall_granularity=1000 +jmeter.reportgenerator.exporter.html.series_filter=^([0-9]*_.*)(-success|-failure)?$ + + +proxy.cert.validity=31 +node.name=Node1 + +#Cookie management +CookieManager.check.cookies=false + +jmeter.save.saveservice.assertion_results_failure_message=true +jmeter.save.saveservice.response_message=true +jmeter.save.saveservice.successful=true +jmeter.save.saveservice.thread_name=true +jmeter.save.saveservice.time=true +jmeter.save.saveservice.timestamp_format=ms +jmeter.save.saveservice.assertion_results=all diff --git a/contrib/executor/jmeterd/examples/jmeter.log b/contrib/executor/jmeterd/examples/jmeter.log new file mode 100644 index 00000000000..dc8b8dc8510 --- /dev/null +++ b/contrib/executor/jmeterd/examples/jmeter.log @@ -0,0 +1,62 @@ +2022-11-03 09:23:58,163 INFO o.a.j.u.JMeterUtils: Setting Locale to en_EN +2022-11-03 09:23:58,175 INFO o.a.j.JMeter: Loading user properties from: /opt/homebrew/Cellar/jmeter/5.5/libexec/bin/user.properties +2022-11-03 09:23:58,175 INFO o.a.j.JMeter: Loading system properties from: /opt/homebrew/Cellar/jmeter/5.5/libexec/bin/system.properties +2022-11-03 09:23:58,178 INFO o.a.j.JMeter: Copyright (c) 1998-2022 The Apache Software Foundation +2022-11-03 09:23:58,178 INFO o.a.j.JMeter: Version 5.5 +2022-11-03 09:23:58,178 INFO o.a.j.JMeter: java.version=19 +2022-11-03 09:23:58,178 INFO o.a.j.JMeter: java.vm.name=OpenJDK 64-Bit Server VM +2022-11-03 09:23:58,178 INFO o.a.j.JMeter: os.name=Mac OS X +2022-11-03 09:23:58,178 INFO o.a.j.JMeter: os.arch=aarch64 +2022-11-03 09:23:58,178 INFO o.a.j.JMeter: os.version=12.4 +2022-11-03 09:23:58,178 INFO o.a.j.JMeter: file.encoding=UTF-8 +2022-11-03 09:23:58,178 INFO o.a.j.JMeter: java.awt.headless=true +2022-11-03 09:23:58,178 INFO o.a.j.JMeter: Max memory =1073741824 +2022-11-03 09:23:58,178 INFO o.a.j.JMeter: Available Processors =10 +2022-11-03 09:23:58,183 INFO o.a.j.JMeter: Default Locale=English (EN) +2022-11-03 09:23:58,183 INFO o.a.j.JMeter: JMeter Locale=English (EN) +2022-11-03 09:23:58,183 INFO o.a.j.JMeter: JMeterHome=/opt/homebrew/Cellar/jmeter/5.5/libexec +2022-11-03 09:23:58,183 INFO o.a.j.JMeter: user.dir =/Users/exu/code/kube/testkube-executor-jmeter/examples +2022-11-03 09:23:58,183 INFO o.a.j.JMeter: PWD =/Users/exu/code/kube/testkube-executor-jmeter/examples +2022-11-03 09:23:58,185 INFO o.a.j.JMeter: IP: 192.168.1.7 Name: 192.168.1.7 FullName: 192.168.1.7 +2022-11-03 09:23:58,190 INFO o.a.j.s.FileServer: Default base='/Users/exu/code/kube/testkube-executor-jmeter/examples' +2022-11-03 09:23:58,190 INFO o.a.j.s.FileServer: Set new base='/Users/exu/code/kube/testkube-executor-jmeter/examples' +2022-11-03 09:23:58,291 INFO o.a.j.s.SaveService: Testplan (JMX) version: 2.2. Testlog (JTL) version: 2.2 +2022-11-03 09:23:58,307 INFO o.a.j.s.SaveService: Using SaveService properties version 5.0 +2022-11-03 09:23:58,309 INFO o.a.j.s.SaveService: Using SaveService properties file encoding UTF-8 +2022-11-03 09:23:58,310 INFO o.a.j.s.SaveService: Loading file: kubeshop.jmx +2022-11-03 09:23:58,333 INFO o.a.j.p.h.s.HTTPSamplerBase: Parser for text/html is org.apache.jmeter.protocol.http.parser.LagartoBasedHtmlParser +2022-11-03 09:23:58,333 INFO o.a.j.p.h.s.HTTPSamplerBase: Parser for application/xhtml+xml is org.apache.jmeter.protocol.http.parser.LagartoBasedHtmlParser +2022-11-03 09:23:58,333 INFO o.a.j.p.h.s.HTTPSamplerBase: Parser for application/xml is org.apache.jmeter.protocol.http.parser.LagartoBasedHtmlParser +2022-11-03 09:23:58,333 INFO o.a.j.p.h.s.HTTPSamplerBase: Parser for text/xml is org.apache.jmeter.protocol.http.parser.LagartoBasedHtmlParser +2022-11-03 09:23:58,333 INFO o.a.j.p.h.s.HTTPSamplerBase: Parser for text/vnd.wap.wml is org.apache.jmeter.protocol.http.parser.RegexpHTMLParser +2022-11-03 09:23:58,334 INFO o.a.j.p.h.s.HTTPSamplerBase: Parser for text/css is org.apache.jmeter.protocol.http.parser.CssParser +2022-11-03 09:23:58,340 INFO o.a.j.JMeter: Creating summariser +2022-11-03 09:23:58,349 INFO o.a.j.e.StandardJMeterEngine: Running the test! +2022-11-03 09:23:58,350 INFO o.a.j.s.SampleEvent: List of sample_variables: [] +2022-11-03 09:23:58,350 INFO o.a.j.s.SampleEvent: List of sample_variables: [] +2022-11-03 09:23:58,352 INFO o.a.j.e.u.CompoundVariable: Note: Function class names must contain the string: '.functions.' +2022-11-03 09:23:58,352 INFO o.a.j.e.u.CompoundVariable: Note: Function class names must not contain the string: '.gui.' +2022-11-03 09:23:58,382 INFO o.a.j.JMeter: Running test (1667463838382) +2022-11-03 09:23:58,395 INFO o.a.j.e.StandardJMeterEngine: Starting ThreadGroup: 1 : Thread Group +2022-11-03 09:23:58,395 INFO o.a.j.e.StandardJMeterEngine: Starting 1 threads for group Thread Group. +2022-11-03 09:23:58,395 INFO o.a.j.e.StandardJMeterEngine: Thread will continue on error +2022-11-03 09:23:58,395 INFO o.a.j.t.ThreadGroup: Starting thread group... number=1 threads=1 ramp-up=1 delayedStart=false +2022-11-03 09:23:58,397 INFO o.a.j.t.ThreadGroup: Started thread group number 1 +2022-11-03 09:23:58,397 INFO o.a.j.e.StandardJMeterEngine: All thread groups have been started +2022-11-03 09:23:58,398 INFO o.a.j.t.JMeterThread: Thread started: Thread Group 1-1 +2022-11-03 09:23:58,406 INFO o.a.j.p.h.s.HTTPHCAbstractImpl: Local host = 192.168.1.7 +2022-11-03 09:23:58,409 INFO o.a.j.p.h.s.HTTPHC4Impl: HTTP request retry count = 0 +2022-11-03 09:23:58,410 INFO o.a.j.s.SampleResult: Note: Sample TimeStamps are START times +2022-11-03 09:23:58,410 INFO o.a.j.s.SampleResult: sampleresult.default.encoding is set to ISO-8859-1 +2022-11-03 09:23:58,410 INFO o.a.j.s.SampleResult: sampleresult.useNanoTime=true +2022-11-03 09:23:58,410 INFO o.a.j.s.SampleResult: sampleresult.nanoThreadSleep=5000 +2022-11-03 09:23:58,456 INFO o.a.j.p.h.s.h.LazyLayeredConnectionSocketFactory: Setting up HTTPS TrustAll Socket Factory +2022-11-03 09:23:58,459 INFO o.a.j.u.JsseSSLManager: Using default SSL protocol: TLS +2022-11-03 09:23:58,459 INFO o.a.j.u.JsseSSLManager: SSL session context: per-thread +2022-11-03 09:23:58,538 INFO o.a.j.u.SSLManager: JmeterKeyStore Location: type JKS +2022-11-03 09:23:58,540 INFO o.a.j.u.SSLManager: KeyStore created OK +2022-11-03 09:23:58,540 WARN o.a.j.u.SSLManager: Keystore file not found, loading empty keystore +2022-11-03 09:23:58,813 INFO o.a.j.t.JMeterThread: Thread is done: Thread Group 1-1 +2022-11-03 09:23:58,813 INFO o.a.j.t.JMeterThread: Thread finished: Thread Group 1-1 +2022-11-03 09:23:58,815 INFO o.a.j.e.StandardJMeterEngine: Notifying test listeners of end of test +2022-11-03 09:23:58,816 INFO o.a.j.r.Summariser: summary = 1 in 00:00:00 = 2.3/s Avg: 362 Min: 362 Max: 362 Err: 0 (0.00%) diff --git a/contrib/executor/jmeterd/examples/kubeshop.jmx b/contrib/executor/jmeterd/examples/kubeshop.jmx new file mode 100644 index 00000000000..11e991a820f --- /dev/null +++ b/contrib/executor/jmeterd/examples/kubeshop.jmx @@ -0,0 +1,76 @@ + + + + + Kubeshop site simple perf test + false + true + false + + + + PATH + /pricing + = + + + + + + + + continue + + false + 1 + + 1 + 1 + false + + + true + + + + + + + false + $PATH + = + true + PATH + + + + testkube.io + 80 + https + + https://testkube.io + GET + true + false + true + false + + + + + + + + Testkube + + + Assertion.response_data + false + 16 + + + + + + + diff --git a/contrib/executor/jmeterd/examples/kubeshop_failed.jmx b/contrib/executor/jmeterd/examples/kubeshop_failed.jmx new file mode 100644 index 00000000000..908b3ebe828 --- /dev/null +++ b/contrib/executor/jmeterd/examples/kubeshop_failed.jmx @@ -0,0 +1,76 @@ + + + + + Kubeshop site simple perf test + false + true + false + + + + PATH + /pricing + = + + + + + + + + continue + + false + 1 + + 1 + 1 + false + + + true + + + + + + + false + $PATH + = + true + PATH + + + + testkube.io + 80 + https + + https://testkube.io + GET + true + false + true + false + + + + + + + + SOME_NONExisting_String + + + Assertion.response_data + false + 16 + + + + + + + diff --git a/contrib/executor/jmeterd/examples/results.jtl b/contrib/executor/jmeterd/examples/results.jtl new file mode 100644 index 00000000000..4a595e7d018 --- /dev/null +++ b/contrib/executor/jmeterd/examples/results.jtl @@ -0,0 +1,4 @@ +timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect +1667463814102,382,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,326,0,235 +1667463836936,365,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,309,0,222 +1667463838447,362,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,309,0,219 diff --git a/contrib/executor/jmeterd/pkg/README.md b/contrib/executor/jmeterd/pkg/README.md new file mode 100644 index 00000000000..633b8777e6e --- /dev/null +++ b/contrib/executor/jmeterd/pkg/README.md @@ -0,0 +1,59 @@ +# `/pkg` + +Library code that's ok to use by external applications (e.g., `/pkg/mypubliclib`). Other projects will import these libraries expecting them to work, so think twice before you put something here :-) Note that the `internal` directory is a better way to ensure your private packages are not importable because it's enforced by Go. The `/pkg` directory is still a good way to explicitly communicate that the code in that directory is safe for use by others. The [`I'll take pkg over internal`](https://travisjeffery.com/b/2019/11/i-ll-take-pkg-over-internal/) blog post by Travis Jeffery provides a good overview of the `pkg` and `internal` directories and when it might make sense to use them. + +It's also a way to group Go code in one place when your root directory contains lots of non-Go components and directories making it easier to run various Go tools (as mentioned in these talks: [`Best Practices for Industrial Programming`](https://www.youtube.com/watch?v=PTE4VJIdHPg) from GopherCon EU 2018, [GopherCon 2018: Kat Zien - How Do You Structure Your Go Apps](https://www.youtube.com/watch?v=oL6JBUk6tj0) and [GoLab 2018 - Massimiliano Pippi - Project layout patterns in Go](https://www.youtube.com/watch?v=3gQa1LWwuzk)). + +Note that this is not a universally accepted pattern and for every popular repo that uses it you can find 10 that don't. It's up to you to decide if you want to use this pattern or not. Regardless of whether or not it's a good pattern more people will know what you mean than not. It might a bit confusing for some of the new Go devs, but it's a pretty simple confusion to resolve and that's one of the goals for this project layout repo. + +Ok not to use it if your app project is really small and where an extra level of nesting doesn't add much value (unless you really want to). Think about it when it's getting big enough and your root directory gets pretty busy (especially if you have a lot of non-Go app components). + +The `pkg` directory origins: The old Go source code used to use `pkg` for its packages and then various Go projects in the community started copying the pattern (see [`this`](https://twitter.com/bradfitz/status/1039512487538970624) Brad Fitzpatrick's tweet for more context). + + +Examples: + +* https://github.com/prometheus/prometheus/tree/master/pkg +* https://github.com/jaegertracing/jaeger/tree/master/pkg +* https://github.com/istio/istio/tree/master/pkg +* https://github.com/GoogleContainerTools/kaniko/tree/master/pkg +* https://github.com/google/gvisor/tree/master/pkg +* https://github.com/google/syzkaller/tree/master/pkg +* https://github.com/perkeep/perkeep/tree/master/pkg +* https://github.com/minio/minio/tree/master/pkg +* https://github.com/heptio/ark/tree/master/pkg +* https://github.com/argoproj/argo/tree/master/pkg +* https://github.com/heptio/sonobuoy/tree/master/pkg +* https://github.com/helm/helm/tree/master/pkg +* https://github.com/kubernetes/kubernetes/tree/master/pkg +* https://github.com/kubernetes/kops/tree/master/pkg +* https://github.com/moby/moby/tree/master/pkg +* https://github.com/grafana/grafana/tree/master/pkg +* https://github.com/influxdata/influxdb/tree/master/pkg +* https://github.com/cockroachdb/cockroach/tree/master/pkg +* https://github.com/derekparker/delve/tree/master/pkg +* https://github.com/etcd-io/etcd/tree/master/pkg +* https://github.com/oklog/oklog/tree/master/pkg +* https://github.com/flynn/flynn/tree/master/pkg +* https://github.com/jesseduffield/lazygit/tree/master/pkg +* https://github.com/gopasspw/gopass/tree/master/pkg +* https://github.com/sosedoff/pgweb/tree/master/pkg +* https://github.com/GoogleContainerTools/skaffold/tree/master/pkg +* https://github.com/knative/serving/tree/master/pkg +* https://github.com/grafana/loki/tree/master/pkg +* https://github.com/bloomberg/goldpinger/tree/master/pkg +* https://github.com/Ne0nd0g/merlin/tree/master/pkg +* https://github.com/jenkins-x/jx/tree/master/pkg +* https://github.com/DataDog/datadog-agent/tree/master/pkg +* https://github.com/dapr/dapr/tree/master/pkg +* https://github.com/cortexproject/cortex/tree/master/pkg +* https://github.com/dexidp/dex/tree/master/pkg +* https://github.com/pusher/oauth2_proxy/tree/master/pkg +* https://github.com/pdfcpu/pdfcpu/tree/master/pkg +* https://github.com/weaveworks/kured/tree/master/pkg +* https://github.com/weaveworks/footloose/tree/master/pkg +* https://github.com/weaveworks/ignite/tree/master/pkg +* https://github.com/tmrts/boilr/tree/master/pkg +* https://github.com/kata-containers/runtime/tree/master/pkg +* https://github.com/okteto/okteto/tree/master/pkg +* https://github.com/solo-io/squash/tree/master/pkg diff --git a/contrib/executor/jmeterd/pkg/jmeterenv/env.go b/contrib/executor/jmeterd/pkg/jmeterenv/env.go new file mode 100644 index 00000000000..fb968ab1b38 --- /dev/null +++ b/contrib/executor/jmeterd/pkg/jmeterenv/env.go @@ -0,0 +1,37 @@ +package jmeterenv + +import ( + "strings" + + "github.com/kubeshop/testkube/pkg/api/v1/testkube" +) + +const ( + MasterOverrideJvmArgs = "MASTER_OVERRIDE_JVM_ARGS" + MasterAdditionalJvmArgs = "MASTER_ADDITIONAL_JVM_ARGS" + SlavesOverrideJvmArgs = "SLAVES_OVERRIDE_JVM_ARGS" + SlavesAdditionalJvmArgs = "SLAVES_ADDITIONAL_JVM_ARGS" + SlavesAdditionalJmeterArgs = "SLAVES_ADDITIONAL_JMETER_ARGS" + SlavesCount = "SLAVES_COUNT" + MasterPrefix = "MASTER_" + SlavesPrefix = "SLAVES_" +) + +// ExtractSlaveEnvVariables removes slave environment variables from the given map and returns them separately. +func ExtractSlaveEnvVariables(variables map[string]testkube.Variable) map[string]testkube.Variable { + slaveVariables := make(map[string]testkube.Variable) + + // Iterate through the variables to extract slave environment variables. + for k, v := range variables { + switch { + case strings.HasPrefix(k, SlavesPrefix): + slaveVariables[k] = v + delete(variables, k) // Remove slave variable from the main variables map. + case strings.HasPrefix(k, MasterPrefix): + continue + default: + slaveVariables[k] = v + } + } + return slaveVariables +} diff --git a/contrib/executor/jmeterd/pkg/runner/mapper.go b/contrib/executor/jmeterd/pkg/runner/mapper.go new file mode 100644 index 00000000000..078cf287113 --- /dev/null +++ b/contrib/executor/jmeterd/pkg/runner/mapper.go @@ -0,0 +1,82 @@ +package runner + +import ( + "fmt" + + "github.com/kubeshop/testkube/contrib/executor/jmeter/pkg/parser" + "github.com/kubeshop/testkube/pkg/api/v1/testkube" +) + +func mapResultsToExecutionResults(out []byte, results parser.Results) (result testkube.ExecutionResult) { + result.Status = testkube.ExecutionStatusPassed + if results.HasError { + result.Status = testkube.ExecutionStatusFailed + result.ErrorMessage = results.LastErrorMessage + } + + result.Output = string(out) + result.OutputType = "text/plain" + + for _, r := range results.Results { + result.Steps = append( + result.Steps, + testkube.ExecutionStepResult{ + Name: r.Label, + Duration: r.Duration.String(), + Status: mapResultStatus(r), + AssertionResults: []testkube.AssertionResult{{ + Name: r.Label, + Status: mapResultStatus(r), + }}, + }) + } + + return result +} + +func mapTestResultsToExecutionResults(out []byte, results parser.TestResults) (result testkube.ExecutionResult) { + result.Status = testkube.ExecutionStatusPassed + + result.Output = string(out) + result.OutputType = "text/plain" + + samples := append(results.HTTPSamples, results.Samples...) + for _, r := range samples { + if !r.Success { + result.Status = testkube.ExecutionStatusFailed + if r.AssertionResult != nil { + result.ErrorMessage = r.AssertionResult.FailureMessage + } + } + + result.Steps = append( + result.Steps, + testkube.ExecutionStepResult{ + Name: r.Label, + Duration: fmt.Sprintf("%dms", r.Time), + Status: mapTestResultStatus(r.Success), + AssertionResults: []testkube.AssertionResult{{ + Name: r.Label, + Status: mapTestResultStatus(r.Success), + }}, + }) + } + + return result +} + +func mapResultStatus(result parser.Result) string { + if result.Success { + return string(testkube.PASSED_ExecutionStatus) + } + + return string(testkube.FAILED_ExecutionStatus) +} + +func mapTestResultStatus(success bool) string { + if success { + return string(testkube.PASSED_ExecutionStatus) + } + + return string(testkube.FAILED_ExecutionStatus) +} diff --git a/contrib/executor/jmeterd/pkg/runner/runner.go b/contrib/executor/jmeterd/pkg/runner/runner.go new file mode 100644 index 00000000000..6192863b588 --- /dev/null +++ b/contrib/executor/jmeterd/pkg/runner/runner.go @@ -0,0 +1,280 @@ +package runner + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/kubeshop/testkube/contrib/executor/jmeter/pkg/parser" + "github.com/kubeshop/testkube/contrib/executor/jmeterd/pkg/jmeterenv" + "github.com/kubeshop/testkube/contrib/executor/jmeterd/pkg/slaves" + "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" + "github.com/kubeshop/testkube/pkg/executor/content" + "github.com/kubeshop/testkube/pkg/executor/env" + "github.com/kubeshop/testkube/pkg/executor/output" + "github.com/kubeshop/testkube/pkg/executor/runner" + "github.com/kubeshop/testkube/pkg/executor/scraper" + "github.com/kubeshop/testkube/pkg/executor/scraper/factory" + "github.com/kubeshop/testkube/pkg/ui" +) + +func NewRunner(ctx context.Context, params envs.Params) (*JMeterDRunner, error) { + output.PrintLogf("%s Preparing test runner", ui.IconTruck) + + var err error + r := &JMeterDRunner{ + Params: params, + } + + r.Scraper, err = factory.TryGetScrapper(ctx, params) + if err != nil { + return nil, err + } + + slavesConfigs := executor.SlavesConfigs{} + if err := json.Unmarshal([]byte(params.SlavesConfigs), &slavesConfigs); err != nil { + return nil, errors.Wrap(err, "error unmarshalling slaves configs") + } + r.SlavesConfigs = slavesConfigs + + return r, nil +} + +// JMeterDRunner runner +type JMeterDRunner struct { + Params envs.Params + Scraper scraper.Scraper + SlavesConfigs executor.SlavesConfigs +} + +var _ runner.Runner = &JMeterDRunner{} + +func (r *JMeterDRunner) Run(ctx context.Context, execution testkube.Execution) (result testkube.ExecutionResult, err error) { + if r.Scraper != nil { + defer r.Scraper.Close() + } + output.PrintEvent( + fmt.Sprintf("%s Running with config", ui.IconTruck), + "scraperEnabled", r.Params.ScrapperEnabled, + "dataDir", r.Params.DataDir, + "SSL", r.Params.Ssl, + "endpoint", r.Params.Endpoint, + ) + + envManager := env.NewManagerWithVars(execution.Variables) + envManager.GetReferenceVars(envManager.Variables) + + path, workingDir, err := content.GetPathAndWorkingDir(execution.Content, r.Params.DataDir) + if err != nil { + output.PrintLogf("%s Failed to resolve absolute directory for %s, using the path directly", ui.IconWarning, r.Params.DataDir) + } + + fileInfo, err := os.Stat(path) + if err != nil { + return result, err + } + + if fileInfo.IsDir() { + scriptName := execution.Args[len(execution.Args)-1] + if workingDir != "" { + path = filepath.Join(r.Params.DataDir, "repo") + if execution.Content != nil && execution.Content.Repository != nil { + scriptName = filepath.Join(execution.Content.Repository.Path, scriptName) + } + } + + execution.Args = execution.Args[:len(execution.Args)-1] + output.PrintLogf("%s It is a directory test - trying to find file from the last executor argument %s in directory %s", ui.IconWorld, scriptName, path) + + // sanity checking for test script + scriptFile := filepath.Join(path, workingDir, scriptName) + fileInfo, errFile := os.Stat(scriptFile) + if errors.Is(errFile, os.ErrNotExist) || fileInfo.IsDir() { + output.PrintLogf("%s Could not find file %s in the directory, error: %s", ui.IconCross, scriptName, errFile) + return *result.Err(errors.Errorf("could not find file %s in the directory: %v", scriptName, errFile)), nil + } + path = scriptFile + } + + slavesEnvVariables := jmeterenv.ExtractSlaveEnvVariables(envManager.Variables) + // compose parameters passed to JMeter with -J + params := make([]string, 0, len(envManager.Variables)) + for _, value := range envManager.Variables { + if value.Name == jmeterenv.MasterOverrideJvmArgs || value.Name == jmeterenv.MasterAdditionalJvmArgs { + //Skip JVM ARGS to be appended in the command + continue + } + params = append(params, fmt.Sprintf("-J%s=%s", value.Name, value.Value)) + + } + + runPath := r.Params.DataDir + if workingDir != "" { + runPath = workingDir + } + + parentTestFolder := filepath.Join(filepath.Dir(path)) + // Set env plugin env variable to set custom plugin directory + // with this path custom plugin will be copied to jmeter's plugin directory + err = os.Setenv("JMETER_PARENT_TEST_FOLDER", parentTestFolder) + if err != nil { + output.PrintLogf("%s Failed to set parent test folder directory %s", ui.IconWarning, parentTestFolder) + } + // Add user plugins folder in slaves env variables + slavesEnvVariables["JMETER_PARENT_TEST_FOLDER"] = testkube.NewBasicVariable("JMETER_PARENT_TEST_FOLDER", parentTestFolder) + + outputDir := filepath.Join(runPath, "output") + // clean output directory it already exists, only useful for local development + _, err = os.Stat(outputDir) + if err == nil { + if err = os.RemoveAll(outputDir); err != nil { + output.PrintLogf("%s Failed to clean output directory %s", ui.IconWarning, outputDir) + } + } + // recreate output directory with wide permissions so JMeter can create report files + if err = os.Mkdir(outputDir, 0777); err != nil { + return *result.Err(errors.Wrapf(err, "error creating directory %s", runPath)), nil + } + + jtlPath := filepath.Join(outputDir, "report.jtl") + reportPath := filepath.Join(outputDir, "report") + jmeterLogPath := filepath.Join(outputDir, "jmeter.log") + args := execution.Args + for i := range args { + if args[i] == "" { + args[i] = path + } + + if args[i] == "" { + args[i] = jtlPath + } + + if args[i] == "" { + args[i] = reportPath + } + + if args[i] == "" { + args[i] = jmeterLogPath + } + } + + slaveClient, err := slaves.NewClient(execution, r.SlavesConfigs, r.Params, slavesEnvVariables) + if err != nil { + return *result.WithErrors(errors.Wrap(err, "error creating slaves client")), nil + } + + //creating slaves provided in SLAVES_COUNT env variable + slaveMeta, err := slaveClient.CreateSlaves(ctx) + if err != nil { + return *result.WithErrors(errors.Wrap(err, "error creating slaves")), nil + } + defer slaveClient.DeleteSlaves(ctx, slaveMeta) + + args = append(args, fmt.Sprintf("-R %v", slaveMeta.ToIPString())) + + for i := range args { + if args[i] == "" { + newArgs := make([]string, len(args)+len(params)-1) + copy(newArgs, args[:i]) + copy(newArgs[i:], params) + copy(newArgs[i+len(params):], args[i+1:]) + args = newArgs + break + } + } + + output.PrintLogf("%s Using arguments: %v", ui.IconWorld, args) + + entryPoint := getEntryPoint() + for i := range execution.Command { + if execution.Command[i] == "" { + execution.Command[i] = entryPoint + } + } + + command, args := executor.MergeCommandAndArgs(execution.Command, args) + // run JMeter inside repo directory ignore execution error in case of failed test + output.PrintLogf("%s Test run command %s %s", ui.IconRocket, command, strings.Join(args, " ")) + out, err := executor.Run(runPath, command, envManager, args...) + if err != nil { + return *result.WithErrors(errors.Errorf("jmeter run error: %v", err)), nil + } + out = envManager.ObfuscateSecrets(out) + + output.PrintLogf("%s Getting report %s", ui.IconFile, jtlPath) + f, err := os.Open(jtlPath) + if err != nil { + return *result.WithErrors(errors.Errorf("getting jtl report error: %v", err)), nil + } + + results, err := parser.ParseCSV(f) + f.Close() + + var executionResult testkube.ExecutionResult + if err != nil { + data, err := os.ReadFile(jtlPath) + if err != nil { + return *result.WithErrors(errors.Errorf("getting jtl report error: %v", err)), nil + } + + testResults, err := parser.ParseXML(data) + if err != nil { + return *result.WithErrors(errors.Errorf("parsing jtl report error: %v", err)), nil + } + + executionResult = mapTestResultsToExecutionResults(out, testResults) + } else { + executionResult = mapResultsToExecutionResults(out, results) + } + + output.PrintLogf("%s Mapped JMeter results to Execution Results...", ui.IconCheckMark) + + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + output.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + output.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + + // scrape artifacts first even if there are errors above + if r.Params.ScrapperEnabled { + directories := []string{ + outputDir, + } + if execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + directories = append(directories, execution.ArtifactRequest.Dirs...) + } + + output.PrintLogf("Scraping directories: %v", directories) + if err := r.Scraper.Scrape(ctx, directories, execution); err != nil { + return *executionResult.Err(err), errors.Wrap(err, "error scraping artifacts for JMeter executor") + } + } + + return executionResult, nil +} + +func getEntryPoint() (entrypoint string) { + if entrypoint = os.Getenv("ENTRYPOINT_CMD"); entrypoint != "" { + return entrypoint + } + wd, err := os.Getwd() + if err != nil { + wd = "." + } + return filepath.Join(wd, "scripts/entrypoint.sh") +} + +// GetType returns runner type +func (r *JMeterDRunner) GetType() runner.Type { + return runner.TypeMain +} diff --git a/contrib/executor/jmeterd/pkg/slaves/client.go b/contrib/executor/jmeterd/pkg/slaves/client.go new file mode 100644 index 00000000000..e246e256d37 --- /dev/null +++ b/contrib/executor/jmeterd/pkg/slaves/client.go @@ -0,0 +1,259 @@ +package slaves + +import ( + "context" + "encoding/json" + "time" + + batchv1 "k8s.io/api/batch/v1" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + + "github.com/kubeshop/testkube/contrib/executor/jmeterd/pkg/jmeterenv" + "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/output" + "github.com/kubeshop/testkube/pkg/k8sclient" +) + +const ( + podsTimeout = 5 * time.Minute + job = "Job" + batchV1 = "batch/v1" +) + +type Client struct { + clientSet *kubernetes.Clientset + slavesConfigs executor.SlavesConfigs + namespace string + execution testkube.Execution + envParams envs.Params + envVariables map[string]testkube.Variable +} + +// NewClient is a method to create new slave client +func NewClient(execution testkube.Execution, slavesConfigs executor.SlavesConfigs, envParams envs.Params, slavesEnvVariables map[string]testkube.Variable) (*Client, error) { + clientSet, err := k8sclient.ConnectToK8s() + if err != nil { + return nil, err + } + + return &Client{ + clientSet: clientSet, + slavesConfigs: slavesConfigs, + namespace: execution.TestNamespace, + execution: execution, + envParams: envParams, + envVariables: slavesEnvVariables, + }, nil +} + +// CreateSlaves creates slaves as per count provided in the SLAVES_COUNT env variable. +// Default SLAVES_COUNT would be 1 if not provided in the env variables +func (c *Client) CreateSlaves(ctx context.Context) (SlaveMeta, error) { + slavesCount, err := getSlavesCount(c.envVariables[jmeterenv.SlavesCount]) + if err != nil { + return nil, errors.Wrap(err, "error getting slaves count from SLAVES_COUNT environment variable") + } + + output.PrintLogf("Creating slave pods: %d", slavesCount) + podIPAddressChan := make(chan map[string]string, slavesCount) + errorChan := make(chan error, slavesCount) + podIPAddresses := make(map[string]string) + + for i := 1; i <= slavesCount; i++ { + go c.createSlavePod(ctx, i, podIPAddressChan, errorChan) + } + + for i := 0; i < slavesCount; i++ { + select { + case ipAddress := <-podIPAddressChan: + for podName, podIp := range ipAddress { + podIPAddresses[podName] = podIp + } + case err := <-errorChan: + if err != nil { + return nil, errors.Wrap(err, "error while creating and resolving slave pod IP addresses") + } + } + } + + output.PrintLog("Successfully resolved slave pods IP addresses") + + slaveMeta := SlaveMeta(podIPAddresses) + return slaveMeta, nil +} + +// createSlavePod creates a slave pod and sends its IP address on the podIPAddressChan +// channel when the pod is in the ready state. +func (c *Client) createSlavePod(ctx context.Context, currentSlavesCount int, podIPAddressChan chan<- map[string]string, errorChan chan<- error) { + slavePod, err := c.getSlavePodConfiguration(ctx, currentSlavesCount) + if err != nil { + errorChan <- err + return + } + + p, err := c.clientSet.CoreV1().Pods(c.namespace).Create(ctx, slavePod, metav1.CreateOptions{}) + if err != nil { + errorChan <- err + return + } + + // Wait for the pod to become ready + conditionFunc := isPodReady(c.clientSet, p.Name, c.namespace) + + if err = wait.PollUntilContextTimeout(ctx, time.Second, podsTimeout, true, conditionFunc); err != nil { + errorChan <- err + return + } + + p, err = c.clientSet.CoreV1().Pods(c.namespace).Get(ctx, p.Name, metav1.GetOptions{}) + if err != nil { + errorChan <- err + return + } + podNameIPMap := map[string]string{ + p.Name: p.Status.PodIP, + } + podIPAddressChan <- podNameIPMap +} + +func (c *Client) getSlavePodConfiguration(ctx context.Context, currentSlavesCount int) (*v1.Pod, error) { + runnerExecutionStr, err := json.Marshal(c.execution) + if err != nil { + return nil, errors.Wrap(err, "error marshalling runner execution") + } + + podName := ValidateAndGetSlavePodName(c.execution.Name, c.execution.Id, currentSlavesCount) + if err != nil { + return nil, errors.Wrap(err, "error validating slave pod name") + } + + executorJob, err := c.clientSet.BatchV1().Jobs(c.namespace).Get(ctx, c.execution.Id, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrap(err, "error getting executor job") + } + + return c.createSlavePodObject(runnerExecutionStr, podName, executorJob), nil +} + +func (c *Client) createSlavePodObject(runnerExecutionStr []byte, podName string, executorJob *batchv1.Job) *v1.Pod { + labels := map[string]string{ + // Execution ID is the only unique field in case of multiple runs of the same test + // So this is the only field which can tag the slave pods to actual job of jmeterd executor + "testkube.io/managed-by": c.execution.Id, + "testkube.io/test-name": c.execution.TestName, + } + ownerReference := []metav1.OwnerReference{ + { + Kind: job, + APIVersion: batchV1, + Name: executorJob.Name, + UID: executorJob.UID, + }, + } + initContainers := []v1.Container{ + { + Name: "init", + Image: c.slavesConfigs.Images.Init, + Command: []string{"/bin/runner", string(runnerExecutionStr)}, + Env: getSlaveRunnerEnv(c.envParams, c.execution), + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: []v1.VolumeMount{ + { + MountPath: "/data", + Name: "data-volume", + }, + }, + }, + } + volumes := []v1.Volume{ + { + Name: "data-volume", + VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}, + }, + } + mainContainerLivenessProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.FromInt32(serverPort), + }, + }, + FailureThreshold: 3, + PeriodSeconds: 5, + SuccessThreshold: 1, + TimeoutSeconds: 1, + } + mainContainerReadinessProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.FromInt32(serverPort), + }, + }, + FailureThreshold: 3, + InitialDelaySeconds: 10, + PeriodSeconds: 5, + TimeoutSeconds: 1, + } + mainContainerPorts := []v1.ContainerPort{ + { + ContainerPort: serverPort, + Name: "server-port", + }, { + ContainerPort: localPort, + Name: "local-port", + }, + } + mainContainerVolumeMounts := []v1.VolumeMount{ + { + MountPath: "/data", + Name: "data-volume", + }, + } + containers := []v1.Container{ + { + Name: "main", + Image: c.slavesConfigs.Images.Slave, + Env: getSlaveConfigurationEnv(c.envVariables), + ImagePullPolicy: v1.PullIfNotPresent, + Ports: mainContainerPorts, + VolumeMounts: mainContainerVolumeMounts, + LivenessProbe: mainContainerLivenessProbe, + ReadinessProbe: mainContainerReadinessProbe, + }, + } + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Labels: labels, + OwnerReferences: ownerReference, + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + InitContainers: initContainers, + Containers: containers, + Volumes: volumes, + }, + } +} + +func (c *Client) DeleteSlaves(ctx context.Context, meta SlaveMeta) error { + for _, name := range meta.Names() { + output.PrintLogf("Deleting slave pod: %v", name) + err := c.clientSet.CoreV1().Pods(c.namespace).Delete(ctx, name, metav1.DeleteOptions{}) + if err != nil { + output.PrintLogf("Error deleting slave pods: %v", err.Error()) + return err + } + + } + return nil +} + +var _ Interface = (*Client)(nil) diff --git a/contrib/executor/jmeterd/pkg/slaves/interface.go b/contrib/executor/jmeterd/pkg/slaves/interface.go new file mode 100644 index 00000000000..e36e4318e4c --- /dev/null +++ b/contrib/executor/jmeterd/pkg/slaves/interface.go @@ -0,0 +1,8 @@ +package slaves + +import "context" + +type Interface interface { + CreateSlaves(context.Context) (SlaveMeta, error) + DeleteSlaves(context.Context, SlaveMeta) error +} diff --git a/contrib/executor/jmeterd/pkg/slaves/meta.go b/contrib/executor/jmeterd/pkg/slaves/meta.go new file mode 100644 index 00000000000..73c1686c96d --- /dev/null +++ b/contrib/executor/jmeterd/pkg/slaves/meta.go @@ -0,0 +1,31 @@ +package slaves + +import ( + "strings" + + "golang.org/x/exp/slices" +) + +type SlaveMeta map[string]string + +func (m *SlaveMeta) Names() []string { + var names []string + for k := range *m { + names = append(names, k) + } + return names +} + +func (m *SlaveMeta) IPs() []string { + var ips []string + for _, v := range *m { + ips = append(ips, v) + } + return ips +} + +func (m *SlaveMeta) ToIPString() string { + ips := m.IPs() + slices.Sort(ips) + return strings.Join(m.IPs(), ",") +} diff --git a/contrib/executor/jmeterd/pkg/slaves/meta_test.go b/contrib/executor/jmeterd/pkg/slaves/meta_test.go new file mode 100644 index 00000000000..a6738b18c66 --- /dev/null +++ b/contrib/executor/jmeterd/pkg/slaves/meta_test.go @@ -0,0 +1,66 @@ +package slaves + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSlaveMeta_Names(t *testing.T) { + t.Parallel() + + meta := SlaveMeta{ + "slave1": "192.168.1.1", + "slave2": "192.168.1.2", + } + names := meta.Names() + assert.Len(t, names, 2) + assert.Contains(t, names, "slave1") + assert.Contains(t, names, "slave2") +} + +func TestSlaveMeta_IPs(t *testing.T) { + t.Parallel() + + meta := SlaveMeta{ + "slave1": "192.168.1.1", + "slave2": "192.168.1.2", + } + ips := meta.IPs() + assert.Len(t, ips, 2) + assert.Contains(t, ips, "192.168.1.1") + assert.Contains(t, ips, "192.168.1.2") +} + +func TestSlaveMeta_ToIPString(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + meta SlaveMeta + expected string + }{ + { + name: "Empty", + meta: SlaveMeta{}, + expected: "", + }, + { + name: "Single", + meta: SlaveMeta{"slave1": "192.168.1.1"}, + expected: "192.168.1.1", + }, + { + name: "Multiple", + meta: SlaveMeta{"slave1": "192.168.1.1", "slave2": "192.168.1.2"}, + expected: "192.168.1.1,192.168.1.2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ipString := tt.meta.ToIPString() + assert.Equal(t, tt.expected, ipString) + }) + } +} diff --git a/contrib/executor/jmeterd/pkg/slaves/utils.go b/contrib/executor/jmeterd/pkg/slaves/utils.go new file mode 100644 index 00000000000..c8427ba23f5 --- /dev/null +++ b/contrib/executor/jmeterd/pkg/slaves/utils.go @@ -0,0 +1,104 @@ +package slaves + +import ( + "context" + "fmt" + "strconv" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + + "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/output" +) + +const ( + defaultSlavesCount = 1 + serverPort = 1099 + localPort = 60001 +) + +func getSlaveRunnerEnv(envParams envs.Params, runnerExecution testkube.Execution) []v1.EnvVar { + var gitEnvs []v1.EnvVar + if runnerExecution.Content.Type_ == "git" && runnerExecution.Content.Repository.UsernameSecret != nil && runnerExecution.Content.Repository.TokenSecret != nil { + gitEnvs = append(gitEnvs, v1.EnvVar{ + Name: "RUNNER_GITUSERNAME", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: runnerExecution.Content.Repository.UsernameSecret.Name, + }, + Key: runnerExecution.Content.Repository.UsernameSecret.Key, + }, + }, + }, v1.EnvVar{ + Name: "RUNNER_GITTOKEN", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: runnerExecution.Content.Repository.TokenSecret.Name, + }, + Key: runnerExecution.Content.Repository.TokenSecret.Key, + }, + }, + }, + ) + } + + return append(executor.RunnerEnvVars, gitEnvs...) +} + +func getSlaveConfigurationEnv(slaveEnv map[string]testkube.Variable) []v1.EnvVar { + var envVars []v1.EnvVar + for envKey, t := range slaveEnv { + envVars = append(envVars, v1.EnvVar{Name: envKey, Value: t.Value}) + } + return envVars +} + +func isPodReady(c kubernetes.Interface, podName, namespace string) wait.ConditionWithContextFunc { + return func(ctx context.Context) (bool, error) { + pod, err := c.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return false, err + } + + for _, condition := range pod.Status.Conditions { + isReadyType := condition.Type == v1.PodReady + isConditionTrue := condition.Status == v1.ConditionTrue + isRunningPhase := pod.Status.Phase == v1.PodRunning + ipNotEmpty := pod.Status.PodIP != "" + if isReadyType && isConditionTrue && isRunningPhase && ipNotEmpty { + return true, nil + } + } + return false, nil + } +} + +func getSlavesCount(count testkube.Variable) (int, error) { + if count.Value == "" { + output.PrintLogf("Slaves count not provided in the SLAVES_COUNT env variable. Defaulting to %v slaves", defaultSlavesCount) + return defaultSlavesCount, nil + } + + replicaCount, err := strconv.Atoi(count.Value) + if err != nil { + return 0, err + } + return replicaCount, err +} + +func ValidateAndGetSlavePodName(testName string, executionId string, currentSlaveCount int) string { + slavePodName := fmt.Sprintf("%s-slave-%v-%s", testName, currentSlaveCount, executionId) + if len(slavePodName) > 64 { + //Get first 20 chars from testName name if pod name > 64 + shortExecutionName := testName[:20] + slavePodName = fmt.Sprintf("%s-slave-%v-%s", shortExecutionName, currentSlaveCount, executionId) + } + return slavePodName +} diff --git a/contrib/executor/jmeterd/pkg/slaves/utils_test.go b/contrib/executor/jmeterd/pkg/slaves/utils_test.go new file mode 100644 index 00000000000..0850ffefd6e --- /dev/null +++ b/contrib/executor/jmeterd/pkg/slaves/utils_test.go @@ -0,0 +1,162 @@ +package slaves + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes/fake" + + "github.com/kubeshop/testkube/pkg/api/v1/testkube" +) + +func TestGetSlavesCount(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input testkube.Variable + want int + wantErr bool + }{ + { + name: "Empty Value", + input: testkube.Variable{Value: ""}, + want: defaultSlavesCount, + wantErr: false, + }, + { + name: "Valid Value", + input: testkube.Variable{Value: "10"}, + want: 10, + wantErr: false, + }, + { + name: "Invalid Value", + input: testkube.Variable{Value: "abc"}, + want: 0, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getSlavesCount(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("getSlavesCount() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.Equal(t, tt.want, got) + }) + } +} + +func TestValidateAndGetSlavePodName(t *testing.T) { + t.Parallel() + + tests := []struct { + testName string + executionId string + currentSlaveCount int + expectedOutput string + }{ + { + testName: "aVeryLongTestNameThatExceedsTheLimitWhenConcatenated", + executionId: "exec123", + currentSlaveCount: 5, + expectedOutput: "aVeryLongTestNameTha-slave-5-exec123", + }, + { + testName: "shortName", + executionId: "exec123", + currentSlaveCount: 5, + expectedOutput: "shortName-slave-5-exec123", + }, + } + + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + actualOutput := ValidateAndGetSlavePodName(tt.testName, tt.executionId, tt.currentSlaveCount) + if actualOutput != tt.expectedOutput { + t.Errorf("expected %v, got %v", tt.expectedOutput, actualOutput) + } + }) + } +} + +func TestIsPodReady(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("poll until pod is ready", func(t *testing.T) { + clientset := fake.NewSimpleClientset() + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + }, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + Conditions: []v1.PodCondition{ + { + Type: v1.PodReady, + Status: v1.ConditionTrue, + }, + }, + PodIP: "192.168.1.1", + }, + } + + _, err := clientset.CoreV1().Pods("default").Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("error injecting pod add: %v", err) + } + + conditionFunc := isPodReady(clientset, "test-pod", "default") + + // Use PollImmediate to repeatedly evaluate condition + err = wait.PollUntilContextTimeout(ctx, time.Millisecond*5, time.Second*3, true, conditionFunc) + if err != nil { + t.Fatalf("error waiting for pod to be ready: %v", err) + } + }) + + t.Run("poll times out", func(t *testing.T) { + clientset := fake.NewSimpleClientset() + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + }, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + Conditions: []v1.PodCondition{ + { + Type: v1.PodInitialized, + Status: v1.ConditionFalse, + }, + }, + PodIP: "192.168.1.1", + }, + } + + _, err := clientset.CoreV1().Pods("default").Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("error injecting pod add: %v", err) + } + + conditionFunc := isPodReady(clientset, "test-pod", "default") + + // Use PollImmediate to repeatedly evaluate condition + err = wait.PollUntilContextTimeout(ctx, time.Millisecond*50, time.Millisecond*160, true, conditionFunc) + assert.ErrorContains(t, err, "context deadline exceeded") + }) + +} diff --git a/contrib/executor/jmeterd/scripts/entrypoint.sh b/contrib/executor/jmeterd/scripts/entrypoint.sh new file mode 100755 index 00000000000..25e844b147d --- /dev/null +++ b/contrib/executor/jmeterd/scripts/entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +EXECUTOR_CUSTOM_PLUGINS_FOLDER="${RUNNER_DATADIR}/uploads/plugins" + +if [ -d $EXECUTOR_CUSTOM_PLUGINS_FOLDER ]; +then + echo "Copying custom plugins from ${EXECUTOR_CUSTOM_PLUGINS_FOLDER} to ${JMETER_HOME}/lib/ext" + for plugin in ${EXECUTOR_CUSTOM_PLUGINS_FOLDER}/*.jar; do + echo "Copying plugin: $plugin" + cp $plugin ${JMETER_HOME}/lib/ext + done; +else + echo "No custom plugins found at ${EXECUTOR_CUSTOM_PLUGINS_FOLDER}" +fi + +if [ -f "/executor_entrypoint_master.sh" ]; +then + echo "Executing custom entrypoint script at /entrypoint.sh" + /executor_entrypoint_master.sh $@ +else + echo "Executing JMeter command directly: jmeter $@" + jmeter $@ +fi + diff --git a/contrib/executor/jmeterd/scripts/jmeter-master.sh b/contrib/executor/jmeterd/scripts/jmeter-master.sh new file mode 100755 index 00000000000..640a69f5681 --- /dev/null +++ b/contrib/executor/jmeterd/scripts/jmeter-master.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +echo "********************************************************" +echo "* Installing JMeter Plugins *" +echo "********************************************************" +echo + + + +if [ -d $JMETER_CUSTOM_PLUGINS_FOLDER ] +then + echo "Installing custom plugins from ${JMETER_CUSTOM_PLUGINS_FOLDER}" + for plugin in ${JMETER_CUSTOM_PLUGINS_FOLDER}/*.jar; do + echo "Copying plugin $plugin to ${JMETER_HOME}/lib/ext/${plugin}" + cp $plugin ${JMETER_HOME}/lib/ext + done; +else + echo "No custom plugins found in ${JMETER_CUSTOM_PLUGINS_FOLDER}" +fi +echo + + + + +if [ -d ${JMETER_PARENT_TEST_FOLDER}/plugins ] +then + echo "Installing user plugins from ${JMETER_PARENT_TEST_FOLDER}/plugins" + for plugin in ${JMETER_PARENT_TEST_FOLDER}/plugins/*.jar; do + echo "Copying plugin $plugin to ${JMETER_HOME}/lib/ext/" + cp $plugin ${JMETER_HOME}/lib/ext + done; +else + echo "No user plugins provided as directory ${JMETER_PARENT_TEST_FOLDER}/plugins is not present" +fi +echo + +if [ -f ${JMETER_PARENT_TEST_FOLDER}/user.properties ] +then + echo "Copying user properties file from ${JMETER_PARENT_TEST_FOLDER}/user.properties" + cp ${JMETER_PARENT_TEST_FOLDER}/user.properties ${JMETER_HOME}/bin/ +else + echo "File user.properties not present in ${JMETER_PARENT_TEST_FOLDER}" +fi +echo + + +echo "********************************************************" +echo "* Initializing JMeter Master *" +echo "********************************************************" +echo + +freeMem=$(awk '/MemAvailable/ { print int($2/1024) }' /proc/meminfo) + +[[ -z ${JVM_XMN} ]] && JVM_XMN=$(($freeMem*2/10)) +[[ -z ${JVM_XMS} ]] && JVM_XMS=$(($freeMem*8/10)) +[[ -z ${JVM_XMX} ]] && JVM_XMX=$(($freeMem*8/10)) + +echo "Setting default JVM_ARGS=-Xmn${JVM_XMN}m -Xms${JVM_XMS}m -Xmx${JVM_XMX}m" +export JVM_ARGS="-Xmn${JVM_XMN}m -Xms${JVM_XMS}m -Xmx${JVM_XMX}m" + +if [ -n "$MASTER_OVERRIDE_JVM_ARGS" ]; then + echo "Overriding JVM_ARGS=${MASTER_OVERRIDE_JVM_ARGS}" + export JVM_ARGS="${MASTER_OVERRIDE_JVM_ARGS}" +fi + +if [ -n "$MASTER_ADDITIONAL_JVM_ARGS" ]; then + echo "Appending additional JVM args: ${MASTER_ADDITIONAL_JVM_ARGS}" + export JVM_ARGS="${JVM_ARGS} ${MASTER_ADDITIONAL_JVM_ARGS}" +fi + +echo "Available memory: ${freeMem} MB" +echo "Configured JVM_ARGS=${JVM_ARGS}" +echo + +echo "********************************************************" +echo "* Preparing JMeter Test Execution *" +echo "********************************************************" +echo + +# Keep entrypoint simple: we must pass the standard JMeter arguments +EXTRA_ARGS=-Dlog4j2.formatMsgNoLookups=true + + +echo "********************************************************" +echo "* Executing JMeter tests *" +echo "********************************************************" +echo + +if [ -z "$SSL_DISABLED" ]; then + SSL_DISABLED=true +fi + +CONN_ARGS="-Jserver.rmi.ssl.disable=${SSL_DISABLED}" +echo "Executing command: jmeter $@ ${CONN_ARGS} " +echo +echo "Started CMD" +jmeter $@ ${CONN_ARGS} + +echo "END Finished JMeter test on $(date) for test ${file}" +echo + +echo "********************************************************" +echo "* JMeter test executions finished *" +echo "********************************************************" +echo diff --git a/contrib/executor/jmeterd/scripts/jmeter-slaves.sh b/contrib/executor/jmeterd/scripts/jmeter-slaves.sh new file mode 100644 index 00000000000..ed2a7aa6087 --- /dev/null +++ b/contrib/executor/jmeterd/scripts/jmeter-slaves.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +echo "********************************************************" +echo "* Installing JMeter Plugins *" +echo "********************************************************" +echo + +if [ -d $JMETER_CUSTOM_PLUGINS_FOLDER ] +then + echo "Installing custom plugins from ${JMETER_CUSTOM_PLUGINS_FOLDER}" + for plugin in ${JMETER_CUSTOM_PLUGINS_FOLDER}/*.jar; do + echo "Copying plugin $plugin to ${JMETER_HOME}/lib/ext/${plugin}" + cp $plugin ${JMETER_HOME}/lib/ext + done; +else + echo "No custom plugins found in ${JMETER_CUSTOM_PLUGINS_FOLDER}" +fi +echo + +if [ -d ${JMETER_PARENT_TEST_FOLDER}/plugins ] +then + echo "Installing user plugins from ${JMETER_PARENT_TEST_FOLDER}/plugins" + for plugin in ${JMETER_PARENT_TEST_FOLDER}/plugins/*.jar; do + echo "Copying plugin $plugin to ${JMETER_HOME}/lib/ext/" + cp $plugin ${JMETER_HOME}/lib/ext + done; +else + echo "No user plugins provided as directory ${JMETER_PARENT_TEST_FOLDER}/plugins is not present" +fi +echo + +echo + +echo "********************************************************" +echo "* Initializing JMeter Master *" +echo "********************************************************" +echo + +freeMem=`awk '/MemAvailable/ { print int($2/1024) }' /proc/meminfo` + +[[ -z ${JVM_XMN} ]] && JVM_XMN=$(($freeMem/10*2)) +[[ -z ${JVM_XMS} ]] && JVM_XMS=$(($freeMem/10*8)) +[[ -z ${JVM_XMX} ]] && JVM_XMX=$(($freeMem/10*8)) + +echo "Setting default JVM_ARGS=-Xmn${JVM_XMN}m -Xms${JVM_XMS}m -Xmx${JVM_XMX}m" +export JVM_ARGS="-Xmn${JVM_XMN}m -Xms${JVM_XMS}m -Xmx${JVM_XMX}m" + +if [ -n "$OVERRIDE_JVM_ARGS" ]; then + echo "Overriding JVM_ARGS=${OVERRIDE_JVM_ARGS}" + export JVM_ARGS="${OVERRIDE_JVM_ARGS}" +fi + +if [ -n "$ADDITIONAL_JVM_ARGS" ]; then + echo "Appending additional JVM args: ${ADDITIONAL_JVM_ARGS}" + export JVM_ARGS="${JVM_ARGS} ${ADDITIONAL_JVM_ARGS}" +fi + +echo "Available memory: ${freeMem} MB" +echo "Configured JVM_ARGS=${JVM_ARGS}" +echo + +echo "********************************************************" +echo "* Starting JMeter Server *" +echo "********************************************************" +echo + +SERVER_ARGS="-Dserver.rmi.localport=60001 -Dserver_port=1099 -Jserver.rmi.ssl.disable=${SSL_DISABLED}" +echo "Running command: jmeter-server ${SERVER_ARGS} ${SLAVES_ADDITIONAL_JMETER_ARGS}" +echo + +jmeter-server ${SERVER_ARGS} ${SLAVES_ADDITIONAL_JMETER_ARGS} \ No newline at end of file diff --git a/contrib/executor/k6/pkg/runner/runner.go b/contrib/executor/k6/pkg/runner/runner.go index 37669472b71..8e64c950fb2 100644 --- a/contrib/executor/k6/pkg/runner/runner.go +++ b/contrib/executor/k6/pkg/runner/runner.go @@ -13,6 +13,7 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/env" outputPkg "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" @@ -174,6 +175,14 @@ func (r *K6Runner) Run(ctx context.Context, execution testkube.Execution) (resul output, err := executor.Run(runPath, command, envManager, args...) output = envManager.ObfuscateSecrets(output) + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + outputPkg.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + outputPkg.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + // scrape artifacts first even if there are errors above if r.Params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { outputPkg.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) diff --git a/contrib/executor/kubepug/pkg/runner/runner.go b/contrib/executor/kubepug/pkg/runner/runner.go index 677a333bbb8..a3bf3c95e5d 100644 --- a/contrib/executor/kubepug/pkg/runner/runner.go +++ b/contrib/executor/kubepug/pkg/runner/runner.go @@ -12,6 +12,7 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/content" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" @@ -90,6 +91,14 @@ func (r *KubepugRunner) Run(ctx context.Context, execution testkube.Execution) ( return testkube.ExecutionResult{}, fmt.Errorf("could not execute kubepug: %w", err) } + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + output.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + output.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + // scrape artifacts first even if there are errors above if r.params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { output.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) diff --git a/contrib/executor/maven/pkg/runner/runner.go b/contrib/executor/maven/pkg/runner/runner.go index f512f3ab3ac..c36f420ce51 100644 --- a/contrib/executor/maven/pkg/runner/runner.go +++ b/contrib/executor/maven/pkg/runner/runner.go @@ -14,6 +14,7 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/content" "github.com/kubeshop/testkube/pkg/executor/env" outputPkg "github.com/kubeshop/testkube/pkg/executor/output" @@ -180,6 +181,14 @@ func (r *MavenRunner) Run(ctx context.Context, execution testkube.Execution) (re } } + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + outputPkg.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + outputPkg.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + // scrape artifacts first even if there are errors above if r.params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { outputPkg.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) diff --git a/contrib/executor/playwright/pkg/runner/playwright.go b/contrib/executor/playwright/pkg/runner/playwright.go index 05e569c1187..f2be7a1ecc3 100644 --- a/contrib/executor/playwright/pkg/runner/playwright.go +++ b/contrib/executor/playwright/pkg/runner/playwright.go @@ -11,6 +11,7 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" @@ -121,6 +122,14 @@ func (r *PlaywrightRunner) Run(ctx context.Context, execution testkube.Execution } } + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + output.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + output.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + if r.Params.ScrapperEnabled { reportFile := "playwright-report" if err = scrapeArtifacts(ctx, r, execution, reportFile); err != nil { diff --git a/contrib/executor/postman/pkg/runner/newman/newman.go b/contrib/executor/postman/pkg/runner/newman/newman.go index 2d194ac2a4e..7346a769bdd 100644 --- a/contrib/executor/postman/pkg/runner/newman/newman.go +++ b/contrib/executor/postman/pkg/runner/newman/newman.go @@ -8,9 +8,12 @@ import ( "path/filepath" "strings" + "github.com/pkg/errors" + "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/content" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" @@ -64,7 +67,25 @@ func (r *NewmanRunner) Run(ctx context.Context, execution testkube.Execution) (r } if fileInfo.IsDir() { - return result, testkube.ErrTestContentTypeNotFile + scriptName := execution.Args[len(execution.Args)-1] + if workingDir != "" { + path = filepath.Join(r.Params.DataDir, "repo") + if execution.Content != nil && execution.Content.Repository != nil { + scriptName = filepath.Join(execution.Content.Repository.Path, scriptName) + } + } + + execution.Args = execution.Args[:len(execution.Args)-1] + output.PrintLogf("%s It is a directory test - trying to find file from the last executor argument %s in directory %s", ui.IconWorld, scriptName, path) + + // sanity checking for test script + scriptFile := filepath.Join(path, workingDir, scriptName) + fileInfo, errFile := os.Stat(scriptFile) + if errors.Is(errFile, os.ErrNotExist) || fileInfo.IsDir() { + output.PrintLogf("%s Could not find file %s in the directory, error: %s", ui.IconCross, scriptName, errFile) + return *result.Err(errors.Errorf("could not find file %s in the directory: %v", scriptName, errFile)), nil + } + path = scriptFile } envManager := env.NewManagerWithVars(execution.Variables) @@ -128,6 +149,14 @@ func (r *NewmanRunner) Run(ctx context.Context, execution testkube.Execution) (r result = MapMetadataToResult(newmanResult) output.PrintLog(fmt.Sprintf("%s Mapped Newman result successfully", ui.IconCheckMark)) + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + output.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + output.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + // scrape artifacts first even if there are errors above if r.Params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { output.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) diff --git a/contrib/executor/soapui/pkg/runner/runner.go b/contrib/executor/soapui/pkg/runner/runner.go index 2360590849f..e17da2c0707 100644 --- a/contrib/executor/soapui/pkg/runner/runner.go +++ b/contrib/executor/soapui/pkg/runner/runner.go @@ -2,9 +2,12 @@ package runner import ( "context" + "fmt" "os" + "path/filepath" "strings" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/scraper" "github.com/kubeshop/testkube/pkg/executor/scraper/factory" @@ -61,20 +64,46 @@ func (r *SoapUIRunner) Run(ctx context.Context, execution testkube.Execution) (r output.PrintLogf("%s Failed to resolve absolute directory for %s, using the path directly", ui.IconWarning, r.Params.DataDir) } - setUpEnvironment(execution.Args, testFile) - fileInfo, err := os.Stat(testFile) if err != nil { return result, err } if fileInfo.IsDir() { - return testkube.ExecutionResult{}, errors.New("SoapUI executor only tests one project per execution, a directory of projects was given") + scriptName := execution.Args[len(execution.Args)-1] + if workingDir != "" { + testFile = filepath.Join(r.Params.DataDir, "repo") + if execution.Content != nil && execution.Content.Repository != nil { + scriptName = filepath.Join(execution.Content.Repository.Path, scriptName) + } + } + + execution.Args = execution.Args[:len(execution.Args)-1] + output.PrintLogf("%s It is a directory test - trying to find file from the last executor argument %s in directory %s", ui.IconWorld, scriptName, testFile) + + // sanity checking for test script + scriptFile := filepath.Join(testFile, workingDir, scriptName) + fileInfo, errFile := os.Stat(scriptFile) + if errors.Is(errFile, os.ErrNotExist) || fileInfo.IsDir() { + output.PrintLogf("%s Could not find file %s in the directory, error: %s", ui.IconCross, scriptName, errFile) + return *result.Err(errors.Errorf("could not find file %s in the directory: %v", scriptName, errFile)), nil + } + testFile = scriptFile } + setUpEnvironment(execution.Args, testFile) + output.PrintLogf("%s Running SoapUI tests", ui.IconMicroscope) result = r.runSoapUI(&execution, workingDir) + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + output.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + output.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + if r.Params.ScrapperEnabled { directories := []string{r.SoapUILogsPath} if execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { diff --git a/contrib/executor/tracetest/pkg/runner/runner.go b/contrib/executor/tracetest/pkg/runner/runner.go index 87d5ae6def3..33614b565d3 100644 --- a/contrib/executor/tracetest/pkg/runner/runner.go +++ b/contrib/executor/tracetest/pkg/runner/runner.go @@ -9,6 +9,7 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/content" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" @@ -86,12 +87,20 @@ func (r *TracetestRunner) Run(ctx context.Context, execution testkube.Execution) output, err := executor.Run("", command, envManager, args...) runResult := model.Result{Output: string(output), ServerEndpoint: te, OutputEndpoint: toe} + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + outputPkg.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + outputPkg.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + // scrape artifacts first even if there are errors above if r.Params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { outputPkg.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) if err := r.Scraper.Scrape(ctx, execution.ArtifactRequest.Dirs, execution); err != nil { - return testkube.ExecutionResult{}, fmt.Errorf("could not scrape kubepug directories: %w", err) + return testkube.ExecutionResult{}, fmt.Errorf("could not scrape tracetest directories: %w", err) } } diff --git a/contrib/executor/zap/pkg/runner/runner.go b/contrib/executor/zap/pkg/runner/runner.go index 1958cd2856a..cb06b59b200 100644 --- a/contrib/executor/zap/pkg/runner/runner.go +++ b/contrib/executor/zap/pkg/runner/runner.go @@ -12,6 +12,7 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/content" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" @@ -162,6 +163,14 @@ func (r *ZapRunner) Run(ctx context.Context, execution testkube.Execution) (resu } } + if execution.PostRunScript != "" && execution.ExecutePostRunScriptBeforeScraping { + output.PrintLog(fmt.Sprintf("%s Running post run script...", ui.IconCheckMark)) + + if err = agent.RunScript(execution.PostRunScript); err != nil { + output.PrintLogf("%s Failed to execute post run script %s", ui.IconWarning, err) + } + } + if r.Params.ScrapperEnabled { directories := []string{reportFolder} if execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { diff --git a/contrib/executor/zap/pkg/runner/runner_test.go b/contrib/executor/zap/pkg/runner/runner_test.go index e99fc77ef95..3aeb02053fd 100644 --- a/contrib/executor/zap/pkg/runner/runner_test.go +++ b/contrib/executor/zap/pkg/runner/runner_test.go @@ -2,7 +2,6 @@ package runner import ( "context" - "io/ioutil" "os" "path/filepath" "testing" @@ -19,7 +18,7 @@ func TestRun(t *testing.T) { t.Run("Run successful API scan", func(t *testing.T) { // given - tempDir, err := ioutil.TempDir(os.TempDir(), "") + tempDir, err := os.MkdirTemp(os.TempDir(), "") assert.NoError(t, err) runner, err := NewRunner(context.TODO(), envs.Params{ DataDir: tempDir, @@ -50,7 +49,7 @@ func TestRun(t *testing.T) { t.Run("Run API scan with PASS and WARN", func(t *testing.T) { // given - tempDir, err := ioutil.TempDir(os.TempDir(), "") + tempDir, err := os.MkdirTemp(os.TempDir(), "") assert.NoError(t, err) runner, err := NewRunner(context.TODO(), envs.Params{ DataDir: tempDir, @@ -81,7 +80,7 @@ func TestRun(t *testing.T) { t.Run("Run API scan with WARN and FailOnWarn", func(t *testing.T) { // given - tempDir, err := ioutil.TempDir(os.TempDir(), "") + tempDir, err := os.MkdirTemp(os.TempDir(), "") assert.NoError(t, err) runner, err := NewRunner(context.TODO(), envs.Params{ DataDir: tempDir, @@ -112,7 +111,7 @@ func TestRun(t *testing.T) { t.Run("Run API scan with FAIL", func(t *testing.T) { // given - tempDir, err := ioutil.TempDir(os.TempDir(), "") + tempDir, err := os.MkdirTemp(os.TempDir(), "") assert.NoError(t, err) runner, err := NewRunner(context.TODO(), envs.Params{ DataDir: tempDir, @@ -143,7 +142,7 @@ func TestRun(t *testing.T) { t.Run("Run Baseline scan with PASS", func(t *testing.T) { // given - tempDir, err := ioutil.TempDir(os.TempDir(), "") + tempDir, err := os.MkdirTemp(os.TempDir(), "") assert.NoError(t, err) runner, err := NewRunner(context.TODO(), envs.Params{ DataDir: tempDir, @@ -172,7 +171,7 @@ func TestRun(t *testing.T) { t.Run("Run Baseline scan with WARN", func(t *testing.T) { // given - tempDir, err := ioutil.TempDir(os.TempDir(), "") + tempDir, err := os.MkdirTemp(os.TempDir(), "") assert.NoError(t, err) runner, err := NewRunner(context.TODO(), envs.Params{ DataDir: tempDir, @@ -202,7 +201,7 @@ func TestRun(t *testing.T) { t.Run("Run Full scan with FAIL", func(t *testing.T) { // given - tempDir, err := ioutil.TempDir(os.TempDir(), "") + tempDir, err := os.MkdirTemp(os.TempDir(), "") assert.NoError(t, err) runner, err := NewRunner(context.TODO(), envs.Params{ DataDir: tempDir, diff --git a/docs/docs/articles/crds.md b/docs/docs/articles/crds.md index d4295c783c4..dd83e1f99b1 100644 --- a/docs/docs/articles/crds.md +++ b/docs/docs/articles/crds.md @@ -12,6 +12,7 @@ kubectl get crds -n testkube NAME CREATED AT executors.executor.testkube.io 2023-06-15T14:49:11Z scripts.tests.testkube.io 2023-06-15T14:49:11Z +templates.tests.testkube.io 2023-06-15T14:49:11Z testexecutions.tests.testkube.io 2023-06-15T14:49:11Z tests.tests.testkube.io 2023-06-15T14:49:11Z testsources.tests.testkube.io 2023-06-15T14:49:11Z diff --git a/docs/docs/articles/creating-test-suites.md b/docs/docs/articles/creating-test-suites.md index bc5aae6e22e..b5cce347721 100644 --- a/docs/docs/articles/creating-test-suites.md +++ b/docs/docs/articles/creating-test-suites.md @@ -7,7 +7,7 @@ A QA leader is responsible for release trains and wants to be sure that before t This is easily done with Testkube. Each team can run their tests against clusters on their own, and the QA manager can create test resources and add tests written by all teams. -`Test Suites` stands for the orchestration of different test steps, which can run sequentially or/and in parallel. +`Test Suites` stands for the orchestration of different test steps, which can run sequentially and/or in parallel. On each batch step you can define either one or multiple steps such as test execution, delay, or other (future) steps. By default the concurrency level for parallel tests is set to 10, you can redefine it using `--concurency` option for CLI command. diff --git a/docs/docs/articles/creating-tests.md b/docs/docs/articles/creating-tests.md index 1660a99269c..1dbc4c4ac6c 100644 --- a/docs/docs/articles/creating-tests.md +++ b/docs/docs/articles/creating-tests.md @@ -11,9 +11,9 @@ As Testkube was designed with flexibility in mind, you can add your own executor Tests can be currently created from multiple sources: 1. A simple `file` with the test content. For example, with Postman collections, we're exporting the collection as a JSON file. For cURL executors, we're passing a JSON file with the configured cURL command. -2. String - we can also define the content of the test as a string -3. A Git directory - we can pass `repository`, `path` and `branch` where our tests are stored. This is used in the Cypress executor as Cypress tests are more like npm-based projects which can have a lot of files. We are handling sparse checkouts which are fast even in the case of huge mono-repos. -4. A Git file - similarly to Git directories, we can use files located on Git by specifying `git-uri` and `branch`. +2. String - We can also define the content of the test as a string. +3. A Git directory - We can pass `repository`, `path` and `branch` where our tests are stored. This is used in the Cypress executor as Cypress tests are more like npm-based projects which can have a lot of files. We are handling sparse checkouts which are fast even in the case of huge mono-repos. +4. A Git file - Similarly to Git directories, we can use files located on Git by specifying `git-uri` and `branch`. :::note Not all executors support all input types. Please refer to the individual executors' documentation to see which options are available. @@ -87,6 +87,7 @@ kubectl get tests -n testkube test-example -oyaml name: test type_: postman/collection +description: some test description content: |- { "info": { @@ -302,6 +303,7 @@ metadata: resourceVersion: "225162" uid: f0d856aa-04fc-4238-bb4c-156ff82b4741 spec: + description: some test description repository: branch: main path: examples @@ -349,9 +351,9 @@ By default, there is a 10 second timeout limit on all requests on the client sid Some of the executors offer the option to set a special file using the flag `--variables-file` on both test creation and test run. For the Postman executor, this expects an environment file, for Maven it is `settings.xml`. There are many differences between `--variables-file` and `--copy-files`. The former one sets this file directly as the configuration file. With the latter, there is an additional need to set the path explicitly on the arguments level. Another difference is that for variables files smaller than 128KB, this will be set on the CRD level and not uploaded to the object storage. This limitation comes from linux-based systems where this is the default maximum length of arguments. -### Redefining the Prebuilt Executor command and arguments +### Redefining the Prebuilt Executor Command and Arguments -Each of Testkube Prebuilt executors has a default command and arguments it uses to execute the test. They are provided as a part of Executor CRD and can be either ovveriden or appended during test creation or execution, for example: +Each of Testkube Prebuilt executors has a default command and arguments it uses to execute the test. They are provided as a part of Executor CRD and can be either overidden or appended during test creation or execution, for example: ```sh testkube create test --name maven-example-test --git-uri https://github.com/kubeshop/testkube-executor-maven.git --git-path examples/hello-maven --type maven/test --git-branch main --command "mvn" --args-mode "override" --executor-args="--settings -Duser.home " @@ -363,7 +365,7 @@ Test created maven-example-test 🥇 ### Changing the Default Job Template Used for Test Execution -You can always create your own custom executor with its own job template definition used for test execution. But sometimes you just need to adjust an existing job template of a standard Testkube executor with a few parameters. In this case you can use additional parameter `--job-template` when you create or run the test: +You can always create your own custom executor with its own job template definition used for test execution. But sometimes you just need to adjust an existing job template of a standard Testkube executor with a few parameters. In this case you can use the additional parameter `--job-template` when you create or run the test: ```sh testkube create test --git-branch main --git-uri https://github.com/kubeshop/testkube-example-cypress-project.git --git-path "cypress" --name template-test --type cypress/project --job-template job.yaml @@ -403,7 +405,7 @@ spec: ``` We also provide special helper methods to use in the job template: -`vartypeptrtostring` - method to convert a pointer to a variable type to a string type +`vartypeptrtostring` is the method to convert a pointer to a variable type to a string type. Usage example: ```yaml @@ -415,7 +417,7 @@ Usage example: {{- end }} ``` -Add `imagePullSecrets` option if you use your own Image Registry. This will add the secret for both `init` and `executor` containers. +Add the `imagePullSecrets` option if you use your own Image Registry. This will add the secret for both `init` and `executor` containers. ### Changing the Default CronJob Template Used for Scheduled Test Execution @@ -436,7 +438,7 @@ metadata: test: kube ``` -When such a test is created you will see additional annotations for its cron job, when the default cron job template doesn't have any annotations. +When such a test is created you will see additional annotations for its cron job, if the default cron job template doesn't have any annotations. ### Executing a Prerun/Postrun Script @@ -464,9 +466,36 @@ Provide the script when you create or run the test using `--prerun-script` and ` testkube create test --file test/postman/LocalHealth.postman_collection.json --name script-test --type postman/collection --prerun-script pre_script.sh --postrun-script post_script.sh --secret-env SSL_CERT=your-k8s-secret ``` +### Adjusting Scraping Parameters + +For any executor type you can specify additional scraping parameters using CLI or CRD definition. For example, below we request to scrape report directories, use a custom bucket to store test artifacts and ask to avoid using separate artifact folders for each test execution + +```yaml +apiVersion: tests.testkube.io/v3 +kind: Test +metadata: + name: jmeter-smoke-test + namespace: testkube +spec: + type: jmeter/test + content: + type: git + repository: + type: git + uri: https://github.com/kubeshop/testkube.git + branch: main + path: test/jmeter/executor-tests/jmeter-executor-smoke.jmx + executionRequest: + artifactRequest: + dirs: + - test/reports + storageBucket: jmeter-artifacts + omitFolderPerExecution: true +``` + ### Changing the Default Scraper Job Template Used for Container Executor Tests -When you use container executor tests generating artifacts for scraping, we launch 2 sequential Kubernetes jobs, one is for test execution and other one is for scraping test results. Sometimes you need to adjust an existing scraper job template of a standard Testkube scraper with a few parameters. In this case you can use the additional parameter `--scraper-template` when you create or run the test: +When you use container executor tests generating artifacts for scraping, we launch 2 sequential Kubernetes jobs. One is for test execution and other is for scraping test results. Sometimes you need to adjust an existing scraper job template of a standard Testkube scraper with a few parameters. In this case you can use the additional parameter `--scraper-template` when you create or run the test: ```sh testkube create test --name scraper-test --type scraper/test --artifact-storage-class-name standard --artifact-volume-mount-path /share --artifact-dir test/files --scraper-template scraper.yaml @@ -499,11 +528,11 @@ spec: memory: 512Mi ``` -When you run such a test, you will face a memory limit for the scraper pod, when the default scraper job template doesn't have any resource constraints. +When you run such a test, you will face a memory limit for the scraper pod, if the default scraper job template doesn't have any resource constraints. ### Mounting ConfigMap and Secret to Executor Pod -If you need to mount your ConfigMap and Secret to your executor environment, then you can provide them as additional +If you need to mount your ConfigMap and Secret to your executor environment, you can provide them as additional parameters when you create or run the test using the `--mount-configmap` and `--mount-secret` options: ```sh diff --git a/docs/docs/articles/deploying-in-aws.md b/docs/docs/articles/deploying-in-aws.md index 9c2a2470ee2..54416880e90 100644 --- a/docs/docs/articles/deploying-in-aws.md +++ b/docs/docs/articles/deploying-in-aws.md @@ -12,6 +12,12 @@ Another important point is [ExternalDNS](https://github.com/kubernetes-sigs/exte And last, but not least - install the Testkube CLI. You can download a binary file from our [installation](./step1-installing-cli) page. For how to deploy Testkube to your cluster with all the necessary changes, please see the next section. +:::caution + +Please mind that is it necessary to install [EBS CSI driver](https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html) to mount PV into your k8s cluster. + +::: + ## Ingress and Service Resources Configuration To deploy and expose Testkube to the outside world, you will need to create two ingresses - Testkube's API and Testkube's Dashboard. In this tutorial, we will be updating `values.yaml` that later will be passed to the `helm install` command. diff --git a/docs/docs/articles/getting-started-overview.md b/docs/docs/articles/getting-started-overview.md index dc645b9c3a4..6a1e470e761 100644 --- a/docs/docs/articles/getting-started-overview.md +++ b/docs/docs/articles/getting-started-overview.md @@ -2,18 +2,10 @@ In this section you will: -1. [Install the Testkube CLI](./step1-installing-cli). +1. [Install the Testkube CLI](./step1-installing-cli.mdx). 2. [Install the Testkube Agent](./step2-installing-cluster-components.md). 3. [Creating your first Test](./step3-creating-first-test.md). -You can also see the full installation video from our product experts: [Testkube Installation Video](https://www.youtube.com/watch?v=bjQboi3Etys): +You can also see the full installation video from our product experts: - - -In summary, you now have Testkube setup in your Kubernetes cluster, ready to discover the Testkube features: -- The Testkube CLI allows you to locally port forward to the kube dashboard deployment. -- The Testkube CLI allows you to interact with the Testkube server components through k8s CRs or REST API calls made through the [kube apiserver proxy](https://kubernetes.io/docs/concepts/cluster-administration/proxies/). - -As usage of Testkube grows within your team, you may choose to: -* Leverage [managed Testkube cloud](../testkube-cloud/articles/intro.md). -* [Move to production](./going-to-production.md) with your own Testkube installation. \ No newline at end of file + \ No newline at end of file diff --git a/docs/docs/articles/run-tests-with-github-actions.md b/docs/docs/articles/run-tests-with-github-actions.md index b0f99a8c913..4a212a99e67 100644 --- a/docs/docs/articles/run-tests-with-github-actions.md +++ b/docs/docs/articles/run-tests-with-github-actions.md @@ -101,7 +101,7 @@ with: OTHER_ENV="${{ secrets.ExternalToken }}" ``` -#### Real-life examples +#### Real-life Examples `testkube-run-action` is also used for running Testkube internal tests with Testkube. Workflow for Testkube Dashboard E2E tests can be found [here](https://github.com/kubeshop/testkube-dashboard/blob/develop/.github/workflows/pr_checks.yml#L28) ## Inputs diff --git a/docs/docs/articles/running-parallel-tests-with-test-suite.md b/docs/docs/articles/running-parallel-tests-with-test-suite.md new file mode 100644 index 00000000000..4b3efc972bc --- /dev/null +++ b/docs/docs/articles/running-parallel-tests-with-test-suite.md @@ -0,0 +1,29 @@ +# Advanced Test Orchestration + +Creating Test Suites with Tracetest allows for the orchestration of tests. Individual tests that can be run at the same time, in parallel, helps to speed up overall testing. + +## Running Parallel Tests in a Test Suite + +### Create a Test Suite + +In the Testkube Dashboard, on the Test Suite screen, select **Add a new test suite**: + +![Add New Test Suite](../img/add-new-testsuite.png) + +### Add Tests + +The new Test Suite will be empty. Click on **Add your first test**: + +![Add First Test](../img/add-first-test.png) + +Continue to add tests to you test suite. You can also add a **delay** when necessary to specify he length of time between certain tests. + +![Add Additional Test or Delay](../img/add-additional-test-or-delay.png) + +For this test suite, we have added 5 tests that all run in parallel: + +![Tests in Test Suite](../img/tests-in-test-suite.png) + +Here is an example of a Test Suite sequence with 2 tests running in parallel and, when they complete, a single test runs, then 2 addtional parallel tests: + +![Test and Order of Execution](../img/test-and-order-of-execution.png) \ No newline at end of file diff --git a/docs/docs/articles/running-tests.md b/docs/docs/articles/running-tests.md index 827973d6cba..2c57fe0f03c 100644 --- a/docs/docs/articles/running-tests.md +++ b/docs/docs/articles/running-tests.md @@ -109,7 +109,7 @@ Let's assume that our example Cypress test needs the `testparam` parameter with This is done by using the `-p` parameter. If you need to pass more parameters, simply pass multiple `-p` flags. -It's possible to pass parameters securely to the executed test. It's necessary to use `--secret` flag, +It's possible to pass parameters securely to the executed test. It's necessary to use the `--secret` flag, which contains a key value pair - a name of the Kubernetes secret and a secret key. It can be passed multiple times if needed. @@ -206,8 +206,8 @@ Local files can be set on the execution of a Testkube Test. Pass the file in the testkube run test maven-example-file-test --copy-files "/Users/local_user/local_maven_settings.xml:/tmp/settings.xml" --args "--settings" --args "/tmp/settings.xml" -v "TESTKUBE_MAVEN=true" ``` -By default, there is a 10 second timeout limit on all requests on the client side, and a 1 GB body size limit on the server side. To update the timeout, use `--upload-timeout` with [Go-compatible duration formats](https://pkg.go.dev/time#ParseDuration). +By default, there is a 10 second timeout limit on all requests on the client side and a 1 GB body size limit on the server side. To update the timeout, use `--upload-timeout` with [Go-compatible duration formats](https://pkg.go.dev/time#ParseDuration). ## Summary -As we can see, running tests in Kubernetes cluster is really easy with use of the Testkube kubectl plugin! +As we can see, running tests in a Kubernetes cluster is really easy with use of the Testkube kubectl plugin! diff --git a/docs/docs/articles/scheduling-tests.md b/docs/docs/articles/scheduling-tests.mdx similarity index 74% rename from docs/docs/articles/scheduling-tests.md rename to docs/docs/articles/scheduling-tests.mdx index 25a045d5e47..851482503b7 100644 --- a/docs/docs/articles/scheduling-tests.md +++ b/docs/docs/articles/scheduling-tests.mdx @@ -1,14 +1,62 @@ +<<<<<<<< HEAD:docs/docs/articles/scheduling-tests.mdx +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Scheduling Tests + + + + +## Create a Scheduled Test or Test Suite from the Testkube Dashboard +======== # Scheduling Tests +>>>>>>>> main:docs/docs/articles/scheduling-tests.md In order to run Tests and Test Suites on a regular basis, we support a scheduling mechanism for these objects. -CRDs both for tests and test suites contain a **schedule** field used to define rules for launching them in time. +CRDs both for Tests and Test Suites contain a **schedule** field used to define rules for launching them in time. Testkube's schedule data format is the same that is used to define Kubernetes Cron jobs (check Wikipedia Cron format for details ). ## Scheduling Architecture Testkube uses the scheduling engine from Kubernetes Cron jobs. +<<<<<<<< HEAD:docs/docs/articles/scheduling-tests.mdx +In fact, for each scheduled Test or Test Suite, a special cron job is created from this [template](ttps://github.com/kubeshop/helm-charts/tree/main/charts/testkube-operator/cronjob-template.yml). + +Technically, it is a callback to the Testkube API server method, launching either Test or Test Suite execution. + +This works similarly to scheduled Test and Test Suite executions done by external scheduling platforms. + +In the Testkube Dashboard, when you have selected a Test or Test Suite, in the **Settings** tab, click on **Scheduling** to create a schedule for the Test or Test Suite. + +## Tests + +![Schedule Tests](../img/schedule-tests-1.14.png) + +![Schedule Tests Options](../img/schedule-tests-options-1.14.png) + +## Test Suites + +![Schedule Test Suites](../img/schedule-test-suites-1.14.png) + +![Schedule Test Suites Options](../img/schedule-test-suites-options-1.14.png) + + + + + +In order to run Tests and Test Suites on a regular basis, we support a scheduling mechanism for these objects. +CRDs both for Tests and Test Suites contain a **schedule** field used to define rules for launching them in time. +Testkube's schedule data format is the same that is used to define Kubernetes Cron jobs (check [Wikipedia Cron format](https://en.wikipedia.org/wiki/Cron) for details. + +## Scheduling Architecture + +Testkube uses the scheduling engine from Kubernetes Cron jobs. +In fact, for each scheduled Test or Test Suite, a special cron job is created from this [template](https://github.com/kubeshop/helm-charts/tree/main/charts/testkube-operator/cronjob-template.yml). + +======== In fact, for each scheduled Test or Test Suite, a special cron job is created from this template: . +>>>>>>>> main:docs/docs/articles/scheduling-tests.md Technically, it is a callback to the Testkube API server method, launching either Test or Test Suite execution. This works similarly to scheduled Test and Test Suite executions done by external scheduling platforms. @@ -143,7 +191,7 @@ kubectl testkube get testsuites scheduled-testsuite | Run test several times | 2 | | */1 * * * * | | ``` -The scheduled test suite was created and successfully scheduled for execution. +The scheduled Test Suite was created and successfully scheduled for execution. We will skip the Cron job details, they are fully similar to test one described above. ## Getting Scheduled Test Suite Results @@ -163,3 +211,6 @@ kubectl testkube get tse ``` The Test Suite is successfully executed on the stated schedule. + + + diff --git a/docs/docs/articles/step2-installing-cluster-components.md b/docs/docs/articles/step2-installing-cluster-components.md index 294ff57d1fc..432a7300690 100644 --- a/docs/docs/articles/step2-installing-cluster-components.md +++ b/docs/docs/articles/step2-installing-cluster-components.md @@ -8,7 +8,7 @@ To get started, sign into [Testkube](https://cloud.testkube.io) and create an ac ## Installation Steps -1. After signing in, create your first environment +1. After signing in, create your first environment: ![Create Environment](../img/create-first-environment.png) @@ -28,7 +28,7 @@ You will need *Helm* installed and `kubectl` configured with access to your Kube ![Install Steps 1](../img/install-steps.png) -5. After some time, you should see the Helm installation notice: +5. When the install is complete, you will see the Helm installation notice: ![Install Steps 2](../img/install-steps-2.png) @@ -42,7 +42,7 @@ Testkube Cloud will notify if the installation is successful. ![Validate Install](../img/validate-install.png) -In case of a RED status you can try to debug the issues with the command below: +In the case of a RED status you can try to debug the issues with the command below: ```sh testkube agent debug diff --git a/docs/docs/articles/step3-creating-first-test.md b/docs/docs/articles/step3-creating-first-test.md index 310883b1948..cc9f759a18d 100644 --- a/docs/docs/articles/step3-creating-first-test.md +++ b/docs/docs/articles/step3-creating-first-test.md @@ -55,9 +55,10 @@ Trigger test execution manually on the Dashboard: #### CLI You can run tests manually from your machine using the CLI as well, or from your CI/CD. Visit [here](https://docs.testkube.io/articles/cicd-overview) for examples on how to setup our CI/CD system to trigger your tests. -image +image + +##### Changing the Output Format -**Changing the Output Format** For lists and details, you can use different output formats via the `--output` flag. The following formats are currently supported: - `RAW` - Raw output from the given executor (e.g., for Postman collection, it's terminal text with colors and tables). diff --git a/docs/docs/articles/templates.mdx b/docs/docs/articles/templates.mdx new file mode 100644 index 00000000000..df2b0b83c9e --- /dev/null +++ b/docs/docs/articles/templates.mdx @@ -0,0 +1,112 @@ +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Templates + +Templates allow you to store templates for other resources used in Testkube. We support a list of templates job | container | cronjob | scraper | pvc | webhook. To define templates in Testkube, you'll need to provide a template body (in Golang template format) and a type of the template. + +TestKube provides access to the [Sprig](http://masterminds.github.io/sprig/) functions library in templates. + +## Creating a Template +The template can be created using the API, CLI, or a Custom Resource. + + + +If you prefer to use the API for creating a template, please visit the API spec for templates in the doc below. + +[OpenAPI spec](../openapi.md) + + + + +Templates can be created with the Testkube CLI using the `create template` command. + +```sh +kubectl testkube create template --name job-template --template-type job --body job-template.yaml +``` + +`--name` - Your template name (in this case `job-template`). +`--template-type` - Your template type (in this case `job` for prebuilt executors). +`--body` - A path to the file with job template content + + + + + +```yaml title="template.yaml" +apiVersion: tests.testkube.io/v1 +kind: Template +metadata: + name: example-webhook + namespace: testkube +spec: + type: job + body: +``` + +Where should be replaced with the Kubernetes job definition in Golang template format. + +And then apply with: + +```sh +kubectl apply -f template.yaml +``` + + + + + +### Using Templates +You will need to refer to a template in the corresponding reference field of the resource. + + + + +Check templateReference fields in API spec. For example, Test -> executionRequest -> jobTemplateReference field. +[OpenAPI spec](../openapi.md) + + + + + +Templates can be created with the Testkube CLI using the `create template` command. + +```sh +kubectl testkube create test --name template-test --type k6/script --job-template-reference=job-template --test-content-type git --git-uri "https://github.com/kubeshop/testkube.git" --git-branch main --git-path test/k6/executor-tests/k6-smoke-test.js +``` + +`--name` - Your test name (in this case `template-test`). +`--type` - Your test type (in this case `k6/script`). +`--job-template-reference` - Job template reference (in this case `job-template`). +`--test-content-type` - Test content type (in this case `git`). +`--git-uri` - Git uri to repository (in this case `https://github.com/kubeshop/testkube.git`). +`--git-branch` - Git branch to use (in this case `main`). +`--git-path` - Git path to the test (in this case `test/k6/executor-tests/k6-smoke-test.js`). + + + + + +```yaml title="test.yaml" +apiVersion: tests.testkube.io/v3 +kind: Test +metadata: + name: template-test + namespace: testkube +spec: + type: k6/script + content: + type: git + repository: + type: git + uri: https://github.com/kubeshop/testkube.git + branch: main + path: test/k6/executor-tests/k6-smoke-test.js + authType: basic + executionRequest: + jobTemplateReference: job-template +``` + + + + diff --git a/docs/docs/articles/test-triggers.mdx b/docs/docs/articles/test-triggers.mdx new file mode 100644 index 00000000000..f309c691c91 --- /dev/null +++ b/docs/docs/articles/test-triggers.mdx @@ -0,0 +1,195 @@ +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Triggers + + + + +Testkube allows you to automate running tests and test suites by defining triggers on certain events for various Kubernetes resources. + +## What is a Testkube Test Trigger? + +In generic terms, a _Trigger_ defines an _action_ which will be executed for a given _execution_ when a certain _event_ on a specific _resource_ occurs. For example, we could define a _TestTrigger_ which _runs_ a _Test_ when a _ConfigMap_ gets _modified_. + +Watch our [video guide](#video-tutorial) on using Testkube Test Triggers to perform **Asynchronous Testing in Kubernetes**. + +## Creating Test Triggers in the Testkube Dashboard + +Click on the lightening bolt icon on the left of the Testkube IDE to open the dialog to create test triggers. Any current test triggers will be listed and the `Create a new trigger` button is at the top right of the screen. + +![Trigger Screen](../img/trigger-screen-1.14.png) + +The `Create new trigger` dialog opens: + +![Create Trigger](../img/create-trigger-1.14.png) + +Input the condition that will cause the trigger and click `Next`. + +Input the action that will be the result of the trigger condition happening and click `Create`. + +![Create Trigger Action](../img/create-trigger-action-1.14.png) + +Get all the details in the video below! + +## Video Tutorial + + + + + + + + + +Testkube allows you to automate running tests and test suites by defining triggers on certain events for various Kubernetes resources. + +## What is a Testkube Test Trigger? + +In generic terms, a _Trigger_ defines an _action_ which will be executed for a given _execution_ when a certain _event_ on a specific _resource_ occurs having a given _concurrencyPolicy_. For example, we could define a _TestTrigger_ which _runs_ a _Test_ when a _ConfigMap_ gets _modified_. + +Watch our [video guide](#video-tutorial) on using Testkube Test Triggers to perform **Asynchronous Testing in Kubernetes**. + +## Custom Resource Definition Model +### Selectors + +The `resourceSelector` and `testSelector` fields support selecting resources either by name or using +the Kubernetes [Label Selector](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#resources-that-support-set-based-requirements). + +Each selector should specify the `namespace` of the object, otherwise the namespace defaults to `testkube`. + +``` +selector := resourceSelector | testSelector +``` + +#### Name Selector + +Name selectors are used when we want to select a specific resource in a specific namespace. + +```yaml +selector: + name: Kubernetes object name + namespace: Kubernetes object namespace (default is **testkube**) +``` + +#### Label Selector + +Label selectors are used when we want to select a group of resources in a specific namespace. + +```yaml +selector: + namespace: Kubernetes object namespace (default is **testkube**) + labelSelector: + matchLabels: map of key-value pairs + matchExpressions: + - key: label name + operator: [In | NotIn | Exists | DoesNotExist + values: list of values +``` + +### Resource Conditions + +Resource Conditions allows triggers to be defined based on the status conditions for a specific resource. + +```yaml +conditionSpec: + timeout: duration in seconds the test trigger waits for conditions, until its stopped + delay: duration in seconds the test trigger waits between condition checks + conditions: + - type: test trigger condition type + status: test trigger condition status, supported values - True, False, Unknown + reason: test trigger condition reason + ttl: test trigger condition ttl +``` + +### Resource Probes + +Resource Probes allows triggers to be defined based on the probe status. + +```yaml +probeSpec: + timeout: duration in seconds the test trigger waits for probes, until its stopped + delay: duration in seconds the test trigger waits between probes + probes: + - scheme: test trigger condition probe scheme to connect to host, default is http + host: test trigger condition probe host, default is pod ip or service name + path: test trigger condition probe path to check, default is / + port: test trigger condition probe port to connect + headers: test trigger condition probe headers to submit +``` + +### Supported Values +* **Resource** - pod, deployment, statefulset, daemonset, service, ingress, event, configmap +* **Action** - run +* **Event** - created, modified, deleted +* **Execution** - test, testsuite +* **ConcurrencyPolicy** - allow, forbid, replace + +**NOTE**: All resources support the above-mentioned events, a list of finer-grained events is in the works, stay tuned... + +## Example + +Here is an example for a **Test Trigger** *default/testtrigger-example* which runs the **TestSuite** *frontend/sanity-test* +when a **deployment** containing the label **testkube.io/tier: backend** gets **modified** and also has the conditions **Progressing: True: NewReplicaSetAvailable** and **Available: True**. + +```yaml +apiVersion: tests.testkube.io/v1 +kind: TestTrigger +metadata: + name: testtrigger-example + namespace: default +spec: + resource: deployment + resourceSelector: + labelSelector: + matchLabels: + testkube.io/tier: backend + event: modified + conditionSpec: + timeout: 100 + delay: 2 + conditions: + - type: Progressing + status: "True" + reason: "NewReplicaSetAvailable" + ttl: 60 + - type: Available + status: "True" + probeSpec: + timeout: 50 + delay: 1 + probes: + - scheme: http + host: testkube-api-server + path: /health + port: 8088 + headers: + X-Token: "12345" + - host: testkube-dashboard + port: 8080 + action: run + execution: testsuite + concurrencyPolicy: allow + testSelector: + name: sanity-test + namespace: frontend +``` + +## Architecture + +Testkube uses [Informers](https://pkg.go.dev/k8s.io/client-go/informers) to watch Kubernetes resources and register handlers +on certain actions on the watched Kubernetes resources. + +Informers are a reliable, scalable and fault-tolerant Kubernetes concept where each informer registers handlers with the +Kubernetes API and gets notified by Kubernetes on each event on the watched resources. + +## API + +Testkube exposes CRUD operations on test triggers in the REST API. Check out the [Open API](../openapi.md) docs for more info. + +## Video Tutorial + + + + + diff --git a/docs/docs/articles/testkube-dashboard-explore.md b/docs/docs/articles/testkube-dashboard-explore.md index 3b1e8b9f07c..22863e831a9 100644 --- a/docs/docs/articles/testkube-dashboard-explore.md +++ b/docs/docs/articles/testkube-dashboard-explore.md @@ -2,28 +2,28 @@ The Testkube Dashboard displays the current status of Tests and Test Suites executed in your environment. -![Test List](../img/test-list-1.6.png) +![Test List](../img/test-list-1.14.png) -![Test Suites List](../img/test-suite-list-1.6.png) +![Test Suites List](../img/test-suite-list-1.14.png) After selecting Tests or Test Suites in the left bar, the list of recent runs is displayed. At the top of the list, a Search field and filters for Labels and Status make finding tests in a large list easier: -![Search & Filter](../img/search-filter-1.6.png) +![Search & Filter](../img/search-filter-1.14.png) Select any Test or Test Suite to see the recent executions and their statuses. -![Execution Status](../img/execution-status-1.6.png) +![Execution Status](../img/execution-status-1.14.png) The execution statistics of the chosen Test or Test Suite are at the top of the screen, along with a graph of success or failure for the executions. The **Recent executions** tab has the list of executions. A green checkmark denotes a successful execution, a red 'x' denotes a failed execution and circling dots denotes a current run of a Test or Test Suite. -![Recent executions](../img/recent-executions-1.6.png) +![Recent executions](../img/recent-executions-1.14.png) The **CLI Commands** tab shows the commands used to perform the selected test: -![CLI Commands](../img/CLI-commands-1.6.png) +![CLI Commands](../img/CLI-commands-1.14.png) -The **Settings** tab contains 3 types of information about the Test or Test Suite. +Use the **Settings** to view or change the local settings of the Test or Test Suite. -![Setting](../img/settings-1.6.png) +![Setting](../img/settings-1.14.png) diff --git a/docs/docs/articles/testkube-dashboard-general-settings.md b/docs/docs/articles/testkube-dashboard-general-settings.md index 1e8ad89a2b7..baa8e778aba 100644 --- a/docs/docs/articles/testkube-dashboard-general-settings.md +++ b/docs/docs/articles/testkube-dashboard-general-settings.md @@ -2,30 +2,46 @@ Clicking the **General** box under the **Settings** tab displays the **Test name & description** and **Labels** for the Test or Test Suite: -![Settings General](../img/settings-general-1.9.png) +![Settings General](../img/settings-general-1.14.png) It is also the place to configure a Timeout or Failure Handling or delete a Test or Test Suite: -![Settings General Delete](../img/settings-general-delete-1.9.png) +![Settings General Delete](../img/settings-general-delete-1.14.png) ## Test Clicking **Test** will display more details for the selected Test: -![Settings Test](../img/settings-test-1.9.png) +![Settings Test](../img/settings-test-1.14.png) -If you have selected a Test Suite, the Tests contained in that Test Suite will be shown. +If you have selected a Test Suite, the Tests contained in that Test Suite will be shown. In this view, you will also see which test are run in parallel and which are run sequentially. Please see the [Scheduling Tests](./scheduling-tests.mdx) doc to learn more about how to schedule tests. -![Settings Test for Test Suite](../img/settings-test-suite-1.9.png) +![Settings Test for Test Suite](../img/settings-test-suite-1.14.png) + +## Execution + +Testkube allows for the Pre-Run or Post-Run of commands for a test. + +In the Execution section of the Settings tab, you can set up a command or a script (relative to your source root) which will be executed before or after the test itself is started. + +![Execution](../img/execution-1.14.png) ## Variables & Secrets -![Variable Tab](../img/variable-tab-1.6.png) +![Variable Tab](../img/variable-tab-1.14.png) + +Visit [Using Test Variables](./adding-tests-variables.md) for a description of adding Variables and Secrets. + +## Scheduling + +Add a cronjob-like schedule for your test which will then be executed automatically. + +![Scheduling](../img/scheduling-1.14.png) -Visit [Using Tests Variables](./adding-tests-variables.md) for a description of adding Variables and Secrets. +![Scheduling Options](../img/scheduling-options-1.14.png) ## Definition -Clicking the **Definition** box under the **Settings** tab allows the validation and export of the configuration for the Test or Test Suite: +Clicking the **Definition** section under the **Settings** tab allows the validation and export of the configuration for the Test or Test Suite: -![Settings Definition](../img/settings-definition-1.9.png) +![Settings Definition](../img/settings-definition-1.14.png) diff --git a/docs/docs/articles/testkube-dashboard.md b/docs/docs/articles/testkube-dashboard.md index 003a3da8d41..3fa61961ef3 100644 --- a/docs/docs/articles/testkube-dashboard.md +++ b/docs/docs/articles/testkube-dashboard.md @@ -2,7 +2,7 @@ The Testkube Dashboard provides a simple web-based user interface for monitoring Testkube test results via a web browser. -![img.png](../img/dashboard-1.6.png) +![img.png](../img/dashboard-1.14.png) The URL to access the Testkube Dashboard is [https://demo.testkube.io](https://demo.testkube.io), which, when first loaded, will prompt for the results endpoint of your Testkube installation. Click the **Settings** icon at the bottom left of the screen to return to change the Testkube API endpoint. diff --git a/docs/docs/articles/webhooks.md b/docs/docs/articles/webhooks.md deleted file mode 100644 index d2ed20339f3..00000000000 --- a/docs/docs/articles/webhooks.md +++ /dev/null @@ -1,28 +0,0 @@ -# Webhooks - -[Webhooks](https://docs.github.com/en/webhooks-and-events/webhooks/about-webhooks) allow you to build or set up integrations and send HTTP POST payloads (your Testkube Execution and its current state) whenever an event is triggered. In this case, when your Tests start or finish. - -To set them up when using Testkube, you'll need to create your webhook as shown in the following format example: - -```yaml title="webhook.yaml" -apiVersion: executor.testkube.io/v1 -kind: Webhook -metadata: - name: example-webhook - namespace: testkube -spec: - uri: http://localhost:8080/events - events: - - start-test - - end-test - - end-test-success - - end-test-failed -``` - -And then apply with: - -```sh -kubectl apply -f webhook.yaml -``` - -Here you'll be able to pass events depending on which webhooks you want to be triggered. Testkube will pass `Event` which can have `type` and `testExecution` or `testsuiteExecution` fields. diff --git a/docs/docs/articles/webhooks.mdx b/docs/docs/articles/webhooks.mdx index 77cfe991eb2..2c041df12ec 100644 --- a/docs/docs/articles/webhooks.mdx +++ b/docs/docs/articles/webhooks.mdx @@ -5,13 +5,136 @@ import TabItem from "@theme/TabItem"; Webhooks allow you to integrate Testkube with external systems by sending HTTP POST payloads containing information about Testkube executions and their current state when specific events occur. To set up webhooks in Testkube, you'll need to have an HTTPS endpoint to receive the events and a payload template to be sent along with the data. -Here's an example format for creating a webhook in Testkube using either the CLI or custom response: +## Creating a Webhook +The webhook can be created using the Dashboard, CLI, or a Custom Resource. + +If you prefer to use the Dashboard, you can view existing webhooks by going to the Webhooks tab. + +![Dashboard menu - webhooks icon](../img/dashboard-webhooks-icon.png) + +Here you can also create a new webhook by clicking the `Create a new webhook` button. + +Then, fill in the webhook details: + +![Dashboard webhook - create dialog 1](../img/dashboard-create-webhook-1.png) + +- Name - your webhook name (in this case `example-webhook`) +- Resource identifier - the resource (or resources) selected by `label` for which the webhook can be triggered (in the example: `test-type:postman-collection` - any postman test) +- Triggered events - events that will trigger the webhook (in this case `start-test`, `end-test-success`, and `end-test-failed`). All available trigger events can be found in the [Supported Event types](#supported-event-types) section. + +![Dashboard webhook - create dialog 2](../img/dashboard-create-webhook-2.png) + +Set your webhook URI - the HTTPS endpoint where you want to receive the webhook events. +After the webhook is created, the custom payload and headers can be set in Settings->Action. + + +Webhooks can be created with Testkube CLI using the `create webhook` command. + +```sh +testkube create webhook --name example-webhook --events start-test --events end-test-success --events end-test-failed --uri +``` +`--name` - Your webhook name (in this case `example-webhook`). +`--events` - Event that will trigger a webhook. Multiple `--events` can be defined (in this case `--events start-test --events end-test-success --events end-test-failed`). All available trigger events can be found in the [Supported Event types](#supported-event-types) section. +`--uri` - The HTTPS endpoint where you want to receive the webhook events. + + + + + +```yaml title="webhook.yaml" +apiVersion: executor.testkube.io/v1 +kind: Webhook +metadata: + name: example-webhook + namespace: testkube +spec: + uri: + events: + - start-test + - end-test-success + - end-test-failed + selector: "" +``` +Where should be replaced with the HTTPS endpoint URL where you want to receive the webhook events. + +And then apply with: + +```sh +kubectl apply -f webhook.yaml +``` + + + + + +### Resource Selector (labels) +In order to limit webhook triggers to a specific resource, or resources, the Resource Selector can be used. It allows you to select the specific resource by label, or labels. + + + -Create a webhook template payload file: +![Dashboard webhook - resource identifier](../img/dashboard-create-webhook-resource-identifier.png) + + + + + +The Resource Selector can be set with `--selector`. +For example, `--selector test-type=postman-collection` will limit the resources to the postman tests (label: `test-type=postman-collection`) + + + + + +```yaml +spec: + selector: test-type=postman-collection +``` + +So, the complete definition may look like this: + +```yaml title="webhook.yaml" +apiVersion: executor.testkube.io/v1 +kind: Webhook +metadata: + name: example-webhook + namespace: testkube +spec: + uri: + events: + - start-test + - end-test-success + - end-test-failed + selector: test-type=postman-collection +``` + + + + + +### Webhook Payload + +Webhook payload can be configured - in this example, `event id`: +``` +{"text": "event id {{ .Id }}"} +``` + + + +When you have selected an existing Webhook, its payload can be configured in Webhook Settings->Action. + +![Dashboard webhook - webhook settings action`](../img/dashboard-webhook-settings-action.png) + +![Dashboard webhook - webhook payload](../img/dashboard-webhook-payload.png) + + + + +Create a webhook payload template file: ```json title="template.json" { @@ -19,9 +142,13 @@ Create a webhook template payload file: } ``` +And set it with `--payload-template template.json`. + ```sh -testkube create webhook --name example-webhook --events start-test --events end-test-failed --payload-template template.json --uri +testkube create webhook --name example-webhook --events start-test --events end-test-passed --events end-test-failed --payload-template template.json --uri ``` +Where should be replaced with the HTTPS endpoint URL where you want to receive the webhook events. + ```sh title="Expected output:" Webhook created example-webhook 🥇 @@ -31,7 +158,14 @@ Webhook created example-webhook 🥇 -```yaml title="webhook.yaml" +Payload template can be configured with `spec.payloadTemplate`. +``` + payloadTemplate: | + {"text": "event id {{ .Id }}"} +``` + +Example: +``` apiVersion: executor.testkube.io/v1 kind: Webhook metadata: @@ -47,29 +181,192 @@ spec: payloadObjectField: "" payloadTemplate: | {"text": "event id {{ .Id }}"} - headers: - X-Token: "12345" ``` -And then apply with: + -```sh -kubectl apply -f webhook.yaml + + +### Webhook Payload Variables +Webhook payload can contain **event-specific** variables - they will be replaced with actual data when the events occurs. In the above examples, only the event `Id` is being sent. +However, any of these [supported Event Variables](#supported-event-variables) can be used. + +For example, the following payload: +``` +{"text": "Event {{ .Type_ }} - Test '{{ .TestExecution.TestName }}' execution ({{ .TestExecution.Number }}) finished with '{{ .TestExecution.ExecutionResult.Status }}' status"} +``` +will result in: +``` +{"text": "Event end-test-success - Test 'postman-executor-smoke' execution (948) finished with 'passed' status"} +``` + +#### testkube-api-server ENV variables +In addition to event-specific variables, it's also possible to pass testkube-api-server ENV variables: + +```sh title="template.txt" +TESTKUBE_CLOUD_URL: {{ index .Envs "TESTKUBE_CLOUD_URL" }} ``` +### URI and HTTP Headers +You can add additional HTTP headers like `Authorization` or `x-api-key` to have a secret token. +It's possible to use golang based template string as header or uri value. + + + + +Webhook headers can be configured in Webhook Settings->Action. + +![Dashboard webhook - webhook settings action`](../img/dashboard-webhook-settings-action.png) + +![Dashboard webhook - webhook headers](../img/dashboard-webhook-headers.png) + + +Custom headers can be set using `--header` - for example: + +`--header X-Token="12345"` + + + + + +```yaml +spec: + headers: + X-Token: "12345" +``` + +```yaml title="webhook.yaml" +apiVersion: executor.testkube.io/v1 +kind: Webhook +metadata: + name: example-webhook + namespace: testkube +spec: + uri: + events: + - start-test + - end-test-success + - end-test-failed + selector: "" + headers: + X-Token: "12345" +``` + + -In the example above, replace with the HTTPS endpoint URL where you want to receive the webhook events. The payload template can be customized to include additional information. In the above example, only the event `Id` is being sent. The template's variables will be replaced with data when events occur. +## Supported Event types +Webhooks can be triggered on any of the following events: +- start-test +- end-test-success +- end-test-failed +- end-test-aborted +- end-test-timeout +- start-testsuite +- end-testsuite-success +- end-testsuite-failed +- end-testsuite-aborted +- end-testsuite-timeout +- created +- updated +- deleted -You can add additional HTTP headers like `Authorization` or `x-api-key` to have a secret token. +They can be triggered by the following resources: +- test +- testsuite +- executor +- trigger +- webhook +- testexecution +- testsuiteexecution -It's possible to get access to env variables of testkube-api-server pod in webhook template: +## Supported Event Variables -```sh title="template.txt" -TESTKUBE_CLOUD_URL: {{ index .Envs "TESTKUBE_CLOUD_URL" }} +### Event-specific variables: +- `Id` - event ID (for example, `2a20c7da-3b77-4ea9-a33d-403187d3e9e6`) +- `Resource` +- `ResourceId` +- `Type_` - event Type (for example, `start-test`, `end-test,success`, etc. All available trigger events can be found in the [Supported Event types](#supported-event-types) section). +- `TestExecution` - test execution details (example: [TestExecution (Execution)](#testexecution-execution) section) +- `TestSuiteExecution` - test suite execution details (example: [TestSuiteExecution](#testsuiteexecution) section) +- `ClusterName` - cluster name +- `Envs` (API-server ENV variables) - list of Testkube API-Server ENV variables + +The full Event Data Model can be found [here](https://github.com/kubeshop/testkube/blob/main/pkg/api/v1/testkube/model_event.go). + +### TestExecution (Execution): +- `Id` - Execution ID (for example, `64f8cf3c712890925aea51ce`) +- `TestName` - Test Name (for example, `postman-executor-smoke`) +- `TestSuiteName` - Test Suite name (if run as a part of a Test Suite) +- `TestNamespace` - Execution namespace, where testkube is installed (for example, `testkube`) +- `TestType` - Test type (for example, `postman/collection`) +- `Name` - Execution name (for example, `postman-executor-smoke-937) +- `Number` - Execution number (for example, `937`) +- `Envs` - List of ENV variables for specific Test (if defined) +- `Command` - Command executed inside the Pod (for example, `newman`) +- `Args` - Command arguments (for example, `run -e --reporters cli,json --reporter-json-export `) +- `Variables` - List of variables +- `Content` - Test content +- `StartTime` - Test start time (for example, `2023-09-06 19:23:34.543433547 +0000 UTC`) +- `EndTime` - Time when the test execution finished (for example, `2023-09-06 19:23:42.221493031 +0000 UTC`) +- `Duration` - Test duration in seconds (for example, `7.68s`) +- `DurationMs` - Test duration in miliseconds (for example, `7678`) +- `ExecutionResult` - Execution result (https://github.com/kubeshop/testkube/blob/main/pkg/api/v1/testkube/model_event.go) +- `Labels` Test labels (for example, `[core-tests:executors executor:postman-executor test-type:postman-collection],`) +- `RunningContext` - Running context - how the test has been triggered (for example, `user-ui`) + +The full Execution data model can be found [here](https://github.com/kubeshop/testkube/blob/main/pkg/api/v1/testkube/model_execution.go). + +### TestSuiteExecution: + +- `Id` - TestSuiteExecution ID (for example, `64f8d5b2712890925aea51dc`) +- `Name` - TestSuite name (for example, `ts-executor-postman-smoke-tests-472`) +- `Status` - TestSuite execution status (for example, `running` or `passed`) +- `Envs` - List of ENV variables +- `Variables` - List of variables +- `StartTime` - Test start time (for example, `2023-09-06 19:23:34.543433547 +0000 UTC`) +- `EndTime` - Time when the test execution finished (for example, `2023-09-06 19:23:42.221493031 +0000 UTC`) +- `Duration` - Test duration in seconds (for example, `7.68s`) +- `DurationMs` - Test duration in miliseconds (for example, `7678`) +- `StepResults` +- `Labels` - TestSuite labels (for example, `[app:testkube testsuite:executor-postman-smoke-tests]`) +- `RunningContext` - Running context - how the TestSuite has been triggered (for example, `user-ui`) + +The full TestSuiteExecution data model can be found [here](https://github.com/kubeshop/testkube/blob/main/pkg/api/v1/testkube/model_test_suite_execution.go). + +## Additional Examples + +### Microsoft Teams +Webhooks can also be used to send messages to Microsoft Teams channels. +First, you need to create an incoming webhook in Teams for a specific channel. You can see how to do it in the Teams Docs [here](https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook?tabs=dotnet#create-incoming-webhooks-1). After your Teams incoming webhook is created, you can use it with Testkube webhooks - just use the URL provided (it will probably look like this: `https://xxxxx.webhook.office.com/xxxxxxxxx`). + +In order to send the message when test execution finishes, the following Webhook can be used: ``` +apiVersion: executor.testkube.io/v1 +kind: Webhook +metadata: + name: example-webhook-teams + namespace: testkube +spec: + events: + - end-test-success + - end-test-failed + - end-test-aborted + - end-test-timeout + uri: https://xxxxx.webhook.office.com/xxxxxxxxx + payloadTemplate: "{\"text\": \"Test '{{ .TestExecution.TestName }}' execution ({{ .TestExecution.Number }}) finished with '{{ .TestExecution.ExecutionResult.Status }}' status\"}\n" +``` + +It will result in: + +``` +{"text": "Test 'postman-executor-smoke' execution (949) finished with 'passed' status"} +``` + +and the message: +`Test 'postman-executor-smoke' execution (949) finished with 'passed' status"` being displayed. ## Testing Webhooks diff --git a/docs/docs/cli/testkube_cloud.md b/docs/docs/cli/testkube_cloud.md index d6aa6eaba56..5d2ce9660bb 100644 --- a/docs/docs/cli/testkube_cloud.md +++ b/docs/docs/cli/testkube_cloud.md @@ -27,6 +27,6 @@ testkube cloud [flags] * [testkube](testkube.md) - Testkube entrypoint for kubectl plugin * [testkube cloud connect](testkube_cloud_connect.md) - Testkube Cloud connect * [testkube cloud disconnect](testkube_cloud_disconnect.md) - Switch back to Testkube OSS mode, based on active .kube/config file -* [testkube cloud init](testkube_cloud_init.md) - Install Helm chart registry in current kubectl context and update dependencies +* [testkube cloud init](testkube_cloud_init.md) - Install Testkube Cloud Agent and connect to Testkube Cloud environment * [testkube cloud login](testkube_cloud_login.md) - Login to Testkube Cloud diff --git a/docs/docs/cli/testkube_cloud_init.md b/docs/docs/cli/testkube_cloud_init.md index e814ab12e38..a637a476b14 100644 --- a/docs/docs/cli/testkube_cloud_init.md +++ b/docs/docs/cli/testkube_cloud_init.md @@ -1,6 +1,6 @@ ## testkube cloud init -Install Helm chart registry in current kubectl context and update dependencies +Install Testkube Cloud Agent and connect to Testkube Cloud environment ``` testkube cloud init [flags] @@ -15,9 +15,11 @@ testkube cloud init [flags] --dry-run dry run mode - only print commands that would be executed --env-id string Testkube Cloud environment id -h, --help help for init + --multi-namespace multi namespace mode --name string installation name (usually you don't need to change it) (default "testkube") --namespace string namespace where to install (default "testkube") --no-confirm don't ask for confirmation - unatended installation mode + --no-operator should operator be installed (for more instances in multi namespace mode it should be set to true) --org-id string Testkube Cloud organization id --values string path to Helm values file ``` diff --git a/docs/docs/cli/testkube_completion_zsh.md b/docs/docs/cli/testkube_completion_zsh.md index df3fe922767..e8c92863953 100644 --- a/docs/docs/cli/testkube_completion_zsh.md +++ b/docs/docs/cli/testkube_completion_zsh.md @@ -13,7 +13,7 @@ to enable it. You can execute the following once: To load completions in your current shell session: - source <(testkube completion zsh); compdef _testkube testkube + source <(testkube completion zsh) To load completions for every new session, execute once: diff --git a/docs/docs/cli/testkube_create.md b/docs/docs/cli/testkube_create.md index 63a7f60f50d..87bcdc9163f 100644 --- a/docs/docs/cli/testkube_create.md +++ b/docs/docs/cli/testkube_create.md @@ -27,6 +27,7 @@ testkube create [flags] * [testkube](testkube.md) - Testkube entrypoint for kubectl plugin * [testkube create executor](testkube_create_executor.md) - Create new Executor +* [testkube create template](testkube_create_template.md) - Create a new Template. * [testkube create test](testkube_create_test.md) - Create new Test * [testkube create testsource](testkube_create_testsource.md) - Create new TestSource * [testkube create testsuite](testkube_create_testsuite.md) - Create new TestSuite diff --git a/docs/docs/cli/testkube_create_executor.md b/docs/docs/cli/testkube_create_executor.md index 673c1351fc2..a9ee9595402 100644 --- a/docs/docs/cli/testkube_create_executor.md +++ b/docs/docs/cli/testkube_create_executor.md @@ -24,10 +24,12 @@ testkube create executor [flags] --image string image used for executor --image-pull-secrets stringArray secret name used to pull the image in executor -j, --job-template string if executor needs to be launched using custom job specification, then a path to template file should be provided + --job-template-reference string reference to job template for using with executor -l, --label stringToString label key value pair: --label key1=value1 (default []) -n, --name string unique executor name - mandatory --tooltip stringToString tooltip key value pair: --tooltip key1=value1 (default []) -t, --types stringArray test types handled by executor + --update update, if executor already exists -u, --uri string if resource need to be loaded from URI ``` diff --git a/docs/docs/cli/testkube_create_template.md b/docs/docs/cli/testkube_create_template.md new file mode 100644 index 00000000000..9ebeeb299b2 --- /dev/null +++ b/docs/docs/cli/testkube_create_template.md @@ -0,0 +1,38 @@ +## testkube create template + +Create a new Template. + +### Synopsis + +Create a new Template Custom Resource. + +``` +testkube create template [flags] +``` + +### Options + +``` + --body string a path to template file to use as template body + -h, --help help for template + -l, --label stringToString label key value pair: --label key1=value1 (default []) + -n, --name string unique template name - mandatory + --template-type string template type one of job|container|cronjob|scraper|pvc|webhook + --update update, if template already exists +``` + +### Options inherited from parent commands + +``` + -a, --api-uri string api uri, default value read from config if set (default "https://demo.testkube.io/results/v1") + -c, --client string client used for connecting to Testkube API one of proxy|direct (default "proxy") + --crd-only generate only crd + --namespace string Kubernetes namespace, default value read from config if set (default "testkube") + --oauth-enabled enable oauth + --verbose show additional debug messages +``` + +### SEE ALSO + +* [testkube create](testkube_create.md) - Create resource + diff --git a/docs/docs/cli/testkube_create_test.md b/docs/docs/cli/testkube_create_test.md index 87687442b07..ac8ce7f1052 100644 --- a/docs/docs/cli/testkube_create_test.md +++ b/docs/docs/cli/testkube_create_test.md @@ -15,12 +15,17 @@ testkube create test [flags] ``` --args-mode string usage mode for arguments. one of append|override (default "append") --artifact-dir stringArray artifact dirs for scraping + --artifact-omit-folder-per-execution don't store artifacts in execution folder + --artifact-storage-bucket string artifact storage class name for container executor --artifact-storage-class-name string artifact storage class name for container executor --artifact-volume-mount-path string artifact volume mount path for container executor --command stringArray command passed to image in executor --copy-files stringArray file path mappings from host to pod of form source:destination --cronjob-template string cron job template file path for extensions to cron job template + --cronjob-template-reference string reference to cron job template to use for the test + --description string test description --env stringToString envs in a form of name1=val1 passed to executor (default []) + --execute-postrun-script-before-scraping whether to execute postrun scipt before scraping or not (prebuilt executor only) --execution-name string execution name, if empty will be autogenerated --executor-args stringArray executor binary additional arguments -f, --file string test file - will be read from stdin if not specified @@ -41,6 +46,7 @@ testkube create test [flags] --image string image for container executor --image-pull-secrets stringArray secret name used to pull the image in container executor --job-template string job template file path for extensions to job template + --job-template-reference string reference to job template to use for the test -l, --label stringToString label key value pair: --label key1=value1 (default []) --mount-configmap stringToString config map value pair for mounting it to executor pod: --mount-configmap configmap_name=configmap_mountpath (default []) --mount-secret stringToString secret value pair for mounting it to executor pod: --mount-secret secret_name=secret_mountpath (default []) @@ -48,8 +54,11 @@ testkube create test [flags] --negative-test negative test, if enabled, makes failure an expected and correct test result. If the test fails the result will be set to success, and vice versa --postrun-script string path to script to be run after test execution --prerun-script string path to script to be run before test execution + --pvc-template string pvc template file path for extensions to pvc template + --pvc-template-reference string reference to pvc template to use for the test --schedule string test schedule in a cron job form: * * * * * --scraper-template string scraper template file path for extensions to scraper template + --scraper-template-reference string reference to scraper template to use for the test --secret-env stringToString secret envs in a form of secret_key1=secret_name1 passed to executor (default []) -s, --secret-variable stringToString secret variable key value pair: --secret-variable key1=value1 (default []) --secret-variable-reference stringToString secret variable references in a form name1=secret_name1=secret_key1 (default []) @@ -57,6 +66,7 @@ testkube create test [flags] --test-content-type string content type of test one of string|file-uri|git --timeout int duration in seconds for test to timeout. 0 disables timeout. -t, --type string test type + --update update, if test already exists --upload-timeout string timeout to use when uploading files, example: 30s --uri string URI of resource - will be loaded by http GET -v, --variable stringToString variable key value pair: --variable key1=value1 (default []) diff --git a/docs/docs/cli/testkube_create_testsource.md b/docs/docs/cli/testkube_create_testsource.md index d7df2179afa..c5f126958af 100644 --- a/docs/docs/cli/testkube_create_testsource.md +++ b/docs/docs/cli/testkube_create_testsource.md @@ -29,7 +29,8 @@ testkube create testsource [flags] -l, --label stringToString label key value pair: --label key1=value1 (default []) -n, --name string unique test source name - mandatory --source-type string source type of test one of string|file-uri|git - -u, --uri string URI which should be called when given event occurs + --update update, if test source already exists + -u, --uri string URI which should be called to get test content ``` ### Options inherited from parent commands diff --git a/docs/docs/cli/testkube_create_testsuite.md b/docs/docs/cli/testkube_create_testsuite.md index 98a83ff2a95..61695bfdeea 100644 --- a/docs/docs/cli/testkube_create_testsuite.md +++ b/docs/docs/cli/testkube_create_testsuite.md @@ -14,17 +14,25 @@ testkube create testsuite [flags] ``` --cronjob-template string cron job template file path for extensions to cron job template + --cronjob-template-reference string reference to cron job template to use for the test --execution-name string execution name, if empty will be autogenerated -f, --file string JSON test suite file - will be read from stdin if not specified, look at testkube.TestUpsertRequest -h, --help help for testsuite --http-proxy string http proxy for executor containers --https-proxy string https proxy for executor containers + --job-template string job template file path for extensions to job template + --job-template-reference string reference to job template to use for the test -l, --label stringToString label key value pair: --label key1=value1 (default []) --name string Set/Override test suite name + --pvc-template string pvc template file path for extensions to pvc template + --pvc-template-reference string reference to pvc template to use for the test --schedule string test suite schedule in a cron job form: * * * * * + --scraper-template string scraper template file path for extensions to scraper template + --scraper-template-reference string reference to scraper template to use for the test -s, --secret-variable stringToString secret variable key value pair: --secret-variable key1=value1 (default []) --secret-variable-reference stringToString secret variable references in a form name1=secret_name1=secret_key1 (default []) --timeout int32 duration in seconds for test suite to timeout. 0 disables timeout. + --update update, if test suite already exists -v, --variable stringToString param key value pair: --variable key1=value1 (default []) ``` diff --git a/docs/docs/cli/testkube_create_webhook.md b/docs/docs/cli/testkube_create_webhook.md index ae50d764a27..603491e2362 100644 --- a/docs/docs/cli/testkube_create_webhook.md +++ b/docs/docs/cli/testkube_create_webhook.md @@ -13,15 +13,17 @@ testkube create webhook [flags] ### Options ``` - -e, --events stringArray event types handled by webhook e.g. start-test|end-test - --header stringToString webhook header value pair: --header Content-Type=application/xml (default []) - -h, --help help for webhook - -l, --label stringToString label key value pair: --label key1=value1 (default []) - -n, --name string unique webhook name - mandatory - --payload-field string field to use for notification object payload - --payload-template string if webhook needs to send a custom notification, then a path to template file should be provided - --selector string expression to select tests and test suites for webhook events: --selector app=backend - -u, --uri string URI which should be called when given event occurs + -e, --events stringArray event types handled by webhook e.g. start-test|end-test + --header stringToString webhook header value pair (golang template supported): --header Content-Type=application/xml (default []) + -h, --help help for webhook + -l, --label stringToString label key value pair: --label key1=value1 (default []) + -n, --name string unique webhook name - mandatory + --payload-field string field to use for notification object payload + --payload-template string if webhook needs to send a custom notification, then a path to template file should be provided + --payload-template-reference string reference to payload template to use for the webhook + --selector string expression to select tests and test suites for webhook events: --selector app=backend + --update update, if webhook already exists + -u, --uri string URI which should be called when given event occurs (golang template supported) ``` ### Options inherited from parent commands diff --git a/docs/docs/cli/testkube_delete.md b/docs/docs/cli/testkube_delete.md index 9c214cd836f..1512aa3bc7b 100644 --- a/docs/docs/cli/testkube_delete.md +++ b/docs/docs/cli/testkube_delete.md @@ -26,6 +26,7 @@ testkube delete [flags] * [testkube](testkube.md) - Testkube entrypoint for kubectl plugin * [testkube delete executor](testkube_delete_executor.md) - Delete Executor +* [testkube delete template](testkube_delete_template.md) - Delete a template. * [testkube delete test](testkube_delete_test.md) - Delete Test * [testkube delete testsource](testkube_delete_testsource.md) - Delete test source * [testkube delete testsuite](testkube_delete_testsuite.md) - Delete test suite diff --git a/docs/docs/cli/testkube_delete_template.md b/docs/docs/cli/testkube_delete_template.md new file mode 100644 index 00000000000..265a9ff458f --- /dev/null +++ b/docs/docs/cli/testkube_delete_template.md @@ -0,0 +1,34 @@ +## testkube delete template + +Delete a template. + +### Synopsis + +Delete a template and pass the template name to be deleted. + +``` +testkube delete template [flags] +``` + +### Options + +``` + -h, --help help for template + -l, --label strings label key value pair: --label key1=value1 + -n, --name string unique template name, you can also pass it as first argument +``` + +### Options inherited from parent commands + +``` + -a, --api-uri string api uri, default value read from config if set (default "https://demo.testkube.io/results/v1") + -c, --client string Client used for connecting to testkube API one of proxy|direct (default "proxy") + --namespace string Kubernetes namespace, default value read from config if set (default "testkube") + --oauth-enabled enable oauth + --verbose should I show additional debug messages +``` + +### SEE ALSO + +* [testkube delete](testkube_delete.md) - Delete resources + diff --git a/docs/docs/cli/testkube_generate_tests-crds.md b/docs/docs/cli/testkube_generate_tests-crds.md index 31ec11fb7a6..36cf66d9c73 100644 --- a/docs/docs/cli/testkube_generate_tests-crds.md +++ b/docs/docs/cli/testkube_generate_tests-crds.md @@ -15,12 +15,17 @@ testkube generate tests-crds [flags] ``` --args-mode string usage mode for arguments. one of append|override (default "append") --artifact-dir stringArray artifact dirs for scraping + --artifact-omit-folder-per-execution don't store artifacts in execution folder + --artifact-storage-bucket string artifact storage class name for container executor --artifact-storage-class-name string artifact storage class name for container executor --artifact-volume-mount-path string artifact volume mount path for container executor --command stringArray command passed to image in executor --copy-files stringArray file path mappings from host to pod of form source:destination --cronjob-template string cron job template file path for extensions to cron job template + --cronjob-template-reference string reference to cron job template to use for the test + --description string test description --env stringToString envs in a form of name1=val1 passed to executor (default []) + --execute-postrun-script-before-scraping whether to execute postrun scipt before scraping or not (prebuilt executor only) --execution-name string execution name, if empty will be autogenerated --executor-args stringArray executor binary additional arguments -h, --help help for tests-crds @@ -29,14 +34,18 @@ testkube generate tests-crds [flags] --image string image for container executor --image-pull-secrets stringArray secret name used to pull the image in container executor --job-template string job template file path for extensions to job template + --job-template-reference string reference to job template to use for the test -l, --label stringToString label key value pair: --label key1=value1 (default []) --mount-configmap stringToString config map value pair for mounting it to executor pod: --mount-configmap configmap_name=configmap_mountpath (default []) --mount-secret stringToString secret value pair for mounting it to executor pod: --mount-secret secret_name=secret_mountpath (default []) --negative-test negative test, if enabled, makes failure an expected and correct test result. If the test fails the result will be set to success, and vice versa --postrun-script string path to script to be run after test execution --prerun-script string path to script to be run before test execution + --pvc-template string pvc template file path for extensions to pvc template + --pvc-template-reference string reference to pvc template to use for the test --schedule string test schedule in a cron job form: * * * * * --scraper-template string scraper template file path for extensions to scraper template + --scraper-template-reference string reference to scraper template to use for the test --secret-env stringToString secret envs in a form of secret_key1=secret_name1 passed to executor (default []) -s, --secret-variable stringToString secret variable key value pair: --secret-variable key1=value1 (default []) --secret-variable-reference stringToString secret variable references in a form name1=secret_name1=secret_key1 (default []) diff --git a/docs/docs/cli/testkube_get.md b/docs/docs/cli/testkube_get.md index d98c5523897..010e0d0899f 100644 --- a/docs/docs/cli/testkube_get.md +++ b/docs/docs/cli/testkube_get.md @@ -35,6 +35,7 @@ testkube get [flags] * [testkube get context](testkube_get_context.md) - Set context for Testkube Cloud * [testkube get execution](testkube_get_execution.md) - Lists or gets test executions * [testkube get executor](testkube_get_executor.md) - Gets executor details +* [testkube get template](testkube_get_template.md) - Get template details. * [testkube get test](testkube_get_test.md) - Get all available tests * [testkube get testsource](testkube_get_testsource.md) - Get test source details * [testkube get testsuite](testkube_get_testsuite.md) - Get test suite by name diff --git a/docs/docs/cli/testkube_get_template.md b/docs/docs/cli/testkube_get_template.md new file mode 100644 index 00000000000..11b7e72a7c8 --- /dev/null +++ b/docs/docs/cli/testkube_get_template.md @@ -0,0 +1,37 @@ +## testkube get template + +Get template details. + +### Synopsis + +Get template allows you to change the output format. To get single details, pass the template name as the first argument. + +``` +testkube get template [flags] +``` + +### Options + +``` + --crd-only show only test crd + -h, --help help for template + -l, --label strings label key value pair: --label key1=value1 + -n, --name string unique template name, you can also pass it as argument +``` + +### Options inherited from parent commands + +``` + -a, --api-uri string api uri, default value read from config if set (default "https://demo.testkube.io/results/v1") + -c, --client string client used for connecting to Testkube API one of proxy|direct (default "proxy") + --go-template string go template to render (default "{{.}}") + --namespace string Kubernetes namespace, default value read from config if set (default "testkube") + --oauth-enabled enable oauth + -o, --output string output type can be one of json|yaml|pretty|go-template (default "pretty") + --verbose show additional debug messages +``` + +### SEE ALSO + +* [testkube get](testkube_get.md) - Get resources + diff --git a/docs/docs/cli/testkube_run_test.md b/docs/docs/cli/testkube_run_test.md index a18c814fe32..524310dc172 100644 --- a/docs/docs/cli/testkube_run_test.md +++ b/docs/docs/cli/testkube_run_test.md @@ -16,6 +16,8 @@ testkube run test [flags] --args stringArray executor binary additional arguments --args-mode string usage mode for argumnets. one of append|override (default "append") --artifact-dir stringArray artifact dirs for scraping + --artifact-omit-folder-per-execution don't store artifacts in execution folder + --artifact-storage-bucket string artifact storage class name for container executor --artifact-storage-class-name string artifact storage class name for container executor --artifact-volume-mount-path string artifact volume mount path for container executor --command stringArray command passed to image in executor @@ -24,6 +26,7 @@ testkube run test [flags] --copy-files stringArray file path mappings from host to pod of form source:destination -d, --download-artifacts downlaod artifacts automatically --download-dir string download dir (default "artifacts") + --execute-postrun-script-before-scraping whether to execute postrun scipt before scraping or not (prebuilt executor only) --execution-label stringToString execution-label key value pair: --execution-label key1=value1 (default []) --format string data format for storing files, one of folder|archive (default "folder") --git-branch string if uri is git repository we can set additional branch parameter @@ -36,6 +39,7 @@ testkube run test [flags] --image string execution variable passed to executor --iterations int how many times to run the test (default 1) --job-template string job template file path for extensions to job template + --job-template-reference string reference to job template to use for the test -l, --label strings label key value pair: --label key1=value1 --mask stringArray regexp to filter downloaded files, single or comma separated, like report/.* or .*\.json,.*\.js$ --mount-configmap stringToString config map value pair for mounting it to executor pod: --mount-configmap configmap_name=configmap_mountpath (default []) @@ -44,7 +48,10 @@ testkube run test [flags] --negative-test negative test, if enabled, makes failure an expected and correct test result. If the test fails the result will be set to success, and vice versa --postrun-script string path to script to be run after test execution --prerun-script string path to script to be run before test execution + --pvc-template string pvc template file path for extensions to pvc template + --pvc-template-reference string reference to pvc template to use for the test --scraper-template string scraper template file path for extensions to scraper template + --scraper-template-reference string reference to scraper template to use for the test -s, --secret-variable stringToString execution secret variable passed to executor (default []) --secret-variable-reference stringToString secret variable references in a form name1=secret_name1=secret_key1 (default []) --upload-timeout string timeout to use when uploading files, example: 30s diff --git a/docs/docs/cli/testkube_run_testsuite.md b/docs/docs/cli/testkube_run_testsuite.md index 487a76d331e..e1138fcf6aa 100644 --- a/docs/docs/cli/testkube_run_testsuite.md +++ b/docs/docs/cli/testkube_run_testsuite.md @@ -23,8 +23,14 @@ testkube run testsuite [flags] -h, --help help for testsuite --http-proxy string http proxy for executor containers --https-proxy string https proxy for executor containers + --job-template string job template file path for extensions to job template + --job-template-reference string reference to job template to use for the test -l, --label strings label key value pair: --label key1=value1 -n, --name string execution name, if empty will be autogenerated + --pvc-template string pvc template file path for extensions to pvc template + --pvc-template-reference string reference to pvc template to use for the test + --scraper-template string scraper template file path for extensions to scraper template + --scraper-template-reference string reference to scraper template to use for the test -s, --secret-variable stringToString execution variables passed to executor (default []) --secret-variable-reference stringToString secret variable references in a form name1=secret_name1=secret_key1 (default []) -v, --variable stringToString execution variables passed to executor (default []) diff --git a/docs/docs/cli/testkube_update.md b/docs/docs/cli/testkube_update.md index 7a3de7cc45b..9da43b9874f 100644 --- a/docs/docs/cli/testkube_update.md +++ b/docs/docs/cli/testkube_update.md @@ -26,6 +26,7 @@ testkube update [flags] * [testkube](testkube.md) - Testkube entrypoint for kubectl plugin * [testkube update executor](testkube_update_executor.md) - Update Executor +* [testkube update template](testkube_update_template.md) - Update Template * [testkube update test](testkube_update_test.md) - Update test * [testkube update testsource](testkube_update_testsource.md) - Update TestSource * [testkube update testsuite](testkube_update_testsuite.md) - Update Test Suite diff --git a/docs/docs/cli/testkube_update_executor.md b/docs/docs/cli/testkube_update_executor.md index 2f8cf5c90b9..c65aece5c9a 100644 --- a/docs/docs/cli/testkube_update_executor.md +++ b/docs/docs/cli/testkube_update_executor.md @@ -24,6 +24,7 @@ testkube update executor [flags] --image string image used for executor --image-pull-secrets stringArray secret name used to pull the image in executor -j, --job-template string if executor needs to be launched using custom job specification, then a path to template file should be provided + --job-template-reference string reference to job template for using with executor -l, --label stringToString label key value pair: --label key1=value1 (default []) -n, --name string unique executor name - mandatory --tooltip stringToString tooltip key value pair: --tooltip key1=value1 (default []) diff --git a/docs/docs/cli/testkube_update_template.md b/docs/docs/cli/testkube_update_template.md new file mode 100644 index 00000000000..aedaa7e8bdd --- /dev/null +++ b/docs/docs/cli/testkube_update_template.md @@ -0,0 +1,36 @@ +## testkube update template + +Update Template + +### Synopsis + +Update Template Custom Resource. + +``` +testkube update template [flags] +``` + +### Options + +``` + --body string a path to template file to use as template body + -h, --help help for template + -l, --label stringToString label key value pair: --label key1=value1 (default []) + -n, --name string unique template name - mandatory + --template-type string template type one of job|container|cronjob|scraper|pvc|webhook +``` + +### Options inherited from parent commands + +``` + -a, --api-uri string api uri, default value read from config if set (default "https://demo.testkube.io/results/v1") + -c, --client string client used for connecting to Testkube API one of proxy|direct (default "proxy") + --namespace string Kubernetes namespace, default value read from config if set (default "testkube") + --oauth-enabled enable oauth + --verbose show additional debug messages +``` + +### SEE ALSO + +* [testkube update](testkube_update.md) - Update resource + diff --git a/docs/docs/cli/testkube_update_test.md b/docs/docs/cli/testkube_update_test.md index fd51cb86d55..66806eba65b 100644 --- a/docs/docs/cli/testkube_update_test.md +++ b/docs/docs/cli/testkube_update_test.md @@ -15,11 +15,17 @@ testkube update test [flags] ``` --args-mode string usage mode for arguments. one of append|override (default "append") --artifact-dir stringArray artifact dirs for scraping + --artifact-omit-folder-per-execution don't store artifacts in execution folder + --artifact-storage-bucket string artifact storage class name for container executor --artifact-storage-class-name string artifact storage class name for container executor --artifact-volume-mount-path string artifact volume mount path for container executor --command stringArray command passed to image in executor --copy-files stringArray file path mappings from host to pod of form source:destination --cronjob-template string cron job template file path for extensions to cron job template + --cronjob-template-reference string reference to cron job template to use for the test + --description string test description + --env stringToString envs in a form of name1=val1 passed to executor (default []) + --execute-postrun-script-before-scraping whether to execute postrun scipt before scraping or not (prebuilt executor only) --execution-name string execution name, if empty will be autogenerated --executor-args stringArray executor binary additional arguments -f, --file string test file - will try to read content from stdin if not specified @@ -37,9 +43,10 @@ testkube update test [flags] -h, --help help for test --http-proxy string http proxy for executor containers --https-proxy string https proxy for executor containers - -i, --image string image for container executor + --image string image for container executor --image-pull-secrets stringArray secret name used to pull the image in container executor --job-template string job template file path for extensions to job template + --job-template-reference string reference to job template to use for the test -l, --label stringToString label key value pair: --label key1=value1 (default []) --mount-configmap stringToString config map value pair for mounting it to executor pod: --mount-configmap configmap_name=configmap_mountpath (default []) --mount-secret stringToString secret value pair for mounting it to executor pod: --mount-secret secret_name=secret_mountpath (default []) @@ -47,16 +54,21 @@ testkube update test [flags] --negative-test negative test, if enabled, makes failure an expected and correct test result. If the test fails the result will be set to success, and vice versa --postrun-script string path to script to be run after test execution --prerun-script string path to script to be run before test execution + --pvc-template string pvc template file path for extensions to pvc template + --pvc-template-reference string reference to pvc template to use for the test --schedule string test schedule in a cron job form: * * * * * --scraper-template string scraper template file path for extensions to scraper template - -s, --secret-variable stringToString secret variable key value pair: -s key1=value1 (default []) + --scraper-template-reference string reference to scraper template to use for the test + --secret-env stringToString secret envs in a form of secret_key1=secret_name1 passed to executor (default []) + -s, --secret-variable stringToString secret variable key value pair: --secret-variable key1=value1 (default []) --secret-variable-reference stringToString secret variable references in a form name1=secret_name1=secret_key1 (default []) --source string source name - will be used together with content parameters --test-content-type string content type of test one of string|file-uri|git --timeout int duration in seconds for test to timeout. 0 disables timeout. - -t, --type string test type (defaults to postman-collection) + -t, --type string test type + --upload-timeout string timeout to use when uploading files, example: 30s --uri string URI of resource - will be loaded by http GET - -v, --variable stringToString variable key value pair: -v key1=value1 (default []) + -v, --variable stringToString variable key value pair: --variable key1=value1 (default []) --variable-configmap stringArray config map name used to map all keys to basis variables --variable-secret stringArray secret name used to map all keys to secret variables --variables-file string variables file path, e.g. postman env file - will be passed to executor if supported diff --git a/docs/docs/cli/testkube_update_testsource.md b/docs/docs/cli/testkube_update_testsource.md index c2f8f68e4b9..3a3ec27d03b 100644 --- a/docs/docs/cli/testkube_update_testsource.md +++ b/docs/docs/cli/testkube_update_testsource.md @@ -29,7 +29,7 @@ testkube update testsource [flags] -l, --label stringToString label key value pair: --label key1=value1 (default []) -n, --name string unique test source name - mandatory --source-type string source type of test one of string|file-uri|git - -u, --uri string URI which should be called when given event occurs + -u, --uri string URI which should be called to get test content ``` ### Options inherited from parent commands diff --git a/docs/docs/cli/testkube_update_testsuite.md b/docs/docs/cli/testkube_update_testsuite.md index f8c6173602e..10253abae79 100644 --- a/docs/docs/cli/testkube_update_testsuite.md +++ b/docs/docs/cli/testkube_update_testsuite.md @@ -14,14 +14,21 @@ testkube update testsuite [flags] ``` --cronjob-template string cron job template file path for extensions to cron job template + --cronjob-template-reference string reference to cron job template to use for the test --execution-name string execution name, if empty will be autogenerated -f, --file string JSON test file - will be read from stdin if not specified, look at testkube.TestUpsertRequest -h, --help help for testsuite --http-proxy string http proxy for executor containers --https-proxy string https proxy for executor containers + --job-template string job template file path for extensions to job template + --job-template-reference string reference to job template to use for the test -l, --label stringToString label key value pair: --label key1=value1 (default []) --name string Set/Override test suite name + --pvc-template string pvc template file path for extensions to pvc template + --pvc-template-reference string reference to pvc template to use for the test --schedule string test suite schedule in a cron job form: * * * * * + --scraper-template string scraper template file path for extensions to scraper template + --scraper-template-reference string reference to scraper template to use for the test -s, --secret-variable stringToString secret variable key value pair: --secret-variable key1=value1 (default []) --secret-variable-reference stringToString secret variable references in a form name1=secret_name1=secret_key1 (default []) --timeout int32 duration in seconds for test suite to timeout. 0 disables timeout. diff --git a/docs/docs/cli/testkube_update_webhook.md b/docs/docs/cli/testkube_update_webhook.md index 08772132bf6..fac9d8ae14f 100644 --- a/docs/docs/cli/testkube_update_webhook.md +++ b/docs/docs/cli/testkube_update_webhook.md @@ -13,15 +13,16 @@ testkube update webhook [flags] ### Options ``` - -e, --events stringArray event types handled by webhook e.g. start-test|end-test - --header stringToString webhook header value pair: --header Content-Type=application/xml (default []) - -h, --help help for webhook - -l, --label stringToString label key value pair: --label key1=value1 (default []) - -n, --name string unique webhook name - mandatory - --payload-field string field to use for notification object payload - --payload-template string if webhook needs to send a custom notification, then a path to template file should be provided - --selector string expression to select tests and test suites for webhook events: --selector app=backend - -u, --uri string URI which should be called when given event occurs + -e, --events stringArray event types handled by webhook e.g. start-test|end-test + --header stringToString webhook header value pair (golang template supported): --header Content-Type=application/xml (default []) + -h, --help help for webhook + -l, --label stringToString label key value pair: --label key1=value1 (default []) + -n, --name string unique webhook name - mandatory + --payload-field string field to use for notification object payload + --payload-template string if webhook needs to send a custom notification, then a path to template file should be provided + --payload-template-reference string reference to payload template to use for the webhook + --selector string expression to select tests and test suites for webhook events: --selector app=backend + -u, --uri string URI which should be called when given event occurs (golang template supported) ``` ### Options inherited from parent commands diff --git a/docs/docs/img/AI-analysis-results.png b/docs/docs/img/AI-analysis-results.png new file mode 100644 index 00000000000..bb8c6ea618b Binary files /dev/null and b/docs/docs/img/AI-analysis-results.png differ diff --git a/docs/docs/img/CLI-commands-1.14.png b/docs/docs/img/CLI-commands-1.14.png new file mode 100644 index 00000000000..b635690255b Binary files /dev/null and b/docs/docs/img/CLI-commands-1.14.png differ diff --git a/docs/docs/img/add-additional-test-or-delay.png b/docs/docs/img/add-additional-test-or-delay.png new file mode 100644 index 00000000000..7dd90b1a078 Binary files /dev/null and b/docs/docs/img/add-additional-test-or-delay.png differ diff --git a/docs/docs/img/add-first-test.png b/docs/docs/img/add-first-test.png new file mode 100644 index 00000000000..f7f65a8e6b5 Binary files /dev/null and b/docs/docs/img/add-first-test.png differ diff --git a/docs/docs/img/add-new-testsuite.png b/docs/docs/img/add-new-testsuite.png new file mode 100644 index 00000000000..f602da5f58d Binary files /dev/null and b/docs/docs/img/add-new-testsuite.png differ diff --git a/docs/docs/img/create-a-test.png b/docs/docs/img/create-a-test.png new file mode 100644 index 00000000000..a54f97a47db Binary files /dev/null and b/docs/docs/img/create-a-test.png differ diff --git a/docs/docs/img/create-trigger-1.14.png b/docs/docs/img/create-trigger-1.14.png new file mode 100644 index 00000000000..d488bb290f9 Binary files /dev/null and b/docs/docs/img/create-trigger-1.14.png differ diff --git a/docs/docs/img/create-trigger-action-1.14.png b/docs/docs/img/create-trigger-action-1.14.png new file mode 100644 index 00000000000..78964d1cba2 Binary files /dev/null and b/docs/docs/img/create-trigger-action-1.14.png differ diff --git a/docs/docs/img/create-trigger-action.png b/docs/docs/img/create-trigger-action.png new file mode 100644 index 00000000000..6e9fa90fd0f Binary files /dev/null and b/docs/docs/img/create-trigger-action.png differ diff --git a/docs/docs/img/create-trigger.png b/docs/docs/img/create-trigger.png new file mode 100644 index 00000000000..a80be7bfca6 Binary files /dev/null and b/docs/docs/img/create-trigger.png differ diff --git a/docs/docs/img/dashboard-1.14.png b/docs/docs/img/dashboard-1.14.png new file mode 100644 index 00000000000..fc22c2fb166 Binary files /dev/null and b/docs/docs/img/dashboard-1.14.png differ diff --git a/docs/docs/img/dashboard-create-webhook-1.png b/docs/docs/img/dashboard-create-webhook-1.png new file mode 100644 index 00000000000..5b6f568773b Binary files /dev/null and b/docs/docs/img/dashboard-create-webhook-1.png differ diff --git a/docs/docs/img/dashboard-create-webhook-2.png b/docs/docs/img/dashboard-create-webhook-2.png new file mode 100644 index 00000000000..a7df407e856 Binary files /dev/null and b/docs/docs/img/dashboard-create-webhook-2.png differ diff --git a/docs/docs/img/dashboard-create-webhook-resource-identifier.png b/docs/docs/img/dashboard-create-webhook-resource-identifier.png new file mode 100644 index 00000000000..46b4b664d16 Binary files /dev/null and b/docs/docs/img/dashboard-create-webhook-resource-identifier.png differ diff --git a/docs/docs/img/dashboard-webhook-headers.png b/docs/docs/img/dashboard-webhook-headers.png new file mode 100644 index 00000000000..f37ca8bbddf Binary files /dev/null and b/docs/docs/img/dashboard-webhook-headers.png differ diff --git a/docs/docs/img/dashboard-webhook-payload.png b/docs/docs/img/dashboard-webhook-payload.png new file mode 100644 index 00000000000..ca344879707 Binary files /dev/null and b/docs/docs/img/dashboard-webhook-payload.png differ diff --git a/docs/docs/img/dashboard-webhook-settings-action.png b/docs/docs/img/dashboard-webhook-settings-action.png new file mode 100644 index 00000000000..01044ed2e58 Binary files /dev/null and b/docs/docs/img/dashboard-webhook-settings-action.png differ diff --git a/docs/docs/img/dashboard-webhooks-icon.png b/docs/docs/img/dashboard-webhooks-icon.png new file mode 100644 index 00000000000..619d58e8918 Binary files /dev/null and b/docs/docs/img/dashboard-webhooks-icon.png differ diff --git a/docs/docs/img/execution-1.14.png b/docs/docs/img/execution-1.14.png new file mode 100644 index 00000000000..ad47a655028 Binary files /dev/null and b/docs/docs/img/execution-1.14.png differ diff --git a/docs/docs/img/execution-status-1.14.png b/docs/docs/img/execution-status-1.14.png new file mode 100644 index 00000000000..07ff7ef7806 Binary files /dev/null and b/docs/docs/img/execution-status-1.14.png differ diff --git a/docs/docs/img/failed-test.png b/docs/docs/img/failed-test.png new file mode 100644 index 00000000000..98ba59e774f Binary files /dev/null and b/docs/docs/img/failed-test.png differ diff --git a/docs/docs/img/incidents-edit.png b/docs/docs/img/incidents-edit.png new file mode 100644 index 00000000000..1d3a6145244 Binary files /dev/null and b/docs/docs/img/incidents-edit.png differ diff --git a/docs/docs/img/incidents-main.png b/docs/docs/img/incidents-main.png new file mode 100644 index 00000000000..a85b75bbb23 Binary files /dev/null and b/docs/docs/img/incidents-main.png differ diff --git a/docs/docs/img/log-output.png b/docs/docs/img/log-output.png new file mode 100644 index 00000000000..32d239fc507 Binary files /dev/null and b/docs/docs/img/log-output.png differ diff --git a/docs/docs/img/offline-list.png b/docs/docs/img/offline-list.png new file mode 100644 index 00000000000..5a0cc118d2d Binary files /dev/null and b/docs/docs/img/offline-list.png differ diff --git a/docs/docs/img/passed-test.png b/docs/docs/img/passed-test.png new file mode 100644 index 00000000000..4b8fc00ce7f Binary files /dev/null and b/docs/docs/img/passed-test.png differ diff --git a/docs/docs/img/recent-executions-1.14.png b/docs/docs/img/recent-executions-1.14.png new file mode 100644 index 00000000000..5682e6b3b59 Binary files /dev/null and b/docs/docs/img/recent-executions-1.14.png differ diff --git a/docs/docs/img/schedule-test-suites-1.14.png b/docs/docs/img/schedule-test-suites-1.14.png new file mode 100644 index 00000000000..c6d16623883 Binary files /dev/null and b/docs/docs/img/schedule-test-suites-1.14.png differ diff --git a/docs/docs/img/schedule-test-suites-options-1.14.png b/docs/docs/img/schedule-test-suites-options-1.14.png new file mode 100644 index 00000000000..8cce027913f Binary files /dev/null and b/docs/docs/img/schedule-test-suites-options-1.14.png differ diff --git a/docs/docs/img/schedule-test-suites.png b/docs/docs/img/schedule-test-suites.png new file mode 100644 index 00000000000..2d23dadf309 Binary files /dev/null and b/docs/docs/img/schedule-test-suites.png differ diff --git a/docs/docs/img/schedule-tests-1.14.png b/docs/docs/img/schedule-tests-1.14.png new file mode 100644 index 00000000000..3aced40c46c Binary files /dev/null and b/docs/docs/img/schedule-tests-1.14.png differ diff --git a/docs/docs/img/schedule-tests-options-1.14.png b/docs/docs/img/schedule-tests-options-1.14.png new file mode 100644 index 00000000000..e761ac5829f Binary files /dev/null and b/docs/docs/img/schedule-tests-options-1.14.png differ diff --git a/docs/docs/img/schedule-tests.png b/docs/docs/img/schedule-tests.png new file mode 100644 index 00000000000..4e081537906 Binary files /dev/null and b/docs/docs/img/schedule-tests.png differ diff --git a/docs/docs/img/scheduling-1.14.png b/docs/docs/img/scheduling-1.14.png new file mode 100644 index 00000000000..9b552f83a2f Binary files /dev/null and b/docs/docs/img/scheduling-1.14.png differ diff --git a/docs/docs/img/scheduling-options-1.14.png b/docs/docs/img/scheduling-options-1.14.png new file mode 100644 index 00000000000..1bd27fc01e1 Binary files /dev/null and b/docs/docs/img/scheduling-options-1.14.png differ diff --git a/docs/docs/img/search-filter-1.14.png b/docs/docs/img/search-filter-1.14.png new file mode 100644 index 00000000000..5b794974469 Binary files /dev/null and b/docs/docs/img/search-filter-1.14.png differ diff --git a/docs/docs/img/settings-1.14.png b/docs/docs/img/settings-1.14.png new file mode 100644 index 00000000000..107f3299060 Binary files /dev/null and b/docs/docs/img/settings-1.14.png differ diff --git a/docs/docs/img/settings-definition-1.14.png b/docs/docs/img/settings-definition-1.14.png new file mode 100644 index 00000000000..8b1fa39fc06 Binary files /dev/null and b/docs/docs/img/settings-definition-1.14.png differ diff --git a/docs/docs/img/settings-general-1.14.png b/docs/docs/img/settings-general-1.14.png new file mode 100644 index 00000000000..90201d5eebf Binary files /dev/null and b/docs/docs/img/settings-general-1.14.png differ diff --git a/docs/docs/img/settings-general-delete-1.14.png b/docs/docs/img/settings-general-delete-1.14.png new file mode 100644 index 00000000000..2ac943fa068 Binary files /dev/null and b/docs/docs/img/settings-general-delete-1.14.png differ diff --git a/docs/docs/img/settings-test-1.14.png b/docs/docs/img/settings-test-1.14.png new file mode 100644 index 00000000000..e367f442dc8 Binary files /dev/null and b/docs/docs/img/settings-test-1.14.png differ diff --git a/docs/docs/img/settings-test-suite-1.14.png b/docs/docs/img/settings-test-suite-1.14.png new file mode 100644 index 00000000000..51c530e679d Binary files /dev/null and b/docs/docs/img/settings-test-suite-1.14.png differ diff --git a/docs/docs/img/sp-create-test.png b/docs/docs/img/sp-create-test.png new file mode 100644 index 00000000000..626a0a9aa18 Binary files /dev/null and b/docs/docs/img/sp-create-test.png differ diff --git a/docs/docs/img/sp-schedule.png b/docs/docs/img/sp-schedule.png new file mode 100644 index 00000000000..30e16182082 Binary files /dev/null and b/docs/docs/img/sp-schedule.png differ diff --git a/docs/docs/img/sp-test-ran.png b/docs/docs/img/sp-test-ran.png new file mode 100644 index 00000000000..506c5d246ef Binary files /dev/null and b/docs/docs/img/sp-test-ran.png differ diff --git a/docs/docs/img/status-page-edit-services.png b/docs/docs/img/status-page-edit-services.png new file mode 100644 index 00000000000..9c9ff6709ec Binary files /dev/null and b/docs/docs/img/status-page-edit-services.png differ diff --git a/docs/docs/img/status-page-edit.png b/docs/docs/img/status-page-edit.png new file mode 100644 index 00000000000..80d3bd8f459 Binary files /dev/null and b/docs/docs/img/status-page-edit.png differ diff --git a/docs/docs/img/status-page-main.png b/docs/docs/img/status-page-main.png new file mode 100644 index 00000000000..dbcbb221799 Binary files /dev/null and b/docs/docs/img/status-page-main.png differ diff --git a/docs/docs/img/status-page-service.png b/docs/docs/img/status-page-service.png new file mode 100644 index 00000000000..ed35a88f95f Binary files /dev/null and b/docs/docs/img/status-page-service.png differ diff --git a/docs/docs/img/test-and-order-of-execution.png b/docs/docs/img/test-and-order-of-execution.png new file mode 100644 index 00000000000..17233d87695 Binary files /dev/null and b/docs/docs/img/test-and-order-of-execution.png differ diff --git a/docs/docs/img/test-list-1.14.png b/docs/docs/img/test-list-1.14.png new file mode 100644 index 00000000000..b6a2463ebce Binary files /dev/null and b/docs/docs/img/test-list-1.14.png differ diff --git a/docs/docs/img/test-suite-list-1.14.png b/docs/docs/img/test-suite-list-1.14.png new file mode 100644 index 00000000000..5c08d08f6fd Binary files /dev/null and b/docs/docs/img/test-suite-list-1.14.png differ diff --git a/docs/docs/img/tests-in-test-suite.png b/docs/docs/img/tests-in-test-suite.png new file mode 100644 index 00000000000..22edc64f293 Binary files /dev/null and b/docs/docs/img/tests-in-test-suite.png differ diff --git a/docs/docs/img/trigger-screen-1.14.png b/docs/docs/img/trigger-screen-1.14.png new file mode 100644 index 00000000000..d06bb1a8f3a Binary files /dev/null and b/docs/docs/img/trigger-screen-1.14.png differ diff --git a/docs/docs/img/trigger-screen.png b/docs/docs/img/trigger-screen.png new file mode 100644 index 00000000000..907bd190cc5 Binary files /dev/null and b/docs/docs/img/trigger-screen.png differ diff --git a/docs/docs/img/variable-tab-1.14.png b/docs/docs/img/variable-tab-1.14.png new file mode 100644 index 00000000000..ed7613a5923 Binary files /dev/null and b/docs/docs/img/variable-tab-1.14.png differ diff --git a/docs/docs/index.mdx b/docs/docs/index.mdx index 8b6fb63cceb..cb003437a4f 100644 --- a/docs/docs/index.mdx +++ b/docs/docs/index.mdx @@ -122,10 +122,33 @@ export const ToolList = (input) => ( ## Advantages of Using Testkube + +

⚡ Testing Made Faster

+- Simple network access and configuration for your testing tools by running your tests inside the cluster. +- Leverage native scalability of K8s to reduce CI/CD bottlenecks. +- What used to take 2 hours now takes 2 minutes. + +

✨ Testing Made Easier

+ +- Store and retrieve test artifacts generated by your testing tools. +- Get a common place for reporting test results across all your testing tools. +- Run tests using multiple versions of your testing tools. + +

🧑‍💻 Testing Made Modern

+ +- Integrated testing with GitOps and event-driven workflows. +- Create tests from our official Test Executor list or bring your own testing tool. +- Vendor neutrality to CI/CD vendors and testing tools. + + + + diff --git a/docs/docs/test-types/executor-artillery.mdx b/docs/docs/test-types/executor-artillery.mdx index daf7801b06a..284355af2b7 100644 --- a/docs/docs/test-types/executor-artillery.mdx +++ b/docs/docs/test-types/executor-artillery.mdx @@ -5,9 +5,9 @@ import Admonition from "@theme/Admonition"; # Artillery.io The Artillery executor allows you to run Artillery tests with Testkube. -Default command for this executor: artillery -Default arguments for this executor command: run <runPath> --dotenv <envFile> -o <reportFile> -(parameters in <> are calculated at test execution) +* Default command for this executor: `artillery` +* Default arguments for this executor command: `run` `` `--dotenv` `` `-o` `` +(parameters in `<>` are calculated at test execution) export const ExecutorInfo = () => { return ( diff --git a/docs/docs/test-types/executor-curl.mdx b/docs/docs/test-types/executor-curl.mdx index 84f9311c007..82dbf4f681e 100644 --- a/docs/docs/test-types/executor-curl.mdx +++ b/docs/docs/test-types/executor-curl.mdx @@ -4,8 +4,8 @@ import TabItem from "@theme/TabItem"; # cURL Testkube is able to run cURL commands as tests. -Default command for this executor: curl -Default arguments for this executor command: -is +Default command for this executor: `curl` +Default arguments for this executor command: `-is` ## Abstraction over cURL Testkube executor provides an abstraction over cURL that allow you to create JSON-based cURL test files. They allow you to combine a cURL command with expected results: diff --git a/docs/docs/test-types/executor-cypress.mdx b/docs/docs/test-types/executor-cypress.mdx index 632c6bd5d88..74e8b160e6c 100644 --- a/docs/docs/test-types/executor-cypress.mdx +++ b/docs/docs/test-types/executor-cypress.mdx @@ -6,9 +6,9 @@ import Admonition from "@theme/Admonition"; Our dedicated Cypress executor allows running Cypress tests with Testkube - directly from your Git repository. -Default command for this executor: ./node_modules/cypress/bin/cypress -Default arguments for this executor command: run --reporter junit --reporter-options mochaFile=<reportFile>,toConsole=false --project <projectPath> --env <envVars> -(parameters in <> are calculated at test execution) +* Default command for this executor: `./node_modules/cypress/bin/cypress` +* Default arguments for this executor command: `run` `--reporter` `junit` `--reporter-options` `mochaFile=,toConsole=false` `--project` `` `--env` `` +(parameters in `<>` are calculated at test execution) export const ExecutorInfo = () => { return ( diff --git a/docs/docs/test-types/executor-distributed-jmeter.md b/docs/docs/test-types/executor-distributed-jmeter.md new file mode 100644 index 00000000000..49044ecfc2a --- /dev/null +++ b/docs/docs/test-types/executor-distributed-jmeter.md @@ -0,0 +1,156 @@ +# Distributed JMeter Executor + +The Distributed JMeter Executor is an extension of JMeter Executor which can run the JMeter Tests in distributed mode by creating slave pods and distributing the test among them. + +## What is an Executor? + +An Executor is nothing more than a program wrapped into a Docker container which gets a JSON (testube.Execution) OpenAPI based document as an input and returns a stream of JSON output lines (testkube.ExecutorOutput), where each output line is simply wrapped in this JSON, similar to the structured logging idea. + +## Features + +This executor is an extension of the JMeter executor and has all the features of the JMeter executor. In addition to that, it has the following features: + +- Can run JMeter tests in distributed mode by creating slave pods and distributing the test among them. +- Supports defining plugins for a test in a Git repo by placing plugins in a directory named `plugins` in the test folder. +- Supports overriding the JMeter `user.properties` file by placing a custom `user.properties` file in the test folder. + +## Usage + +### Supported Environment Variables + +1. **MASTER_OVERRIDE_JVM_ARGS/SLAVES_OVERRIDE_JVM_ARGS**: Used to override default memory options for JMeter master/slaves. Example: `MASTER_OVERRIDE_JVM_ARGS=-Xmn256m -Xms512m -Xmx512m`. + +2. **SLAVES_COUNT**: Specifies the number of slave pods required for Distributed JMeter tests. Example: `SLAVES_COUNT=3`. Default value of `SLAVES_COUNT` is 1. + +3. **MASTER_ADDITIONAL_JVM_ARGS/SLAVES_ADDITIONAL_JMETER_ARGS**: Allows exporting additional JVM arguments for slaves/master. Example: `MASTER_ADDITIONAL_JVM_ARGS=-Xmx1024m -Xms512m -XX:MaxMetaspaceSize=256m`. + +4. **SLAVES_ADDITIONAL_JMETER_ARGS**: Provides additional JVM arguments for JMeter server/slaves. Example: `SLAVES_ADDITIONAL_JMETER_ARGS=jmeter-server -Jserver.rmi.ssl.disable=true -Dserver_port=60000`. + +### Guide +The guide below will provide you the details about how to run a Jmeter test in a distributed environment. + +1. File option: + When you provide a test (.jmx) file to `Distributed JMeter (JMeter in distributed mode)`, the executor of `Distributed JMeter` will spawn number of slaves pods specified by user through the `SLAVES_COUNT` environment variable as described above and run the test on all the slave pods. + +2. Git Option: + Using Git flow of the executor, we can use advanced features of the `Distributed JMeter` executor which is not possible with the JMeter executor: + + - Additional files required by a particular test like a CSV or JSON file can be provided through Git repo. There is an example of using a CSV file by the test (.jmx) file in the `example` folder of `Distributed JMeter`. + - Dynamic plugins are required for a test by keeping the plugins inside the test folder in a directory named `plugins` in the Git repo. + - Overriding the JMeter `user.properties` can be provided by using custom `user.properties` file in the Git repo. + +To use the Git option and to take advantage of all the above features, the user should have the following directory structure in the Git repo: + +``` + github.com/`/`/--- + + |-test1/--- + |- testfile1.jmx + |- userdata.csv ( or any other additional file ) + |- user.properties + |- plugins/--- + |- plugin-manager.jar + |- + + |-test2/--- + |- testfile2.jmx + |- userdata.json ( or any other additional file ) + |- user.properties + |- plugins/--- + |- plugin-manager.jar + |- + ``` + +For additional info, see the [GitFlow Example test for Distributed JMeter](https://github.com/kubeshop/testkube/blob/develop/contrib/executor/jmeterd/examples/gitflow/README.md). + +## Prerequisites + +Make sure the following tools are installed on your machine and available in your PATH: + +- [JMeter](https://jmeter.apache.org/download_jmeter.cgi) - A pure Java application designed to load test functional behavior and measure performance. + +### Setup + +1. Create a directory called `data/` where JMeter will run and store results (the best practice is to create it in the project root because it is git-ignored). +2. Create a JMeter XML project file and save it as a file named `test-content` in the newly created `data/` directory. +3. Create an execution JSON file and save it as a file named `execution.json` based on the template below (the best practice is to save it in the temp/ folder in the project root because it is git-ignored). +``` +{ + "id": "jmeterd-test", + "args": [], + "variables": {}, + "content": { + "type": "string" + } +} +``` +4. You need to provide the `RUNNER_SCRAPPERENABLED`, `RUNNER_SSL` and `RUNNER_DATADIR` environment variables and run the Executor using the `make run run_args="-f|--file "` make command where `-f|--file ` argument is the path to the `execution.json` file you created in step 3. +``` +RUNNER_SCRAPPERENABLED=false RUNNER_SSL=false RUNNER_DATADIR="./data" make run run_args="-f temp/execution.json" +``` + +#### Execution JSON + +Execution JSON stores information required for an Executor to run the configured tests. + +Breakdown of the Execution JSON: +``` +{ + "args": ["-n", "-t", "test.jmx"], + "variables": { + "example": { + "type": "basic", + "name": "example", + "value": "some-value" + } + }, + "content": { + "type": "string" + } +} +``` +- **args** - An array of strings which will be passed to JMeter as arguments. + - `example: ["-n", "-t", "test.jmx"]` +- **variables** - The map of variables which will be passed to JMeter as arguments. + - example: `{"example": {"type": "basic", "name": "example", "value": "some-value"}}` +- **content.type** - Used to specify that JMeter XML is provided as a text file. + +#### Environment Variables +``` +RUNNER_SSL=false # used if storage backend is behind HTTPS, should be set to false for local development +RUNNER_SCRAPPERENABLED=false # used to enable/disable scrapper, should be set to false for local development +RUNNER_DATADIR= # path to the data/ directory where JMeter will run and store results +``` + +## Testing in Kubernetes + +### Prerequisites + +- Kubernetes cluster with Testkube installed (the best practice is to install it in the testkube namespace). + +### Guide + +After validating locally that the Executor changes work as expected, the next step is to test whether Testkube can successfully schedule a Test using the new Executor image. + +:::note +The following commands assume that Testkube is installed in the `testkube` namespace, if you have it installed in a different namespace, please adjust the `--namespace` flag accordingly. +::: + +The following steps need to be executed in order for Testkube to use the new Executor image: + +1. Build the new Executor image using the `make docker-build-local` command. By default, the image will be tagged as `kubeshop/testkube-executor-jmeter:999.0.0` unless a `LOCAL_TAG` environment variable is provided before the command. +2. Now you need to make the image accessible in Kubernetes, there are a couple of approaches: + - *kind* - `kind load docker-image --name ` (e.g. `kind load docker-image testkube-executor-jmeter:999.0.0 --name testkube-k8s-cluster`) + - *minikube* - `minikube image load --profile (e.g. minikube image load testkube-executor-jmeter:999.0.0 --profile k8s-cluster-test)` + - *Docker Desktop* - Simply by building the image locally, it becomes accessible in the Docker Desktop Kubernetes cluster. + - *other* - You can push the image to a registry and then Testkube will pull it into Kubernetes (assuming it has credentials for it, if needed). +3. Edit the Job Template and change the `imagePullPolicy` to `IfNotPresent`. + - Edit the ConfigMap `testkube-api-server` either by running `kubectl edit configmap testkube-api-server --namespace testkube` or by using a tool like Monokle. + - Find the `job-template.yml` key and change the `imagePullPolicy` field in the `containers` section to `IfNotPresent`. +4. Edit the Executors configuration and change the base image to use the newly created image: + - Edit the ConfigMap `testkube-api-server` either by running `kubectl edit configmap testkube-api-server --namespace testkube` or by using a tool like Monokle. + - Find the `executors.json` key and change the `executor.image` field to use the newly created image for the JMeter Executor (`name` field is `jmeter-executor`). +5. Restart the API Server by running `kubectl rollout restart deployment testkube-api-server --namespace testkube`. + +Testkube should now use the new image for the Executor and you can schedule a Test with your preferred method. + diff --git a/docs/docs/test-types/executor-ginkgo.md b/docs/docs/test-types/executor-ginkgo.md index 53ad6be2a31..15520ab8e4a 100644 --- a/docs/docs/test-types/executor-ginkgo.md +++ b/docs/docs/test-types/executor-ginkgo.md @@ -4,10 +4,11 @@ import Admonition from "@theme/Admonition"; Our dedicated Ginkgo executor allows running Ginkgo tests with Testkube - directly from your Git repository. -Default command for this executor: ginkgo -Default arguments for this executor command: -r -p --randomize-all --randomize-suites --keep-going --trace --junit-report <reportFile> <envVars> <runPath> +* Default command for this executor: `ginkgo` +* Default arguments for this executor command: `-r` `-p` `--randomize-all` `--randomize-suites` `--keep-going` `--trace` `--junit-report` `` `` `` (parameters in <> are calculated at test execution) + export const ExecutorInfo = () => { return (
diff --git a/docs/docs/test-types/executor-gradle.md b/docs/docs/test-types/executor-gradle.md index e25d7223ee3..39efa39c9c2 100644 --- a/docs/docs/test-types/executor-gradle.md +++ b/docs/docs/test-types/executor-gradle.md @@ -2,9 +2,9 @@ Testkube allows running Gradle based tasks that could also be tests. For example, we can easily run JUnit tests in Testkube. -Default command for this executor: gradle -Default arguments for this executor command: --no-daemon <taskName> -p <projectDir> -(parameters in <> are calculated at test execution) +* Default command for this executor: `gradle` +* Default arguments for this executor command: `--no-daemon` `` `-p` `` +(parameters in `<>` are calculated at test execution) ## **Test Environment** diff --git a/docs/docs/test-types/executor-jmeter.md b/docs/docs/test-types/executor-jmeter.md index 80a20c1c7a4..9c85d38d63c 100644 --- a/docs/docs/test-types/executor-jmeter.md +++ b/docs/docs/test-types/executor-jmeter.md @@ -4,9 +4,9 @@ import Admonition from "@theme/Admonition"; [JMeter](https://jmeter.apache.org/) is an integral part of Testkube. The Testkube JMeter executor is installed by default during the Testkube installation. -Default command for this executor: <entryPoint> -Default arguments for this executor command: -n -j <logFile> -t <runPath> -l <jtlFile> -e -o <reportFile> <envVars> -(parameters in <> are calculated at test execution) +* Default command for this executor: `` +* Default arguments for this executor command: `-n` `-j` `` `-t` `` `-l` `` `-e` `-o` `` `` +(parameters in `<>` are calculated at test execution) diff --git a/docs/docs/test-types/executor-k6.mdx b/docs/docs/test-types/executor-k6.mdx index 9fd4b364c0c..f0ccdd41d72 100644 --- a/docs/docs/test-types/executor-k6.mdx +++ b/docs/docs/test-types/executor-k6.mdx @@ -6,9 +6,9 @@ import Admonition from "@theme/Admonition"; Testkube's k6 executor provides a convenient way of running k6 tests. -Default command for this executor: k6 -Default arguments for this executor command: <k6Command> <envVars> <runPath> -(parameters in <> are calculated at test execution) +* Default command for this executor: `k6` +* Default arguments for this executor command: `` `` `` +(parameters in `<>` are calculated at test execution)