DBP-1012-setup-loadtest-in-k8s (#7)

* added workflow dispatch and cleaned inputs * switching helm install to other namespace * switched to branch dependand repo checkout * added hardcoded branch name for testing * changed cronjob values * removed secret mount * updated k6 version due to CVEs * updated k6 version * test rollout to cluster * chnage GHA * fix gha 2 * fix gha 3 * fix gha 4 * fix gha 5 * test it * test GHA 7 * test GHA 8 * test GHA 9 * test GHA 10 * test GHA 11 * test GHA 12 * test GHA 13 * test GHA 14 * test GHA 15 * test GHA 16 --------- Co-authored-by: M_Westerholz <[email protected]> Co-authored-by: M. Westerholz <[email protected]>
dBildungsplattform · Oct 16, 2024 · e655e04 · e655e04
1 parent 892fc8e
commit e655e04
Show file tree

Hide file tree

Showing 14 changed files with 117 additions and 75 deletions.
diff --git a/.github/workflows/install-and-run-spsh-loadtest.yml b/.github/workflows/install-and-run-spsh-loadtest.yml
@@ -1,14 +1,15 @@
 ---
-name: Install and Run SPSH loadtest Action 
+name: Install and Run SPSH Loadtest Action 
 
-on:
-  workflow_dispatch:
-
+on: 
   workflow_call:
     inputs:
-      release_tag:
+      branch:
         type: string
-        required: true
+        required: false
+        # default: main
+        default: DBP-1012-setup-loadtest-env
+
     secrets:
       KUBECONFIG:
         required: true
@@ -17,7 +18,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: spsh-k6-loadtest-rollout-run
+  group: schulportal-load-tests-rollout-run
   cancel-in-progress: true
 
 jobs:
@@ -29,8 +30,8 @@ jobs:
         with:
           repository: 'dBildungsplattform/schulportal-load-tests'
           # change this ref to a branch to test changes
-          ref: "${{ inputs.release_tag }}"
-          # ref: "<branch_name>"
+          # ref: "${{ inputs.branch }}"
+          ref: "DBP-1012-setup-loadtest-env"
 
       - name: Install kubectl and Helm
         uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f #v4.0.0
@@ -41,15 +42,12 @@ jobs:
           echo "${{ secrets.KUBECONFIG }}" > /home/runner/.kube/config
           chmod 600 /home/runner/.kube/config
 
-      - name: Helm add repo dbilungsplattform
-        run: |
-          helm repo add dbildungsplattform https://dbildungsplattform.github.io/helm-charts-registry/
-
       - name: Rollout schulportal-load-tests Helm Chart 
         run: |
             helm upgrade --install \
             schulportal-load-tests \
             ./charts/schulportal-load-tests \
-            --namespace loadtest \
+            --namespace spsh \
             --kubeconfig /home/runner/.kube/config \
+            --set branch="${{ inputs.branch }}"\
             --wait
diff --git a/.github/workflows/publish-image-on-push-to-branch.yml b/.github/workflows/publish-image-on-push-to-branch.yml
@@ -10,7 +10,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_and_push:
+  build_and_push_image:
     permissions:
       packages: write
       security-events: write
@@ -21,4 +21,4 @@ jobs:
       run_trivy_scan: true
       image_tag_generation: "ticket_from_branch"
       add_latest_tag: false
-      container_registry: "ghcr.io"
+      container_registry: "ghcr.io"
diff --git a/...rkflows/publish-image-and-helm-on-tag.yml → .github/workflows/publish-image-on-tag.yml b/...rkflows/publish-image-and-helm-on-tag.yml → .github/workflows/publish-image-on-tag.yml
@@ -21,13 +21,4 @@ jobs:
       run_trivy_scan: true
       image_tag_generation: "version_git_tag"
       add_latest_tag: true
-      container_registry: "ghcr.io"
-  publish_helm:
-    uses: dBildungsplattform/dbp-github-workflows/.github/workflows/[email protected]
-    secrets: inherit
-    with:
-      chart_name: schulportal-load-tests
-      helm_chart_version_generation: specified
-      image_tag_generation: specified
-      helm_chart_version: ${{ github.ref_name }}
-      image_tag: ${{ github.ref_name }}
+      container_registry: "ghcr.io"
diff --git a/.github/workflows/trigger-loadtest.yml b/.github/workflows/trigger-loadtest.yml
@@ -0,0 +1,20 @@
+---
+name: Trigger Loadtest
+
+on:
+  workflow_dispatch:
+  push:
+    branch: DBP-1012-setup-loadtest-env
+
+jobs:
+  run_loadtest:
+    permissions:
+      packages: write
+      security-events: write
+      contents: read
+    uses: ./.github/workflows/install-and-run-spsh-loadtest.yml
+    with:
+      branch: DBP-1012-setup-loadtest-env #
+    secrets:
+      # token: ${{ secrets.GITHUB_TOKEN }} # not sure whether this is needed
+      KUBECONFIG: ${{ secrets.KUBECONFIG }}
diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,5 @@
 # Create image for running k6 with output for Prometheus
-FROM grafana/k6:0.43.1
+FROM grafana/k6:0.54.0
 USER root
 
-COPY wrapper-script.sh wrapper-script.sh
-RUN apk add git && \
-    chmod +x wrapper-script.sh
-
-ENTRYPOINT ["./wrapper-script.sh"]
+RUN apk add git
diff --git a/charts/schulportal-load-tests/Chart.yaml b/charts/schulportal-load-tests/Chart.yaml
@@ -1,5 +1,5 @@
 apiVersion: v1
 name: schulportal-load-tests
-description: A Helm chart for distributed k6 loadtests in kubernetes.
+description: A Helm chart for distributed k6 loadtests
 version: 0.0.1
 appVersion: "0.0.1"
diff --git a/charts/schulportal-load-tests/templates/cronjob.yaml b/charts/schulportal-load-tests/templates/cronjob.yaml
@@ -3,48 +3,64 @@ apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: loadtest-spsh-{{ $job_name }}
-  namespace: loadtest
+  namespace: spsh
 spec:
   schedule: {{ $job_options.schedule }}
+  startingDeadlineSeconds: 300
   suspend: true
   jobTemplate:
     spec:
       completions: {{ $job_options.jobsParallelism }}
       parallelism: {{ $job_options.jobsParallelism }}
+      backoffLimit: 0
       template:
         metadata:
           labels:
             test: loadtest-{{ $job_options.serviceName }}
             pod: {{ $job_name }}
         spec:
+          automountServiceAccountToken: false
           containers:
           - name: {{ $job_name }}
-            image: {{ $job_options.image }}
+            image: ghcr.io/dbildungsplattform/schulportal-load-tests:{{ $job_options.imageTag }}
+            imagePullPolicy: Always
             securityContext:
               allowPrivilegeEscalation: false
-            imagePullPolicy: Always
             env:
               - name: OPTIONS_FILE_PATH
                 value: testoption.json
               - name: BRANCH
                 value: main
-            args:
-              ['run', '-u', '0',
-                         './loadtest/tests/{{ $job_options.scriptPath }}',
-                         '--tag','NAME={{ $job_options.serviceName }}',
-                         #'--out', 'experimental-prometheus-rw',
-                         '-e', 'ENVIRONMENT={{ $job_options.environment }}']
+              - name: SPSH_BASE
+                value: https://main.dev.spsh.dbildungsplattform.de
+              - name: CONFIG
+                value: stress
+            command: ["sh", "-c", 
+                        "git clone https://github.com/dBildungsplattform/schulportal-load-tests &&
+                        cd schulportal-load-tests && 
+                        git checkout {{ $.Values.branch }} && 
+                        cp /data/json/users.json /home/k6/schulportal-load-tests/loadtest/data/users.json &&
+                        chmod +x run.sh &&
+                        sleep 3600 &&
+                        ./run.sh
+                        # k6 run -u 0 ./loadtest/usecases/{{ $job_options.scriptPath }} --compatibility-mode=experimental_enhanced"]
             volumeMounts:
-            - name: secret-volume
-              mountPath: /secrets
-              readOnly: true
+              - name: secret-volume
+                mountPath: /secrets
+                readOnly: true
+              - name: secret-volume-json
+                mountPath: /data/json
+                readOnly: true
             ports:
             - containerPort: {{ $job_options.port }}
               name: loadtest-pod
           volumes:
           - name: secret-volume
             secret:
-              secretName: login-secret
-          restartPolicy: OnFailure
+              secretName: spsh-loadtest-secret
+          - name: secret-volume-json
+            secret:
+              secretName: spsh-loadtest-secret-json
+          restartPolicy: Never
 ---
 {{- end}}
diff --git a/charts/schulportal-load-tests/templates/login-secret.yaml b/charts/schulportal-load-tests/templates/login-secret.yaml
diff --git a/charts/schulportal-load-tests/templates/spsh-loadtest-secret-json.yaml b/charts/schulportal-load-tests/templates/spsh-loadtest-secret-json.yaml
@@ -0,0 +1,9 @@
+apiVersion: onepassword.com/v1
+kind: OnePasswordItem
+metadata:
+  name: spsh-loadtest-secret-json
+  namespace: spsh
+  labels: 
+    app.kubernetes.io/managed-by: helm
+spec:
+  itemPath: "vaults/spsh-dev/items/spsh-loadtest-secret-json"
diff --git a/charts/schulportal-load-tests/templates/spsh-loadtest-secret.yaml b/charts/schulportal-load-tests/templates/spsh-loadtest-secret.yaml
@@ -0,0 +1,9 @@
+apiVersion: onepassword.com/v1
+kind: OnePasswordItem
+metadata:
+  name: spsh-loadtest-secret
+  namespace: spsh
+  labels: 
+    app.kubernetes.io/managed-by: helm
+spec:
+  itemPath: "vaults/spsh-dev/items/spsh-loadtest-secret"
diff --git a/charts/schulportal-load-tests/values.yaml b/charts/schulportal-load-tests/values.yaml
@@ -1,9 +1,26 @@
+branch: "main"
+
 cronjobs:
   dev-scenario:
-    scriptPath: test.js
+    scriptPath: 1_goto-sp-oidc.ts # 01, 02, 03 Files
     schedule: 0 8 * * *
     port: 5656
     serviceName: dev-scenario
-    image: ghcr.io/dbildungsplattform/spsh-k6-loadtest:latest
-    environment: spsh.loadtest.dbildungscloud.dev
-
+    # image-tag: latest
+    imageTag: DBP-1012
+    environment: main.dev.spsh.dbildungsplattform.de
+    # jobsParallelism: not used yet but available? test it
+  # staging-scenario:
+  #   scriptPath: test.js
+  #   schedule: 0 8 * * *
+  #   port: 5656
+  #   serviceName: staging-scenario
+  #   image: ghcr.io/dbildungsplattform/schulportal-load-tests:latest
+  #   environment: spsh.staging.dbildungsplattform.de
+  # prod-scenario:
+  #   scriptPath: test.js
+  #   schedule: 0 8 * * *
+  #   port: 5656
+  #   serviceName: prod-scenario
+  #   image: ghcr.io/dbildungsplattform/schulportal-load-tests:latest
+  #   environment: spsh.dbildungsplattform.de
diff --git a/loadtest/util/config.ts b/loadtest/util/config.ts
@@ -21,21 +21,21 @@ export function getDefaultOptions() {
     case CONFIG.SPIKE:
       return {
         stages: [
-          { duration: "30s", target: 100 },
+          { duration: "30s", target: 10 },
           { duration: "10s", target: 0 },
         ],
       };
     case CONFIG.STRESS:
       return {
         stages: [
-          { duration: "1m", target: 100 },
+          { duration: "1m", target: 10 },
           // { duration: "5m", target: 100 },
           { duration: "1m", target: 0 },
         ],
       };
     case CONFIG.BREAKPOINT:
       return {
-        stages: [{ duration: "5m", target: 100 }],
+        stages: [{ duration: "5m", target: 10 }],
         thresholds: {
           http_req_failed: [{ threshold: "rate<0.10", abortOnFail: true }],
           http_req_duration: [{ threshold: "p(95)<2000", abortOnFail: true }],

diff --git a/run.sh b/run.sh
@@ -9,8 +9,7 @@ if [[ -z "$PATTERN" ]]; then
     PATTERN="*"
 fi
 
-# KC_BASE=$2 # not needed yet
-
+# only for local development
 SKIP_SSL=""
 if [[ "$SPSH_BASE" =~ "localhost" ]]; then
     SKIP_SSL="--insecure-skip-tls-verify"
@@ -21,11 +20,16 @@ if [[ ! -d output/ ]]; then
     mkdir output/
 fi
 
+
+# einmal Grafana ohne out mit der CSV Datei ()
+# funktinonieren die Ergebnisse in den Logs auch wenn die Parallel laufen? 
+
 for uc in loadtest/usecases/*; do
     if [[ "$uc" =~ "$PATTERN" ]]; then
         filename=${uc##*/}
         csv="output/${filename%.ts}.csv"
         touch "$csv"
-        k6 run --compatibility-mode=experimental_enhanced --out csv="$csv" "$SKIP_SSL" -e SPSH_BASE="$SPSH_BASE" -e CONFIG="$CONFIG" -e KC_BASE="$KC_BASE" "$uc"
+        echo k6 run --compatibility-mode=experimental_enhanced --out csv="$csv" "$SKIP_SSL" -e SPSH_BASE="$SPSH_BASE" -e CONFIG="$CONFIG" -e KC_BASE="$KC_BASE" "$uc"
+        # k6 run --compatibility-mode=experimental_enhanced --out csv="$csv" "$SKIP_SSL" -e SPSH_BASE="$SPSH_BASE" -e CONFIG="$CONFIG" "$uc"
     fi
 done
diff --git a/wrapper-script.sh b/wrapper-script.sh