From f411e5f346d147fbdc2b72b8e872266442d393af Mon Sep 17 00:00:00 2001 From: Daniel Aaron Salwerowicz Date: Thu, 23 May 2024 14:19:54 +0200 Subject: [PATCH 1/2] MOL-0003 Add GitHub actions to the project (#1) * Create test flows in the `.github/test-flows` folder, keeping them just in case * Add a test version of the actual deploy flow, needs to be tested and fixed * Update Kotlin to version 2.0.0 --- .github/test-flows/package-project.yaml | 38 +++++ .github/test-flows/test-project.yaml | 33 ++++ .github/workflows/deploy-project.yaml | 211 ++++++++++++++++++++++++ .m2/settings.xml | 22 +++ k8s/prod/wls.yml | 96 +++++++++++ k8s/stage/wls.yml | 96 +++++++++++ pom.xml | 2 +- 7 files changed, 497 insertions(+), 1 deletion(-) create mode 100644 .github/test-flows/package-project.yaml create mode 100644 .github/test-flows/test-project.yaml create mode 100644 .github/workflows/deploy-project.yaml create mode 100644 .m2/settings.xml create mode 100644 k8s/prod/wls.yml create mode 100644 k8s/stage/wls.yml diff --git a/.github/test-flows/package-project.yaml b/.github/test-flows/package-project.yaml new file mode 100644 index 00000000..a4f2bbab --- /dev/null +++ b/.github/test-flows/package-project.yaml @@ -0,0 +1,38 @@ +name: Package Project +run-name: ${{ github.actor }} is packaging the project +on: + # This event should allow for manual workflow triggering + workflow_dispatch: + # Run on push to main branch + push: + branches: + - main + # Runs every time a pull request is created or updated + pull_request: +concurrency: + # Define concurrency group name which will then be used to determine duplicate workflow runs + group: ${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }} + cancel-in-progress: true +jobs: + package-project: + runs-on: "ubuntu-latest" + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Java JDK + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + architecture: x64 + # Cache maven dependencies for quicker runs + cache: maven + - name: Package the Project + run: mvn package + - name: Upload an Artifact + uses: actions/upload-artifact@v4 + with: + name: packaged-project + path: target/wls-*.jar + if-no-files-found: error + overwrite: true diff --git a/.github/test-flows/test-project.yaml b/.github/test-flows/test-project.yaml new file mode 100644 index 00000000..a92977c5 --- /dev/null +++ b/.github/test-flows/test-project.yaml @@ -0,0 +1,33 @@ +name: Test Project +run-name: ${{ github.actor }} is testing the project +on: + # Run on push to main branch + push: + branches: + - main + # Runs every time a pull request is created or updated + pull_request: +concurrency: + # Define concurrency group name which will then be used to determine duplicate workflow runs + group: ${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }} + cancel-in-progress: true +jobs: + test-project: + runs-on: "ubuntu-latest" + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Java JDK + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + architecture: x64 + # Cache maven dependencies for quicker runs + cache: maven + - name: Test the Project + run: mvn verify --fail-at-end + - name: Publish Test Report + uses: scacap/action-surefire-report@v1 + with: + fail_on_test_failures: true diff --git a/.github/workflows/deploy-project.yaml b/.github/workflows/deploy-project.yaml new file mode 100644 index 00000000..21970d40 --- /dev/null +++ b/.github/workflows/deploy-project.yaml @@ -0,0 +1,211 @@ +name: Deploy Project +run-name: ${{ github.actor }} is testing the project + +on: + # Run on push to main branch and tags + push: + branches: + - main + tags: + - "v*.*.*" + # Runs every time a pull request is created or updated + pull_request: + +env: + # Run in batch mode, produce errors, use settings file, set local repo + MAVEN_CONFIG: -B -e -s .m2/settings.xml -Dmaven.repo.local=.m2/repository + +concurrency: + # Define concurrency group name which will then be used to determine duplicate workflow runs + group: ${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + test-project: + name: Build and Test the Project + runs-on: self-hosted-linux + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Java JDK + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + architecture: x64 + # Cache maven dependencies for quicker runs + cache: maven + + - name: Cache Maven Packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + + - name: Import Secrets + uses: hashicorp/vault-action@v3 + with: + url: ${{ secrets.VAULT_URL }} + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + secrets: secret/v1/application/k8s/mlt/proxy * + + - name: Test the Project + run: | + echo ${{ env.HTTP_PROXY_HOST }} + echo ${{ env.HTTP_PROXY_PORT }} + echo ${{ env.HTTPS_PROXY_HOST }} + echo ${{ env.HTTPS_PROXY_PORT }} + envsubst < .m2/settings.xml + mvn ${{ env.MAVEN_CONFIG }} verify --fail-at-end + + - name: Publish Test Report + uses: scacap/action-surefire-report@v1 + with: + fail_on_test_failures: true + + - name: Upload the JAR File + uses: actions/upload-artifact@v4 + with: + name: packaged-project + path: target/wls-*.jar + if-no-files-found: error + overwrite: true + + - name: Cache Maven Repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + + publish-image: + name: Build and Publish the Docker Image + needs: test-project + if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') + runs-on: self-hosted-linux + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker + + - name: Import Secrets + uses: hashicorp/vault-action@v3 + with: + url: ${{ secrets.VAULT_URL }} + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + secrets: secret/v1/application/k8s/mlt/harbor * + + - name: Log in to Harbor Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.HARBOR_URL }} + username: ${{ env.HARBOR_USERNAME }} + password: ${{ env.HARBOR_PASSWORD }} + + - name: Extract Metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: mlt/wls # TODO: Figure out what image name will be generated + tags: | + type=semver,pattern={{version}} + type=ref,event=branch + type=ref,event=pr + + - name: Download the JAR File + uses: actions/download-artifact@v4 + with: + name: packaged-project + + - name: Build the Docker Image + uses: docker/build-push-action@v5 + with: + push: true + context: . + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + deploy-to-stage: + name: Deploy to Kubernetes Stage + needs: publish-image + if: github.ref == 'refs/heads/main' + runs-on: self-hosted-linux + environment: stage + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Import Secrets + uses: hashicorp/vault-action@v3 + with: + url: ${{ secrets.VAULT_URL }} + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + secrets: secret/v1/application/k8s/mlt/k8s-stage * + + - name: Setup Kubectl + uses: azure/setup-kubectl@v4 # TODO: Check if it needs some more config + + - name: Deploy to Stage + run: | # TODO: Fix this to fit our config and needs, as I've "borrowed" this from Tekst team + echo "Deploying to stage version ${{ env.APP_VERSION }}" + sed -i "s//${{ env.APP_VERSION }}/g" k8s/stage/wls.yml + sed -i "s//${{ env.K8S_HOST_URL }}/g" k8s/stage/wls.yml + kubectl config set-cluster stagecl --server=${{ env.K8S_STAGE_SERVER }} + kubectl config set clusters.stagecl.certificate-authority-data ${{ env.K8S_STAGE_NB_NO_CA }} + kubectl config set-credentials ${{ env.K8S_STAGE_USER }} --token=${{ env.K8S_STAGE_NB_NO_TOKEN }} + kubectl config set-context mlt --cluster=stagecl --user=${{ env.K8S_STAGE_USER }} --namespace=mlt-stage + kubectl config use-context mlt + kubectl config view + kubectl version + kubectl apply -f k8s/stage/wls.yml + kubectl rollout restart deploy/wls + + deploy-to-prod: + name: Deploy to Kubernetes Prod + needs: publish-image + if: startsWith(github.event.ref, 'refs/tags/v') + runs-on: self-hosted-linux + environment: prod + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Import Secrets + uses: hashicorp/vault-action@v3 + with: + url: ${{ secrets.VAULT_URL }} + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + secrets: | + secret/v1/application/k8s/mlt/k8s-prod * + + - name: Setup Kubectl + uses: azure/setup-kubectl@v4 # TODO: Check if it needs some more config + + - name: Deploy to Prod + run: | # TODO: Fix this to fit our config and needs, as I've "borrowed" this from Tekst team + echo "Deploying to production version ${{ env.APP_VERSION }}" + sed -i "s//${{ env.APP_VERSION }}/g" k8s/prod/wls.yml + sed -i "s//${{ env.K8S_HOST_URL }}/g" k8s/prod/wls.yml + kubectl config set-cluster prodcl --server=${{ env.K8S_PROD_SERVER }} + kubectl config set clusters.prodcl.certificate-authority-data ${{ env.K8S_PROD_NB_NO_CA }} + kubectl config set-credentials ${{ env.K8S_PROD_USER }} --token=${{ env.K8S_PROD_NB_NO_TOKEN }} + kubectl config set-context mlt --cluster=prodcl --user=${{ env.K8S_PROD_USER }} --namespace=mlt-prod + kubectl config use-context mlt + kubectl config view + kubectl version + kubectl apply -f k8s/prod/wls.yml + kubectl rollout restart deploy/wls \ No newline at end of file diff --git a/.m2/settings.xml b/.m2/settings.xml new file mode 100644 index 00000000..da344e93 --- /dev/null +++ b/.m2/settings.xml @@ -0,0 +1,22 @@ + + + + + nb-proxy-http + true + http + ${HTTP_PROXY_HOST} + ${HTTP_PROXY_PORT} + localhost|127.0.0.1|docker + + + nb-proxy-https + true + https + ${HTTPS_PROXY_HOST} + ${HTTPS_PROXY_PORT} + localhost|127.0.0.1|docker + + + \ No newline at end of file diff --git a/k8s/prod/wls.yml b/k8s/prod/wls.yml new file mode 100644 index 00000000..354873d6 --- /dev/null +++ b/k8s/prod/wls.yml @@ -0,0 +1,96 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: wls + namespace: mlt-prod +spec: + replicas: 3 + selector: + matchLabels: + app: wls + template: + metadata: + labels: + app: wls + spec: + priorityClassName: medium-priority + containers: + - name: app + image: harbor.nb.no/mlt/wls: + ports: + - name: app-port + containerPort: 8080 + - name: actuator-port + containerPort: 8888 + env: + - name: INFO_ENVIRONMENT + value: PROD + - name: SPRING_PROFILES_ACTIVE + value: "prod" + - name: HTTP_PROXY_HOST + valueFrom: + secretKeyRef: + name: wls-secrets + key: http_proxy_host + - name: HTTP_PROXY_PORT + valueFrom: + secretKeyRef: + name: wls-secrets + key: http_proxy_port + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: actuator-port + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 3 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: actuator-port + initialDelaySeconds: 30 + periodSeconds: 15 + failureThreshold: 3 + timeoutSeconds: 1 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: wls + namespace: mlt-prod +spec: + ports: + - port: 8080 + name: rest + targetPort: 8080 + - port: 8888 + name: http + targetPort: 8888 + selector: + app: wls + type: ClusterIP + +--- + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: wls-ingress + namespace: mlt-prod +spec: + ingressClassName: nginx + rules: + - host: + http: + paths: + - backend: + service: + name: wls + port: + number: 8080 + path: /bifrost + pathType: Prefix \ No newline at end of file diff --git a/k8s/stage/wls.yml b/k8s/stage/wls.yml new file mode 100644 index 00000000..fc594209 --- /dev/null +++ b/k8s/stage/wls.yml @@ -0,0 +1,96 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: wls + namespace: mlt-stage +spec: + replicas: 1 + selector: + matchLabels: + app: wls + template: + metadata: + labels: + app: wls + spec: + priorityClassName: medium-priority + containers: + - name: app + image: harbor.nb.no/mlt/wls: + ports: + - name: app-port + containerPort: 8080 + - name: actuator-port + containerPort: 8888 + env: + - name: INFO_ENVIRONMENT + value: STAGE + - name: SPRING_PROFILES_ACTIVE + value: "stage" + - name: HTTP_PROXY_HOST + valueFrom: + secretKeyRef: + name: wls-secrets + key: http_proxy_host + - name: HTTP_PROXY_PORT + valueFrom: + secretKeyRef: + name: wls-secrets + key: http_proxy_port + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: actuator-port + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 3 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: actuator-port + initialDelaySeconds: 30 + periodSeconds: 15 + failureThreshold: 3 + timeoutSeconds: 1 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: wls + namespace: mlt-stage +spec: + ports: + - port: 8080 + name: rest + targetPort: 8080 + - port: 8888 + name: http + targetPort: 8888 + selector: + app: wls + type: ClusterIP + +--- + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: wls-ingress + namespace: mlt-stage +spec: + ingressClassName: nginx + rules: + - host: + http: + paths: + - backend: + service: + name: wls + port: + number: 8080 + path: /bifrost + pathType: Prefix \ No newline at end of file diff --git a/pom.xml b/pom.xml index 751087eb..148b58b6 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ Warehouse Logistics Service functioning as a middleware between NLBs catalogues and storage systems 21 - 1.9.23 + 2.0.0 2023.0.1 From 44e3d50acaa632e4b005f0b02612491c3f9409e2 Mon Sep 17 00:00:00 2001 From: Daniel Aaron Salwerowicz Date: Thu, 23 May 2024 15:16:31 +0200 Subject: [PATCH 2/2] Add missing whitespace --- .github/workflows/deploy-project.yaml | 2 +- .m2/settings.xml | 2 +- k8s/prod/wls.yml | 2 +- k8s/stage/wls.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-project.yaml b/.github/workflows/deploy-project.yaml index 21970d40..92efbbd7 100644 --- a/.github/workflows/deploy-project.yaml +++ b/.github/workflows/deploy-project.yaml @@ -208,4 +208,4 @@ jobs: kubectl config view kubectl version kubectl apply -f k8s/prod/wls.yml - kubectl rollout restart deploy/wls \ No newline at end of file + kubectl rollout restart deploy/wls diff --git a/.m2/settings.xml b/.m2/settings.xml index da344e93..38610980 100644 --- a/.m2/settings.xml +++ b/.m2/settings.xml @@ -19,4 +19,4 @@ localhost|127.0.0.1|docker - \ No newline at end of file + diff --git a/k8s/prod/wls.yml b/k8s/prod/wls.yml index 354873d6..8e8f6e09 100644 --- a/k8s/prod/wls.yml +++ b/k8s/prod/wls.yml @@ -93,4 +93,4 @@ spec: port: number: 8080 path: /bifrost - pathType: Prefix \ No newline at end of file + pathType: Prefix diff --git a/k8s/stage/wls.yml b/k8s/stage/wls.yml index fc594209..a694c760 100644 --- a/k8s/stage/wls.yml +++ b/k8s/stage/wls.yml @@ -93,4 +93,4 @@ spec: port: number: 8080 path: /bifrost - pathType: Prefix \ No newline at end of file + pathType: Prefix