Merge branch 'dev' into bal-3137

ballerine-io · Jan 4, 2025 · 64a72fb · 64a72fb
2 parents be6480d + 995102f
commit 64a72fb
Show file tree

Hide file tree

Showing 19 changed files with 477 additions and 64 deletions.
diff --git a/.github/workflows/build-preview-environment.yml b/.github/workflows/build-preview-environment.yml
@@ -11,15 +11,16 @@ on:
 
 permissions:
   id-token: write
-  contents: read
-  pull-requests: read
+  contents: write
+  pull-requests: write
+  packages: write
 
 env:
   REF: ${{ github.event_name == 'workflow_dispatch' && github.ref_name || github.event_name == 'pull_request' && github.event.pull_request.head.ref }}
 
 jobs:
   deploy-dev-pr-environment:
-    if: contains(github.event.pull_request.labels.*.name, 'deploy-pr')
+    if: contains(github.event.pull_request.labels.*.name, 'deploy-pr') || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     outputs:
       env_name: ${{ steps.env-name.outputs.PR_ENV_NAME }}
@@ -45,7 +46,7 @@ jobs:
         run: |
           SANITIZED_BRANCH_NAME=$(echo -n "${{ steps.clean-ref.outputs.ref }}" | tr "/" "-")
           echo "Sanitized branch name: $SANITIZED_BRANCH_NAME"
-          TRIMMED_BRANCH_NAME=$(echo -n "$SANITIZED_BRANCH_NAME" | cut -c 1-8)
+          TRIMMED_BRANCH_NAME=$(echo -n "$SANITIZED_BRANCH_NAME" | cut -c 1-7)
           echo "sanitized_env_name=$SANITIZED_BRANCH_NAME" >> $GITHUB_OUTPUT;
           echo "trimmed_env_name=$TRIMMED_BRANCH_NAME" >> $GITHUB_OUTPUT;
         
@@ -65,39 +66,92 @@ jobs:
       image_name: workflows-service
       ref: ${{ needs.deploy-dev-pr-environment.outputs.ref }}
       tag: ${{ needs.deploy-dev-pr-environment.outputs.env_name }}
+      file: 'services/workflows-service/Dockerfile'
+
+  build-wf-service-ee:
+    needs: [deploy-dev-pr-environment,build-wf-service]
+    uses: ./.github/workflows/build-push-docker-images.yml
+    with:
+      registry: ghcr.io/${{ github.repository_owner }}
+      context: services/workflows-service
+      image_name: workflows-service-ee
+      ref: ${{ needs.deploy-dev-pr-environment.outputs.ref }}
+      tag: ${{ needs.deploy-dev-pr-environment.outputs.env_name }}
+      file: 'services/workflows-service/Dockerfile.ee'
 
   build-backoffice:
-    needs: deploy-dev-pr-environment
+    needs: [deploy-dev-pr-environment]
     uses: ./.github/workflows/build-push-docker-images.yml
     with:
       registry: ghcr.io/${{ github.repository_owner }}
       context: apps/backoffice-v2
       image_name: backoffice
       ref: ${{ needs.deploy-dev-pr-environment.outputs.ref }}
       tag: ${{ needs.deploy-dev-pr-environment.outputs.env_name }}
+      file: 'apps/backoffice-v2/Dockerfile.preview'
 
   build-kyb:
-    needs: deploy-dev-pr-environment
+    needs: [deploy-dev-pr-environment]
     uses: ./.github/workflows/build-push-docker-images.yml
     with:
       registry: ghcr.io/${{ github.repository_owner }}
       context: apps/kyb-app
       image_name: kyb-app
       ref: ${{ needs.deploy-dev-pr-environment.outputs.ref }}
       tag: ${{ needs.deploy-dev-pr-environment.outputs.env_name }}
+      file: 'apps/kyb-app/Dockerfile.preview'
 
   build-dashboard:
-    needs: deploy-dev-pr-environment
+    needs: [deploy-dev-pr-environment]
     uses: ./.github/workflows/build-push-docker-images.yml
     with:
       registry: ghcr.io/${{ github.repository_owner }}
       context: apps/workflows-dashboard
       image_name: workflows-dashboard
       ref: ${{ needs.deploy-dev-pr-environment.outputs.ref }}
       tag: ${{ needs.deploy-dev-pr-environment.outputs.env_name }}
+      file: 'apps/workflows-dashboard/Dockerfile.preview'
+
+  build-unified-api:
+    runs-on: ubuntu-latest
+    needs: [deploy-dev-pr-environment]
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ vars.PREVIEW_OIDC_ROLE }}
+          aws-region: ${{ vars.PREVIEW_AWS_REGION }}
+
+      # Access the secret
+      - name: Retrieve secret from Secrets Manager
+        id: get-secret
+        run: |
+          secret_value=$(aws secretsmanager get-secret-value --secret-id ${{ vars.PREVIEW_SECRET }} --query 'SecretString' --output text | jq -r '.SUBMODULE_SECRET')
+          echo "SUBMODULE_SECRET=$secret_value" >> $GITHUB_ENV
+          echo "SUBMODULE_SECRET=$secret_value" >> $GITHUB_OUTPUT
+
+      - name: Log in to the container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ghcr.io/${{ github.repository_owner }}
+          username: ${{ github.actor }}
+          password: ${{ steps.get-secret.outputs.SUBMODULE_SECRET }}
+
+      - name: Checkout repository
+        run: |
+          docker pull ghcr.io/${{ github.repository_owner }}/${{ vars.UNIFIED_IMAGE_NAME }}:latest
+          docker tag ghcr.io/${{ github.repository_owner }}/${{ vars.UNIFIED_IMAGE_NAME }}:latest ghcr.io/${{ github.repository_owner }}/${{ vars.UNIFIED_IMAGE_NAME }}:${{ needs.deploy-dev-pr-environment.outputs.env_name }}
+          docker push ghcr.io/${{ github.repository_owner }}/${{ vars.UNIFIED_IMAGE_NAME }}:${{ needs.deploy-dev-pr-environment.outputs.env_name }} 
 
   deploy-preview:
-    needs: [deploy-dev-pr-environment,build-wf-service,build-backoffice,build-kyb,build-dashboard]
+    needs: [deploy-dev-pr-environment,build-wf-service,build-wf-service-ee,build-backoffice,build-kyb,build-dashboard,build-unified-api]
     runs-on: ubuntu-latest
     steps:
       - name: Trigger workflow in another repo

diff --git a/.github/workflows/build-push-docker-images.yml b/.github/workflows/build-push-docker-images.yml
@@ -23,10 +23,15 @@ on:
         required: true
         description: "Tag name of the Preview Image"
         type: string
+      file:
+        required: true
+        description: "File name for the Preview Image"
+        type: string
 
 permissions:
   id-token: write
   contents: write
+  packages: write
   pull-requests: write
 
 jobs:
@@ -40,18 +45,56 @@ jobs:
           ref: ${{ inputs.ref }}
           fetch-depth: 1
           persist-credentials: false
-          sparse-checkout: |
-            ${{ inputs.context }}
-          sparse-checkout-cone-mode: true
 
-      - name: Get tags
-        run: git fetch --tags origin
+      - name: Configure AWS credentials
+        if: inputs.image_name == 'workflows-service-ee'
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ vars.PREVIEW_OIDC_ROLE }}
+          aws-region: ${{ vars.PREVIEW_AWS_REGION }}
+
+      # Access the secret
+      - name: Retrieve secret from Secrets Manager
+        if: inputs.image_name == 'workflows-service-ee'
+        id: get-secret
+        run: |
+          echo ${{ inputs.image_name }}
+          secret_value=$(aws secretsmanager get-secret-value --secret-id ${{ vars.PREVIEW_SECRET }} --query 'SecretString' --output text | jq -r '.SUBMODULE_SECRET')
+          echo "SUBMODULE_SECRET=$secret_value" >> $GITHUB_ENV
+          echo "SUBMODULE_SECRET=$secret_value" >> $GITHUB_OUTPUT
+      
+      - name: Checkout wf-data-migration
+        id: wf-migration-code
+        if: inputs.image_name == 'workflows-service-ee'
+        uses: actions/checkout@v4
+        with:
+          repository: ballerine-io/wf-data-migration
+          token: ${{ steps.get-secret.outputs.SUBMODULE_SECRET }}
+          ref: dev
+          fetch-depth: 1
+          path: services/workflows-service/prisma/data-migrations
+
+      - name: Get Latest Commit ID
+        if: inputs.image_name == 'workflows-service-ee'
+        id: lastcommit
+        uses: nmbgeek/github-action-get-latest-commit@main
+        with:
+          owner: ${{ github.repository_owner }}
+          token: ${{ steps.get-secret.outputs.SUBMODULE_SECRET }}
+          repo: wf-data-migration
+          branch: dev
+
+      # - name: Get tags
+      #   if: ${{ inputs.image_name }} != 'workflows-service-ee'
+      #   run: git fetch --tags origin
 
       - name: Get version
-        if: ${{ inputs.image_name }} == 'workflows-service'
+        if: ${{ inputs.image_name == 'workflows-service' }}
         id: version
         run: |
-          TAG=$(git tag -l "$(echo ${{ inputs.image_name }}@)*" | sort -V -r | head -n 1)
+          echo ${{ inputs.image_name }}
+          git fetch --tags origin
+          TAG=$(git tag -l "$(echo workflow-service@)*" | sort -V -r | head -n 1)
           echo "tag=$TAG"
           echo "tag=$TAG" >> "$GITHUB_OUTPUT"
           echo "TAG=$TAG" >> "$GITHUB_ENV"
@@ -61,7 +104,7 @@ jobs:
       
       - name: Bump version
         id: bump-version
-        if: ${{ inputs.image_name }} == 'workflows-service'
+        if: ${{ inputs.image_name == 'workflows-service' }}
         uses: ./.github/actions/bump-version
         with:
           tag: ${{ steps.version.outputs.tag }}
@@ -103,7 +146,7 @@ jobs:
       - name: Print docker version outputs
         run: |
           echo "Metadata: ${{ steps.docker_meta.outputs.tags }}"
-          if [[ "${{ inputs.image_name }}" == "workflows-service" ]]; then
+          if [[ "${{ inputs.image_name }}" == "workflows-service" && "${{ inputs.image_name }}" != "workflows-service-ee" ]]; then
             echo "sha_short: ${{ steps.version.outputs.sha_short }}"
             echo "bump-version-version: ${{ steps.bump-version.outputs.version }}"
             echo "bump-version-tag: ${{ steps.bump-version.outputs.tag }}"
@@ -118,17 +161,6 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
           tags: ${{ steps.docker_meta.outputs.tags }}
+          file: ${{ inputs.file }}
           build-args: |
-            ${{ inputs.image_name == 'workflows-service' && format('"RELEASE={0}"\n"SHORT_SHA={1}"', steps.version.outputs.tag, steps.version.outputs.sha_short) || '' }}
-
-      - name: Scan Docker Image
-        uses: aquasecurity/trivy-action@master
-        continue-on-error: true
-        with:
-          image-ref: ${{ steps.docker_meta.outputs.tags }}
-          format: 'table'
-          ignore-unfixed: true
-          exit-code: 1
-          vuln-type: 'os,library'
-          severity: 'CRITICAL,HIGH'
-          timeout: '5m'
+            ${{ (inputs.image_name == 'workflows-service' && format('"RELEASE={0}"\n"SHORT_SHA={1}"', steps.version.outputs.tag, steps.version.outputs.sha_short)) || (inputs.image_name == 'workflows-service-ee' && format('"BASE_IMAGE=ghcr.io/ballerine-io/workflows-service:{0}"', inputs.tag)) || '' }}
diff --git a/.github/workflows/deploy-wf-service.yml b/.github/workflows/deploy-wf-service.yml
@@ -111,6 +111,7 @@ jobs:
             docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:${{ env.SHORT_HASH }}-sb ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:${{ env.SHORT_HASH }}-${{ inputs.environment }}
             docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:${{ env.SHORT_HASH }}-sb ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:${{ inputs.environment }}
             docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:${{ env.SHORT_HASH }}-sb ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:latest
+            docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:latest
           else
             docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:${{ env.SHORT_HASH }}-dev
             docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:${{ env.SHORT_HASH }}-dev ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:${{ env.SHORT_HASH }}-${{ inputs.environment }}

diff --git a/.github/workflows/destroy-preview-environment.yml b/.github/workflows/destroy-preview-environment.yml
@@ -18,7 +18,12 @@ env:
 
 jobs:
   deploy-dev-pr-environment:
-    if: contains(github.event.pull_request.labels.*.name, 'deploy-pr')
+    if: |
+      (github.event_name == 'pull_request' && github.event.action == 'unlabeled' && github.event.label.name == 'deploy-pr') 
+      ||
+      (github.event_name == 'pull_request' && github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'deploy-pr'))
+      ||
+      github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     outputs:
       env_name: ${{ steps.env-name.outputs.PR_ENV_NAME }}
@@ -43,7 +48,7 @@ jobs:
         run: |
           SANITIZED_BRANCH_NAME=$(echo -n ${{ steps.clean-ref.outputs.ref }} | tr "/" "-")
           echo "Sanitized branch name: $SANITIZED_BRANCH_NAME"
-          TRIMMED_BRANCH_NAME=$(echo -n "$SANITIZED_BRANCH_NAME" | cut -c 1-8)
+          TRIMMED_BRANCH_NAME=$(echo -n "$SANITIZED_BRANCH_NAME" | cut -c 1-7)
           echo "sanitized_env_name=$SANITIZED_BRANCH_NAME" >> $GITHUB_OUTPUT;
           echo "trimmed_env_name=$TRIMMED_BRANCH_NAME" >> $GITHUB_OUTPUT;
         
@@ -60,6 +65,8 @@ jobs:
       (github.event_name == 'pull_request' && github.event.action == 'unlabeled' && github.event.label.name == 'deploy-pr') 
       ||
       (github.event_name == 'pull_request' && github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'deploy-pr'))
+      ||
+      github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     steps:
       - name: Trigger workflow in another repo

diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@
 
   <a href="https://docs.ballerine.com/">Documentation</a>
     ·
-   <a href="https://join.slack.com/t/ballerine-oss/shared_invite/zt-1iu6otkok-OqBF3TrcpUmFd9oUjNs2iw">Slack</a>
+   <a href="https://join.slack.com/t/ballerine-oss/shared_invite/zt-1il7txerq-K0YrXtlzMttGgD3XXYxlfw">Slack</a>
     ·
     <a href="https://www.ballerine.com/">Website</a>
     ·

diff --git a/apps/backoffice-v2/Dockerfile.preview b/apps/backoffice-v2/Dockerfile.preview
@@ -0,0 +1,17 @@
+FROM node:18.17.1-bullseye-slim
+
+WORKDIR /app
+
+COPY ./package.json .
+
+RUN npm install --legacy-peer-deps
+
+COPY . .
+
+RUN npm run build --verbose
+
+ENV PATH="$PATH:/app/node_modules/.bin"
+
+EXPOSE 5137
+
+CMD ["npm", "run", "prod:next", "--host"]
diff --git a/apps/backoffice-v2/package.json b/apps/backoffice-v2/package.json
@@ -42,6 +42,7 @@
     "start": "vite",
     "dev": "vite",
     "build": "cross-env NODE_OPTIONS=--max-old-space-size=32768 vite build",
+    "prod:next": "vite build && vite --host",
     "test": "vitest run --passWithNoTests",
     "test:unit": "vitest run --passWithNoTests",
     "test:e2e": "playwright test",

diff --git a/apps/kyb-app/Dockerfile.preview b/apps/kyb-app/Dockerfile.preview
@@ -0,0 +1,19 @@
+FROM node:18.17.1-bullseye-slim
+
+WORKDIR /app
+
+RUN apt update -y && apt install xdg-utils -y
+
+COPY ./package.json .
+
+RUN npm install --legacy-peer-deps
+
+COPY . .
+
+RUN npm run build  --verbose
+
+ENV PATH="$PATH:./node_modules/.bin"
+
+EXPOSE 5201
+
+CMD ["npm", "run", "prod:next", "--host"]
diff --git a/apps/kyb-app/package.json b/apps/kyb-app/package.json
@@ -7,6 +7,7 @@
     "dev": "vite",
     "start": "vite",
     "build": "tsc && vite build",
+    "prod:next": "vite build && vite --host",
     "lint": "eslint . --fix",
     "format": "prettier --write .",
     "format:check": "prettier --check .",

diff --git a/apps/workflows-dashboard/Dockerfile.preview b/apps/workflows-dashboard/Dockerfile.preview
@@ -0,0 +1,17 @@
+FROM node:18.17.1-bullseye-slim
+
+WORKDIR /app
+
+COPY ./package.json .
+
+RUN npm install --legacy-peer-deps
+
+COPY . .
+
+RUN npm run build --verbose
+
+ENV PATH="$PATH:/app/node_modules/.bin"
+
+EXPOSE 5200
+
+CMD ["npm", "run", "prod:next", "--host"]
diff --git a/apps/workflows-dashboard/package.json b/apps/workflows-dashboard/package.json
@@ -8,6 +8,7 @@
     "dev": "vite --host",
     "start": "vite",
     "build": "tsc && vite build",
+    "prod:next": "vite build && vite --host",
     "lint": "eslint . --fix",
     "format": "prettier --write .",
     "format:check": "prettier --check .",

diff --git a/services/workflows-service/package.json b/services/workflows-service/package.json
@@ -12,6 +12,7 @@
     "start": "nest start",
     "dev": "npm run start:watch",
     "start:prod": "node dist/src/main",
+    "start:preview": "npm run db:migrate-up && npm run db:data-migration:migrate && npm run db:data-sync && npm run start:prod",
     "prod": "npm run db:migrate-up && npm run start:prod",
     "prod:next": "npm run db:migrate-up && npm run db:data-sync && npm run start:prod",
     "start:watch": "nest start --watch",

diff --git a/services/workflows-service/src/common/types.ts b/services/workflows-service/src/common/types.ts
@@ -11,7 +11,7 @@ export type TDocumentsWithoutPageType = TDocumentWithoutPageType[];
 export const SubscriptionSchema = z.discriminatedUnion('type', [
   z
     .object({
-      type: z.literal('webhook'),
+      type: z.enum(['webhook', 'email']),
       url: z.string().url(),
       events: z.array(z.string()),
       config: z