Merge branch 'main' into audioqa

opea-project · Aug 16, 2024 · 29ab751 · 29ab751
2 parents 5d4e821 + 1c07a38
commit 29ab751
Show file tree

Hide file tree

Showing 63 changed files with 1,139 additions and 997 deletions.
diff --git a/.github/workflows/_example-workflow.yml b/.github/workflows/_example-workflow.yml
@@ -6,6 +6,9 @@ permissions: read-all
 on:
   workflow_call:
     inputs:
+      node:
+        required: true
+        type: string
       example:
         required: true
         type: string
@@ -29,96 +32,94 @@ on:
         default: false
         required: false
         type: boolean
-      publish:
-        default: false
-        required: false
-        type: boolean
-      publish_tags:
-        default: "latest"
+      GenAIComps_branch:
+        default: "main"
         required: false
         type: string
 jobs:
 ####################################################################################################
 # Image Build
 ####################################################################################################
   build-images:
-    if: ${{ fromJSON(inputs.build) }}
-    strategy:
-      matrix:
-        node: ["docker-build-xeon", "docker-build-gaudi"]
-    runs-on: ${{ matrix.node }}
+    runs-on: "docker-build-${{ inputs.node }}"
     continue-on-error: true
     steps:
       - name: Clean Up Working Directory
-        run: |
-          sudo rm -rf ${{github.workspace}}/*
+        run: sudo rm -rf ${{github.workspace}}/*
 
       - name: Checkout out Repo
         uses: actions/checkout@v4
 
+      - name: Clone required Repo
+        run: |
+          cd ${{ github.workspace }}/${{ inputs.example }}/docker
+          build_compose_path=${{ github.workspace }}/${{ inputs.example }}/docker/docker_build_compose.yaml
+          if [[ $(grep -c "tei-gaudi:" ${docker_compose_path}) != 0 ]]; then
+              git clone https://github.com/huggingface/tei-gaudi.git
+          fi
+          if [[ $(grep -c "vllm:" ${docker_compose_path}) != 0 ]]; then
+              git clone https://github.com/huggingface/tei-gaudi.git
+          fi
+          git clone https://github.com/opea-project/GenAIComps.git
+          cd GenAIComps && checkout ${{ inputs.GenAIComps_branch }} && cd ../
+
       - name: Build Image
+        if: ${{ fromJSON(inputs.build) }}
         uses: opea-project/validation/actions/image-build@main
         with:
-          work_dir: ${{ github.workspace }}/${{ inputs.example }}
-          docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.example }}-compose.yaml
+          work_dir: ${{ github.workspace }}/${{ inputs.example }}/docker
+          docker_compose_path: ${{ github.workspace }}/${{ inputs.example }}/docker/docker_build_compose.yaml
           registry: ${OPEA_IMAGE_REPO}opea
           tag: ${{ inputs.tag }}
 
 ####################################################################################################
 # Trivy Scan
 ####################################################################################################
-  image-list:
-    needs: [ build-images ]
-    if: ${{ fromJSON(inputs.scan) }}
+  get-image-list:
+    needs: [build-images]
+    if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi' }}
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.scan-matrix.outputs.matrix }}
     steps:
-      - name: Harden Runner
-        uses: step-security/[email protected]
-        with:
-          egress-policy: audit
-
       - name: Checkout out Repo
         uses: actions/checkout@v4
 
       - name: Set Matrix
         id: scan-matrix
         run: |
           pip install yq
-          compose_path=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.example }}-compose.yaml
+          compose_path=${{ github.workspace }}/${{ inputs.example }}/docker/docker_build_compose.yaml
           echo "matrix=$(cat ${compose_path} | yq -r '.[]' | jq 'keys' | jq -c '.')" >> $GITHUB_OUTPUT
 
   scan-images:
-    needs: [image-list]
-    if: ${{ fromJSON(inputs.scan) }}
-    runs-on: "docker-build-gaudi"
+    needs: [get-image-list, build-images]
+    if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi'}}
+    runs-on: "docker-build-${{ inputs.node }}"
     strategy:
       matrix:
-        image: ${{ fromJSON(needs.image-list.outputs.matrix) }}
+        image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}
       fail-fast: false
     steps:
-      - name: Harden Runner
-        uses: step-security/[email protected]
-        with:
-          egress-policy: audit
-
       - name: Pull Image
-        run: docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+        run: |
+          docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+          echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV
 
       - name: Scan Container
         uses: opea-project/validation/actions/trivy-scan@main
         with:
-          image-ref: ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
-          output: ${{ inputs.example }}-${{ matrix.image }}-scan.txt
+          image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
+          output: ${{ matrix.image }}-scan.txt
 
       - name: Cleanup
         if: always()
         run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+
       - uses: actions/[email protected]
         with:
-          name: ${{ inputs.example }}-scan
-          path: ${{ inputs.example }}-${{ matrix.image }}-scan.txt
+          name: ${{ matrix.image }}-scan
+          path: ${{ matrix.image }}-scan.txt
           overwrite: true
 
 ####################################################################################################
@@ -127,15 +128,11 @@ jobs:
   test-example-compose:
     needs: [build-images]
     if: ${{ fromJSON(inputs.test_compose) }}
-    strategy:
-      matrix:
-        hardware: ["xeon", "gaudi"]
-      fail-fast: false
     uses: ./.github/workflows/_run-docker-compose.yml
     with:
       tag: ${{ inputs.tag }}
       example: ${{ inputs.example }}
-      hardware: ${{ matrix.hardware }}
+      hardware: ${{ inputs.node }}
     secrets: inherit
 
 
@@ -145,32 +142,9 @@ jobs:
   test-k8s-manifest:
     needs: [build-images]
     if: ${{ fromJSON(inputs.test_k8s) }}
-    strategy:
-      matrix:
-        hardware: ["xeon", "gaudi"]
-      fail-fast: false
     uses: ./.github/workflows/_manifest-e2e.yml
     with:
       example: ${{ inputs.example }}
-      hardware: ${{ matrix.hardware }}
+      hardware: ${{ inputs.node }}
       tag: ${{ inputs.tag }}
     secrets: inherit
-
-
-####################################################################################################
-# Publish
-####################################################################################################
-  publish:
-    needs: [image-list, build-images, scan-images, test-example-compose]
-    if: ${{ fromJSON(inputs.publish) }}
-    strategy:
-      matrix:
-        image: ${{ fromJSON(needs.image-list.outputs.matrix) }}
-    runs-on: "docker-build-gaudi"
-    steps:
-      - name: Image Publish
-        uses: opea-project/validation/actions/image-publish@main
-        with:
-          local_image_ref: ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
-          image_name: opea/${{ matrix.image }}
-          publish_tags: ${{ inputs.publish_tags }}
diff --git a/.github/workflows/_run-docker-compose.yml b/.github/workflows/_run-docker-compose.yml
@@ -72,6 +72,7 @@ jobs:
           sudo rm -rf ${{github.workspace}}/* || true
           docker system prune -f
           docker rmi $(docker images --filter reference="*/*/*:latest" -q) || true
+          docker rmi $(docker images --filter reference="*/*:ci" -q) || true
 
       - name: Checkout out Repo
         uses: actions/checkout@v4
@@ -93,7 +94,7 @@ jobs:
           test_case: ${{ matrix.test_case }}
         run: |
           cd ${{ github.workspace }}/$example/tests
-          export IMAGE_REPO=${OPEA_IMAGE_REPO}
+          if [[ "$IMAGE_REPO" == "" ]]; then export IMAGE_REPO="${OPEA_IMAGE_REPO}opea"; fi
           if [ -f ${test_case} ]; then timeout 30m bash ${test_case}; else echo "Test script {${test_case}} not found, skip test!"; fi
 
       - name: Clean up container

diff --git a/.github/workflows/manual-bom-scan.yml b/.github/workflows/manual-bom-scan.yml
@@ -0,0 +1,74 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Examples docker images BoM scan
+on:
+  workflow_dispatch:
+    inputs:
+      node:
+        default: "gaudi"
+        description: "Hardware to run test"
+        required: true
+        type: string
+      examples:
+        default: "ChatQnA"
+        description: 'List of examples to test [AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation]'
+        required: true
+        type: string
+      tag:
+        default: "latest"
+        description: "Tag to apply to images"
+        required: true
+        type: string
+
+permissions: read-all
+jobs:
+  get-image-list:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.scan-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Set Matrix
+        id: scan-matrix
+        run: |
+          pip install yq
+          examples=($(echo ${{ inputs.examples }} | tr ',' ' '))
+          image_list=[]
+          for example in ${examples[@]}
+          do
+              images=$(cat ${{ github.workspace }}/${example}/docker/docker_build_compose.yaml | yq -r '.[]' | jq 'keys' | jq -c '.')
+              image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images}))
+          done
+          echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT
+
+  scan-license:
+    needs: get-image-list
+    runs-on: "docker-build-${{ inputs.node }}"
+    strategy:
+      matrix:
+        image: ${{ fromJson(needs.get-image-list.outputs.matrix) }}
+      fail-fast: false
+    steps:
+      - name: Pull Image
+        run: |
+          docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+          echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV
+
+      - name: Scan Container
+        uses: opea-project/validation/actions/license-scan@main # TODO
+        with:
+          image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
+          output: ${{ matrix.image }}-scan.txt
+
+      - name: Cleanup
+        if: always()
+        run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+
+      - uses: actions/[email protected]
+        with:
+          name: ${{ matrix.image }}-scan
+          path: ${{ matrix.image }}-scan.txt
+          overwrite: true
diff --git a/.github/workflows/manual-docker-publish.yml b/.github/workflows/manual-docker-publish.yml
@@ -0,0 +1,68 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Examples CD workflow on manual event
+on:
+  workflow_dispatch:
+    inputs:
+      nodes:
+        default: "gaudi"
+        description: "Hardware to run test"
+        required: true
+        type: string
+      examples:
+        default: "ChatQnA"
+        description: 'List of examples to test [AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation]'
+        required: true
+        type: string
+      tag:
+        default: "latest"
+        description: "Tag to apply to images"
+        required: true
+        type: string
+      publish:
+        default: false
+        description: 'Publish images to docker hub'
+        required: false
+        type: boolean
+      publish_tags:
+        default: "latest,v1.0"
+        description: 'Tag list apply to publish images'
+        required: false
+        type: string
+
+permissions: read-all
+jobs:
+  get-image-list:
+    runs-on: ${{ inputs.node }}
+    outputs:
+      matrix: ${{ steps.scan-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Set Matrix
+        id: scan-matrix
+        run: |
+          examples=($(echo ${{ inputs.examples }} | tr ',' ' '))
+          image_list=[]
+          for example in ${examples[@]}
+          do
+              images=$(cat ${{ github.workspace }}/${example}/docker/docker_build_compose.yaml | yq -r '.[]' | jq 'keys' | jq -c '.')
+              image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images}))
+          done
+          echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT
+
+  publish:
+    needs: [get-image-list]
+    strategy:
+      matrix:
+        image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}
+    runs-on: "docker-build-${{ inputs.node }}"
+    steps:
+      - name: Image Publish
+        uses: opea-project/validation/actions/image-publish@main
+        with:
+          local_image_ref: ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+          image_name: opea/${{ matrix.image }}
+          publish_tags: ${{ inputs.publish_tags }}