resolve merge conflicts

Signed-off-by: sharanshirodkar7 <[email protected]>

.github/CODEOWNERS

@@ -1,9 +1,21 @@
+/comps/agent/ [email protected]
+/comps/finetuning/ [email protected]
+/comps/guardrails/ [email protected]
 /comps/asr/ [email protected]
+/comps/intent_detection/ [email protected]
+/comps/knowledgegraphs/ [email protected]
 /comps/cores/ [email protected]
 /comps/dataprep/ [email protected]
 /comps/embeddings/ [email protected]
 /comps/guardrails/ [email protected]
 /comps/llms/ [email protected]
+/comps/lvms/ [email protected]
+/comps/nginx/ [email protected]
+/comps/prompt_registry/ [email protected]
+/comps/chathistory/ [email protected]
 /comps/reranks/ [email protected]
 /comps/retrievers/ [email protected]
 /comps/tts/ [email protected]
+/comps/ragas/ [email protected]
+/comps/vectorstores/ [email protected]
+/comps/web_retrievers/ [email protected]

.github/workflows/_comps-workflow.yml

@@ -0,0 +1,95 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Comps jobs
+permissions: read-all
+on:
+  workflow_call:
+    inputs:
+      node:
+        required: true
+        type: string
+      service:
+        required: true
+        type: string
+      tag:
+        default: "comps"
+        required: false
+        type: string
+      build:
+        default: true
+        required: false
+        type: boolean
+      test:
+        default: true
+        description: "Test comps with docker compose"
+        required: false
+        type: boolean
+      mode:
+        default: "CD"
+        description: "Whether the test range is CI or CD"
+        required: false
+        type: string
+
+jobs:
+  ####################################################################################################
+  # Image Build
+  ####################################################################################################
+  build-images:
+    runs-on: "docker-build-${{ inputs.node }}"
+    continue-on-error: true
+    outputs:
+      file_exists: ${{ steps.get-yaml-path.outputs.file_exists }}
+    steps:
+      - name: Clean Up Working Directory
+        run: sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Clone required Repo
+        id: get-yaml-path
+        run: |
+          cd ${{ github.workspace }}/.github/workflows/docker/compose
+          # service=$(echo ${{ inputs.service }} | cut -d'_' -f1)
+          if [[ "${{ inputs.mode }}" == "CD" ]]; then
+            docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml
+          else
+            docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
+          fi
+          echo "docker_compose_path=${docker_compose_yml}" >> $GITHUB_OUTPUT
+          if [ -f "$docker_compose_yml" ]; then
+            echo "file_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "There is no ${{ inputs.mode }} part of ${{ inputs.service }} that needs to be executed."
+            echo "file_exists=false" >> $GITHUB_OUTPUT
+          fi
+
+          if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then
+              git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4
+          fi
+          if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then
+              git clone https://github.com/vllm-project/vllm.git vllm-openvino
+          fi
+
+      - name: Build Image
+        if: ${{ fromJSON(inputs.build) && steps.get-yaml-path.outputs.file_exists == 'true' }}
+        uses: opea-project/validation/actions/image-build@main
+        with:
+          work_dir: ${{ github.workspace }}
+          docker_compose_path: ${{ steps.get-yaml-path.outputs.docker_compose_path }}
+          registry: ${OPEA_IMAGE_REPO}opea
+          tag: ${{ inputs.tag }}
+
+  ####################################################################################################
+  # Docker Compose Test
+  ####################################################################################################
+  test-service-compose:
+    needs: [build-images]
+    if: ${{ fromJSON(inputs.test) && needs.build-images.outputs.file_exists == 'true' }}
+    uses: ./.github/workflows/_run-docker-compose.yml
+    with:
+      tag: ${{ inputs.tag }}
+      service: ${{ inputs.service }}
+      hardware: ${{ inputs.node }}
+    secrets: inherit

.github/workflows/reuse-get-test-matrix.yml → .github/workflows/_get-test-matrix.yml

@@ -25,7 +25,6 @@ jobs:
           else
             echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
           fi
-          echo "checkout ref ${{ env.CHECKOUT_REF }}"
 
       - name: Checkout out Repo
         uses: actions/checkout@v4
@@ -38,14 +37,17 @@ jobs:
         run: |
           set -xe
           if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
-            base_commit=${{ github.event.pull_request.base.sha }}
+            LATEST_COMMIT_SHA=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/repos/opea-project/GenAIComps/commits?sha=${{ github.event.pull_request.base.ref }}" | jq -r '.[0].sha')
+            echo "Latest commit SHA is $LATEST_COMMIT_SHA"
+            base_commit=$LATEST_COMMIT_SHA
           else
             base_commit=$(git rev-parse HEAD~1) # push event
           fi
           merged_commit=$(git log -1 --format='%H')
 
           changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | \
-          grep 'comps/' | grep -vE '*.md|*.txt|comps/cores')" || true
+          grep 'comps/' | grep -vE '*.md|comps/cores')" || true
           services=$(printf '%s\n' "${changed_files[@]}" | cut -d'/' -f2 | grep -vE '*.py' | sort -u) || true
           run_matrix="{\"include\":["
           for service in ${services}; do

.github/workflows/_run-docker-compose.yml

@@ -0,0 +1,111 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Image Build
+permissions: read-all
+on:
+  workflow_call:
+    inputs:
+      registry:
+        description: Container Registry URL
+        required: false
+        default: ""
+        type: string
+      tag:
+        description: Container Tag
+        required: false
+        default: "latest"
+        type: string
+      service:
+        description: Example to test
+        required: true
+        type: string
+      hardware:
+        description: Hardware to run the test on
+        required: true
+        type: string
+jobs:
+  get-test-case:
+    runs-on: ubuntu-latest
+    outputs:
+      test_cases: ${{ steps.test-case-matrix.outputs.test_cases }}
+      CHECKOUT_REF: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
+    steps:
+      - name: Get checkout ref
+        id: get-checkout-ref
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
+            CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
+          else
+            CHECKOUT_REF=${{ github.ref }}
+          fi
+          echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
+          echo "checkout ref ${CHECKOUT_REF}"
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
+          fetch-depth: 0
+
+      - name: Get test matrix
+        shell: bash
+        id: test-case-matrix
+        run: |
+          set -x
+          service_l=$(echo ${{ inputs.service }} | tr '[:upper:]' '[:lower:]')
+          cd ${{ github.workspace }}/tests
+          test_cases=$(find . -type f -name "test_${service_l}*.sh" -print | cut -d/ -f2 | jq -R '.' | jq -sc '.')
+          echo "test_cases=$test_cases" >> $GITHUB_OUTPUT
+
+  run-test:
+    needs: [get-test-case]
+    strategy:
+      matrix:
+        test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }}
+      fail-fast: false
+    runs-on: ${{ inputs.hardware }}
+    continue-on-error: true
+    steps:
+      - name: Clean up Working Directory
+        run: |
+          sudo rm -rf ${{github.workspace}}/*
+          docker system prune -f
+          docker rmi $(docker images --filter reference="*/*:comps" -q) || true
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.get-test-case.outputs.CHECKOUT_REF }}
+          fetch-depth: 0
+
+      - name: Run test
+        shell: bash
+        env:
+          HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
+          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+          PINECONE_KEY: ${{ secrets.PINECONE_KEY }}
+          service: ${{ inputs.service }}
+          hardware: ${{ inputs.hardware }}
+          test_case: ${{ matrix.test_case }}
+        run: |
+          cd ${{ github.workspace }}/tests
+          service=$(echo "${test_case}" | sed 's/test_\(.*\)\.sh/\1/')
+          echo "service=${service}" >> $GITHUB_ENV
+          if [ -f ${test_case} ]; then timeout 30m bash ${test_case}; else echo "Test script {${test_case}} not found, skip test!"; fi
+
+      - name: Clean up container
+        if: cancelled() || failure()
+        run: |
+          cid=$(docker ps -aq --filter "name=test-comps-*")
+          if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
+          docker system prune -f
+
+      - name: Publish pipeline artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.service }}
+          path: ${{ github.workspace }}/tests/*.log

.github/workflows/docker/compose/agent-compose-cd.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  comps-agent-langchain:
+    build:
+      dockerfile: comps/agent/langchain/docker/Dockerfile
+    image: ${REGISTRY}opea/comps-agent-langchain:${TAG:-latest}

.github/workflows/docker/compose/chathistory-compose-cd.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  chathistory-mongo-server:
+    build:
+      dockerfile: comps/chathistory/mongo/docker/Dockerfile
+    image: ${REGISTRY}opea/chathistory-mongo-server:${TAG:-latest}

.github/workflows/docker/compose/dataprep-compose-cd.yaml

@@ -0,0 +1,25 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# others: dataprep-redis-llama-index,dataprep-on-ray-redis
+services:
+  dataprep-redis-llama-index:
+    build:
+      dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}
+  dataprep-on-ray-redis:
+    build:
+      dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest}
+  dataprep-milvus:
+    build:
+      dockerfile: comps/dataprep/milvus/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-milvus:${TAG:-latest}
+  dataprep-pgvector:
+    build:
+      dockerfile: comps/dataprep/pgvector/langchain/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-pgvector:${TAG:-latest}
+  dataprep-pinecone:
+    build:
+      dockerfile: comps/dataprep/pinecone/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-pinecone:${TAG:-latest}

.github/workflows/docker/compose/dataprep-compose.yaml

@@ -13,11 +13,3 @@ services:
     build:
       dockerfile: comps/dataprep/qdrant/docker/Dockerfile
     image: ${REGISTRY}opea/dataprep-qdrant:${TAG:-latest}
-  dataprep-redis-llama-index:
-    build:
-      dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
-    image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}
-  dataprep-on-ray-redis:
-    build:
-      dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
-    image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest}

.github/workflows/docker/compose/embeddings-compose-cd.yaml

@@ -0,0 +1,16 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  embedding-langchain-mosec-endpoint:
+    build:
+      dockerfile: comps/embeddings/langchain-mosec/mosec-docker/Dockerfile
+    image: ${REGISTRY}opea/embedding-langchain-mosec-endpoint:${TAG:-latest}
+  embedding-langchain-mosec:
+    build:
+      dockerfile: comps/embeddings/langchain-mosec/docker/Dockerfile
+    image: ${REGISTRY}opea/embedding-langchain-mosec:${TAG:-latest}
+  embedding-tei-llama-index:
+    build:
+      dockerfile: comps/embeddings/llama_index/docker/Dockerfile
+    image: ${REGISTRY}opea/embedding-tei-llama-index:${TAG:-latest}

.github/workflows/docker/compose/guardrails-compose-cd.yaml

@@ -0,0 +1,8 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  guardrails-pii-detection:
+    build:
+      dockerfile: comps/guardrails/pii_detection/docker/Dockerfile
+    image: ${REGISTRY}opea/guardrails-pii-detection:${TAG:-latest}

.github/workflows/docker/compose/llms-compose-cd.yaml

@@ -0,0 +1,8 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  llm-native:
+    build:
+      dockerfile: comps/llms/text-generation/native/docker/Dockerfile
+    image: ${REGISTRY}opea/llm-native:${TAG:-latest}

.github/workflows/docker/compose/llms-compose.yaml

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # this file should be run in the root of the repo
-# images used by GenAIExamples: llm-tgi,llm-ollama,llm-docsum-tgi,llm-faqgen-tgi,llm-vllm,llm-vllm-ray
+# images used by GenAIExamples: llm-tgi,llm-ollama,llm-docsum-tgi,llm-faqgen-tgi,llm-vllm,llm-vllm-hpu,llm-vllm-ray,llm-vllm-ray-hpu
 services:
   llm-tgi:
     build:
@@ -24,6 +24,10 @@ services:
     build:
       dockerfile: comps/llms/text-generation/vllm/docker/Dockerfile.microservice
     image: ${REGISTRY}opea/llm-vllm:${TAG:-latest}
+  llm-vllm-hpu:
+    build:
+      dockerfile: comps/llms/text-generation/vllm/docker/Dockerfile.hpu
+    image: ${REGISTRY}opea/llm-vllm-hpu:${TAG:-latest}
   llm-vllm-ray:
     build:
       dockerfile: comps/llms/text-generation/vllm-ray/docker/Dockerfile.microservice
@@ -36,3 +40,7 @@ services:
     build:
       dockerfile: comps/llms/text-generation/predictionguard/docker/Dockerfile
     image: ${REGISTRY}opea/llm-predictionguard:${TAG:-latest}
+  llm-vllm-ray-hpu:
+    build:
+      dockerfile: comps/llms/text-generation/vllm-ray/docker/Dockerfile.vllmray
+    image: ${REGISTRY}opea/llm-vllm-ray-hpu:${TAG:-latest}