resolving conflict

Signed-off-by: sharanshirodkar7 <[email protected]>

.github/CODEOWNERS

@@ -1,9 +1,21 @@
+/comps/agent/ [email protected]
+/comps/finetuning/ [email protected]
+/comps/guardrails/ [email protected]
 /comps/asr/ [email protected]
+/comps/intent_detection/ [email protected]
+/comps/knowledgegraphs/ [email protected]
 /comps/cores/ [email protected]
 /comps/dataprep/ [email protected]
 /comps/embeddings/ [email protected]
 /comps/guardrails/ [email protected]
 /comps/llms/ [email protected]
+/comps/lvms/ [email protected]
+/comps/nginx/ [email protected]
+/comps/prompt_registry/ [email protected]
+/comps/chathistory/ [email protected]
 /comps/reranks/ [email protected]
 /comps/retrievers/ [email protected]
 /comps/tts/ [email protected]
+/comps/ragas/ [email protected]
+/comps/vectorstores/ [email protected]
+/comps/web_retrievers/ [email protected]

.github/workflows/_comps-workflow.yml

@@ -20,17 +20,26 @@ on:
         default: true
         required: false
         type: boolean
-      # scan:
-      #   default: true
-      #   required: false
-      #   type: boolean
+      test:
+        default: true
+        description: "Test comps with docker compose"
+        required: false
+        type: boolean
+      mode:
+        default: "CD"
+        description: "Whether the test range is CI or CD"
+        required: false
+        type: string
+
 jobs:
   ####################################################################################################
   # Image Build
   ####################################################################################################
   build-images:
     runs-on: "docker-build-${{ inputs.node }}"
     continue-on-error: true
+    outputs:
+      file_exists: ${{ steps.get-yaml-path.outputs.file_exists }}
     steps:
       - name: Clean Up Working Directory
         run: sudo rm -rf ${{github.workspace}}/*
@@ -39,82 +48,45 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Clone required Repo
+        id: get-yaml-path
         run: |
           cd ${{ github.workspace }}/.github/workflows/docker/compose
           # service=$(echo ${{ inputs.service }} | cut -d'_' -f1)
-          docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
-          echo ${docker_compose_yml}
+          if [[ "${{ inputs.mode }}" == "CD" ]]; then
+            docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml
+          else
+            docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
+          fi
+          echo "docker_compose_path=${docker_compose_yml}" >> $GITHUB_OUTPUT
+          if [ -f "$docker_compose_yml" ]; then
+            echo "file_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "There is no ${{ inputs.mode }} part of ${{ inputs.service }} that needs to be executed."
+            echo "file_exists=false" >> $GITHUB_OUTPUT
+          fi
+
           if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then
               git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4
           fi
           if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then
               git clone https://github.com/vllm-project/vllm.git vllm-openvino
           fi
-          # echo "service=$service" >> $GITHUB_ENV
 
       - name: Build Image
-        if: ${{ fromJSON(inputs.build) }}
+        if: ${{ fromJSON(inputs.build) && steps.get-yaml-path.outputs.file_exists == 'true' }}
         uses: opea-project/validation/actions/image-build@main
         with:
-          work_dir: ${{ github.workspace }}/
-          docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
+          work_dir: ${{ github.workspace }}
+          docker_compose_path: ${{ steps.get-yaml-path.outputs.docker_compose_path }}
           registry: ${OPEA_IMAGE_REPO}opea
           tag: ${{ inputs.tag }}
-  # ####################################################################################################
-  # # Trivy Scan
-  # ####################################################################################################
-  #   get-image-list:
-  #     needs: [build-images]
-  #     if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi' }}
-  #     runs-on: ubuntu-latest
-  #     outputs:
-  #       matrix: ${{ steps.scan-matrix.outputs.matrix }}
-  #     steps:
-  #       - name: Checkout out Repo
-  #         uses: actions/checkout@v4
-
-  #       - name: Set Matrix
-  #         id: scan-matrix
-  #         run: |
-  #           pip install yq
-  #           compose_path=${{ github.workspace }}/${{ inputs.example }}/docker/docker_build_compose.yaml
-  #           echo "matrix=$(cat ${compose_path} | yq -r '.[]' | jq 'keys' | jq -c '.')" >> $GITHUB_OUTPUT
-
-  #   scan-images:
-  #     needs: [get-image-list, build-images]
-  #     if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi'}}
-  #     runs-on: "docker-build-${{ inputs.node }}"
-  #     strategy:
-  #       matrix:
-  #         image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}
-  #       fail-fast: false
-  #     steps:
-  #       - name: Pull Image
-  #         run: |
-  #           docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
-  #           echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV
-
-  #       - name: Scan Container
-  #         uses: opea-project/validation/actions/trivy-scan@main
-  #         with:
-  #           image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
-  #           output: ${{ matrix.image }}-scan.txt
-
-  #       - name: Cleanup
-  #         if: always()
-  #         run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
-
-  #       - uses: actions/[email protected]
-  #         with:
-  #           name: ${{ matrix.image }}-scan
-  #           path: ${{ matrix.image }}-scan.txt
-  #           overwrite: true
 
   ####################################################################################################
   # Docker Compose Test
   ####################################################################################################
   test-service-compose:
     needs: [build-images]
+    if: ${{ fromJSON(inputs.test) && needs.build-images.outputs.file_exists == 'true' }}
     uses: ./.github/workflows/_run-docker-compose.yml
     with:
       tag: ${{ inputs.tag }}

.github/workflows/_get-test-matrix.yml

@@ -25,7 +25,6 @@ jobs:
           else
             echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
           fi
-          echo "checkout ref ${{ env.CHECKOUT_REF }}"
 
       - name: Checkout out Repo
         uses: actions/checkout@v4
@@ -39,7 +38,7 @@ jobs:
           set -xe
           if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
             LATEST_COMMIT_SHA=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-            "https://api.github.com/repos/opea-project/GenAIComps/commits?sha=main" | jq -r '.[0].sha')
+            "https://api.github.com/repos/opea-project/GenAIComps/commits?sha=${{ github.event.pull_request.base.ref }}" | jq -r '.[0].sha')
             echo "Latest commit SHA is $LATEST_COMMIT_SHA"
             base_commit=$LATEST_COMMIT_SHA
           else

.github/workflows/docker/compose/agent-compose-cd.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  comps-agent-langchain:
+    build:
+      dockerfile: comps/agent/langchain/docker/Dockerfile
+    image: ${REGISTRY}opea/comps-agent-langchain:${TAG:-latest}

.github/workflows/docker/compose/chathistory-compose-cd.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  chathistory-mongo-server:
+    build:
+      dockerfile: comps/chathistory/mongo/docker/Dockerfile
+    image: ${REGISTRY}opea/chathistory-mongo-server:${TAG:-latest}

.github/workflows/docker/compose/dataprep-compose-cd.yaml

@@ -0,0 +1,25 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# others: dataprep-redis-llama-index,dataprep-on-ray-redis
+services:
+  dataprep-redis-llama-index:
+    build:
+      dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}
+  dataprep-on-ray-redis:
+    build:
+      dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest}
+  dataprep-milvus:
+    build:
+      dockerfile: comps/dataprep/milvus/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-milvus:${TAG:-latest}
+  dataprep-pgvector:
+    build:
+      dockerfile: comps/dataprep/pgvector/langchain/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-pgvector:${TAG:-latest}
+  dataprep-pinecone:
+    build:
+      dockerfile: comps/dataprep/pinecone/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-pinecone:${TAG:-latest}

.github/workflows/docker/compose/dataprep-compose.yaml

@@ -13,11 +13,3 @@ services:
     build:
       dockerfile: comps/dataprep/qdrant/docker/Dockerfile
     image: ${REGISTRY}opea/dataprep-qdrant:${TAG:-latest}
-  dataprep-redis-llama-index:
-    build:
-      dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
-    image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}
-  dataprep-on-ray-redis:
-    build:
-      dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
-    image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest}

.github/workflows/docker/compose/embeddings-compose-cd.yaml

@@ -0,0 +1,16 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  embedding-langchain-mosec-endpoint:
+    build:
+      dockerfile: comps/embeddings/langchain-mosec/mosec-docker/Dockerfile
+    image: ${REGISTRY}opea/embedding-langchain-mosec-endpoint:${TAG:-latest}
+  embedding-langchain-mosec:
+    build:
+      dockerfile: comps/embeddings/langchain-mosec/docker/Dockerfile
+    image: ${REGISTRY}opea/embedding-langchain-mosec:${TAG:-latest}
+  embedding-tei-llama-index:
+    build:
+      dockerfile: comps/embeddings/llama_index/docker/Dockerfile
+    image: ${REGISTRY}opea/embedding-tei-llama-index:${TAG:-latest}

.github/workflows/docker/compose/guardrails-compose-cd.yaml

@@ -0,0 +1,8 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  guardrails-pii-detection:
+    build:
+      dockerfile: comps/guardrails/pii_detection/docker/Dockerfile
+    image: ${REGISTRY}opea/guardrails-pii-detection:${TAG:-latest}

.github/workflows/docker/compose/llms-compose-cd.yaml

@@ -0,0 +1,8 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  llm-native:
+    build:
+      dockerfile: comps/llms/text-generation/native/docker/Dockerfile
+    image: ${REGISTRY}opea/llm-native:${TAG:-latest}

.github/workflows/docker/compose/lvms-compose-cd.yaml

@@ -0,0 +1,23 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  lvm:
+    build:
+      dockerfile: comps/lvms/Dockerfile
+    image: ${REGISTRY}opea/lvm:${TAG:-latest}
+  # Xeon CPU
+  llava:
+    build:
+      dockerfile: comps/lvms/llava/Dockerfile
+    image: ${REGISTRY}opea/llava:${TAG:-latest}
+  # Gaudi2 HPU
+  llava_hpu:
+    build:
+      dockerfile: comps/lvms/llava/Dockerfile_hpu
+    image: ${REGISTRY}opea/llava_hpu:${TAG:-latest}
+  lvm-tgi:
+    build:
+      dockerfile: comps/lvms/Dockerfile_tgi
+    image: ${REGISTRY}opea/lvm-tgi:${TAG:-latest}

.github/workflows/docker/compose/nginx-compose-cd.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  nginx:
+    build:
+      dockerfile: comps/nginx/docker/Dockerfile
+    image: ${REGISTRY}opea/nginx:${TAG:-latest}

.github/workflows/docker/compose/prompt_registry-compose-cd.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  promptregistry-mongo-server:
+    build:
+      dockerfile: comps/prompt_registry/mongo/docker/Dockerfile
+    image: ${REGISTRY}opea/promptregistry-mongo-server:${TAG:-latest}

.github/workflows/docker/compose/reranks-compose-cd.yaml

@@ -0,0 +1,16 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  reranking-fastrag:
+    build:
+      dockerfile: comps/reranks/fastrag/docker/Dockerfile
+    image: ${REGISTRY}opea/reranking-fastrag:${TAG:-latest}
+  reranking-langchain-mosec-endpoint:
+    build:
+      dockerfile: comps/reranks/langchain-mosec/mosec-docker/Dockerfile
+    image: ${REGISTRY}opea/reranking-langchain-mosec-endpoint:${TAG:-latest}
+  reranking-langchain-mosec:
+    build:
+      dockerfile: comps/reranks/langchain-mosec/docker/Dockerfile
+    image: ${REGISTRY}opea/reranking-langchain-mosec:${TAG:-latest}

.github/workflows/docker/compose/retrievers-compose.yaml

@@ -11,3 +11,19 @@ services:
     build:
       dockerfile: comps/retrievers/haystack/qdrant/docker/Dockerfile
     image: ${REGISTRY}opea/retriever-qdrant:${TAG:-latest}
+  retriever-pgvector:
+    build:
+      dockerfile: comps/retrievers/langchain/pgvector/docker/Dockerfile
+    image: ${REGISTRY}opea/retriever-qdrant:${TAG:-latest}
+  retriever-pinecone:
+    build:
+      dockerfile: comps/retrievers/langchain/pinecone/docker/Dockerfile
+    image: ${REGISTRY}opea/retriever-pinecone:${TAG:-latest}
+  retriever-milvus:
+    build:
+      dockerfile: comps/retrievers/langchain/milvus/docker/Dockerfile
+    image: ${REGISTRY}opea/retriever-milvus:${TAG:-latest}
+  retriever-redis-llamaindex:
+    build:
+      dockerfile: comps/retrievers/llamaindex/docker/Dockerfile
+    image: ${REGISTRY}opea/retriever-redis-llamaindex:${TAG:-latest}