Merge branch 'main' into xinyuye/sd

.github/CODEOWNERS

@@ -0,0 +1,9 @@
+/comps/asr/ [email protected]
+/comps/cores/ [email protected]
+/comps/dataprep/ [email protected]
+/comps/embeddings/ [email protected]
+/comps/guardrails/ [email protected]
+/comps/llms/ [email protected]
+/comps/reranks/ [email protected]
+/comps/retrievers/ [email protected]
+/comps/tts/ [email protected]

.github/workflows/reuse-get-test-matrix.yml → .github/workflows/_get-test-matrix.yml

@@ -45,7 +45,7 @@ jobs:
           merged_commit=$(git log -1 --format='%H')
 
           changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | \
-          grep 'comps/' | grep -vE '*.md|*.txt|comps/cores')" || true
+          grep 'comps/' | grep -vE '*.md|comps/cores')" || true
           services=$(printf '%s\n' "${changed_files[@]}" | cut -d'/' -f2 | grep -vE '*.py' | sort -u) || true
           run_matrix="{\"include\":["
           for service in ${services}; do

.github/workflows/docker/code-scan.dockerfile

@@ -4,7 +4,7 @@
 ARG UBUNTU_VER=22.04
 FROM ubuntu:${UBUNTU_VER} as devel
 
-ENV LANG C.UTF-8
+ENV LANG=C.UTF-8
 
 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
     aspell \

.github/workflows/docker/compose/asr-compose.yaml

@@ -0,0 +1,18 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+# images used by GenAIExamples: asr,whisper,whisper-gaudi
+services:
+  asr:
+    build:
+      dockerfile: comps/asr/Dockerfile
+    image: ${REGISTRY}opea/asr:${TAG:-latest}
+  whisper:
+    build:
+      dockerfile: comps/asr/whisper/Dockerfile
+    image: ${REGISTRY}opea/whisper:${TAG:-latest}
+  whisper-gaudi:
+    build:
+      dockerfile: comps/asr/whisper/Dockerfile_hpu
+    image: ${REGISTRY}opea/whisper-gaudi:${TAG:-latest}

.github/workflows/docker/compose/dataprep-compose.yaml

@@ -0,0 +1,23 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+# images used by GenAIExamples: dataprep,dataprep-qdrant
+# others: dataprep-redis-llama-index,dataprep-on-ray-redis
+services:
+  dataprep-redis:
+    build:
+      dockerfile: comps/dataprep/redis/langchain/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-redis:${TAG:-latest}
+  dataprep-qdrant:
+    build:
+      dockerfile: comps/dataprep/qdrant/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-qdrant:${TAG:-latest}
+  dataprep-redis-llama-index:
+    build:
+      dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}
+  dataprep-on-ray-redis:
+    build:
+      dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest}

.github/workflows/docker/compose/embeddings-compose.yaml

@@ -0,0 +1,10 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+# images used by GenAIExamples: embedding-tei
+services:
+  embedding-tei:
+    build:
+      dockerfile: comps/embeddings/langchain/docker/Dockerfile
+    image: ${REGISTRY}opea/embedding-tei:${TAG:-latest}

.github/workflows/docker/compose/guardrails-compose.yaml

@@ -0,0 +1,15 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+# images used by GenAIExamples: guardrails-tgi
+# others: guardrails-pii-detection
+services:
+  guardrails-tgi:
+    build:
+      dockerfile: comps/guardrails/llama_guard/docker/Dockerfile
+    image: ${REGISTRY}opea/guardrails-tgi:${TAG:-latest}
+  guardrails-pii-detection:
+    build:
+      dockerfile: comps/guardrails/pii_detection/docker/Dockerfile
+    image: ${REGISTRY}opea/guardrails-pii-detection:${TAG:-latest}

.github/workflows/docker/compose/llms-compose.yaml

@@ -0,0 +1,38 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+# images used by GenAIExamples: llm-tgi,llm-ollama,llm-docsum-tgi,llm-faqgen-tgi,llm-vllm,llm-vllm-hpu,llm-vllm-ray,llm-vllm-ray-hpu
+services:
+  llm-tgi:
+    build:
+      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+    image: ${REGISTRY}opea/llm-tgi:${TAG:-latest}
+  llm-ollama:
+    build:
+      dockerfile: comps/llms/text-generation/ollama/Dockerfile
+    image: ${REGISTRY}opea/llm-ollama:${TAG:-latest}
+  llm-docsum-tgi:
+    build:
+      dockerfile: comps/llms/summarization/tgi/Dockerfile
+    image: ${REGISTRY}opea/llm-docsum-tgi:${TAG:-latest}
+  llm-faqgen-tgi:
+    build:
+      dockerfile: comps/llms/faq-generation/tgi/Dockerfile
+    image: ${REGISTRY}opea/llm-faqgen-tgi:${TAG:-latest}
+  llm-vllm:
+    build:
+      dockerfile: comps/llms/text-generation/vllm/docker/Dockerfile.microservice
+    image: ${REGISTRY}opea/llm-vllm:${TAG:-latest}
+  llm-vllm-hpu:
+    build:
+      dockerfile: comps/llms/text-generation/vllm/docker/Dockerfile.hpu
+    image: ${REGISTRY}opea/llm-vllm-hpu:${TAG:-latest}
+  llm-vllm-ray:
+    build:
+      dockerfile: comps/llms/text-generation/vllm-ray/docker/Dockerfile.microservice
+    image: ${REGISTRY}opea/llm-vllm-ray:${TAG:-latest}
+  llm-vllm-ray-hpu:
+    build:
+      dockerfile: comps/llms/text-generation/vllm-ray/docker/Dockerfile.vllmray
+    image: ${REGISTRY}opea/llm-vllm-ray-hpu:${TAG:-latest}

.github/workflows/docker/compose/reranks-compose.yaml

@@ -0,0 +1,10 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+# images used by GenAIExamples: reranking-tei
+services:
+  reranking-tei:
+    build:
+      dockerfile: comps/reranks/tei/docker/Dockerfile
+    image: ${REGISTRY}opea/reranking-tei:${TAG:-latest}

.github/workflows/docker/compose/retrievers-compose.yaml

@@ -0,0 +1,13 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  retriever-redis:
+    build:
+      dockerfile: comps/retrievers/langchain/redis/docker/Dockerfile
+    image: ${REGISTRY}opea/retriever-redis:${TAG:-latest}
+  retriever-qdrant:
+    build:
+      dockerfile: comps/retrievers/haystack/qdrant/docker/Dockerfile
+    image: ${REGISTRY}opea/retriever-qdrant:${TAG:-latest}

.github/workflows/docker/compose/tts-compose.yaml

@@ -0,0 +1,18 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+# images used by GenAIExamples: reranking-tei
+services:
+  asr:
+    build:
+      dockerfile: comps/tts/Dockerfile
+    image: ${REGISTRY}opea/asr:${TAG:-latest}
+  speecht5:
+    build:
+      dockerfile: comps/tts/speecht5/Dockerfile
+    image: ${REGISTRY}opea/speecht5:${TAG:-latest}
+  speecht5-gaudi:
+    build:
+      dockerfile: comps/tts/speecht5/Dockerfile_hpu
+    image: ${REGISTRY}opea/speecht5-gaudi:${TAG:-latest}

.github/workflows/docker/compose/web_retrievers-compose.yaml

@@ -0,0 +1,10 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+# images used by GenAIExamples: web-retriever-chroma
+services:
+  web-retriever-chroma:
+    build:
+      dockerfile: comps/web_retrievers/langchain/chroma/docker/Dockerfile
+    image: ${REGISTRY}opea/web-retriever-chroma:${TAG:-latest}

.github/workflows/docker/ut.dockerfile

@@ -4,7 +4,7 @@
 ARG UBUNTU_VER=22.04
 FROM ubuntu:${UBUNTU_VER} as devel
 
-ENV LANG C.UTF-8
+ENV LANG=C.UTF-8
 
 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
     aspell \

.github/workflows/manual-image-build.yml

@@ -0,0 +1,67 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Build latest images on manual event
+
+on:
+  workflow_dispatch:
+    inputs:
+      services:
+        default: "asr,dataprep"
+        description: "List of services to build including [asr,dataprep,embeddings,guardrails,llms,reranks,retrievers,tts,web_retrievers]"
+        required: true
+        type: string
+      tag:
+        default: "latest"
+        description: "Tag to apply to images"
+        required: true
+        type: string
+      nodes:
+        default: "docker-build-xeon,docker-build-gaudi"
+        description: "List of nodes to run the build including [docker-build-xeon,docker-build-gaudi]"
+        required: true
+        type: string
+
+jobs:
+  get-build-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      services: ${{ steps.get-services.outputs.services }}
+      nodes: ${{ steps.get-services.outputs.nodes }}
+    steps:
+      - name: Get test Services
+        id: get-services
+        run: |
+          set -x
+          service_list=($(echo ${{ github.event.inputs.services }} | tr ',' ' '))
+          services=$(printf '%s\n' "${service_list[@]}" | sort -u | jq -R '.' | jq -sc '.')
+          echo "services=$services" >> $GITHUB_OUTPUT
+          node_list=($(echo ${{ github.event.inputs.nodes }} | tr ',' ' '))
+          nodes=$(printf '%s\n' "${node_list[@]}" | sort -u | jq -R '.' | jq -sc '.')
+          echo "nodes=$nodes" >> $GITHUB_OUTPUT
+
+  image-build:
+    needs: get-build-matrix
+    strategy:
+      matrix:
+        service: ${{ fromJSON(needs.get-build-matrix.outputs.services) }}
+        node: ${{ fromJSON(needs.get-build-matrix.outputs.nodes) }}
+    runs-on: ${{ matrix.node }}
+    continue-on-error: true
+    steps:
+      - name: Clean Up Working Directory
+        run: |
+          sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Build image
+        env:
+          service: ${{ matrix.service }}
+        uses: opea-project/validation/actions/image-build@main
+        with:
+          work_dir: ${{ github.workspace }}
+          docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml
+          registry: ${OPEA_IMAGE_REPO}
+          tag: ${{ github.event.inputs.tag }}