Refactor multimodal embedding code

.github/workflows/docker/compose/embeddings-compose.yaml

@@ -9,15 +9,15 @@ services:
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
   embedding-multimodal-clip:
     build:
-      dockerfile: comps/embeddings/multimodal_clip/Dockerfile
+      dockerfile: comps/embeddings/native/multimodal/clip/Dockerfile
     image: ${REGISTRY:-opea}/embedding-multimodal-clip:${TAG:-latest}
   embedding-multimodal-bridgetower:
     build:
-      dockerfile: comps/embeddings/multimodal/bridgetower/Dockerfile
+      dockerfile: comps/embeddings/native/multimodal/bridgetower/dependency/Dockerfile
     image: ${REGISTRY:-opea}/embedding-multimodal-bridgetower:${TAG:-latest}
   embedding-multimodal:
     build:
-      dockerfile: comps/embeddings/multimodal/multimodal_langchain/Dockerfile
+      dockerfile: comps/embeddings/native/multimodal/bridgetower/wrapper/Dockerfile
     image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest}
   embedding-langchain-mosec-endpoint:
     build:
@@ -33,7 +33,7 @@ services:
     image: ${REGISTRY:-opea}/embedding-tei-llama-index:${TAG:-latest}
   embedding-multimodal-bridgetower-gaudi:
     build:
-      dockerfile: comps/embeddings/multimodal/bridgetower/Dockerfile.intel_hpu
+      dockerfile: comps/embeddings/native/multimodal/bridgetower/dependency/Dockerfile.intel_hpu
     image: ${REGISTRY:-opea}/embedding-multimodal-bridgetower-gaudi:${TAG:-latest}
   embedding-predictionguard:
     build:

comps/embeddings/README.md

@@ -24,13 +24,13 @@ For details, please refer to [langchain readme](tei/langchain/README.md) or [lla
 
 For details, please refer to this [readme](mosec/langchain/README.md).
 
-## Embeddings Microservice with Multimodal
+## Embeddings Microservice with Multimodal BridgePower
 
-For details, please refer to this [readme](multimodal/README.md).
+For details, please refer to this [readme](native/multimodal/bridgetower/README.md).
 
 ## Embeddings Microservice with Multimodal Clip
 
-For details, please refer to this [readme](multimodal_clip/README.md).
+For details, please refer to this [readme](native/multimodal/clip/README.md).
 
 ## Embeddings Microservice with Prediction Guard
 

comps/embeddings/multimodal/README.md → comps/embeddings/native/multimodal/bridgetower/README.md

@@ -1,4 +1,4 @@
-# Multimodal Embeddings Microservice
+# Multimodal Embeddings Microservice with BridgePower
 
 The Multimodal Embedding Microservice is designed to efficiently convert pairs of textual string and image into vectorized embeddings, facilitating seamless integration into various machine learning and data processing workflows. This service utilizes advanced algorithms to generate high-quality embeddings that capture the joint semantic essence of the input text-and-image pairs, making it ideal for applications in multi-modal data processing, information retrieval, and similar fields.
 
@@ -26,8 +26,8 @@ For both of the implementations, you need to install requirements first.
 ### 1.1 Install Requirements
 
 ```bash
-# run with langchain
-pip install -r multimodal_langchain/requirements.txt
+# install the dependencies
+pip install -r wrapper/requirements.txt
 ```
 
 ### 1.2 Start Embedding Service
@@ -51,19 +51,19 @@ Currently, we employ [**BridgeTower**](https://huggingface.co/BridgeTower/bridge
 - Gaudi2 HPU
 
 ```bash
-cd ../../..
-docker build -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/bridgetower/Dockerfile.intel_hpu .
-cd comps/embeddings/multimodal/bridgetower/
-docker compose -f docker_compose_bridgetower_embedding_endpoint.yaml up -d
+cd ../../../../../../
+docker build -t opea/embedding-multimodal-bridgetower-gaudi:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/native/multimodal/bridgetower/dependency/Dockerfile.intel_hpu .
+cd comps/embeddings/native/multimodal/bridgetower/dependency/
+docker compose -f docker_compose_bridgetower_embedding_intel_hpu.yaml up -d
 ```
 
 - Xeon CPU
 
 ```bash
-cd ../../..
-docker build -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/bridgetower/Dockerfile .
-cd comps/embeddings/multimodal/bridgetower/
-docker compose -f docker_compose_bridgetower_embedding_endpoint.yaml up -d
+cd ../../../../../../
+docker build -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/native/multimodal/bridgetower/dependency/Dockerfile .
+cd comps/embeddings/native/multimodal/bridgetower/dependency
+docker compose -f docker_compose_bridgetower_embedding_intel_cpu.yaml up -d
 ```
 
 Then you need to test your MMEI service using the following commands:
@@ -78,8 +78,8 @@ curl http://localhost:$your_mmei_port/v1/encode \
 Start the embedding service with MMEI_EMBEDDING_ENDPOINT.
 
 ```bash
-# run with langchain
-cd multimodal_langchain
+# run with wrapper microservice
+cd ../wrapper/
 export MMEI_EMBEDDING_ENDPOINT="http://localhost:$your_mmei_port/v1/encode"
 export your_embedding_port_microservice=6600
 export MM_EMBEDDING_PORT_MICROSERVICE=$your_embedding_port_microservice
@@ -89,8 +89,8 @@ python mm_embedding_mmei.py
 **Start Embedding Service with Local Model**
 
 ```bash
-# run with langchain
-cd multimodal_langchain
+# run with wrapper microservice
+cd ../wrapper/
 export your_embedding_port_microservice=6600
 export MM_EMBEDDING_PORT_MICROSERVICE=$your_embedding_port_microservice
 python local_mm_embedding.py
@@ -115,19 +115,19 @@ Currently, we employ [**BridgeTower**](https://huggingface.co/BridgeTower/bridge
 - Gaudi2 HPU
 
 ```bash
-cd ../../..
-docker build -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/bridgetower/Dockerfile.intel_hpu .
-cd comps/embeddings/multimodal/bridgetower/
-docker compose -f docker_compose_bridgetower_embedding_endpoint.yaml up -d
+cd ../../../../../../
+docker build -t opea/embedding-multimodal-bridgetower-gaudi:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/native/multimodal/bridgetower/dependency/Dockerfile.intel_hpu .
+cd comps/embeddings/native/multimodal/bridgetower/dependency/
+docker compose -f docker_compose_bridgetower_embedding_intel_hpu.yaml up -d
 ```
 
 - Xeon CPU
 
 ```bash
-cd ../../..
-docker build -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/bridgetower/Dockerfile .
-cd comps/embeddings/multimodal/bridgetower/
-docker compose -f docker_compose_bridgetower_embedding_endpoint.yaml up -d
+cd ../../../../../../
+docker build -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/native/multimodal/bridgetower/dependency/Dockerfile .
+cd comps/embeddings/native/multimodal/bridgetower/dependency
+docker compose -f docker_compose_bridgetower_embedding_intel_cpu.yaml up -d
 ```
 
 Then you need to test your MMEI service using the following commands:
@@ -151,14 +151,14 @@ export MMEI_EMBEDDING_ENDPOINT="http://$ip_address:$your_mmei_port/v1/encode"
 #### Build Langchain Docker
 
 ```bash
-cd ../../..
-docker build -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/multimodal_langchain/Dockerfile .
+cd ../../../../../../
+docker build -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/native/multimodal/bridgetower/wrapper/Dockerfile .
 ```
 
 ### 2.3 Run Docker with Docker Compose
 
 ```bash
-cd multimodal_langchain
+cd comps/embeddings/native/multimodal/bridgetower/wrapper/
 export your_embedding_port_microservice=6600
 export MM_EMBEDDING_PORT_MICROSERVICE=$your_embedding_port_microservice
 docker compose -f docker_compose_multimodal_embedding.yaml up -d

comps/embeddings/multimodal/bridgetower/Dockerfile → comps/embeddings/native/multimodal/bridgetower/dependency/Dockerfile

@@ -13,13 +13,13 @@ ENV PYTHONPATH=/home/user:/usr/lib/habanalabs/:/optimum-habana
 COPY --chown=user comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir -r /home/user/comps/embeddings/multimodal/multimodal_langchain/requirements.txt
+    pip install --no-cache-dir -r /home/user/comps/embeddings/native/multimodal/bridgetower/wrapper/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
 ARG EMBEDDER_PORT=8080
 ENV PORT=$EMBEDDER_PORT
 
-WORKDIR /home/user/comps/embeddings/multimodal/bridgetower
+WORKDIR /home/user/comps/embeddings/native/multimodal/bridgetower/dependency/
 
 ENTRYPOINT ["python", "bridgetower_server.py", "--device", "cpu"]

comps/embeddings/multimodal/bridgetower/Dockerfile.intel_hpu → comps/embeddings/native/multimodal/bridgetower/dependency/Dockerfile.intel_hpu

@@ -17,13 +17,13 @@ COPY --chown=user comps /home/user/comps
 
 # Install requirements and optimum habana
 RUN pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir -r /home/user/comps/embeddings/multimodal/multimodal_langchain/requirements.txt && \
+    pip install --no-cache-dir -r /home/user/comps/embeddings/native/multimodal/bridgetower/wrapper/requirements.txt && \
     pip install --no-cache-dir optimum[habana]
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
 ARG EMBEDDER_PORT=8080
 ENV PORT=$EMBEDDER_PORT
 
-WORKDIR /home/user/comps/embeddings/multimodal/bridgetower
+WORKDIR /home/user/comps/embeddings/native/multimodal/bridgetower/dependency/
 ENTRYPOINT ["python", "bridgetower_server.py", "--device", "hpu"]

comps/embeddings/multimodal/bridgetower/bridgetower_server.py → comps/embeddings/native/multimodal/bridgetower/dependency/bridgetower_server.py

@@ -17,7 +17,7 @@
 from fastapi.responses import JSONResponse, Response
 from utils import build_logger
 
-from comps.embeddings.multimodal.bridgetower import BridgeTowerEmbedding
+from comps.embeddings.native.multimodal.bridgetower import BridgeTowerEmbedding
 
 worker_id = str(uuid.uuid4())[:6]
 print(f"worker_id: {worker_id}")

comps/embeddings/native/multimodal/bridgetower/dependency/docker_compose_bridgetower_embedding_intel_hpu.yaml

@@ -0,0 +1,19 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  embedding-multimodal-bridgetower:
+    image: opea/embedding-multimodal-bridgetower-gaudi:latest
+    container_name: embedding-multimodal-bridgetower
+    ports:
+      - ${EMBEDDER_PORT}:${EMBEDDER_PORT}
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+    restart: unless-stopped
+
+networks:
+  default:
+    driver: bridge

comps/embeddings/multimodal/multimodal_langchain/Dockerfile → comps/embeddings/native/multimodal/bridgetower/wrapper/Dockerfile

@@ -16,10 +16,10 @@ USER user
 COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
-    pip install --no-cache-dir -r /home/user/comps/embeddings/multimodal/multimodal_langchain/requirements.txt
+    pip install --no-cache-dir -r /home/user/comps/embeddings/native/multimodal/bridgetower/wrapper/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
-WORKDIR /home/user/comps/embeddings/multimodal/multimodal_langchain
+WORKDIR /home/user/comps/embeddings/native/multimodal/bridgetower/wrapper
 
 ENTRYPOINT ["python", "mm_embedding_mmei.py"]

comps/embeddings/multimodal/multimodal_langchain/local_mm_embedding.py → comps/embeddings/native/multimodal/bridgetower/wrapper/local_mm_embedding.py

@@ -14,7 +14,7 @@
     opea_microservices,
     register_microservice,
 )
-from comps.embeddings.multimodal.bridgetower import BridgeTowerEmbedding
+from comps.embeddings.native.multimodal.bridgetower import BridgeTowerEmbedding
 
 logger = CustomLogger("local_multimodal_embedding")
 logflag = os.getenv("LOGFLAG", False)

comps/embeddings/multimodal_clip/Dockerfile → comps/embeddings/native/multimodal/clip/Dockerfile

@@ -19,10 +19,10 @@ COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
     if [ ${ARCH} = "cpu" ]; then pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \
-    pip install --no-cache-dir -r /home/user/comps/embeddings/multimodal_clip/requirements.txt
+    pip install --no-cache-dir -r /home/user/comps/embeddings/native/multimodal/clip/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
-WORKDIR /home/user/comps/embeddings/multimodal_clip
+WORKDIR /home/user/comps/embeddings/native/multimodal/clip/
 
 ENTRYPOINT ["python", "embedding_multimodal.py"]

comps/embeddings/multimodal_clip/README.md → comps/embeddings/native/multimodal/clip/README.md

@@ -1,4 +1,4 @@
-# Multimodal CLIP Embeddings Microservice
+# Multimodal Embeddings Microservice with CLIP
 
 The Multimodal CLIP Embedding Microservice is designed to efficiently convert textual strings and images into vectorized embeddings, facilitating seamless integration into various machine learning and data processing workflows. This service utilizes advanced algorithms to generate high-quality embeddings that capture the semantic essence of the input text and images, making it ideal for applications in multi-modal data processing, information retrieval, and similar fields.
 
@@ -21,14 +21,14 @@ Users are albe to configure and build embedding-related services according to th
 #### Build Langchain Docker
 
 ```bash
-cd ../../..
-docker build -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal_clip/Dockerfile .
+cd ../../../../../
+docker build -t opea/embedding-multimodal-clip:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/native/multimodal/clip/Dockerfile .
 ```
 
 ### 1.2 Run Docker with Docker Compose
 
 ```bash
-cd comps/embeddings/multimodal_clip
+cd comps/embeddings/native/multimodal/clip
 docker compose -f docker_compose_embedding.yaml up -d
 ```
 

tests/embeddings/test_embeddings_multimodal.sh → tests/embeddings/test_embeddings_multimodal_bridgetower_on_intel_cpu.sh

@@ -16,7 +16,7 @@ unset http_proxy
 function build_mmei_docker_images() {
     cd $WORKPATH
     echo $(pwd)
-    docker build --no-cache -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/bridgetower/Dockerfile .
+    docker build --no-cache -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/native/multimodal/bridgetower/dependency/Dockerfile  .
 
     if [ $? -ne 0 ]; then
         echo "opea/embedding-multimodal-bridgetower built fail"
@@ -29,7 +29,7 @@ function build_mmei_docker_images() {
 function build_embedding_service_images() {
     cd $WORKPATH
     echo $(pwd)
-    docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/multimodal_langchain/Dockerfile .
+    docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/native/multimodal/bridgetower/wrapper/Dockerfile .
 
     if [ $? -ne 0 ]; then
         echo "opea/embedding-multimodal built fail"
@@ -46,10 +46,10 @@ function build_docker_images() {
 
 function start_service() {
     cd $WORKPATH
-    cd comps/embeddings/multimodal/bridgetower/
-    docker compose -f docker_compose_bridgetower_embedding_endpoint.yaml up -d
+    cd comps/embeddings/native/multimodal/bridgetower/dependency
+    docker compose -f docker_compose_bridgetower_embedding_intel_cpu.yaml up -d
     cd $WORKPATH
-    cd comps/embeddings/multimodal/multimodal_langchain/
+    cd comps/embeddings/native/multimodal/bridgetower/wrapper/
     docker compose -f docker_compose_multimodal_embedding.yaml up -d
     sleep 2m
 }

tests/embeddings/test_embeddings_multimodal_on_intel_hpu.sh → tests/embeddings/test_embeddings_multimodal_bridgetower_on_intel_hpu.sh

@@ -16,20 +16,20 @@ unset http_proxy
 function build_mmei_docker_images() {
     cd $WORKPATH
     echo $(pwd)
-    docker build --no-cache -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/bridgetower/Dockerfile.intel_hpu .
+    docker build --no-cache -t opea/embedding-multimodal-bridgetower-gaudi:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/native/multimodal/bridgetower/dependency/Dockerfile.intel_hpu .
 
     if [ $? -ne 0 ]; then
-        echo "opea/embedding-multimodal-bridgetower built fail"
+        echo "opea/embedding-multimodal-bridgetower-gaudi built fail"
         exit 1
     else
-        echo "opea/embedding-multimodal-bridgetower built successful"
+        echo "opea/embedding-multimodal-bridgetower-gaudi built successful"
     fi
 }
 
 function build_embedding_service_images() {
     cd $WORKPATH
     echo $(pwd)
-    docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/multimodal_langchain/Dockerfile .
+    docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/native/multimodal/bridgetower/wrapper/Dockerfile .
 
     if [ $? -ne 0 ]; then
         echo "opea/embedding-multimodal built fail"
@@ -46,10 +46,10 @@ function build_docker_images() {
 
 function start_service() {
     cd $WORKPATH
-    cd comps/embeddings/multimodal/bridgetower/
-    docker compose -f docker_compose_bridgetower_embedding_endpoint.yaml up -d
+    cd comps/embeddings/native/multimodal/bridgetower/dependency
+    docker compose -f docker_compose_bridgetower_embedding_intel_hpu.yaml up -d
     cd $WORKPATH
-    cd comps/embeddings/multimodal/multimodal_langchain/
+    cd comps/embeddings/native/multimodal/bridgetower/wrapper/
     docker compose -f docker_compose_multimodal_embedding.yaml up -d
     sleep 2m
 }

tests/embeddings/test_embeddings_multimodal_clip.sh

@@ -10,17 +10,17 @@ ip_address=$(hostname -I | awk '{print $1}')
 function build_docker_images() {
     cd $WORKPATH
     echo $(pwd)
-    docker build --no-cache -t opea/embedding-multimodal:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy  -f comps/embeddings/multimodal_clip/Dockerfile .
+    docker build --no-cache -t opea/embedding-multimodal-clip:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy  -f comps/embeddings/native/multimodal/clip/Dockerfile .
     if [ $? -ne 0 ]; then
-        echo "opea/embedding-multimodal built fail"
+        echo "opea/embedding-multimodal-clip built fail"
         exit 1
     else
-        echo "opea/embedding-multimodal built successful"
+        echo "opea/embedding-multimodal-clip built successful"
     fi
 }
 
 function start_service() {
-    docker run -d --name="test-embedding-multimodal-server" -e LOGFLAG=True -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 5038:6000 --ipc=host  opea/embedding-multimodal:comps
+    docker run -d --name="test-embedding-multimodal-server" -e LOGFLAG=True -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 5038:6000 --ipc=host  opea/embedding-multimodal-clip:comps
     sleep 3m
 }