Merge branch 'main' into revertM

.github/workflows/manual-bom-scan.yml

@@ -59,13 +59,15 @@ jobs:
 
       - name: SBOM Scan Container
         uses: anchore/[email protected]
+        if: always()
         with:
           image: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
           output-file: ${{ matrix.image }}-sbom-scan.txt
           format: 'spdx-json'
 
       - name: Security Scan Container
         uses: aquasecurity/[email protected]
+        if: always()
         with:
           image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
           output: ${{ matrix.image }}-trivy-scan.txt
@@ -80,6 +82,7 @@ jobs:
         run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
 
       - uses: actions/[email protected]
+        if: always()
         with:
           name: ${{ matrix.image }}-scan
           path: ${{ matrix.image }}-*-scan.txt

.github/workflows/scripts/freeze_images.sh → .github/workflows/scripts/update_images_tag.sh

@@ -4,14 +4,18 @@
 # SPDX-License-Identifier: Apache-2.0
 
 declare -A dict
-dict["langchain/langchain"]="docker://docker.io/langchain/langchain"
-dict["ghcr.io/huggingface/text-generation-inference"]="docker://ghcr.io/huggingface/text-generation-inference"
+dict["ghcr.io/huggingface/text-generation-inference"]="docker://ghcr.io/huggingface/text-generation-inference:latest-intel-cpu"
 
 function get_latest_version() {
     repo_image=$1
-    versions=$(skopeo list-tags ${dict[$repo_image]} | jq -r '.Tags[]')
-    printf "version list:\n$versions\n"
-    latest_version=$(printf "%s\n" "${versions[@]}" | grep -E '^[\.0-9\-]+$' | sort -V | tail -n 1)
+    if [[ $repo_image == *"huggingface"* ]]; then
+        revision=$(skopeo inspect --config ${dict[$repo_image]} | jq -r '.config.Labels["org.opencontainers.image.revision"][:7]')
+        latest_version="sha-$revision-intel-cpu"
+    else
+        versions=$(skopeo list-tags ${dict[$repo_image]} | jq -r '.Tags[]')
+        printf "version list:\n$versions\n"
+        latest_version=$(printf "%s\n" "${versions[@]}" | grep -E '^[\.0-9\-]+$' | sort -V | tail -n 1)
+    fi
     echo "latest version: $latest_version"
     replace_image_version $repo_image $latest_version
 }
@@ -22,10 +26,10 @@ function replace_image_version() {
     if [[ -z "$version" ]]; then
         echo "version is empty"
     else
-        echo "replace $repo_image:latest with $repo_image:$version"
-        find . -name "Dockerfile" | xargs sed -i "s|$repo_image:latest.*|$repo_image:$version|g"
-        find . -name "*.yaml" | xargs sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g"
-        find . -name "*.md" | xargs sed -i "s|$repo_image:latest[A-Za-z0-9\-]*|$repo_image:$version|g"
+        echo "replace $repo_image:tag with $repo_image:$version"
+        find . -name "Dockerfile" | xargs sed -i "s|$repo_image:sha[A-Za-z0-9\-]*|$repo_image:$version|g"
+        find . -name "*.yaml" | xargs sed -i "s|$repo_image:sha[A-Za-z0-9\-]*|$repo_image:$version|g"
+        find . -name "*.md" | xargs sed -i "s|$repo_image:sha[A-Za-z0-9\-]*|$repo_image:$version|g"
     fi
 }
 

.github/workflows/weekly-update-images.yml

@@ -0,0 +1,54 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Weekly update base images and 3rd party images
+
+on:
+  schedule:
+    - cron: "0 0 * * 0"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  freeze-images:
+    runs-on: ubuntu-latest
+    env:
+      USER_NAME: "NeuralChatBot"
+      USER_EMAIL: "[email protected]"
+      BRANCH_NAME: "update_images_tag"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: "main"
+
+      - name: Install skopeo
+        run: |
+          sudo apt update
+          sudo apt -y install skopeo
+
+      - name: Set up Git
+        run: |
+          git config --global user.name ${{ env.USER_NAME }}
+          git config --global user.email ${{ env.USER_EMAIL }}
+          git remote set-url origin https://${{ env.USER_NAME }}:"${{ secrets.ACTION_TOKEN }}"@github.com/opea-project/GenAIExamples.git
+          git checkout -b ${{ env.BRANCH_NAME }}
+
+      - name: Run script
+        run: |
+          bash .github/workflows/scripts/update_images_tag.sh
+
+      - name: Commit changes
+        run: |
+          git add .
+          git commit -s -m "Update third party images tag"
+          git push --set-upstream origin update_images_tag
+
+      - name: create pull request
+        run: gh pr create -B main -H ${{ env.BRANCH_NAME }} --title 'Update ghcr.io/huggingface/text-generation-inference image tag' --body 'Created by Github action'
+        env:
+          GITHUB_TOKEN: ${{ secrets.ACTION_TOKEN }}

AgentQnA/README.md

@@ -26,50 +26,50 @@ This example showcases a hierarchical multi-agent system for question-answering
 1. Build agent docker image </br>
    First, clone the opea GenAIComps repo
 
-```
-export WORKDIR=<your-work-directory>
-cd $WORKDIR
-git clone https://github.com/opea-project/GenAIComps.git
-```
+   ```
+   export WORKDIR=<your-work-directory>
+   cd $WORKDIR
+   git clone https://github.com/opea-project/GenAIComps.git
+   ```
 
-Then build the agent docker image. Both the supervisor agent and the worker agent will use the same docker image, but when we launch the two agents we will specify different strategies and register different tools.
+   Then build the agent docker image. Both the supervisor agent and the worker agent will use the same docker image, but when we launch the two agents we will specify different strategies and register different tools.
 
-```
-cd GenAIComps
-docker build -t opea/comps-agent-langchain:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/agent/langchain/docker/Dockerfile .
-```
+   ```
+   cd GenAIComps
+   docker build -t opea/comps-agent-langchain:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/agent/langchain/docker/Dockerfile .
+   ```
 
 2. Launch tool services </br>
    In this example, we will use some of the mock APIs provided in the Meta CRAG KDD Challenge to demonstrate the benefits of gaining additional context from mock knowledge graphs.
 
-```
-docker run -d -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0
-```
+   ```
+   docker run -d -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0
+   ```
 
 3. Set up environment for this example </br>
    First, clone this repo
 
-```
-cd $WORKDIR
-git clone https://github.com/opea-project/GenAIExamples.git
-```
+   ```
+   cd $WORKDIR
+   git clone https://github.com/opea-project/GenAIExamples.git
+   ```
 
-Second, set up env vars
+   Second, set up env vars
 
-```
-export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
-# optional: OPANAI_API_KEY
-export OPENAI_API_KEY=<your-openai-key>
-```
+   ```
+   export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
+   # optional: OPANAI_API_KEY
+   export OPENAI_API_KEY=<your-openai-key>
+   ```
 
 4. Launch agent services</br>
    The configurations of the supervisor agent and the worker agent are defined in the docker-compose yaml file. We currently use openAI GPT-4o-mini as LLM, and we plan to add support for llama3.1-70B-instruct (served by TGI-Gaudi) in a subsequent release.
    To use openai llm, run command below.
 
-```
-cd docker/openai/
-bash launch_agent_service_openai.sh
-```
+   ```
+   cd docker/openai/
+   bash launch_agent_service_openai.sh
+   ```
 
 ## Validate services
 

AudioQnA/docker/Dockerfile

@@ -8,7 +8,6 @@ FROM python:3.11-slim
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     libgl1-mesa-glx \
     libjemalloc-dev \
-    vim \
     git
 
 RUN useradd -m -s /bin/bash user && \

ChatQnA/docker/Dockerfile

@@ -8,7 +8,6 @@ FROM python:3.11-slim
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     libgl1-mesa-glx \
     libjemalloc-dev \
-    vim \
     git
 
 RUN useradd -m -s /bin/bash user && \

ChatQnA/docker/Dockerfile_guardrails

@@ -8,7 +8,6 @@ FROM python:3.11-slim
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     libgl1-mesa-glx \
     libjemalloc-dev \
-    vim \
     git
 
 RUN useradd -m -s /bin/bash user && \

ChatQnA/docker/Dockerfile_without_rerank

@@ -8,7 +8,6 @@ FROM python:3.11-slim
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     libgl1-mesa-glx \
     libjemalloc-dev \
-    vim \
     git
 
 RUN useradd -m -s /bin/bash user && \

ChatQnA/docker/ui/svelte/.env

@@ -1,7 +1,7 @@
-CHAT_BASE_URL = 'http://backend_address:8888/v1/chatqna'
+CHAT_BASE_URL = '/v1/chatqna'
 
-UPLOAD_FILE_BASE_URL = 'http://backend_address:6007/v1/dataprep'
+UPLOAD_FILE_BASE_URL = '/v1/dataprep'
 
-GET_FILE = 'http://backend_address:6007/v1/dataprep/get_file'
+GET_FILE = '/v1/dataprep/get_file'
 
-DELETE_FILE = 'http://backend_address:6007/v1/dataprep/delete_file'
+DELETE_FILE = '/v1/dataprep/delete_file'

CodeGen/docker/Dockerfile

@@ -10,7 +10,6 @@ ENV LANG=C.UTF-8
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     libgl1-mesa-glx \
     libjemalloc-dev \
-    vim \
     git
 
 RUN useradd -m -s /bin/bash user && \

CodeTrans/docker/Dockerfile

@@ -8,7 +8,6 @@ FROM python:3.11-slim
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     libgl1-mesa-glx \
     libjemalloc-dev \
-    vim \
     git
 
 RUN useradd -m -s /bin/bash user && \

DocIndexRetriever/docker/Dockerfile

@@ -8,7 +8,6 @@ COPY GenAIComps /home/user/GenAIComps
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
     libgl1-mesa-glx \
     libjemalloc-dev \
-    vim \
     git
 
 RUN useradd -m -s /bin/bash user && \
@@ -27,4 +26,4 @@ USER user
 
 WORKDIR /home/user
 
-ENTRYPOINT ["python", "retrieval_tool.py"]
+ENTRYPOINT ["python", "retrieval_tool.py"]