Merge branch 'main' into main

.github/workflows/tests.yml

@@ -42,19 +42,23 @@ jobs:
       - name: Ensure workspace ownership
         run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
 
+      # https://github.com/actions/checkout/issues/1781
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
 
       - name: Build ragflow:dev-slim
         run: |
           RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
           cp -r ${RUNNER_WORKSPACE_PREFIX}/huggingface.co ${RUNNER_WORKSPACE_PREFIX}/nltk_data ${RUNNER_WORKSPACE_PREFIX}/libssl*.deb  ${RUNNER_WORKSPACE_PREFIX}/tika-server*.jar* ${RUNNER_WORKSPACE_PREFIX}/chrome* ${RUNNER_WORKSPACE_PREFIX}/cl100k_base.tiktoken .
           sudo docker pull ubuntu:22.04
-          sudo docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
+          sudo docker build --progress=plain -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 
       - name: Build ragflow:dev
         run: |
-          sudo docker build -f Dockerfile -t infiniflow/ragflow:dev .
+          sudo docker build --progress=plain -f Dockerfile -t infiniflow/ragflow:dev .
 
       - name: Start ragflow:dev-slim
         run: |

Dockerfile

@@ -75,6 +75,7 @@ RUN current_commit=$(git rev-parse --short HEAD); \
     else \
         version_info="$version_info full"; \
     fi; \
+    echo "RAGFlow version: $version_info"; \
     echo $version_info > /ragflow/VERSION
 
 COPY web web

Dockerfile.slim

@@ -75,6 +75,7 @@ RUN current_commit=$(git rev-parse --short HEAD); \
     else \
         version_info="$version_info full"; \
     fi; \
+    echo "RAGFlow version: $version_info"; \
     echo $version_info > /ragflow/VERSION
 
 COPY web web

README_id.md

@@ -286,7 +286,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:dev .
 
    Tambahkan baris berikut ke `/etc/hosts` untuk memetakan semua host yang ditentukan di **docker/service_conf.yaml** ke `127.0.0.1`:
    ```
-   127.0.0.1       es01 mysql minio redis
+   127.0.0.1       es01 infinity mysql minio redis
    ```  
    Di **docker/service_conf.yaml**, perbarui port mysql ke `5455` dan es ke `1200`, sesuai dengan yang ditentukan di **docker/.env**.
 

api/apps/kb_app.py

@@ -162,6 +162,7 @@ def rm():
                     message="Database error (Document removal)!")
             f2d = File2DocumentService.get_by_document_id(doc.id)
             FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
+            FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kbs[0].name])
             File2DocumentService.delete_by_document_id(doc.id)
 
         if not KnowledgebaseService.delete_by_id(req["kb_id"]):

api/apps/sdk/chat.py

@@ -111,7 +111,7 @@ def create(tenant_id):
         req['prompt_config'] = {}
     for key in key_list_2:
         temp = req['prompt_config'].get(key)
-        if not temp:
+        if (not temp and key == 'system') or (key not in req["prompt_config"]):
             req['prompt_config'][key] = default_prompt[key]
     for p in req['prompt_config']["parameters"]:
         if p["optional"]:

api/apps/sdk/dataset.py

@@ -252,6 +252,8 @@ def delete(tenant_id):
                     File.id == f2d[0].file_id,
                 ]
             )
+            FileService.filter_delete(
+                [File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kbs[0].name])
             File2DocumentService.delete_by_document_id(doc.id)
         if not KnowledgebaseService.delete_by_id(id):
             return get_error_data_result(message="Delete dataset error.(Database error)")

api/apps/sdk/session.py

@@ -15,7 +15,7 @@
 #
 import re
 import json
-from functools import partial
+from copy import deepcopy
 from uuid import uuid4
 from api.db import LLMType
 from flask import request, Response
@@ -155,6 +155,14 @@ def completion(tenant_id, chat_id):
 
     def fillin_conv(ans):
         reference = ans["reference"]
+        temp_reference = deepcopy(ans["reference"])
+        nonlocal conv, message_id
+        if not conv.reference:
+            conv.reference.append(temp_reference)
+        else:
+            conv.reference[-1] = temp_reference
+        conv.message[-1] = {"role": "assistant", "content": ans["answer"],
+                            "id": message_id, "prompt": ans.get("prompt", "")}
         if "chunks" in reference:
             chunks = reference.get("chunks")
             chunk_list = []
@@ -165,21 +173,14 @@ def fillin_conv(ans):
                     "document_id": chunk["doc_id"],
                     "document_name": chunk["docnm_kwd"],
                     "dataset_id": chunk["kb_id"],
-                    "image_id": chunk.get("img_id", ""),
+                    "image_id": chunk.get("image_id", ""),
                     "similarity": chunk["similarity"],
                     "vector_similarity": chunk["vector_similarity"],
                     "term_similarity": chunk["term_similarity"],
                     "positions": chunk.get("positions", []),
                 }
                 chunk_list.append(new_chunk)
             reference["chunks"] = chunk_list
-        nonlocal conv, message_id
-        if not conv.reference:
-            conv.reference.append(ans["reference"])
-        else:
-            conv.reference[-1] = ans["reference"]
-        conv.message[-1] = {"role": "assistant", "content": ans["answer"],
-                            "id": message_id, "prompt": ans.get("prompt", "")}
         ans["id"] = message_id
         ans["session_id"]=session_id
 
@@ -271,6 +272,13 @@ def agent_completion(tenant_id, agent_id):
 
     def fillin_conv(ans):
         reference = ans["reference"]
+        temp_reference = deepcopy(ans["reference"])
+        nonlocal conv, message_id
+        if not conv.reference:
+            conv.reference.append(temp_reference)
+        else:
+            conv.reference[-1] = temp_reference
+        conv.message[-1] = {"role": "assistant", "content": ans["answer"], "id": message_id}
         if "chunks" in reference:
             chunks = reference.get("chunks")
             chunk_list = []
@@ -281,20 +289,14 @@ def fillin_conv(ans):
                     "document_id": chunk["doc_id"],
                     "document_name": chunk["docnm_kwd"],
                     "dataset_id": chunk["kb_id"],
-                    "image_id": chunk["img_id"],
+                    "image_id": chunk["image_id"],
                     "similarity": chunk["similarity"],
                     "vector_similarity": chunk["vector_similarity"],
                     "term_similarity": chunk["term_similarity"],
                     "positions": chunk["positions"],
                 }
                 chunk_list.append(new_chunk)
             reference["chunks"] = chunk_list
-        nonlocal conv, message_id
-        if not conv.reference:
-            conv.reference.append(ans["reference"])
-        else:
-            conv.reference[-1] = ans["reference"]
-        conv.message[-1] = {"role": "assistant", "content": ans["answer"], "id": message_id}
         ans["id"] = message_id
         ans["session_id"] = session_id
 
@@ -413,7 +415,7 @@ def list_session(chat_id,tenant_id):
                                 "document_id": chunk["doc_id"],
                                 "document_name": chunk["docnm_kwd"],
                                 "dataset_id": chunk["kb_id"],
-                                "image_id": chunk["img_id"],
+                                "image_id": chunk["image_id"],
                                 "similarity": chunk["similarity"],
                                 "vector_similarity": chunk["vector_similarity"],
                                 "term_similarity": chunk["term_similarity"],

conf/service_conf.yaml

@@ -18,7 +18,7 @@ es:
   username: 'elastic'
   password: 'infini_rag_flow'
 infinity:
-  uri: 'http://infinity:23817'
+  uri: 'infinity:23817'
   db_name: 'default_db'
 redis:
   db: 1

docker/README.md

@@ -120,7 +120,7 @@ The [.env](./.env) file contains important environment variables for Docker.
 ### Maximum file size
 
 - `MAX_CONTENT_LENGTH`  
-  The maximum file size for each uploaded file, in bytes. You can uncomment this line if you wish to change 128M file size limit.
+  The maximum file size for each uploaded file, in bytes. You can uncomment this line if you wish to change the 128M file size limit.
 
 ## 🐋 Service configuration
 

docs/guides/deploy_local_llm.mdx

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 6
+sidebar_position: 7
 slug: /deploy_local_llm
 ---
 

docs/guides/llm_api_key_setup.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 5
+sidebar_position: 6
 slug: /llm_api_key_setup
 ---
 

docs/guides/manage_files.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 4
+sidebar_position: 5
 slug: /manage_files
 ---
 

docs/guides/manage_team_members.md

@@ -0,0 +1,39 @@
+---
+sidebar_position: 4
+slug: /manage_team_members
+---
+
+# Manage team members
+
+Invite or remove team members, join or leave a team.
+
+By default, each RAGFlow user is assigned a single team named after their name. RAGFlow allows you to invite RAGFlow users to your team. Your team members can help you:
+
+- Upload documents to your datasets.
+- Update document configurations in your datasets.
+- Parse documents in your datasets.
+
+:::tip NOTE
+Team members are currently *not* allowed to invite users to your team, and only you, the team owner, is permitted to do so.
+:::
+
+To enter the **Team** page, click on your avatar on the top right corner of the page **>** Team:
+
+![team](https://github.com/user-attachments/assets/0eac2503-26bc-4568-b3f2-bcd84069a07a)
+
+_On the **Team** page, you can view the information about members of your team and the teams you have joined._
+
+## Invite team members
+
+You are, by default, the owner of your own team and the only person permitted to invite users to join your team or remove team members.
+
+![invite_team_member](https://github.com/user-attachments/assets/75e19d53-3a00-480e-8b16-fe00c23c4486)
+
+## Remove team members
+
+![remove_members](https://github.com/user-attachments/assets/5c1a6ab5-8862-47a0-ad09-77fe88866508)
+
+## Accept or decline team invite
+
+![accept_or_decline_team_invite](https://github.com/user-attachments/assets/6a2cb61f-03d5-4423-9ed1-71df97ff4114)
+

docs/guides/run_health_check.md

@@ -0,0 +1,31 @@
+---
+sidebar_position: 7
+slug: /run_health_check
+---
+
+# Run health check on RAGFlow's dependencies
+
+Double check the health status of RAGFlow's dependencies.
+
+The operation of RAGFlow depends on four services:
+
+- **Elasticsearch** (default) or [Infinity](https://github.com/infiniflow/infinity) as the document engine
+- **MySQL**
+- **Redis**
+- **MinIO** for object storage
+
+If an exception or error occurs related to any of the above services, such as `Exception: Can't connect to ES cluster`, refer to this document to check their health status.
+
+You can also click you avatar on the top right corner of the page **>** System to view the visualized health status of RAGFlow's core services. The following screenshot shows that all services are 'green' (running healthily). The task executor displays the *cumulative* number of completed and failed document parsing tasks from the past 30 minutes:
+
+![system_status_page](https://github.com/user-attachments/assets/b0c1a11e-93e3-4947-b17a-1bfb4cdab6e4)
+
+Services with a yellow or red light are not running properly. The following is a screenshot of the system page after running `docker stop ragflow-es-10`:
+
+![es_failed](https://github.com/user-attachments/assets/06056540-49f5-48bf-9cc9-a7086bc75790)
+
+You can click on a specific 30-second time interval to view the details of completed and failed tasks:
+
+![done_tasks](https://github.com/user-attachments/assets/49b25ec4-03af-48cf-b2e5-c892f6eaa261)
+
+![done_vs_failed](https://github.com/user-attachments/assets/eaa928d0-a31c-4072-adea-046091e04599)

docs/guides/upgrade_ragflow.mdx

@@ -1,25 +1,47 @@
 ---
-sidebar_position: 7
+sidebar_position: 11
 slug: /upgrade_ragflow
 ---
 
 # Upgrade RAGFlow
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-## Upgrade RAGFlow to `dev`, the most recent, tested Docker image
+Upgrade RAGFlow to `dev-slim`/`dev` or the latest, published release.
+
+## Upgrade RAGFlow to `dev-slim`/`dev`, the most recent, tested Docker image
+
+`dev-slim` refers to the RAGFlow Docker image *without* embedding models, while `dev` refers to the RAGFlow Docker image with embedding models. For details on their differences, see **docker/.env**.
 
 1. Clone the repo
 
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    ```
 
-2. Update **ragflow/docker/.env** as follows:
+2. Update **ragflow/docker/.env**:
 
-   ```bash
-   RAGFLOW_IMAGE=infiniflow/ragflow:dev
-   ```
+<Tabs
+  defaultValue="dev-slim"
+  values={[
+    {label: 'dev-slim', value: 'dev-slim'},
+    {label: 'dev', value: 'dev'},
+  ]}>
+  <TabItem value="dev-slim">
+
+```bash
+RAGFLOW_IMAGE=infiniflow/ragflow:dev-slim
+```
+
+  </TabItem>
+  <TabItem value="dev">
+
+```bash
+RAGFLOW_IMAGE=infiniflow/ragflow:dev
+```
+
+  </TabItem>
+</Tabs>
 
 3. Update RAGFlow image and restart RAGFlow:
 
@@ -28,7 +50,7 @@ import TabItem from '@theme/TabItem';
    docker compose -f docker/docker-compose.yml up -d
    ```
 
-## Upgrade RAGFlow to `latest`, the most recent, officially published release
+## Upgrade RAGFlow to the most recent, officially published release
 
 1. Clone the repo
 
@@ -45,7 +67,7 @@ import TabItem from '@theme/TabItem';
 3. Update **ragflow/docker/.env** as follows:
 
    ```bash
-   RAGFLOW_IMAGE=infiniflow/ragflow:latest
+   RAGFLOW_IMAGE=infiniflow/ragflow:v0.14.0
    ```
 
 4. Update the RAGFlow image and restart RAGFlow: