Merge branch 'chroma-core:main' into feature/gcp-defaul-auth-enabled

.github/actions/bandit-scan/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.10-alpine AS base-action
+
+RUN pip3 install -U setuptools pip bandit
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["sh","/entrypoint.sh"]

.github/actions/bandit-scan/action.yaml

@@ -0,0 +1,26 @@
+name: 'Bandit Scan'
+description: 'This action performs a security vulnerability scan of python code using bandit library.'
+inputs:
+  bandit-config:
+    description: 'Bandit configuration file'
+    required: false
+  input-dir:
+    description: 'Directory to scan'
+    required: false
+    default: '.'
+  format:
+    description: 'Output format (txt, csv, json, xml, yaml). Default: json'
+    required: false
+    default: 'json'
+  output-file:
+    description: "The report file to produce. Make sure to align your format with the file extension to avoid confusion."
+    required: false
+    default: "bandit-scan.json"
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  args:
+    - ${{ inputs.format }}
+    - ${{ inputs.bandit-config }}
+    - ${{ inputs.input-dir }}
+    - ${{ inputs.output-file }}

.github/actions/bandit-scan/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+CFG="-c $2"
+if [ -z "$1" ]; then
+    echo "No path to scan provided"
+    exit 1
+fi
+
+if [ -z "$2" ]; then
+    CFG = ""
+fi
+
+bandit -f "$1" ${CFG} -r "$3" -o "$4"
+exit 0 #we want to ignore the exit code of bandit (for now)

.github/workflows/chroma-client-integration-test.yml

@@ -8,7 +8,8 @@ on:
     branches:
       - main
       - '**'
-
+  workflow_dispatch:
+
 jobs:
   test:
     timeout-minutes: 90

.github/workflows/chroma-cluster-test.yml

@@ -0,0 +1,37 @@
+name: Chroma Cluster Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+      - '**'
+  workflow_dispatch:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python: ['3.7']
+        platform: [ubuntu-latest]
+        testfile: ["--ignore-glob 'chromadb/test/property/*' --ignore='chromadb/test/test_cli.py'",
+                   "chromadb/test/property/test_add.py",
+                   "chromadb/test/property/test_collections.py",
+                   "chromadb/test/property/test_embeddings.py",
+                   "chromadb/test/property/test_filtering.py",
+                   "chromadb/test/property/test_persist.py"]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install test dependencies
+      run: python -m pip install -r requirements.txt && python -m pip install -r requirements_dev.txt
+    - name: Integration Test
+      run: bin/cluster-test.sh ${{ matrix.testfile }}
+      continue-on-error: true # Mark the job as successful even if the tests fail for now (Xfail)

.github/workflows/chroma-integration-test.yml

@@ -9,15 +9,17 @@ on:
     branches:
       - main
       - '**'
+  workflow_dispatch:
 
 jobs:
   test:
     strategy:
       matrix:
         python: ['3.7']
         platform: [ubuntu-latest, windows-latest]
-        testfile: ["--ignore-glob 'chromadb/test/property/*'",
+        testfile: ["--ignore-glob 'chromadb/test/property/*' --ignore='chromadb/test/test_cli.py'",
                    "chromadb/test/property/test_add.py",
+                   "chromadb/test/test_cli.py",
                    "chromadb/test/property/test_collections.py",
                    "chromadb/test/property/test_cross_version_persist.py",
                    "chromadb/test/property/test_embeddings.py",

.github/workflows/chroma-release.yml

@@ -8,8 +8,8 @@ on:
       - main
 
 env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: "ghcr.io/chroma-core/chroma"
+  GHCR_IMAGE_NAME: "ghcr.io/chroma-core/chroma"
+  DOCKERHUB_IMAGE_NAME: "chromadb/chroma"
   PLATFORMS: linux/amd64,linux/arm64 #linux/riscv64, linux/arm/v7
 
 jobs:
@@ -27,14 +27,7 @@ jobs:
   build-and-release:
     runs-on: ubuntu-latest
     needs: check_tag
-    if: needs.check_tag.outputs.tag_matches == 'true'
     permissions: write-all
-#      id-token: write
-#      contents: read
-#      deployments: write
-#      packages: write
-#      pull-requests: read
-#      statuses: write
     steps:
     - name: Checkout
       uses: actions/checkout@v3
@@ -57,36 +50,38 @@ jobs:
       run: python -m build
     - name: Test Client Package
       run: bin/test-package.sh dist/*.tar.gz
-    - name: Log in to the Container registry
+    - name: Log in to the Github Container registry
       uses: docker/[email protected]
       with:
-        registry: ${{ env.REGISTRY }}
+        registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Login to DockerHub
+      uses: docker/[email protected]
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
     - name: Install setuptools_scm
       run: python -m pip install setuptools_scm
-    - name: Get Docker Tag
-      id: tag
-      run: echo "tag_name=$IMAGE_NAME:$(bin/version)" >> $GITHUB_OUTPUT
+    - name: Get Release Version
+      id: version
+      run: echo "version=$(python -m setuptools_scm)" >> $GITHUB_OUTPUT
     - name: Build and push prerelease Docker image
-      if: "!startsWith(github.ref, 'refs/tags/')"
+      if: "needs.check_tag.outputs.tag_matches != 'true'"
       uses: docker/[email protected]
       with:
         context: .
         platforms: ${{ env.PLATFORMS }}
         push: true
-        tags: ${{ steps.tag.outputs.tag_name}}
+        tags: "${{ env.GHCR_IMAGE_NAME }}:${{ steps.version.outputs.version }},${{ env.DOCKERHUB_IMAGE_NAME }}:${{ steps.version.outputs.version }}"
     - name: Build and push release Docker image
-      if: "startsWith(github.ref, 'refs/tags/')"
+      if: "needs.check_tag.outputs.tag_matches == 'true'"
       uses: docker/[email protected]
       with:
         context: .
         platforms: ${{ env.PLATFORMS }}
         push: true
-        tags: "${{ steps.tag.outputs.tag_name }},${{ env.IMAGE_NAME }}:latest"
-    - name: Get Release Version
-      id: version
-      run: echo "version=$(python -m setuptools_scm)" >> $GITHUB_OUTPUT
+        tags: "${{ env.GHCR_IMAGE_NAME }}:${{ steps.version.outputs.version }},${{ env.DOCKERHUB_IMAGE_NAME }}:${{ steps.version.outputs.version }},${{ env.GHCR_IMAGE_NAME }}:latest,${{ env.DOCKERHUB_IMAGE_NAME }}:latest"
     - name: Get current date
       id: builddate
       run: echo "builddate=$(date +'%Y-%m-%dT%H:%M')" >> $GITHUB_OUTPUT
@@ -96,7 +91,7 @@ jobs:
         password: ${{ secrets.TEST_PYPI_API_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
     - name: Publish to PyPI
-      if: startsWith(github.ref, 'refs/tags')
+      if: "needs.check_tag.outputs.tag_matches == 'true'"
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.PYPI_API_TOKEN }}
@@ -107,31 +102,32 @@ jobs:
         aws-region: us-east-1
     - name: Generate CloudFormation template
       id: generate-cf
-      if: "startsWith(github.ref, 'refs/tags/')"
+      if: "needs.check_tag.outputs.tag_matches == 'true'"
       run: "pip install boto3 && python bin/generate_cloudformation.py"
     - name: Release Tagged Version
       uses: ncipollo/[email protected]
-      if: "startsWith(github.ref, 'refs/tags/')"
+      if: "needs.check_tag.outputs.tag_matches == 'true'"
       with:
         body: |
           Version: `${{steps.version.outputs.version}}`
           Git ref: `${{github.ref}}`
           Build Date: `${{steps.builddate.outputs.builddate}}`
           PIP Package: `chroma-${{steps.version.outputs.version}}.tar.gz`
-          Docker Image: `${{steps.tag.outputs.tag_name}}`
+          Github Container Registry Image: `${{ env.GHCR_IMAGE_NAME }}:${{ steps.version.outputs.version }}`
+          DockerHub Image: `${{ env.DOCKERHUB_IMAGE_NAME }}:${{ steps.version.outputs.version }}`
         artifacts: "dist/chroma-${{steps.version.outputs.version}}.tar.gz"
         prerelease: true
         generateReleaseNotes: true
     - name: Update Tag
       uses: richardsimko/[email protected]
-      if: "!startsWith(github.ref, 'refs/tags/')"
+      if: "needs.check_tag.outputs.tag_matches != 'true'"
       with:
         tag_name: latest
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Release Latest
       uses: ncipollo/[email protected]
-      if: "!startsWith(github.ref, 'refs/tags/')"
+      if: "needs.check_tag.outputs.tag_matches != 'true'"
       with:
         tag: "latest"
         name: "Latest"
@@ -140,7 +136,8 @@ jobs:
           Git ref: `${{github.ref}}`
           Build Date: `${{steps.builddate.outputs.builddate}}`
           PIP Package: `chroma-${{steps.version.outputs.version}}.tar.gz`
-          Docker Image: `${{steps.tag.outputs.tag_name}}`
+          Github Container Registry Image: `${{ env.GHCR_IMAGE_NAME }}:${{ steps.version.outputs.version }}`
+          DockerHub Image: `${{ env.DOCKERHUB_IMAGE_NAME }}:${{ steps.version.outputs.version }}`
         artifacts: "dist/chroma-${{steps.version.outputs.version}}.tar.gz"
         allowUpdates: true
         prerelease: true

.github/workflows/chroma-test.yml

@@ -9,6 +9,7 @@ on:
     branches:
       - main
       - '**'
+  workflow_dispatch:
 
 jobs:
   test:

.github/workflows/python-vuln.yaml

@@ -0,0 +1,28 @@
+name: Python Vulnerability Scan
+on:
+  push:
+    branches:
+      - '*'
+      - '*/**'
+    paths:
+      - chromadb/**
+      - clients/python/**
+  workflow_dispatch:
+jobs:
+  bandit-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: ./.github/actions/bandit-scan/
+        with:
+          input-dir: '.'
+          format: 'json'
+          bandit-config: 'bandit.yaml'
+          output-file: 'bandit-report.json'
+      - name: Upload Bandit Report
+        uses: actions/upload-artifact@v3
+        with:
+          name: bandit-artifact
+          path: |
+            bandit-report.json

.pre-commit-config.yaml

@@ -1,3 +1,4 @@
+exclude: 'chromadb/proto/chroma_pb2\.(py|pyi|py_grpc\.py)' # Generated files
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
@@ -32,4 +33,4 @@ repos:
     hooks:
       - id: mypy
         args: [--strict, --ignore-missing-imports, --follow-imports=silent, --disable-error-code=type-abstract]
-        additional_dependencies: ["types-requests", "pydantic", "overrides", "hypothesis", "pytest", "pypika", "numpy"]
+        additional_dependencies: ["types-requests", "pydantic", "overrides", "hypothesis", "pytest", "pypika", "numpy", "types-protobuf"]

DEVELOP.md

@@ -43,7 +43,7 @@ print(api.heartbeat())
 
 3. With a persistent backend and a small frontend client
 
-Run `docker-compose up -d --build`
+Run `chroma run --path /chroma_db_path`
 ```python
 import chromadb
 api = chromadb.HttpClient(host="localhost", port="8000")

README.md

@@ -35,7 +35,7 @@
 ```bash
 pip install chromadb # python client
 # for javascript, npm install chromadb!
-# for client-server mode, docker-compose up -d --build
+# for client-server mode, chroma run --path /chroma_db_path
 ```
 
 The core API is only 4 functions (run our [💡 Google Colab](https://colab.research.google.com/drive/1QEzFyqnoFxq7LUGyP1vzR4iLt9PpCDXv?usp=sharing) or [Replit template](https://replit.com/@swyx/BasicChromaStarter?v=1)):

bandit.yaml

@@ -0,0 +1,4 @@
+# FILE: bandit.yaml
+exclude_dirs: [ 'chromadb/test', 'bin', 'build', 'build', '.git', '.venv', 'venv', 'env','.github','examples','clients/js','.vscode' ]
+tests: [ ]
+skips: [ ]

bin/cluster-test.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+
+function cleanup {
+  docker compose -f docker-compose.cluster.test.yml down --rmi local --volumes
+}
+
+trap cleanup EXIT
+
+docker compose -f docker-compose.cluster.test.yml up -d --wait
+
+export CHROMA_CLUSTER_TEST_ONLY=1
+
+echo testing: python -m pytest "$@"
+python -m pytest "$@"

bin/docker_entrypoint.sh

@@ -3,4 +3,4 @@
 echo "Rebuilding hnsw to ensure architecture compatibility"
 pip install --force-reinstall --no-cache-dir chroma-hnswlib
 export IS_PERSISTENT=1
-uvicorn chromadb.app:app --workers 1 --host 0.0.0.0 --port 8000 --proxy-headers --log-config log_config.yml
+uvicorn chromadb.app:app --workers 1 --host 0.0.0.0 --port 8000 --proxy-headers --log-config chromadb/log_config.yml