Merge branch 'main' into bugfix/1013-embedding-datatype-compatibility

.github/actions/bandit-scan/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.10-alpine AS base-action
+
+RUN pip3 install -U setuptools pip bandit
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["sh","/entrypoint.sh"]

.github/actions/bandit-scan/action.yaml

@@ -0,0 +1,26 @@
+name: 'Bandit Scan'
+description: 'This action performs a security vulnerability scan of python code using bandit library.'
+inputs:
+  bandit-config:
+    description: 'Bandit configuration file'
+    required: false
+  input-dir:
+    description: 'Directory to scan'
+    required: false
+    default: '.'
+  format:
+    description: 'Output format (txt, csv, json, xml, yaml). Default: json'
+    required: false
+    default: 'json'
+  output-file:
+    description: "The report file to produce. Make sure to align your format with the file extension to avoid confusion."
+    required: false
+    default: "bandit-scan.json"
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  args:
+    - ${{ inputs.format }}
+    - ${{ inputs.bandit-config }}
+    - ${{ inputs.input-dir }}
+    - ${{ inputs.output-file }}

.github/actions/bandit-scan/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+CFG="-c $2"
+if [ -z "$1" ]; then
+    echo "No path to scan provided"
+    exit 1
+fi
+
+if [ -z "$2" ]; then
+    CFG = ""
+fi
+
+bandit -f "$1" ${CFG} -r "$3" -o "$4"
+exit 0 #we want to ignore the exit code of bandit (for now)

.github/workflows/chroma-client-integration-test.yml

@@ -8,7 +8,8 @@ on:
     branches:
       - main
       - '**'
-
+  workflow_dispatch:
+
 jobs:
   test:
     timeout-minutes: 90

.github/workflows/chroma-cluster-test.yml

@@ -0,0 +1,31 @@
+name: Chroma Cluster Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+      - '**'
+  workflow_dispatch:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python: ['3.7']
+        platform: [ubuntu-latest]
+        testfile: ["chromadb/test/ingest/test_producer_consumer.py"] # Just this one test for now
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install test dependencies
+      run: python -m pip install -r requirements.txt && python -m pip install -r requirements_dev.txt
+    - name: Integration Test
+      run: bin/cluster-test.sh ${{ matrix.testfile }}

.github/workflows/chroma-integration-test.yml

@@ -9,15 +9,17 @@ on:
     branches:
       - main
       - '**'
+  workflow_dispatch:
 
 jobs:
   test:
     strategy:
       matrix:
         python: ['3.7']
         platform: [ubuntu-latest, windows-latest]
-        testfile: ["--ignore-glob 'chromadb/test/property/*'",
+        testfile: ["--ignore-glob 'chromadb/test/property/*' --ignore='chromadb/test/test_cli.py'",
                    "chromadb/test/property/test_add.py",
+                   "chromadb/test/test_cli.py",
                    "chromadb/test/property/test_collections.py",
                    "chromadb/test/property/test_cross_version_persist.py",
                    "chromadb/test/property/test_embeddings.py",

.github/workflows/chroma-js-release.yml

@@ -0,0 +1,42 @@
+name: Chroma Release JS Client
+
+on:
+  push:
+    tags:
+      - 'js_release_*.*.*'  # Match tags in the form js_release_X.Y.Z
+      - 'js_release_alpha_*.*.*'  # Match tags in the form js_release_alpha_X.Y.Z
+
+jobs:
+  build-and-release:
+    runs-on: ubuntu-latest
+    permissions: write-all
+    steps:
+    - name: Check if tag matches the pattern
+      run: |
+        if [[ "${{ github.ref }}" =~ ^refs/tags/js_release_alpha_[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+          echo "Tag matches the pattern js_release_alpha_X.Y.Z"
+          echo "NPM_SCRIPT=release_alpha" >> "$GITHUB_ENV"
+        elif [[ "${{ github.ref }}" =~ ^refs/tags/js_release_[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+          echo "Tag matches the pattern js_release_X.Y.Z"
+          echo "NPM_SCRIPT=release" >> "$GITHUB_ENV"
+        else
+          echo "Tag does not match the release tag pattern, exiting workflow"
+          exit 1
+        fi
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Set up JS
+      uses: actions/setup-node@v3
+      with:
+        node-version: '16.x'
+        registry-url: 'https://registry.npmjs.org'
+    - name: Install Client Dev Dependencies
+      run: npm install
+      working-directory: ./clients/js/
+    - name: npm Test & Publish
+      run: npm run db:run && PORT=8001 npm run $NPM_SCRIPT
+      working-directory: ./clients/js/
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/chroma-release-python-client.yml

@@ -3,14 +3,27 @@ name: Chroma Release Python Client
 on:
   push:
     tags:
-      - '*'
+      - '[0-9]+.[0-9]+.[0-9]+'  # Match tags in the form X.Y.Z
     branches:
       - main
       - hammad/thin_client
 
 jobs:
+  check_tag:
+      runs-on: ubuntu-latest
+      outputs:
+        tag_matches: ${{ steps.check-tag.outputs.tag_matches }}
+      steps:
+        - name: Check Tag
+          id: check-tag
+          run: |
+            if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+                echo "tag_matches=true" >> $GITHUB_OUTPUT
+            fi
   build-and-release:
     runs-on: ubuntu-latest
+    needs: check_tag
+    if: needs.check_tag.outputs.tag_matches == 'true'
     permissions: write-all
     steps:
     - name: Checkout

.github/workflows/chroma-release.yml

@@ -8,24 +8,38 @@ on:
       - main
 
 env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: "ghcr.io/chroma-core/chroma"
+  GHCR_IMAGE_NAME: "ghcr.io/chroma-core/chroma"
+  DOCKERHUB_IMAGE_NAME: "chromadb/chroma"
+  PLATFORMS: linux/amd64,linux/arm64 #linux/riscv64, linux/arm/v7
 
 jobs:
+  check_tag:
+    runs-on: ubuntu-latest
+    outputs:
+      tag_matches: ${{ steps.check-tag.outputs.tag_matches }}
+    steps:
+      - name: Check Tag
+        id: check-tag
+        run: |
+          if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+              echo "tag_matches=true" >> $GITHUB_OUTPUT
+          fi
   build-and-release:
     runs-on: ubuntu-latest
+    needs: check_tag
     permissions: write-all
-#      id-token: write
-#      contents: read
-#      deployments: write
-#      packages: write
-#      pull-requests: read
-#      statuses: write
     steps:
     - name: Checkout
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
+    # https://github.com/docker/setup-qemu-action - for multiplatform builds
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+    # https://github.com/docker/setup-buildx-action - for multiplatform builds
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v2
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
@@ -36,34 +50,38 @@ jobs:
       run: python -m build
     - name: Test Client Package
       run: bin/test-package.sh dist/*.tar.gz
-    - name: Log in to the Container registry
+    - name: Log in to the Github Container registry
       uses: docker/[email protected]
       with:
-        registry: ${{ env.REGISTRY }}
+        registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Login to DockerHub
+      uses: docker/[email protected]
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
     - name: Install setuptools_scm
       run: python -m pip install setuptools_scm
-    - name: Get Docker Tag
-      id: tag
-      run: echo "tag_name=$IMAGE_NAME:$(bin/version)" >> $GITHUB_OUTPUT
+    - name: Get Release Version
+      id: version
+      run: echo "version=$(python -m setuptools_scm)" >> $GITHUB_OUTPUT
     - name: Build and push prerelease Docker image
-      if: "!startsWith(github.ref, 'refs/tags/')"
+      if: "needs.check_tag.outputs.tag_matches != 'true'"
       uses: docker/[email protected]
       with:
         context: .
+        platforms: ${{ env.PLATFORMS }}
         push: true
-        tags: ${{ steps.tag.outputs.tag_name}}
+        tags: "${{ env.GHCR_IMAGE_NAME }}:${{ steps.version.outputs.version }},${{ env.DOCKERHUB_IMAGE_NAME }}:${{ steps.version.outputs.version }}"
     - name: Build and push release Docker image
-      if: "startsWith(github.ref, 'refs/tags/')"
+      if: "needs.check_tag.outputs.tag_matches == 'true'"
       uses: docker/[email protected]
       with:
         context: .
+        platforms: ${{ env.PLATFORMS }}
         push: true
-        tags: "${{ steps.tag.outputs.tag_name }},${{ env.IMAGE_NAME }}:latest"
-    - name: Get Release Version
-      id: version
-      run: echo "version=$(python -m setuptools_scm)" >> $GITHUB_OUTPUT
+        tags: "${{ env.GHCR_IMAGE_NAME }}:${{ steps.version.outputs.version }},${{ env.DOCKERHUB_IMAGE_NAME }}:${{ steps.version.outputs.version }},${{ env.GHCR_IMAGE_NAME }}:latest,${{ env.DOCKERHUB_IMAGE_NAME }}:latest"
     - name: Get current date
       id: builddate
       run: echo "builddate=$(date +'%Y-%m-%dT%H:%M')" >> $GITHUB_OUTPUT
@@ -73,7 +91,7 @@ jobs:
         password: ${{ secrets.TEST_PYPI_API_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
     - name: Publish to PyPI
-      if: startsWith(github.ref, 'refs/tags')
+      if: "needs.check_tag.outputs.tag_matches == 'true'"
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.PYPI_API_TOKEN }}
@@ -84,31 +102,32 @@ jobs:
         aws-region: us-east-1
     - name: Generate CloudFormation template
       id: generate-cf
-      if: "startsWith(github.ref, 'refs/tags/')"
+      if: "needs.check_tag.outputs.tag_matches == 'true'"
       run: "pip install boto3 && python bin/generate_cloudformation.py"
     - name: Release Tagged Version
       uses: ncipollo/[email protected]
-      if: "startsWith(github.ref, 'refs/tags/')"
+      if: "needs.check_tag.outputs.tag_matches == 'true'"
       with:
         body: |
           Version: `${{steps.version.outputs.version}}`
           Git ref: `${{github.ref}}`
           Build Date: `${{steps.builddate.outputs.builddate}}`
           PIP Package: `chroma-${{steps.version.outputs.version}}.tar.gz`
-          Docker Image: `${{steps.tag.outputs.tag_name}}`
+          Github Container Registry Image: `${{ env.GHCR_IMAGE_NAME }}:${{ steps.version.outputs.version }}`
+          DockerHub Image: `${{ env.DOCKERHUB_IMAGE_NAME }}:${{ steps.version.outputs.version }}`
         artifacts: "dist/chroma-${{steps.version.outputs.version}}.tar.gz"
         prerelease: true
         generateReleaseNotes: true
     - name: Update Tag
       uses: richardsimko/[email protected]
-      if: "!startsWith(github.ref, 'refs/tags/')"
+      if: "needs.check_tag.outputs.tag_matches != 'true'"
       with:
         tag_name: latest
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Release Latest
       uses: ncipollo/[email protected]
-      if: "!startsWith(github.ref, 'refs/tags/')"
+      if: "needs.check_tag.outputs.tag_matches != 'true'"
       with:
         tag: "latest"
         name: "Latest"
@@ -117,7 +136,8 @@ jobs:
           Git ref: `${{github.ref}}`
           Build Date: `${{steps.builddate.outputs.builddate}}`
           PIP Package: `chroma-${{steps.version.outputs.version}}.tar.gz`
-          Docker Image: `${{steps.tag.outputs.tag_name}}`
+          Github Container Registry Image: `${{ env.GHCR_IMAGE_NAME }}:${{ steps.version.outputs.version }}`
+          DockerHub Image: `${{ env.DOCKERHUB_IMAGE_NAME }}:${{ steps.version.outputs.version }}`
         artifacts: "dist/chroma-${{steps.version.outputs.version}}.tar.gz"
         allowUpdates: true
         prerelease: true

.github/workflows/chroma-test.yml

@@ -9,6 +9,7 @@ on:
     branches:
       - main
       - '**'
+  workflow_dispatch:
 
 jobs:
   test:

.github/workflows/python-vuln.yaml

@@ -0,0 +1,28 @@
+name: Python Vulnerability Scan
+on:
+  push:
+    branches:
+      - '*'
+      - '*/**'
+    paths:
+      - chromadb/**
+      - clients/python/**
+  workflow_dispatch:
+jobs:
+  bandit-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: ./.github/actions/bandit-scan/
+        with:
+          input-dir: '.'
+          format: 'json'
+          bandit-config: 'bandit.yaml'
+          output-file: 'bandit-report.json'
+      - name: Upload Bandit Report
+        uses: actions/upload-artifact@v3
+        with:
+          name: bandit-artifact
+          path: |
+            bandit-report.json

.gitignore

@@ -11,6 +11,10 @@
 
 index_data
 
+# Default configuration for persist_directory in chromadb/config.py
+# Currently it's located in "./chroma/"
+chroma/
+
 venv
 .env
 .chroma

.pre-commit-config.yaml

@@ -32,4 +32,4 @@ repos:
     hooks:
       - id: mypy
         args: [--strict, --ignore-missing-imports, --follow-imports=silent, --disable-error-code=type-abstract]
-        additional_dependencies: ["types-requests", "pydantic", "overrides", "hypothesis", "pytest", "pypika", "numpy"]
+        additional_dependencies: ["types-requests", "pydantic", "overrides", "hypothesis", "pytest", "pypika", "numpy", "types-protobuf"]