Merge remote-tracking branch 'origin/main' into feature/authz

# Conflicts:
#	chromadb/server/fastapi/__init__.py
#	chromadb/telemetry/opentelemetry/__init__.py

.github/workflows/chroma-client-integration-test.yml

@@ -9,13 +9,13 @@ on:
       - main
       - '**'
   workflow_dispatch:
-  
+
 jobs:
   test:
     timeout-minutes: 90
     strategy:
       matrix:
-        python: ['3.7', '3.8', '3.9', '3.10']
+        python: ['3.8', '3.9', '3.10', '3.11']
         platform: [ubuntu-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:

.github/workflows/chroma-cluster-test.yml

@@ -14,7 +14,7 @@ jobs:
   test:
     strategy:
       matrix:
-        python: ['3.7']
+        python: ['3.8']
         platform: [ubuntu-latest]
         testfile: ["chromadb/test/ingest/test_producer_consumer.py",
                    "chromadb/test/segment/distributed/test_memberlist_provider.py",]

.github/workflows/chroma-integration-test.yml

@@ -15,7 +15,7 @@ jobs:
   test:
     strategy:
       matrix:
-        python: ['3.7']
+        python: ['3.8']
         platform: [ubuntu-latest, windows-latest]
         testfile: ["--ignore-glob 'chromadb/test/property/*' --ignore='chromadb/test/test_cli.py' --ignore='chromadb/test/auth/test_simple_rbac_authz.py'",
                    "chromadb/test/property/test_add.py",

.github/workflows/chroma-test.yml

@@ -16,7 +16,7 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        python: ['3.7', '3.8', '3.9', '3.10']
+        python: ['3.8', '3.9', '3.10', '3.11']
         platform: [ubuntu-latest, windows-latest]
         testfile: ["--ignore-glob 'chromadb/test/property/*' --ignore-glob 'chromadb/test/stress/*'",
                    "chromadb/test/property/test_add.py",
@@ -44,7 +44,7 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        python: ['3.7']
+        python: ['3.8']
         platform: ['16core-64gb-ubuntu-latest', '16core-64gb-windows-latest']
         testfile: ["'chromadb/test/stress/'"]
     runs-on: ${{ matrix.platform }}

bin/cluster-test.sh

@@ -3,11 +3,11 @@
 set -e
 
 function cleanup {
-  # Restore the previous kube context
-  kubectl config use-context $PREV_CHROMA_KUBE_CONTEXT
-  # Kill the tunnel process
-  kill $TUNNEL_PID
-  minikube delete -p chroma-test
+    # Restore the previous kube context
+    kubectl config use-context $PREV_CHROMA_KUBE_CONTEXT
+    # Kill the tunnel process
+    kill $TUNNEL_PID
+    minikube delete -p chroma-test
 }
 
 trap cleanup EXIT
@@ -25,6 +25,7 @@ minikube addons enable ingress-dns -p chroma-test
 # Setup docker to build inside the minikube cluster and build the image
 eval $(minikube -p chroma-test docker-env)
 docker build -t server:latest -f Dockerfile .
+docker build -t chroma-coordinator:latest -f go/coordinator/Dockerfile .
 
 # Apply the kubernetes manifests
 kubectl apply -f k8s/deployment
@@ -36,7 +37,7 @@ kubectl apply -f k8s/test
 kubectl wait --namespace chroma --for=condition=Ready pods --all --timeout=300s
 
 # Run mini kube tunnel in the background to expose the service
-minikube tunnel -p chroma-test &
+minikube tunnel -c true -p chroma-test &
 TUNNEL_PID=$!
 
 # Wait for the tunnel to be ready. There isn't an easy way to check if the tunnel is ready. So we just wait for 10 seconds
@@ -45,8 +46,12 @@ sleep 10
 export CHROMA_CLUSTER_TEST_ONLY=1
 export CHROMA_SERVER_HOST=$(kubectl get svc server -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}')
 export PULSAR_BROKER_URL=$(kubectl get svc pulsar -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}')
+export CHROMA_COORDINATOR_HOST=$(kubectl get svc coordinator -n chroma -o=jsonpath='{.status.loadBalancer.ingress[0].ip}')
+export CHROMA_SERVER_GRPC_PORT="50051"
+
 echo "Chroma Server is running at port $CHROMA_SERVER_HOST"
 echo "Pulsar Broker is running at port $PULSAR_BROKER_URL"
+echo "Chroma Coordinator is running at port $CHROMA_COORDINATOR_HOST"
 
 echo testing: python -m pytest "$@"
 python -m pytest "$@"

bin/reset.sh

@@ -0,0 +1,13 @@
+ #!/usr/bin/env bash
+
+eval $(minikube -p chroma-test docker-env)
+
+docker build -t chroma-coordinator:latest -f go/coordinator/Dockerfile .
+
+kubectl delete deployment coordinator -n chroma
+
+# Apply the kubernetes manifests
+kubectl apply -f k8s/deployment
+kubectl apply -f k8s/crd
+kubectl apply -f k8s/cr
+kubectl apply -f k8s/test

chromadb/__init__.py

@@ -1,8 +1,10 @@
-from typing import Dict
+from typing import Dict, Optional
 import logging
+from chromadb.api.client import Client as ClientCreator
+from chromadb.api.client import AdminClient as AdminClientCreator
 import chromadb.config
-from chromadb.config import Settings, System
-from chromadb.api import API
+from chromadb.config import DEFAULT_DATABASE, DEFAULT_TENANT, Settings
+from chromadb.api import AdminAPI, ClientAPI
 from chromadb.api.models.Collection import Collection
 from chromadb.api.types import (
     CollectionMetadata,
@@ -35,15 +37,12 @@
     "QueryResult",
     "GetResult",
 ]
-from chromadb.telemetry.product.events import ClientStartEvent
-from chromadb.telemetry.product import ProductTelemetryClient
-
 
 logger = logging.getLogger(__name__)
 
 __settings = Settings()
 
-__version__ = "0.4.14"
+__version__ = "0.4.15"
 
 # Workaround to deal with Colab's old sqlite3 version
 try:
@@ -55,7 +54,7 @@
 
 is_client = False
 try:
-    from chromadb.is_thin_client import is_thin_client  # type: ignore
+    from chromadb.is_thin_client import is_thin_client
 
     is_client = is_thin_client
 except ImportError:
@@ -95,37 +94,58 @@ def get_settings() -> Settings:
     return __settings
 
 
-def EphemeralClient(settings: Settings = Settings()) -> API:
+def EphemeralClient(
+    settings: Optional[Settings] = None,
+    tenant: str = DEFAULT_TENANT,
+    database: str = DEFAULT_DATABASE,
+) -> ClientAPI:
     """
     Creates an in-memory instance of Chroma. This is useful for testing and
     development, but not recommended for production use.
+
+    Args:
+        tenant: The tenant to use for this client. Defaults to the default tenant.
+        database: The database to use for this client. Defaults to the default database.
     """
+    if settings is None:
+        settings = Settings()
     settings.is_persistent = False
 
-    return Client(settings)
+    return ClientCreator(settings=settings, tenant=tenant, database=database)
 
 
-def PersistentClient(path: str = "./chroma", settings: Settings = Settings()) -> API:
+def PersistentClient(
+    path: str = "./chroma",
+    settings: Optional[Settings] = None,
+    tenant: str = DEFAULT_TENANT,
+    database: str = DEFAULT_DATABASE,
+) -> ClientAPI:
     """
     Creates a persistent instance of Chroma that saves to disk. This is useful for
     testing and development, but not recommended for production use.
 
     Args:
         path: The directory to save Chroma's data to. Defaults to "./chroma".
+        tenant: The tenant to use for this client. Defaults to the default tenant.
+        database: The database to use for this client. Defaults to the default database.
     """
+    if settings is None:
+        settings = Settings()
     settings.persist_directory = path
     settings.is_persistent = True
 
-    return Client(settings)
+    return ClientCreator(tenant=tenant, database=database, settings=settings)
 
 
 def HttpClient(
     host: str = "localhost",
     port: str = "8000",
     ssl: bool = False,
-    headers: Dict[str, str] = {},
-    settings: Settings = Settings(),
-) -> API:
+    headers: Optional[Dict[str, str]] = None,
+    settings: Optional[Settings] = None,
+    tenant: str = DEFAULT_TENANT,
+    database: str = DEFAULT_DATABASE,
+) -> ClientAPI:
     """
     Creates a client that connects to a remote Chroma server. This supports
     many clients connecting to the same server, and is the recommended way to
@@ -136,27 +156,44 @@ def HttpClient(
         port: The port of the Chroma server. Defaults to "8000".
         ssl: Whether to use SSL to connect to the Chroma server. Defaults to False.
         headers: A dictionary of headers to send to the Chroma server. Defaults to {}.
+        tenant: The tenant to use for this client. Defaults to the default tenant.
+        database: The database to use for this client. Defaults to the default database.
     """
 
+    if headers is None:
+        headers = {}
+    if settings is None:
+        settings = Settings()
+
     settings.chroma_api_impl = "chromadb.api.fastapi.FastAPI"
     settings.chroma_server_host = host
     settings.chroma_server_http_port = port
     settings.chroma_server_ssl_enabled = ssl
     settings.chroma_server_headers = headers
 
-    return Client(settings)
+    return ClientCreator(tenant=tenant, database=database, settings=settings)
+
 
+def Client(
+    settings: Settings = __settings,
+    tenant: str = DEFAULT_TENANT,
+    database: str = DEFAULT_DATABASE,
+) -> ClientAPI:
+    """
+    Return a running chroma.API instance
 
-def Client(settings: Settings = __settings) -> API:
-    """Return a running chroma.API instance"""
+    tenant: The tenant to use for this client. Defaults to the default tenant.
+    database: The database to use for this client. Defaults to the default database.
 
-    system = System(settings)
+    """
 
-    product_telemetry_client = system.instance(ProductTelemetryClient)
-    api = system.instance(API)
+    return ClientCreator(tenant=tenant, database=database, settings=settings)
 
-    system.start()
 
-    product_telemetry_client.capture(ClientStartEvent())
+def AdminClient(settings: Settings = Settings()) -> AdminAPI:
+    """
 
-    return api
+    Creates an admin client that can be used to create tenants and databases.
+
+    """
+    return AdminClientCreator(settings=settings)