Merge branch 'chroma-core:main' into feature/ollama-ef

Tiltfile

@@ -1,26 +1,30 @@
 update_settings(max_parallel_updates=6)
 
 docker_build(
-  'local:migration',
-  context='.',
+  'local:sysdb-migration',
+  '.',
+  only=['go/'],
   dockerfile='./go/Dockerfile.migration'
 )
 
 docker_build(
-  'local:coordinator',
-  context='.',
+  'local:sysdb',
+  '.',
+  only=['go/', 'idl/'],
   dockerfile='./go/Dockerfile'
 )
 
 docker_build(
-  'local:frontend-server',
-  context='.',
+  'local:frontend-service',
+  '.',
+  only=['chromadb/', 'idl/', 'requirements.txt', 'bin/'],
   dockerfile='./Dockerfile',
 )
 
 docker_build(
-  'local:worker',
-  context='.',
+  'local:query-service',
+  '.',
+  only=["rust/", "idl/", "Cargo.toml", "Cargo.lock"],
   dockerfile='./rust/worker/Dockerfile'
 )
 
@@ -40,13 +44,13 @@ k8s_yaml([
 
 # Extra stuff to make debugging and testing easier
 k8s_yaml([
-  'k8s/test/coordinator_service.yaml',
-  'k8s/test/jaeger_service.yaml',
-  'k8s/test/logservice_service.yaml',
+  'k8s/test/sysdb-service.yaml',
+  'k8s/test/jaeger-service.yaml',
+  'k8s/test/pulsar-service.yaml',
+  'k8s/test/logservice-service.yaml',
   'k8s/test/minio.yaml',
-  'k8s/test/pulsar_service.yaml',
-  'k8s/test/worker_service.yaml',
-  'k8s/test/test_memberlist_cr.yaml',
+  'k8s/test/query-service-service.yaml',
+  'k8s/test/test-memberlist-cr.yaml',
 ])
 
 # Lots of things assume the cluster is in a basic state. Get it into a basic
@@ -60,19 +64,19 @@ k8s_resource(
   objects=[
     'pod-watcher:Role',
     'memberlists.chroma.cluster:CustomResourceDefinition',
-    'worker-memberlist:MemberList',
+    'query-service-memberlist:MemberList',
 
-    'coordinator-serviceaccount:serviceaccount',
-    'coordinator-serviceaccount-rolebinding:RoleBinding',
-    'coordinator-worker-memberlist-binding:clusterrolebinding',
+    'sysdb-serviceaccount:serviceaccount',
+    'sysdb-serviceaccount-rolebinding:RoleBinding',
+    'sysdb-query-service-memberlist-binding:clusterrolebinding',
 
     'logservice-serviceaccount:serviceaccount',
 
-    'worker-serviceaccount:serviceaccount',
-    'worker-serviceaccount-rolebinding:RoleBinding',
-    'worker-memberlist-readerwriter:ClusterRole',
-    'worker-worker-memberlist-binding:clusterrolebinding',
-    'worker-memberlist-readerwriter-binding:clusterrolebinding',
+    'query-service-serviceaccount:serviceaccount',
+    'query-service-serviceaccount-rolebinding:RoleBinding',
+    'query-service-memberlist-readerwriter:ClusterRole',
+    'query-service-query-service-memberlist-binding:clusterrolebinding',
+    'query-service-memberlist-readerwriter-binding:clusterrolebinding',
 
     'test-memberlist:MemberList',
     'test-memberlist-reader:ClusterRole',
@@ -84,17 +88,17 @@ k8s_resource(
 )
 
 # Production Chroma
-k8s_resource('postgres', resource_deps=['k8s_setup'], labels=["infrastructure"])
-k8s_resource('pulsar', resource_deps=['k8s_setup'], labels=["infrastructure"], port_forwards=['6650:6650', '8080:8080'])
-k8s_resource('migration', resource_deps=['postgres'], labels=["infrastructure"])
-k8s_resource('logservice', resource_deps=['migration'], labels=["chroma"], port_forwards='50052:50051')
-k8s_resource('coordinator', resource_deps=['pulsar', 'migration'], labels=["chroma"], port_forwards='50051:50051')
-k8s_resource('frontend-server', resource_deps=['pulsar', 'coordinator', 'logservice'],labels=["chroma"], port_forwards='8000:8000')
-k8s_resource('worker', resource_deps=['coordinator', 'pulsar'], labels=["chroma"])
+k8s_resource('postgres', resource_deps=['k8s_setup', 'namespace'], labels=["infrastructure"])
+k8s_resource('pulsar', resource_deps=['k8s_setup', 'namespace'], labels=["infrastructure"], port_forwards=['6650:6650', '8080:8080'])
+k8s_resource('sysdb-migration', resource_deps=['postgres', 'namespace'], labels=["infrastructure"])
+k8s_resource('logservice', resource_deps=['sysdb-migration'], labels=["chroma"], port_forwards='50052:50051')
+k8s_resource('sysdb', resource_deps=['pulsar', 'sysdb-migration'], labels=["chroma"], port_forwards='50051:50051')
+k8s_resource('frontend-service', resource_deps=['pulsar', 'sysdb', 'logservice'],labels=["chroma"], port_forwards='8000:8000')
+k8s_resource('query-service', resource_deps=['sysdb', 'pulsar'], labels=["chroma"])
 
 # I have no idea why these need their own lines but the others don't.
-k8s_resource(objects=['worker:service'], new_name='worker_service', resource_deps=['worker'], labels=["chroma"])
-k8s_resource(objects=['jaeger-lb:Service'], new_name='jaeger_service', resource_deps=['k8s_setup'], labels=["debug"])
+k8s_resource(objects=['query-service:service'], new_name='query-service-service', resource_deps=['query-service'], labels=["chroma"])
+k8s_resource(objects=['jaeger-lb:Service'], new_name='jaeger-service', resource_deps=['k8s_setup'], labels=["debug"])
 
 # Local S3
 k8s_resource('minio-deployment', resource_deps=['k8s_setup'], labels=["debug"], port_forwards='9000:9000')

bin/cluster-test.sh

@@ -11,10 +11,10 @@ echo "Chroma Server is running at port $CHROMA_SERVER_HOST"
 echo "Pulsar Broker is running at port $PULSAR_BROKER_URL"
 echo "Chroma Coordinator is running at port $CHROMA_COORDINATOR_HOST"
 
-kubectl -n chroma port-forward svc/coordinator-lb 50051:50051 &
+kubectl -n chroma port-forward svc/sysdb-lb 50051:50051 &
 kubectl -n chroma port-forward svc/logservice-lb 50052:50051 &
 kubectl -n chroma port-forward svc/pulsar-lb 6650:6650 &
 kubectl -n chroma port-forward svc/pulsar-lb 8080:8080 &
-kubectl -n chroma port-forward svc/frontend-server 8000:8000 &
+kubectl -n chroma port-forward svc/frontend-service 8000:8000 &
 
 "$@"

chromadb/config.py

@@ -112,7 +112,7 @@ class Settings(BaseSettings):  # type: ignore
     chroma_collection_assignment_policy_impl: str = (
         "chromadb.ingest.impl.simple_policy.SimpleAssignmentPolicy"
     )
-    worker_memberlist_name: str = "worker-memberlist"
+    worker_memberlist_name: str = "query-service-memberlist"
     chroma_coordinator_host = "localhost"
 
     chroma_logservice_host = "localhost"
@@ -331,11 +331,11 @@ def __init__(self, settings: Settings):
             and settings["chroma_segment_cache_policy"] != "LRU"
         ):
             logger.error(
-                f"Failed to set chroma_segment_cache_policy: Only LRU is available."
+                "Failed to set chroma_segment_cache_policy: Only LRU is available."
             )
             if settings["chroma_memory_limit_bytes"] == 0:
                 logger.error(
-                    f"Failed to set chroma_segment_cache_policy: chroma_memory_limit_bytes is require."
+                    "Failed to set chroma_segment_cache_policy: chroma_memory_limit_bytes is require."
                 )
 
         # Apply the nofile limit if set

chromadb/logservice/logservice.py

@@ -166,6 +166,7 @@ def pull_logs(
             collection_id=str(collection_id),
             start_from_id=start_id,
             batch_size=batch_size,
+            end_timestamp=-1,
         )
         response = self._log_service_stub.PullLogs(request)
         return response.records  # type: ignore