Feat: add clean logging for api routes (#2928)

* feat: add clean logging for api routes

* nit

* `MULTI_TENANT` must be shared config

* nit

backend/alembic/env.py

@@ -9,7 +9,7 @@
 from sqlalchemy.ext.asyncio import create_async_engine
 from sqlalchemy.sql import text
 
-from danswer.configs.app_configs import MULTI_TENANT
+from shared_configs.configs import MULTI_TENANT
 from danswer.db.engine import build_connection_string
 from danswer.db.models import Base
 from celery.backends.database.session import ResultModelBase  # type: ignore

backend/danswer/auth/users.py

@@ -60,7 +60,6 @@
 from danswer.configs.app_configs import DISABLE_AUTH
 from danswer.configs.app_configs import DISABLE_VERIFICATION
 from danswer.configs.app_configs import EMAIL_FROM
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import REQUIRE_EMAIL_VERIFICATION
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from danswer.configs.app_configs import SMTP_PASS
@@ -94,6 +93,7 @@
 from danswer.utils.telemetry import RecordType
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 

backend/danswer/background/celery/tasks/indexing/tasks.py

@@ -24,7 +24,6 @@
 from danswer.background.indexing.run_indexing import run_indexing_entrypoint
 from danswer.background.indexing.run_indexing import RunIndexingCallbackInterface
 from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.constants import CELERY_INDEXING_LOCK_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DANSWER_REDIS_FUNCTION_LOCK_PREFIX
@@ -56,6 +55,7 @@
 from danswer.utils.variable_functionality import global_version
 from shared_configs.configs import INDEXING_MODEL_SERVER_HOST
 from shared_configs.configs import INDEXING_MODEL_SERVER_PORT
+from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 

backend/danswer/configs/app_configs.py

@@ -461,11 +461,6 @@
 AZURE_DALLE_DEPLOYMENT_NAME = os.environ.get("AZURE_DALLE_DEPLOYMENT_NAME")
 
 
-# Cloud configuration
-
-# Multi-tenancy configuration
-MULTI_TENANT = os.environ.get("MULTI_TENANT", "").lower() == "true"
-
 # Use managed Vespa (Vespa Cloud). If set, must also set VESPA_CLOUD_URL, VESPA_CLOUD_CERT_PATH and VESPA_CLOUD_KEY_PATH
 MANAGED_VESPA = os.environ.get("MANAGED_VESPA", "").lower() == "true"
 

backend/danswer/configs/constants.py

@@ -31,9 +31,6 @@
     "You can still use Danswer as a search engine."
 )
 
-# Prefix used for all tenant ids
-TENANT_ID_PREFIX = "tenant_"
-
 # Postgres connection constants for application_name
 POSTGRES_WEB_APP_NAME = "web"
 POSTGRES_INDEXER_APP_NAME = "indexer"

backend/danswer/db/engine.py

@@ -25,7 +25,6 @@
 
 from danswer.configs.app_configs import LOG_POSTGRES_CONN_COUNTS
 from danswer.configs.app_configs import LOG_POSTGRES_LATENCY
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_OVERFLOW
 from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_SIZE
 from danswer.configs.app_configs import POSTGRES_DB
@@ -37,10 +36,11 @@
 from danswer.configs.app_configs import POSTGRES_USER
 from danswer.configs.app_configs import USER_AUTH_SECRET
 from danswer.configs.constants import POSTGRES_UNKNOWN_APP_NAME
-from danswer.configs.constants import TENANT_ID_PREFIX
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
+from shared_configs.configs import TENANT_ID_PREFIX
 
 logger = setup_logger()
 

backend/danswer/db/swap_index.py

@@ -1,6 +1,5 @@
 from sqlalchemy.orm import Session
 
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.constants import KV_REINDEX_KEY
 from danswer.db.connector_credential_pair import get_connector_credential_pairs
 from danswer.db.connector_credential_pair import resync_cc_pair
@@ -15,6 +14,7 @@
 from danswer.db.search_settings import update_search_settings_status
 from danswer.key_value_store.factory import get_kv_store
 from danswer.utils.logger import setup_logger
+from shared_configs.configs import MULTI_TENANT
 
 
 logger = setup_logger()

backend/danswer/document_index/factory.py

@@ -1,9 +1,9 @@
 from sqlalchemy.orm import Session
 
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.db.search_settings import get_current_search_settings
 from danswer.document_index.interfaces import DocumentIndex
 from danswer.document_index.vespa.index import VespaIndex
+from shared_configs.configs import MULTI_TENANT
 
 
 def get_default_document_index(

backend/danswer/document_index/vespa/index.py

@@ -17,7 +17,6 @@
 import requests  # type: ignore
 
 from danswer.configs.app_configs import DOCUMENT_INDEX_NAME
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.chat_configs import DOC_TIME_DECAY
 from danswer.configs.chat_configs import NUM_RETURNED_HITS
 from danswer.configs.chat_configs import TITLE_CONTENT_RATIO
@@ -73,6 +72,7 @@
 from danswer.search.models import InferenceChunkUncleaned
 from danswer.utils.batching import batch_generator
 from danswer.utils.logger import setup_logger
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.model_server_models import Embedding
 
 

backend/danswer/key_value_store/store.py

@@ -8,7 +8,6 @@
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.engine import is_valid_schema_name
 from danswer.db.models import KVStore
@@ -18,6 +17,7 @@
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 logger = setup_logger()

backend/danswer/main.py

@@ -32,7 +32,6 @@
 from danswer.configs.app_configs import AUTH_TYPE
 from danswer.configs.app_configs import DISABLE_GENERATIVE_AI
 from danswer.configs.app_configs import LOG_ENDPOINT_LATENCY
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import OAUTH_CLIENT_ID
 from danswer.configs.app_configs import OAUTH_CLIENT_SECRET
 from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_OVERFLOW
@@ -94,6 +93,7 @@
 from danswer.utils.variable_functionality import global_version
 from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
 from shared_configs.configs import CORS_ALLOWED_ORIGIN
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import SENTRY_DSN
 
 

backend/danswer/search/preprocessing/preprocessing.py

@@ -1,6 +1,5 @@
 from sqlalchemy.orm import Session
 
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.chat_configs import BASE_RECENCY_DECAY
 from danswer.configs.chat_configs import CONTEXT_CHUNKS_ABOVE
 from danswer.configs.chat_configs import CONTEXT_CHUNKS_BELOW
@@ -31,6 +30,7 @@
 from danswer.utils.threadpool_concurrency import FunctionCall
 from danswer.utils.threadpool_concurrency import run_functions_in_parallel
 from danswer.utils.timing import log_function_time
+from shared_configs.configs import MULTI_TENANT
 
 
 logger = setup_logger()

backend/danswer/server/manage/users.py

@@ -33,7 +33,6 @@
 from danswer.auth.users import optional_user
 from danswer.configs.app_configs import AUTH_TYPE
 from danswer.configs.app_configs import ENABLE_EMAIL_INVITES
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from danswer.configs.app_configs import VALID_EMAIL_DOMAINS
 from danswer.configs.constants import AuthType
@@ -66,6 +65,7 @@
 from ee.danswer.server.tenants.billing import register_tenant_users
 from ee.danswer.server.tenants.provisioning import add_users_to_tenant
 from ee.danswer.server.tenants.provisioning import remove_users_from_tenant
+from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 

backend/danswer/setup.py

@@ -5,7 +5,6 @@
 from danswer.chat.load_yamls import load_chat_yamls
 from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
 from danswer.configs.app_configs import MANAGED_VESPA
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.constants import KV_REINDEX_KEY
 from danswer.configs.constants import KV_SEARCH_SETTINGS
 from danswer.configs.model_configs import FAST_GEN_AI_MODEL_VERSION
@@ -52,6 +51,7 @@
 from shared_configs.configs import ALT_INDEX_SUFFIX
 from shared_configs.configs import MODEL_SERVER_HOST
 from shared_configs.configs import MODEL_SERVER_PORT
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import SUPPORTED_EMBEDDING_MODELS
 from shared_configs.model_server_models import SupportedEmbeddingModel
 

backend/danswer/utils/logger.py

@@ -4,10 +4,14 @@
 from logging.handlers import RotatingFileHandler
 from typing import Any
 
+from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import DEV_LOGGING_ENABLED
 from shared_configs.configs import LOG_FILE_NAME
 from shared_configs.configs import LOG_LEVEL
+from shared_configs.configs import MULTI_TENANT
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.configs import SLACK_CHANNEL_ID
+from shared_configs.configs import TENANT_ID_PREFIX
 
 
 logging.addLevelName(logging.INFO + 5, "NOTICE")
@@ -66,6 +70,18 @@ def process(
         if cc_pair_id is not None:
             msg = f"[CC Pair: {cc_pair_id}] {msg}"
 
+        # Add tenant information if it differs from default
+        # This will always be the case for authenticated API requests
+        if MULTI_TENANT:
+            tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
+            if tenant_id != POSTGRES_DEFAULT_SCHEMA:
+                # Strip tenant_ prefix and take first 8 chars for cleaner logs
+                tenant_display = tenant_id.removeprefix(TENANT_ID_PREFIX)
+                short_tenant = (
+                    tenant_display[:8] if len(tenant_display) > 8 else tenant_display
+                )
+                msg = f"[t:{short_tenant}] {msg}"
+
         # For Slack Bot, logs the channel relevant to the request
         channel_id = self.extra.get(SLACK_CHANNEL_ID) if self.extra else None
         if channel_id:

backend/ee/danswer/background/celery/apps/primary.py

@@ -1,7 +1,6 @@
 from danswer.background.celery.apps.primary import celery_app
 from danswer.background.task_utils import build_celery_task_wrapper
 from danswer.configs.app_configs import JOB_TIMEOUT
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.db.chat import delete_chat_sessions_older_than
 from danswer.db.engine import get_session_with_tenant
 from danswer.server.settings.store import load_settings
@@ -30,6 +29,7 @@
 )
 from ee.danswer.server.reporting.usage_export_generation import create_new_usage_report
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 

backend/ee/danswer/main.py

@@ -5,7 +5,6 @@
 from danswer.auth.users import create_danswer_oauth_router
 from danswer.auth.users import fastapi_users
 from danswer.configs.app_configs import AUTH_TYPE
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import OAUTH_CLIENT_ID
 from danswer.configs.app_configs import OAUTH_CLIENT_SECRET
 from danswer.configs.app_configs import USER_AUTH_SECRET
@@ -43,6 +42,7 @@
 )
 from ee.danswer.server.user_group.api import router as user_group_router
 from ee.danswer.utils.encryption import test_encryption
+from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 

backend/ee/danswer/server/middleware/tenant_tracking.py

@@ -8,10 +8,10 @@
 from fastapi import Request
 from fastapi import Response
 
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import USER_AUTH_SECRET
 from danswer.db.engine import is_valid_schema_name
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 
@@ -21,7 +21,6 @@ async def set_tenant_id(
         request: Request, call_next: Callable[[Request], Awaitable[Response]]
     ) -> Response:
         try:
-            logger.info(f"Request route: {request.url.path}")
             if not MULTI_TENANT:
                 tenant_id = POSTGRES_DEFAULT_SCHEMA
             else:
@@ -53,8 +52,6 @@ async def set_tenant_id(
                     tenant_id = POSTGRES_DEFAULT_SCHEMA
 
             CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
-            logger.info(f"Middleware set current_tenant_id to: {tenant_id}")
-
             response = await call_next(request)
             return response
 

backend/ee/danswer/server/tenants/api.py

@@ -5,7 +5,6 @@
 
 from danswer.auth.users import current_admin_user
 from danswer.auth.users import User
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.notification import create_notification
@@ -25,6 +24,7 @@
 from ee.danswer.server.tenants.provisioning import run_alembic_migrations
 from ee.danswer.server.tenants.provisioning import user_owns_a_tenant
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 
 
 stripe.api_key = STRIPE_SECRET_KEY

backend/shared_configs/configs.py

@@ -129,12 +129,17 @@ def validate_cors_origin(origin: str) -> None:
     CORS_ALLOWED_ORIGIN = ["*"]
 
 
+# Multi-tenancy configuration
+MULTI_TENANT = os.environ.get("MULTI_TENANT", "").lower() == "true"
+
 POSTGRES_DEFAULT_SCHEMA = os.environ.get("POSTGRES_DEFAULT_SCHEMA") or "public"
 
 CURRENT_TENANT_ID_CONTEXTVAR = contextvars.ContextVar(
     "current_tenant_id", default=POSTGRES_DEFAULT_SCHEMA
 )
 
+# Prefix used for all tenant ids
+TENANT_ID_PREFIX = "tenant_"
 
 SUPPORTED_EMBEDDING_MODELS = [
     # Cloud-based models