refactor(tracer): split large tracer file into multiple specific trac…

…er files

TASK: IL-297

Concepts.md

@@ -31,7 +31,7 @@ class Task(ABC, Generic[Input, Output]):
 
 `Input` and `Output` are normal Python datatypes that can be serialized from and to JSON. For this the Intelligence
 Layer relies on [Pydantic](https://docs.pydantic.dev/). The types that can actually be used are defined in form
-of the type-alias [`PydanticSerializable`](src/intelligence_layer/core/tracer.py#L44).
+of the type-alias [`PydanticSerializable`](src/intelligence_layer/core/tracer/tracer.py#L44).
 
 The second parameter `task_span` is used for [tracing](#Trace) which is described below.
 

run.py

@@ -3,8 +3,7 @@
 from dotenv import load_dotenv
 from fastapi import Depends, FastAPI
 
-from intelligence_layer.core.model import ControlModel, LuminousControlModel
-from intelligence_layer.core.tracer import NoOpTracer
+from intelligence_layer.core import ControlModel, LuminousControlModel, NoOpTracer
 from intelligence_layer.use_cases.classify.classify import (
     ClassifyInput,
     SingleLabelClassifyOutput,

src/intelligence_layer/core/__init__.py

@@ -16,6 +16,7 @@
 from .echo import EchoInput as EchoInput
 from .echo import EchoOutput as EchoOutput
 from .echo import EchoTask as EchoTask
+from .echo import TokenWithLogProb as TokenWithLogProb
 from .instruct import Instruct as Instruct
 from .instruct import InstructInput as InstructInput
 from .intelligence_app import (
@@ -34,27 +35,30 @@
 from .prompt_template import PromptTemplate as PromptTemplate
 from .prompt_template import RichPrompt as RichPrompt
 from .prompt_template import TextCursor as TextCursor
+from .task import MAX_CONCURRENCY as MAX_CONCURRENCY
 from .task import Input as Input
 from .task import Output as Output
 from .task import Task as Task
+from .task import Token as Token
 from .text_highlight import ScoredTextHighlight as ScoredTextHighlight
 from .text_highlight import TextHighlight as TextHighlight
 from .text_highlight import TextHighlightInput as TextHighlightInput
 from .text_highlight import TextHighlightOutput as TextHighlightOutput
-from .tracer import CompositeTracer as CompositeTracer
-from .tracer import FileSpan as FileSpan
-from .tracer import FileTaskSpan as FileTaskSpan
-from .tracer import FileTracer as FileTracer
-from .tracer import InMemorySpan as InMemorySpan
-from .tracer import InMemoryTaskSpan as InMemoryTaskSpan
-from .tracer import InMemoryTracer as InMemoryTracer
-from .tracer import LogEntry as LogEntry
-from .tracer import NoOpTracer as NoOpTracer
-from .tracer import OpenTelemetryTracer as OpenTelemetryTracer
-from .tracer import PydanticSerializable as PydanticSerializable
-from .tracer import Span as Span
-from .tracer import TaskSpan as TaskSpan
-from .tracer import Tracer as Tracer
-from .tracer import utc_now as utc_now
+from .tracer.composite_tracer import CompositeTracer as CompositeTracer
+from .tracer.file_tracer import FileSpan as FileSpan
+from .tracer.file_tracer import FileTaskSpan as FileTaskSpan
+from .tracer.file_tracer import FileTracer as FileTracer
+from .tracer.in_memory_tracer import InMemorySpan as InMemorySpan
+from .tracer.in_memory_tracer import InMemoryTaskSpan as InMemoryTaskSpan
+from .tracer.in_memory_tracer import InMemoryTracer as InMemoryTracer
+from .tracer.open_telemetry_tracer import OpenTelemetryTracer as OpenTelemetryTracer
+from .tracer.tracer import JsonSerializer as JsonSerializer
+from .tracer.tracer import LogEntry as LogEntry
+from .tracer.tracer import NoOpTracer as NoOpTracer
+from .tracer.tracer import PydanticSerializable as PydanticSerializable
+from .tracer.tracer import Span as Span
+from .tracer.tracer import TaskSpan as TaskSpan
+from .tracer.tracer import Tracer as Tracer
+from .tracer.tracer import utc_now as utc_now
 
 __all__ = [symbol for symbol in dir()]

src/intelligence_layer/core/chunk.py

@@ -5,7 +5,7 @@
 
 from intelligence_layer.core.model import ControlModel
 from intelligence_layer.core.task import Task
-from intelligence_layer.core.tracer import TaskSpan
+from intelligence_layer.core.tracer.tracer import TaskSpan
 
 Chunk = NewType("Chunk", str)
 """Segment of a larger text.

src/intelligence_layer/core/detect_language.py

@@ -6,7 +6,7 @@
 from pydantic import BaseModel
 
 from intelligence_layer.core.task import Task
-from intelligence_layer.core.tracer import TaskSpan
+from intelligence_layer.core.tracer.tracer import TaskSpan
 
 
 class LanguageNotSupportedError(ValueError):

src/intelligence_layer/core/echo.py

@@ -7,7 +7,7 @@
 from intelligence_layer.core.model import CompleteInput, ControlModel
 from intelligence_layer.core.prompt_template import PromptTemplate
 from intelligence_layer.core.task import Task, Token
-from intelligence_layer.core.tracer import TaskSpan
+from intelligence_layer.core.tracer.tracer import TaskSpan
 
 LogProb = NewType("LogProb", float)
 

src/intelligence_layer/core/instruct.py

@@ -4,7 +4,7 @@
 
 from intelligence_layer.core.model import CompleteInput, CompleteOutput, ControlModel
 from intelligence_layer.core.task import Task
-from intelligence_layer.core.tracer import TaskSpan
+from intelligence_layer.core.tracer.tracer import TaskSpan
 
 
 class InstructInput(BaseModel):

src/intelligence_layer/core/intelligence_app.py

@@ -7,7 +7,7 @@
 from uvicorn import run
 
 from intelligence_layer.core.task import Input, Output, Task
-from intelligence_layer.core.tracer import NoOpTracer, Tracer
+from intelligence_layer.core.tracer.tracer import NoOpTracer, Tracer
 
 T = TypeVar("T")
 

src/intelligence_layer/core/model.py

@@ -17,7 +17,7 @@
 )
 from intelligence_layer.core.prompt_template import PromptTemplate, RichPrompt
 from intelligence_layer.core.task import Task
-from intelligence_layer.core.tracer import TaskSpan, Tracer
+from intelligence_layer.core.tracer.tracer import TaskSpan, Tracer
 
 
 class CompleteInput(BaseModel, CompletionRequest, frozen=True):

src/intelligence_layer/core/task.py

@@ -4,7 +4,7 @@
 
 from pydantic import BaseModel
 
-from intelligence_layer.core.tracer import PydanticSerializable, TaskSpan, Tracer
+from intelligence_layer.core.tracer.tracer import PydanticSerializable, TaskSpan, Tracer
 
 
 class Token(BaseModel):

src/intelligence_layer/core/text_highlight.py

@@ -17,7 +17,7 @@
     TextCursor,
 )
 from intelligence_layer.core.task import Task
-from intelligence_layer.core.tracer import TaskSpan
+from intelligence_layer.core.tracer.tracer import TaskSpan
 
 
 class TextHighlightInput(BaseModel):