IL-262 Finalize separation of core and evaluation package

Aleph-Alpha · Feb 15, 2024 · 16c9a19 · 16c9a19
1 parent 22a4b0a
commit 16c9a19
Show file tree

Hide file tree

Showing 10 changed files with 37 additions and 29 deletions.
diff --git a/src/__init__.py b/src/__init__.py
diff --git a/src/examples/evaluation.ipynb b/src/examples/evaluation.ipynb
@@ -44,7 +44,7 @@
     "from dotenv import load_dotenv\n",
     "\n",
     "from intelligence_layer.connectors import LimitedConcurrencyClient\n",
-    "from intelligence_layer.core import InMemoryEvaluationRepository, InMemoryDatasetRepository, Runner\n",
+    "from intelligence_layer.evaluation import InMemoryEvaluationRepository, InMemoryDatasetRepository, Runner\n",
     "from intelligence_layer.use_cases import SingleLabelClassifyEvaluator, PromptBasedClassify\n",
     "\n",
     "load_dotenv()\n",
@@ -74,7 +74,7 @@
    "source": [
     "from intelligence_layer.core import Chunk, NoOpTracer\n",
     "from intelligence_layer.use_cases import ClassifyInput\n",
-    "from intelligence_layer.core import Example\n",
+    "from intelligence_layer.evaluation import Example\n",
     "\n",
     "\n",
     "classify_input = ClassifyInput(\n",

diff --git a/src/examples/human_evaluation.ipynb b/src/examples/human_evaluation.ipynb
@@ -44,16 +44,9 @@
    "source": [
     "from dotenv import load_dotenv\n",
     "from intelligence_layer.core import (\n",
-    "    ArgillaEvaluator, \n",
-    "    ArgillaEvaluationRepository, \n",
-    "    Example, \n",
     "    InstructInput, \n",
     "    Instruct, \n",
-    "    InMemoryDatasetRepository, \n",
-    "    InMemoryEvaluationRepository, \n",
-    "    PromptOutput, \n",
-    "    Runner,\n",
-    "    SuccessfulExampleOutput\n",
+    "    PromptOutput\n",
     ")\n",
     "from intelligence_layer.connectors import (\n",
     "    LimitedConcurrencyClient, \n",
@@ -63,6 +56,15 @@
     "    Field, \n",
     "    RecordData\n",
     ")\n",
+    "from intelligence_layer.evaluation import (\n",
+    "    ArgillaEvaluator,\n",
+    "    ArgillaEvaluationRepository,\n",
+    "    Example,\n",
+    "    InMemoryDatasetRepository,\n",
+    "    InMemoryEvaluationRepository,\n",
+    "    Runner,\n",
+    "    SuccessfulExampleOutput\n",
+    ")\n",
     "from typing import Iterable, cast, Sequence\n",
     "from datasets import load_dataset\n",
     "import os\n",

diff --git a/src/examples/qa.ipynb b/src/examples/qa.ipynb
@@ -19,7 +19,10 @@
    "outputs": [],
    "source": [
     "from os import getenv\n",
+    "from dotenv import load_dotenv\n",
     "\n",
+    "\n",
+    "load_dotenv()\n",
     "from intelligence_layer.connectors import LimitedConcurrencyClient\n",
     "\n",
     "client = LimitedConcurrencyClient.from_token(getenv(\"AA_TOKEN\"))"

diff --git a/src/examples/quickstart_task.ipynb b/src/examples/quickstart_task.ipynb
@@ -272,13 +272,14 @@
    "source": [
     "from statistics import mean\n",
     "from typing import Iterable\n",
-    "from intelligence_layer.core import (\n",
+    "from intelligence_layer.evaluation import (\n",
     "    EvaluationRepository,\n",
     "    Evaluator,\n",
     "    DatasetRepository\n",
     ")\n",
     "\n",
     "\n",
+    "\n",
     "class KeywordExtractionEvaluator(\n",
     "    Evaluator[\n",
     "        KeywordExtractionInput,\n",
@@ -329,13 +330,19 @@
     "Let's run this on a single example."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from intelligence_layer.core import InMemoryDatasetRepository, InMemoryEvaluationRepository, Runner, Example, NoOpTracer\n",
+    "from intelligence_layer.core import NoOpTracer\n",
+    "from intelligence_layer.evaluation import InMemoryDatasetRepository, InMemoryEvaluationRepository, Runner, Example\n",
     "\n",
     "evaluation_repository = InMemoryEvaluationRepository()\n",
     "dataset_repository = InMemoryDatasetRepository()\n",

diff --git a/src/examples/summarize.ipynb b/src/examples/summarize.ipynb
@@ -20,6 +20,9 @@
    "outputs": [],
    "source": [
     "from os import getenv\n",
+    "from dotenv import load_dotenv\n",
+    "\n",
+    "load_dotenv()\n",
     "\n",
     "from intelligence_layer.connectors import LimitedConcurrencyClient\n",
     "\n",

diff --git a/src/intelligence_layer/connectors/retrievers/base_retriever.py b/src/intelligence_layer/connectors/retrievers/base_retriever.py
@@ -3,8 +3,6 @@
 
 from pydantic import BaseModel
 
-from intelligence_layer.core.chunk import Chunk
-
 
 class Document(BaseModel):
     """A document.
@@ -22,11 +20,11 @@ class DocumentChunk(BaseModel):
     """Part of a :class:`Document`, specifically for retrieval use cases.
 
     Attributes:
-        chunk: Chunk of the document that matched the search query.
+        text: Chunk of the document that matched the search query.
         metadata: Any metadata added to the document.
     """
 
-    text: Chunk
+    text: str
     metadata: Any = None
 
 
@@ -37,9 +35,10 @@ class SearchResult(BaseModel, Generic[ID]):
     """Contains a text alongside its search score.
 
     Attributes:
+        id: Unique identifier of the document
         score: The similarity score between the text and the query that was searched with.
             Will be between 0 and 1, where 0 means no similarity and 1 perfect similarity.
-        document: The document found by search.
+        document_chunk: The document chunk found by search.
     """
 
     id: ID

diff --git a/src/intelligence_layer/connectors/retrievers/document_index_retriever.py b/src/intelligence_layer/connectors/retrievers/document_index_retriever.py
@@ -12,7 +12,6 @@
     DocumentChunk,
     SearchResult,
 )
-from intelligence_layer.core.chunk import Chunk
 
 
 class DocumentIndexRetriever(BaseRetriever[DocumentPath]):
@@ -65,7 +64,7 @@ def get_relevant_documents_with_scores(
                 id=result.document_path,
                 score=result.score,
                 document_chunk=DocumentChunk(
-                    text=Chunk(result.section),
+                    text=result.section,
                 ),
             )
             for result in response

diff --git a/src/intelligence_layer/evaluation/run.py b/src/intelligence_layer/evaluation/run.py
@@ -9,8 +9,8 @@
 from intelligence_layer.connectors.limited_concurrency_client import (
     LimitedConcurrencyClient,
 )
-from intelligence_layer.evaluation.evaluation_repository import FileEvaluationRepository
 from intelligence_layer.evaluation.dataset_repository import FileDatasetRepository
+from intelligence_layer.evaluation.evaluation_repository import FileEvaluationRepository
 from intelligence_layer.evaluation.runner import Runner
 
 

diff --git a/src/intelligence_layer/use_cases/qa/retriever_based_qa.py b/src/intelligence_layer/use_cases/qa/retriever_based_qa.py
@@ -3,9 +3,7 @@
 from pydantic import BaseModel
 
 from intelligence_layer.connectors.retrievers.base_retriever import ID, BaseRetriever
-from intelligence_layer.core.detect_language import Language
-from intelligence_layer.core.task import Task
-from intelligence_layer.core.tracer import TaskSpan
+from intelligence_layer.core import Chunk, Language, Task, TaskSpan
 from intelligence_layer.use_cases.qa.multiple_chunk_qa import Subanswer
 from intelligence_layer.use_cases.qa.single_chunk_qa import (
     SingleChunkQaInput,
@@ -65,11 +63,8 @@ class RetrieverBasedQa(
         `model` provided should be a control-type model.
 
     Args:
-        client: Aleph Alpha client instance for running model related API calls.
         retriever: Used to access and return a set of texts.
-        model: A valid Aleph Alpha model name.
-        allowed_languages: List of languages to which the language detection is limited (ISO619).
-        fallback_language: The default language of the output.
+        qa_task: Any single chunk QA task.
 
     Example:
         >>> import os
@@ -112,7 +107,7 @@ def do_run(
 
         sorted_qa_inputs = [
             SingleChunkQaInput(
-                chunk=output.document_chunk.text,
+                chunk=Chunk(output.document_chunk.text),
                 question=input.question,
                 language=input.language,
             )
@@ -124,7 +119,7 @@ def do_run(
         enriched_answers = [
             EnrichedSubanswer(
                 answer=answer.answer,
-                chunk=input.document_chunk.text,
+                chunk=Chunk(input.document_chunk.text),
                 highlights=answer.highlights,
                 id=input.id,
             )