diff --git a/src/__init__.py b/src/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/examples/evaluation.ipynb b/src/examples/evaluation.ipynb index 8daf3abd3..55295fa6a 100644 --- a/src/examples/evaluation.ipynb +++ b/src/examples/evaluation.ipynb @@ -44,7 +44,7 @@ "from dotenv import load_dotenv\n", "\n", "from intelligence_layer.connectors import LimitedConcurrencyClient\n", - "from intelligence_layer.core import InMemoryEvaluationRepository, InMemoryDatasetRepository, Runner\n", + "from intelligence_layer.evaluation import InMemoryEvaluationRepository, InMemoryDatasetRepository, Runner\n", "from intelligence_layer.use_cases import SingleLabelClassifyEvaluator, PromptBasedClassify\n", "\n", "load_dotenv()\n", @@ -74,7 +74,7 @@ "source": [ "from intelligence_layer.core import Chunk, NoOpTracer\n", "from intelligence_layer.use_cases import ClassifyInput\n", - "from intelligence_layer.core import Example\n", + "from intelligence_layer.evaluation import Example\n", "\n", "\n", "classify_input = ClassifyInput(\n", diff --git a/src/examples/human_evaluation.ipynb b/src/examples/human_evaluation.ipynb index 5ce7da186..13be43884 100644 --- a/src/examples/human_evaluation.ipynb +++ b/src/examples/human_evaluation.ipynb @@ -44,16 +44,9 @@ "source": [ "from dotenv import load_dotenv\n", "from intelligence_layer.core import (\n", - " ArgillaEvaluator, \n", - " ArgillaEvaluationRepository, \n", - " Example, \n", " InstructInput, \n", " Instruct, \n", - " InMemoryDatasetRepository, \n", - " InMemoryEvaluationRepository, \n", - " PromptOutput, \n", - " Runner,\n", - " SuccessfulExampleOutput\n", + " PromptOutput\n", ")\n", "from intelligence_layer.connectors import (\n", " LimitedConcurrencyClient, \n", @@ -63,6 +56,15 @@ " Field, \n", " RecordData\n", ")\n", + "from intelligence_layer.evaluation import (\n", + " ArgillaEvaluator,\n", + " ArgillaEvaluationRepository,\n", + " Example,\n", + " InMemoryDatasetRepository,\n", + " InMemoryEvaluationRepository,\n", + " Runner,\n", + " SuccessfulExampleOutput\n", + ")\n", "from typing import Iterable, cast, Sequence\n", "from datasets import load_dataset\n", "import os\n", diff --git a/src/examples/qa.ipynb b/src/examples/qa.ipynb index 83ed7e4df..ace60726a 100644 --- a/src/examples/qa.ipynb +++ b/src/examples/qa.ipynb @@ -19,7 +19,10 @@ "outputs": [], "source": [ "from os import getenv\n", + "from dotenv import load_dotenv\n", "\n", + "\n", + "load_dotenv()\n", "from intelligence_layer.connectors import LimitedConcurrencyClient\n", "\n", "client = LimitedConcurrencyClient.from_token(getenv(\"AA_TOKEN\"))" diff --git a/src/examples/quickstart_task.ipynb b/src/examples/quickstart_task.ipynb index 90b315093..ab9e44fdb 100644 --- a/src/examples/quickstart_task.ipynb +++ b/src/examples/quickstart_task.ipynb @@ -272,13 +272,14 @@ "source": [ "from statistics import mean\n", "from typing import Iterable\n", - "from intelligence_layer.core import (\n", + "from intelligence_layer.evaluation import (\n", " EvaluationRepository,\n", " Evaluator,\n", " DatasetRepository\n", ")\n", "\n", "\n", + "\n", "class KeywordExtractionEvaluator(\n", " Evaluator[\n", " KeywordExtractionInput,\n", @@ -329,13 +330,19 @@ "Let's run this on a single example." ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "from intelligence_layer.core import InMemoryDatasetRepository, InMemoryEvaluationRepository, Runner, Example, NoOpTracer\n", + "from intelligence_layer.core import NoOpTracer\n", + "from intelligence_layer.evaluation import InMemoryDatasetRepository, InMemoryEvaluationRepository, Runner, Example\n", "\n", "evaluation_repository = InMemoryEvaluationRepository()\n", "dataset_repository = InMemoryDatasetRepository()\n", diff --git a/src/examples/summarize.ipynb b/src/examples/summarize.ipynb index 71ff62bae..ba1d32950 100644 --- a/src/examples/summarize.ipynb +++ b/src/examples/summarize.ipynb @@ -20,6 +20,9 @@ "outputs": [], "source": [ "from os import getenv\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv()\n", "\n", "from intelligence_layer.connectors import LimitedConcurrencyClient\n", "\n", diff --git a/src/intelligence_layer/connectors/retrievers/base_retriever.py b/src/intelligence_layer/connectors/retrievers/base_retriever.py index ba3b8f179..6775d17ae 100644 --- a/src/intelligence_layer/connectors/retrievers/base_retriever.py +++ b/src/intelligence_layer/connectors/retrievers/base_retriever.py @@ -3,8 +3,6 @@ from pydantic import BaseModel -from intelligence_layer.core.chunk import Chunk - class Document(BaseModel): """A document. @@ -22,11 +20,11 @@ class DocumentChunk(BaseModel): """Part of a :class:`Document`, specifically for retrieval use cases. Attributes: - chunk: Chunk of the document that matched the search query. + text: Chunk of the document that matched the search query. metadata: Any metadata added to the document. """ - text: Chunk + text: str metadata: Any = None @@ -37,9 +35,10 @@ class SearchResult(BaseModel, Generic[ID]): """Contains a text alongside its search score. Attributes: + id: Unique identifier of the document score: The similarity score between the text and the query that was searched with. Will be between 0 and 1, where 0 means no similarity and 1 perfect similarity. - document: The document found by search. + document_chunk: The document chunk found by search. """ id: ID diff --git a/src/intelligence_layer/connectors/retrievers/document_index_retriever.py b/src/intelligence_layer/connectors/retrievers/document_index_retriever.py index 66de5a0ee..55f0cd3b9 100644 --- a/src/intelligence_layer/connectors/retrievers/document_index_retriever.py +++ b/src/intelligence_layer/connectors/retrievers/document_index_retriever.py @@ -12,7 +12,6 @@ DocumentChunk, SearchResult, ) -from intelligence_layer.core.chunk import Chunk class DocumentIndexRetriever(BaseRetriever[DocumentPath]): @@ -65,7 +64,7 @@ def get_relevant_documents_with_scores( id=result.document_path, score=result.score, document_chunk=DocumentChunk( - text=Chunk(result.section), + text=result.section, ), ) for result in response diff --git a/src/intelligence_layer/evaluation/run.py b/src/intelligence_layer/evaluation/run.py index 363c64cee..e9c97ae3e 100644 --- a/src/intelligence_layer/evaluation/run.py +++ b/src/intelligence_layer/evaluation/run.py @@ -9,8 +9,8 @@ from intelligence_layer.connectors.limited_concurrency_client import ( LimitedConcurrencyClient, ) -from intelligence_layer.evaluation.evaluation_repository import FileEvaluationRepository from intelligence_layer.evaluation.dataset_repository import FileDatasetRepository +from intelligence_layer.evaluation.evaluation_repository import FileEvaluationRepository from intelligence_layer.evaluation.runner import Runner diff --git a/src/intelligence_layer/use_cases/qa/retriever_based_qa.py b/src/intelligence_layer/use_cases/qa/retriever_based_qa.py index d4dc8dd55..a2d1ce07c 100644 --- a/src/intelligence_layer/use_cases/qa/retriever_based_qa.py +++ b/src/intelligence_layer/use_cases/qa/retriever_based_qa.py @@ -3,9 +3,7 @@ from pydantic import BaseModel from intelligence_layer.connectors.retrievers.base_retriever import ID, BaseRetriever -from intelligence_layer.core.detect_language import Language -from intelligence_layer.core.task import Task -from intelligence_layer.core.tracer import TaskSpan +from intelligence_layer.core import Chunk, Language, Task, TaskSpan from intelligence_layer.use_cases.qa.multiple_chunk_qa import Subanswer from intelligence_layer.use_cases.qa.single_chunk_qa import ( SingleChunkQaInput, @@ -65,11 +63,8 @@ class RetrieverBasedQa( `model` provided should be a control-type model. Args: - client: Aleph Alpha client instance for running model related API calls. retriever: Used to access and return a set of texts. - model: A valid Aleph Alpha model name. - allowed_languages: List of languages to which the language detection is limited (ISO619). - fallback_language: The default language of the output. + qa_task: Any single chunk QA task. Example: >>> import os @@ -112,7 +107,7 @@ def do_run( sorted_qa_inputs = [ SingleChunkQaInput( - chunk=output.document_chunk.text, + chunk=Chunk(output.document_chunk.text), question=input.question, language=input.language, ) @@ -124,7 +119,7 @@ def do_run( enriched_answers = [ EnrichedSubanswer( answer=answer.answer, - chunk=input.document_chunk.text, + chunk=Chunk(input.document_chunk.text), highlights=answer.highlights, id=input.id, )