From da89357ae4d5d6f42980971ace74a5f1db48a0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20K=C3=B6hnecke?= Date: Wed, 26 Jun 2024 15:06:16 +0200 Subject: [PATCH 1/2] fix: SingleChunkQa crashing on specific prompt ordering --- CHANGELOG.md | 2 ++ src/intelligence_layer/core/prompt_template.py | 16 +++++++++++++--- .../examples/qa/single_chunk_qa.py | 4 +++- tests/core/test_prompt_template.py | 17 +++++++++++------ tests/examples/qa/test_single_chunk_qa.py | 13 +++++++++++++ 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6c619156..77b691666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ ### Fixes - Serialization and Deserialization of `Union[SpanAttributes, TaskSpanAttributes]` is working now. +- `PromptTemplate.to_rich_prompt` now always returns an empty list for prompt ranges that are empty. +- `SingleChunkQa` no longer crashes if given an empty input and a specific prompt template. ### Deprecations ... diff --git a/src/intelligence_layer/core/prompt_template.py b/src/intelligence_layer/core/prompt_template.py index ff426ae93..dcb5355ce 100644 --- a/src/intelligence_layer/core/prompt_template.py +++ b/src/intelligence_layer/core/prompt_template.py @@ -394,9 +394,19 @@ def valid_range_for( yield new_prompt_item(placeholder_prompt_item) else: if range_starts.get(placeholder): - placeholder_ranges[placeholder].extend( - valid_range_for(placeholder, end_cursor()) - ) + test_range = list(valid_range_for(placeholder, end_cursor())) + # the test_range is empty if there was never any text and the range is empty + # However, if there has been text previously, an empty text range will have + # a range consisting of start==end. This check is there to filter these as well. + if len(test_range) == 0 or ( + test_range[0].start.item == test_range[0].end.item + and isinstance(test_range[0].end, TextCursor) + and isinstance(test_range[0].start, TextCursor) + and test_range[0].end.position == test_range[0].start.position + ): + placeholder_ranges[placeholder].extend([]) + else: + placeholder_ranges[placeholder].extend(test_range) else: range_starts[placeholder] = initial_start_text_cursor() last_to = placeholder_to diff --git a/src/intelligence_layer/examples/qa/single_chunk_qa.py b/src/intelligence_layer/examples/qa/single_chunk_qa.py index 296172aac..c52bf9d88 100644 --- a/src/intelligence_layer/examples/qa/single_chunk_qa.py +++ b/src/intelligence_layer/examples/qa/single_chunk_qa.py @@ -93,7 +93,7 @@ class SingleChunkQa(Task[SingleChunkQaInput, SingleChunkQaOutput]): model: The model used throughout the task for model related API calls. text_highlight: The task that is used for highlighting that parts of the input that are relevant for the answer. Defaults to :class:`TextHighlight` . - instruction_config: defines instructions for different langaugaes. + instruction_config: defines instructions for different languages. maximum_token: the maximal number of tokens to be generated for an answer. Attributes: @@ -176,6 +176,8 @@ def _shift_highlight_ranges_to_input( self, prompt: RichPrompt, raw_highlights: Sequence[ScoredTextHighlight] ) -> Sequence[ScoredTextHighlight]: # This only works with models that have an 'input' range, e.g. control models. + if "input" not in prompt.ranges or len(prompt.ranges["input"]) == 0: + return raw_highlights input_cursor = prompt.ranges["input"][0].start assert isinstance(input_cursor, TextCursor) input_offset = input_cursor.position diff --git a/tests/core/test_prompt_template.py b/tests/core/test_prompt_template.py index 4716c9cf6..e8d7d8e32 100644 --- a/tests/core/test_prompt_template.py +++ b/tests/core/test_prompt_template.py @@ -193,7 +193,7 @@ def test_to_prompt_resets_template(prompt_image: Image) -> None: assert prompt_with_reset_template != prompt -def test_to_prompt_data_returns_ranges(prompt_image: Image) -> None: +def test_to_rich_prompt_returns_ranges(prompt_image: Image) -> None: embedded_text = "Embedded" prefix_items: list[PromptItem] = [ Text.from_text("Prefix Text Item"), @@ -229,7 +229,7 @@ def test_to_prompt_data_returns_ranges(prompt_image: Image) -> None: ] -def test_to_prompt_data_returns_ranges_for_image_only_prompt( +def test_to_rich_prompt_returns_ranges_for_image_only_prompt( prompt_image: Image, ) -> None: template = PromptTemplate( @@ -250,13 +250,18 @@ def test_to_prompt_data_returns_ranges_for_image_only_prompt( assert r1 == prompt_data.ranges.get("r2") -def test_to_prompt_data_returns_no_range_with_empty_template() -> None: +def test_to_rich_prompt_returns_no_range_with_empty_template() -> None: template = PromptTemplate("{% promptrange r1 %}{% endpromptrange %}") assert template.to_rich_prompt().ranges.get("r1") == [] -def test_to_prompt_data_returns_no_empty_ranges(prompt_image: Image) -> None: +def test_to_rich_prompt_returns_no_range_with_range_with_other_ranges() -> None: + template = PromptTemplate("""text{% promptrange r1 %}{% endpromptrange %}""") + assert template.to_rich_prompt().ranges.get("r1") == [] + + +def test_to_rich_prompt_returns_no_empty_ranges(prompt_image: Image) -> None: template = PromptTemplate( "{{image}}{% promptrange r1 %}{% endpromptrange %} Some Text" ) @@ -276,7 +281,7 @@ def test_prompt_template_raises_liquid_error_on_illeagal_range() -> None: ) -def test_to_prompt_data_returns_multiple_text_ranges_in_for_loop() -> None: +def test_to_rich_prompt_returns_multiple_text_ranges_in_for_loop() -> None: embedded = "Some Content" template = PromptTemplate( "{% for i in (1..4) %}{% promptrange r1 %}{{embedded}}{% endpromptrange %}{% endfor %}" @@ -293,7 +298,7 @@ def test_to_prompt_data_returns_multiple_text_ranges_in_for_loop() -> None: ] -def test_to_prompt_data_returns_multiple_imgae_ranges_in_for_loop( +def test_to_rich_prompt_returns_multiple_image_ranges_in_for_loop( prompt_image: Image, ) -> None: template = PromptTemplate( diff --git a/tests/examples/qa/test_single_chunk_qa.py b/tests/examples/qa/test_single_chunk_qa.py index b76f10dfe..e15e502da 100644 --- a/tests/examples/qa/test_single_chunk_qa.py +++ b/tests/examples/qa/test_single_chunk_qa.py @@ -119,3 +119,16 @@ def test_qa_highlights_will_not_become_out_of_bounds( for highlight in output.highlights: assert highlight.start >= 0 assert 0 < highlight.end <= len(input_text) + + +def test_qa_single_chunk_does_not_crash_when_input_is_empty( + single_chunk_qa: SingleChunkQa, +) -> None: + input_text = "" + input = SingleChunkQaInput( + chunk=TextChunk(input_text), + question="something", + language=Language("de"), + ) + res = single_chunk_qa.run(input, NoOpTracer()) + assert res.answer is None From 1e56939008b3b69faf09afd003d2d9932923ab5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20K=C3=B6hnecke?= Date: Wed, 26 Jun 2024 15:13:42 +0200 Subject: [PATCH 2/2] docs: update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77b691666..17872318c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,9 @@ ... ### Fixes -- Serialization and Deserialization of `Union[SpanAttributes, TaskSpanAttributes]` is working now. +- Serialization and deserialization of `ExportedSpan` and its `attributes` now works as expected. - `PromptTemplate.to_rich_prompt` now always returns an empty list for prompt ranges that are empty. -- `SingleChunkQa` no longer crashes if given an empty input and a specific prompt template. +- `SingleChunkQa` no longer crashes if given an empty input and a specific prompt template. This did not affect users who used models provided in `core`. ### Deprecations ...