Update

app/config.py

@@ -12,6 +12,6 @@ class InferenceConfig(BaseModel):
         # Task.SUMMARISE: LLMType.OPENAI_GPT3_5,
         # Task.PRACTICE: LLMType.OPENAI_GPT4_TURBO
         # Task.PRACTICE: LLMType.COHERE_COMMAND_R_PLUS
-        Task.PRACTICE: LLMType.OPENAI_GPT3_5
-        # Task.PRACTICE: LLMType.GEMINI_PRO
+        # Task.PRACTICE: LLMType.OPENAI_GPT3_5
+        Task.PRACTICE: LLMType.GEMINI_PRO
     }

app/control/post/examiner.py

@@ -53,6 +53,7 @@ def _verify_expected_similarity_and_difference(question: str, answer: str) -> tu
     q_index = 0
     a_index = 0
 
+    only_comments: bool = True
     # Loop through each line until we run out of lines in question
     while q_index < len(question_lines):
         if "TODO" in question_lines[q_index]:
@@ -67,9 +68,14 @@ def _verify_expected_similarity_and_difference(question: str, answer: str) -> tu
 
             # Check for matching lines strictly after TODO
             while a_index < len(answer_lines) and question_lines[q_index] != answer_lines[a_index]:
+                curr_answer_line: str = answer_lines[a_index].strip()
+                if not (curr_answer_line.startswith("#") or curr_answer_line.startswith("//") or curr_answer_line == ""):
+                    only_comments = False
                 a_index += 1  # Skip non-matching lines in the answer until a match is found
 
             if a_index < len(answer_lines) and question_lines[q_index] == answer_lines[a_index]:
+                if only_comments:
+                    raise ValueError("The user input section contains only comments.")
                 q_index += 1
                 a_index += 1  # Increment both to continue matching
             else:

tests/control/post/test_examiner.py

@@ -1,7 +1,7 @@
 import pytest
 
 from app.control.post.examiner import (_determine_question_and_answer,
-                                       _extract_code, post_process)
+                                       _extract_code, _verify_expected_similarity_and_difference, post_process)
 from app.control.post.summariser import _remove_output_wrapper
 from app.exceptions.exception import LogicError
 from app.llm.model import LLMType
@@ -188,6 +188,40 @@ class ExampleModel(pydantic.BaseModel):
 # Validation testing - Check if the model is valid
 assert model.validate()
 
+# Property testing - Check if the model has the expected properties
+assert model.name == "John"
+assert model.age == 30
+""",
+"""
+import pydantic
+
+class ExampleModel(pydantic.BaseModel):
+    name: str
+    age: int
+
+# Create a model instance
+model = ExampleModel(name="John", age=30)
+
+# Validation testing - Check if the model is valid
+assert model.validate()
+
+# Property testing - Check if the model has the expected properties
+assert model.name == "John"
+# TODO: Add the missing line(s) below.
+""",
+"""
+import pydantic
+
+class ExampleModel(pydantic.BaseModel):
+    name: str
+    age: int
+
+# Create a model instance
+model = ExampleModel(name="John", age=30)
+
+# Validation testing - Check if the model is valid
+assert model.validate()
+
 # Property testing - Check if the model has the expected properties
 assert model.name == "John"
 assert model.age == 30
@@ -224,6 +258,40 @@ class ExampleModel(pydantic.BaseModel):
 # Validation testing - Check if the model is valid
 assert model.validate()
 
+# Property testing - Check if the model has the expected properties
+assert model.name == "John"
+assert model.age == 30
+""",
+"""
+import pydantic
+
+class ExampleModel(pydantic.BaseModel):
+    name: str
+    age: int
+
+# Create a model instance
+model = ExampleModel(name="John", age=30)
+
+# Validation testing - Check if the model is valid
+# TODO: Add the missing line(s) below.
+
+# Property testing - Check if the model has the expected properties
+assert model.name == "John"
+assert model.age == 30
+""",
+"""
+import pydantic
+
+class ExampleModel(pydantic.BaseModel):
+    name: str
+    age: int
+
+# Create a model instance
+model = ExampleModel(name="John", age=30)
+
+# Validation testing - Check if the model is valid
+assert model.validate()
+
 # Property testing - Check if the model has the expected properties
 assert model.name == "John"
 assert model.age == 30
@@ -259,6 +327,132 @@ class ExampleModel(pydantic.BaseModel):
 # Property testing - Check if the model has the expected properties
 assert model.name == "John"
 assert model.age == 30
+""",
+"""
+import pydantic
+
+class ExampleModel(pydantic.BaseModel):
+    name: str
+    age: int
+
+# Create a model instance
+model = ExampleModel(name="John", age=30)
+
+# Validation testing - Check if the model is valid
+# TODO: Add the missing line(s) below.
+""",
+"""
+import pydantic
+
+class ExampleModel(pydantic.BaseModel):
+    name: str
+    age: int
+
+# Create a model instance
+model = ExampleModel(name="John", age=30)
+
+# Validation testing - Check if the model is valid
+assert model.validate()
+
+# Property testing - Check if the model has the expected properties
+assert model.name == "John"
+assert model.age == 30
+"""
+    ),
+    (
+"""
+# Initialize an empty list to store the structured data
+data = []
+
+# Iterate over the data points
+for data_point in data_points:
+    # TODO: Add the missing line(s) below.
+    data.append({'id': data_point['id'], 'value': data_point['value']})
+
+# Construct a DataFrame from the list
+df = pd.DataFrame(data)
+
+""",
+"""
+# Initialize an empty list to store the structured data
+data = []
+
+# Iterate over the data points
+for data_point in data_points:
+    # Append a structured data (e.g., dictionary) to the list
+    do_something_productive()
+    data.append({'id': data_point['id'], 'value': data_point['value']})
+
+# Construct a DataFrame from the list
+df = pd.DataFrame(data)
+
+""",
+"""
+# Initialize an empty list to store the structured data
+data = []
+
+# Iterate over the data points
+for data_point in data_points:
+    # TODO: Add the missing line(s) below.
+    data.append({'id': data_point['id'], 'value': data_point['value']})
+
+# Construct a DataFrame from the list
+df = pd.DataFrame(data)
+
+""",
+"""
+# Initialize an empty list to store the structured data
+data = []
+
+# Iterate over the data points
+for data_point in data_points:
+    # Append a structured data (e.g., dictionary) to the list
+    do_something_productive()
+    data.append({'id': data_point['id'], 'value': data_point['value']})
+
+# Construct a DataFrame from the list
+df = pd.DataFrame(data)
+
 """
     )
-]
+]
+
+@pytest.mark.parametrize("question, answer, expected_returned_question, expected_returned_answer", VERIFY_EXPECTED_SIMILARITY_AND_DIFFERENCE_VALID_DATA)
+def test_valid_verify_expected_similarity_and_difference(question, answer, expected_returned_question, expected_returned_answer):
+    assert _verify_expected_similarity_and_difference(question=question, answer=answer) == (expected_returned_question, expected_returned_answer)
+
+VERIFY_EXPECTED_SIMILARITY_AND_DIFFERENCE_INVALID_DATA = [
+    (
+"""
+# Initialize an empty list to store the structured data
+data = []
+
+# Iterate over the data points
+for data_point in data_points:
+    # TODO: Add the missing line(s) below.
+    data.append({'id': data_point['id'], 'value': data_point['value']})
+
+# Construct a DataFrame from the list
+df = pd.DataFrame(data)
+
+""",
+"""
+# Initialize an empty list to store the structured data
+data = []
+
+# Iterate over the data points
+for data_point in data_points:
+    # Append a structured data (e.g., dictionary) to the list
+    data.append({'id': data_point['id'], 'value': data_point['value']})
+
+# Construct a DataFrame from the list
+df = pd.DataFrame(data)
+
+""",
+    )
+]
+
+@pytest.mark.parametrize("question, answer", VERIFY_EXPECTED_SIMILARITY_AND_DIFFERENCE_INVALID_DATA)
+def test_invalid_verify_expected_similarity_and_difference(question, answer):
+    with pytest.raises(ValueError):
+        _verify_expected_similarity_and_difference(question=question, answer=answer)