New Test-Case - Metadata Reading plus some basics image processing

data/samples_o1-mini-2024-09-12_passk_results.csv

@@ -0,0 +1,2 @@
+,pass@1,pass@5,pass@10
+0,0.531578947368421,0.6970899470899472,0.7543859649122807

demo/evaluate_samples.ipynb

@@ -54,7 +54,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "570it [00:00, 11641.82it/s]\n"
+      "570it [00:00, 19356.29it/s]\n"
      ]
     },
     {
@@ -68,21 +68,21 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|████████████████████████████████████████████████████████████████████████████████| 570/570 [01:55<00:00,  4.94it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████| 570/570 [01:49<00:00,  5.21it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Writing results to ../data/samples_gpt-4o-2024-08-06.jsonl_results.jsonl...\n"
+      "Writing results to ../data/samples_o1-mini-2024-09-12.jsonl_results.jsonl...\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████████████████████| 570/570 [00:00<00:00, 20066.92it/s]\n"
+      "100%|█████████████████████████████████████████████████████████████████████████████| 570/570 [00:00<00:00, 24642.37it/s]\n"
      ]
     }
    ],
@@ -124,7 +124,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.14"
+   "version": "3.10.15"
   }
  },
  "nbformat": 4,

demo/inspect_samples.ipynb

@@ -0,0 +1,148 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "cc550e70-608a-47ab-b488-602f424321ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "filename = '../data/samples_deepseek-coder-v2.jsonl_results.jsonl'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "48993e46-ba18-4baf-9f2a-905621bbe88c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5974269914ef446798b36fa87696233b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HBox(children=(VBox(children=(Label(value='task_id'), Label(value='Task ID: ../test_cases/apply…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import json\n",
+    "from ipywidgets import Label, Textarea, Button, HBox, VBox, Layout\n",
+    "from IPython.display import display\n",
+    "\n",
+    "# Load the JSONL file\n",
+    "with open(filename, 'r') as file:\n",
+    "    data = [json.loads(line) for line in file]\n",
+    "\n",
+    "# Initialize index\n",
+    "current_index = 0\n",
+    "\n",
+    "# Create GUI components\n",
+    "label_task_id = Label(layout=Layout(width='50%'))\n",
+    "textarea_result = Textarea(layout=Layout(width='100%', height='200px'))\n",
+    "textarea_completion = Textarea(layout=Layout(width='100%', height='400px'))\n",
+    "textarea_full_response = Textarea(layout=Layout(width='100%', height='400px'))\n",
+    "label_index = Label()\n",
+    "\n",
+    "# Create navigation buttons\n",
+    "button_back_10 = Button(description=\"Jump 10 backward\")\n",
+    "button_previous = Button(description=\"Show previous\")\n",
+    "button_next = Button(description=\"Show next\")\n",
+    "button_forward_10 = Button(description=\"Jump 10 forward\")\n",
+    "\n",
+    "# Define a function to update display\n",
+    "def update_display(index):\n",
+    "    label_task_id.value = f\"Task ID: {data[index]['task_id']}\"\n",
+    "    textarea_completion.value = data[index]['completion']\n",
+    "    textarea_result.value = data[index]['result']\n",
+    "    textarea_full_response.value = data[index]['full_response']\n",
+    "    label_index.value = f\"Index: {index+1}/{len(data)}\"\n",
+    "\n",
+    "# Button functions\n",
+    "def go_back_10(_):\n",
+    "    global current_index\n",
+    "    current_index = max(0, current_index - 10)\n",
+    "    update_display(current_index)\n",
+    "\n",
+    "def show_previous(_):\n",
+    "    global current_index\n",
+    "    current_index = max(0, current_index - 1)\n",
+    "    update_display(current_index)\n",
+    "\n",
+    "def show_next(_):\n",
+    "    global current_index\n",
+    "    current_index = min(len(data) - 1, current_index + 1)\n",
+    "    update_display(current_index)\n",
+    "\n",
+    "def go_forward_10(_):\n",
+    "    global current_index\n",
+    "    current_index = min(len(data) - 1, current_index + 10)\n",
+    "    update_display(current_index)\n",
+    "\n",
+    "# Assign functions to button clicks\n",
+    "button_back_10.on_click(go_back_10)\n",
+    "button_previous.on_click(show_previous)\n",
+    "button_next.on_click(show_next)\n",
+    "button_forward_10.on_click(go_forward_10)\n",
+    "\n",
+    "def Lab(caption, widget):\n",
+    "    return VBox([\n",
+    "        Label(caption),\n",
+    "        widget\n",
+    "    ], layout=Layout(width='100%'))\n",
+    "\n",
+    "# Layout of the GUI\n",
+    "panel_layout = VBox([\n",
+    "    HBox([\n",
+    "        Lab(\"task_id\", label_task_id), \n",
+    "        Lab(\"result\", textarea_result)]),\n",
+    "    HBox([\n",
+    "        Lab(\"completion\", textarea_completion), \n",
+    "        Lab(\"full_response\", textarea_full_response)]),\n",
+    "    HBox([label_index, button_back_10, button_previous, button_next, button_forward_10])\n",
+    "])\n",
+    "\n",
+    "# Display the initial content\n",
+    "update_display(current_index)\n",
+    "\n",
+    "# Render the GUI\n",
+    "display(panel_layout)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1c1208a5-7f20-4f85-be9f-1502060786a4",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

demo/model_order.txt

@@ -1,6 +1,7 @@
 reference
 claude-3-5-sonnet-20240620
 gpt-4o-2024-08-06
+o1-mini-2024-09-12
 gpt-4o-2024-05-13
 gpt-4-turbo-2024-04-09
 claude-3-opus-20240229