Text Extraction Eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04bba52c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import os\n",
    "import random\n",
    "import time\n",
    "import typing\n",
    "\n",
    "import weave\n",
    "\n",
    "weave.use_frontend_devmode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8172d67c",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_labels = {\n",
    "    \"Articles_of_Incorporation_Real_Example_3_txt\": {\n",
    "        \"name\": \"HealthFirst Solutions LLC\",\n",
    "        \"shares\": 500000,\n",
    "    },\n",
    "    \"Articles_of_Incorporation_Real_Example_2_txt\": {\n",
    "        \"name\": \"GreenLeaf LLC\",\n",
    "        \"shares\": None,\n",
    "        \"directors\": [\"Sarah Miller\", \"Daniel Lee\"],\n",
    "    },\n",
    "    \"Articles_of_Incorporation_Real_Example_1_txt\": {\n",
    "        \"name\": \"TechBoost Corp\",\n",
    "        \"shares\": 1000000,\n",
    "    },\n",
    "    \"Highly_Varied_Article_of_Incorporation_10_txt\": {\n",
    "        \"name\": \"Brown, Fernandez and Smith\",\n",
    "        \"shares\": 41141,\n",
    "    },\n",
    "    \"Highly_Varied_Article_of_Incorporation_9_txt\": {\n",
    "        \"name\": \"Ruiz-Goodman\",\n",
    "        \"shares\": 31783,\n",
    "    },\n",
    "    \"Highly_Varied_Article_of_Incorporation_8_txt\": {\n",
    "        \"name\": \"Gibson, Hunt and Davidson\",\n",
    "        \"shares\": 96403,\n",
    "    },\n",
    "    \"Highly_Varied_Article_of_Incorporation_7_txt\": {\n",
    "        \"name\": \"Boyd-Browning\",\n",
    "        \"shares\": 41300,\n",
    "    },\n",
    "    \"Highly_Varied_Article_of_Incorporation_6_txt\": {\n",
    "        \"name\": \"Newton, Moreno and Yang\",\n",
    "        \"shares\": 73981,\n",
    "    },\n",
    "    \"Highly_Varied_Article_of_Incorporation_5_txt\": {\n",
    "        \"name\": \"Matthews and Sons\",\n",
    "        \"shares\": 98608,\n",
    "    },\n",
    "    \"Highly_Varied_Article_of_Incorporation_4_txt\": {\n",
    "        \"name\": \"Moore LLC\",\n",
    "        \"shares\": 5732,\n",
    "    },\n",
    "    \"Highly_Varied_Article_of_Incorporation_3_txt\": {\n",
    "        \"name\": \"Mullen Inc\",\n",
    "        \"shares\": 76197,\n",
    "    },\n",
    "    \"Highly_Varied_Article_of_Incorporation_2_txt\": {\n",
    "        \"name\": \"Ellis and Sons\",\n",
    "        \"shares\": 54183,\n",
    "    },\n",
    "    \"Highly_Varied_Article_of_Incorporation_1_txt\": {\n",
    "        \"name\": \"French, Wyatt and Coleman\",\n",
    "        \"shares\": 78821,\n",
    "    },\n",
    "}\n",
    "\n",
    "\n",
    "def read_dataset(root):\n",
    "    dataset_rows = []\n",
    "    for p in glob.glob(os.path.join(root, \"*.txt\")):\n",
    "        # Have to do replace here because of weave '.' access issues\n",
    "        example_id = os.path.basename(p).replace(\".\", \"_\")\n",
    "        label = raw_labels.get(example_id)\n",
    "        if label:\n",
    "            dataset_rows.append(\n",
    "                {\"id\": example_id, \"example\": open(p).read(), \"label\": label}\n",
    "            )\n",
    "    return dataset_rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb93f2a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Can't just make our own types, server won't deserialize.\n",
    "# A fairly easy fix.\n",
    "@weave.type()\n",
    "class Dataset:\n",
    "    rows: list[typing.Any]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92c6535c",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_dataset = read_dataset(\"/Users/shawn/datasets/aoi\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e803efd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# raw_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc723706",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "dataset = weave.save(Dataset(raw_dataset), \"my_dataset5\")\n",
    "# Now, here I really want to make my own labels in the UI immediately.\n",
    "# where should the added column go? A new version of this dataset?\n",
    "# yeah sure why not.\n",
    "# What's missing for editing to be good?\n",
    "#   - batch editing, ie make a bunch of changes and choose where/how to save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a99df38",
   "metadata": {},
   "outputs": [],
   "source": [
    "# published = weave.publish(Dataset(raw_dataset), 'weave-flow1/my_dataset1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77c71a23",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.rows\n",
    "# Here i went to render labels next to dataset.\n",
    "# I need access to labels in the notebook memory... would be easy enough to pass in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "331d88d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "\n",
    "def split_paragraphs(doc):\n",
    "    lines = [l.strip() for l in doc.split(\"\\n\")]\n",
    "    stripped_doc = \"\\n\".join(lines)\n",
    "    return [p.strip() for p in stripped_doc.split(\"\\n\\n\")]\n",
    "\n",
    "\n",
    "def find_first_numeric(s):\n",
    "    match = re.search(r\"\\d+\", s)\n",
    "    if match is None:\n",
    "        return None\n",
    "    return int(match.group().replace(\",\", \"\"))\n",
    "\n",
    "\n",
    "def predict(dataset_row, config):\n",
    "    paragraphs = split_paragraphs(dataset_row[\"example\"])\n",
    "    capital_paragraph = None\n",
    "    name_paragraph = None\n",
    "    for p in paragraphs:\n",
    "        if \"name\" in p.lower():\n",
    "            name_paragraph = p\n",
    "        if \"share\" in p.lower():\n",
    "            capital_paragraph = p\n",
    "    result = {\n",
    "        \"name\": None,\n",
    "        \"shares\": None,\n",
    "    }\n",
    "    if capital_paragraph:\n",
    "        paragraph_start = config.get(\"shares_skip_chars\", 0)\n",
    "        result[\"shares\"] = find_first_numeric(capital_paragraph[paragraph_start:])\n",
    "    if name_paragraph:\n",
    "        match = re.search(r\"is \", name_paragraph)\n",
    "        if match is not None:\n",
    "            result[\"name\"] = name_paragraph[match.end() :]\n",
    "        if result[\"name\"] and config.get(\"name_up_to_period\"):\n",
    "            match = re.search(r\"\\.\", result[\"name\"])\n",
    "            if match is not None:\n",
    "                result[\"name\"] = result[\"name\"][: match.start()]\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2378174",
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset_row in weave.use(dataset.rows):\n",
    "    print(predict(dataset_row, {\"shares_skip_chars\": 4}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efc63aa7",
   "metadata": {},
   "outputs": [],
   "source": [
    "fields = [\"name\", \"shares\", \"directors\"]\n",
    "\n",
    "\n",
    "def p_r_f1(tp, fp, fn):\n",
    "    # if any denom is zero, then zero. could use NaN instead...\n",
    "    precision = 0\n",
    "    if tp or fp:\n",
    "        precision = tp / (tp + fp)\n",
    "    recall = 0\n",
    "    if tp or fn:\n",
    "        recall = tp / (tp + fn)\n",
    "    f1 = 0\n",
    "    if precision or recall:\n",
    "        f1 = 2 * (precision * recall) / (precision + recall)\n",
    "    return precision, recall, f1\n",
    "\n",
    "\n",
    "def summarize_item(item_result, item_label):\n",
    "    item_summary = {}\n",
    "    for f in fields:\n",
    "        item_summary[f\"{f}_negative\"] = item_result.get(f) is None\n",
    "        item_summary[f\"{f}_correct\"] = item_result.get(f) == item_label.get(f)\n",
    "\n",
    "    item_correct = sum([item_summary[f\"{f}_correct\"] for f in fields])\n",
    "    item_tp = sum(\n",
    "        [item_label.get(f) is not None and item_summary[f\"{f}_correct\"] for f in fields]\n",
    "    )\n",
    "    item_fp = sum(\n",
    "        [\n",
    "            item_label.get(f) is not None and not item_summary[f\"{f}_correct\"]\n",
    "            for f in fields\n",
    "        ]\n",
    "    )\n",
    "    item_tn = sum(\n",
    "        [item_label.get(f) is None and item_summary[f\"{f}_correct\"] for f in fields]\n",
    "    )\n",
    "    item_fn = sum(\n",
    "        [item_label.get(f) is None and not item_summary[f\"{f}_correct\"] for f in fields]\n",
    "    )\n",
    "\n",
    "    item_precision, item_recall, item_f1 = p_r_f1(item_tp, item_fp, item_fn)\n",
    "\n",
    "    return {\n",
    "        **item_summary,\n",
    "        \"correct\": item_correct,\n",
    "        \"tp\": item_tp,\n",
    "        \"fp\": item_fp,\n",
    "        \"tn\": item_tn,\n",
    "        \"fn\": item_fn,\n",
    "        \"precision\": item_precision,\n",
    "        \"recall\": item_recall,\n",
    "        \"f1\": item_f1,\n",
    "    }\n",
    "\n",
    "\n",
    "def field_pr(eval_result, field_name):\n",
    "    tp = sum(\n",
    "        not item[\"summary\"][f\"{field_name}_negative\"]\n",
    "        and item[\"summary\"][f\"{field_name}_correct\"]\n",
    "        for item in eval_result\n",
    "    )\n",
    "    fp = sum(\n",
    "        not item[\"summary\"][f\"{field_name}_negative\"]\n",
    "        and not item[\"summary\"][f\"{field_name}_correct\"]\n",
    "        for item in eval_result\n",
    "    )\n",
    "    tn = sum(\n",
    "        item[\"summary\"][f\"{field_name}_negative\"]\n",
    "        and item[\"summary\"][f\"{field_name}_correct\"]\n",
    "        for item in eval_result\n",
    "    )\n",
    "    fn = sum(\n",
    "        item[\"summary\"][f\"{field_name}_negative\"]\n",
    "        and not item[\"summary\"][f\"{field_name}_correct\"]\n",
    "        for item in eval_result\n",
    "    )\n",
    "\n",
    "    precision, recall, f1 = p_r_f1(tp, fp, fn)\n",
    "    return {\n",
    "        \"tp\": tp,\n",
    "        \"fp\": fp,\n",
    "        \"tn\": tn,\n",
    "        \"fn\": fn,\n",
    "        \"precision\": precision,\n",
    "        \"recall\": recall,\n",
    "        \"f1\": f1,\n",
    "    }\n",
    "\n",
    "\n",
    "def summarize(eval_result):\n",
    "    summary = {}\n",
    "    for field in fields:\n",
    "        summary[f\"field_{field}\"] = field_pr(eval_result, field)\n",
    "    for metric in [\"precision\", \"recall\", \"f1\"]:\n",
    "        summary[f\"avg_{metric}\"] = sum(\n",
    "            summary[f\"field_{f}\"][metric] for f in fields\n",
    "        ) / len(fields)\n",
    "    return summary\n",
    "\n",
    "\n",
    "def evaluate(dataset, predict_config):\n",
    "    eval_result = []\n",
    "    correct_count = 0\n",
    "    count = 0\n",
    "    for dataset_row in dataset:\n",
    "        start_time = time.time()\n",
    "        result = predict(dataset_row, predict_config)\n",
    "        latency = time.time() - start_time\n",
    "        latency = random.gauss(\n",
    "            predict_config[\"latency_mu\"], predict_config[\"latency_sigma\"]\n",
    "        )\n",
    "        item_summary = summarize_item(result, dataset_row[\"label\"])\n",
    "        eval_result.append(\n",
    "            {\n",
    "                \"dataset_id\": dataset_row[\"id\"],\n",
    "                \"result\": result,\n",
    "                \"summary\": {\"latency\": latency, **item_summary},\n",
    "            }\n",
    "        )\n",
    "    return {\n",
    "        \"config\": predict_config,\n",
    "        \"eval_table\": eval_result,\n",
    "        \"summary\": summarize(eval_result),\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2248364",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_results0 = weave.save(\n",
    "    evaluate(weave.use(dataset).rows, {\"latency_mu\": 0.3, \"latency_sigma\": 0.1}),\n",
    "    \"eval_results0\",\n",
    ")\n",
    "eval_results1 = weave.save(\n",
    "    evaluate(\n",
    "        weave.use(dataset).rows,\n",
    "        {\"latency_mu\": 0.3, \"latency_sigma\": 0.1, \"name_up_to_period\": True},\n",
    "    ),\n",
    "    \"eval_results1\",\n",
    ")\n",
    "eval_results2 = weave.save(\n",
    "    evaluate(\n",
    "        weave.use(dataset).rows,\n",
    "        {\n",
    "            \"latency_mu\": 0.3,\n",
    "            \"latency_sigma\": 0.1,\n",
    "            \"name_up_to_period\": True,\n",
    "            \"shares_skip_chars\": 4,\n",
    "        },\n",
    "    ),\n",
    "    \"eval_results2\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93727275",
   "metadata": {},
   "outputs": [],
   "source": [
    "from weave.legacy.panels_py import panel_eval\n",
    "\n",
    "panel_eval.eval_board(dataset.rows, eval_results0, eval_results2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "166e8607",
   "metadata": {},
   "outputs": [],
   "source": [
    "# To add:\n",
    "#   - backed by W&B runs\n",
    "#   - show run code / config comparison\n",
    "#   - show traces of pipelines and compare them\n",
    "#   - add view of N runs instead of 2"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}