demo/evaluate_samples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "599017a5-ebc9-43a1-abd5-60c9f58b30b9",
   "metadata": {},
   "source": [
    "# Evaluate samples\n",
    "This notebook can be used to evaluate samples. \n",
    "\n",
    "Warning: It will execute code snippets generated from LLMs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "726e33d1-047a-46e1-94ea-807f651e3b41",
   "metadata": {},
   "outputs": [],
   "source": [
    "from human_eval.evaluation import evaluate_functional_correctness\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "04d34ad7-a2c1-4341-8a80-5f45d98edbd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "directory = \"../data/\"\n",
    "problem_file = 'human-eval-bia.jsonl'\n",
    "k = [1, 5, 10]\n",
    "timeout = 300\n",
    "n_workers = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "34977768-78de-43b8-a21c-6673674bedef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "570it [00:00, 9892.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running test suites...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 570/570 [02:40<00:00,  3.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing results to ../data/samples_meta-llama-3.1-405b-instruct.jsonl_results.jsonl...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 570/570 [00:00<00:00, 26535.33it/s]\n"
     ]
    }
   ],
   "source": [
    "for filename in os.listdir(directory):\n",
    "    if filename.endswith(\".jsonl\") and \"_results.jsonl\" not in filename and \"human-eval\" not in filename:\n",
    "        sample_file = filename\n",
    "        \n",
    "        results = evaluate_functional_correctness(directory + sample_file, k, n_workers, timeout, directory + problem_file)\n",
    "\n",
    "        results = {k: [v] for k, v in results.items()}\n",
    "        \n",
    "        df = pd.DataFrame(results)\n",
    "        df.to_csv(\"../data/\" + filename.replace(\".jsonl\", \"_passk_results.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3573144e-4b26-41e7-87e6-5cdfd1d574b5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}