plot_image_and_tables.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process z-score, eval score, Generate Tables and Images."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before running this notebook, please run the following scripts:\n",
    "``` bash   \n",
    "    python -m process.process_z --model llama2-7b-chat-4k\n",
    "    python -m process.process_z --model internlm-7b-8k\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import json\n",
    "import re\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "# Record the eval score for different models, modes and missions\n",
    "df_eval = pd.DataFrame(columns=[\"model_name\", \"mode\", \"mission_name\", \"score\"])\n",
    "# Record the TP score for different models, modes and missions\n",
    "df_tp = pd.DataFrame(columns=[\"model_name\", \"mode\", \"mission_name\", \"z_score\", \"true_positive\", \"sum\"])\n",
    "# Record the TN score for different models, modes and missions\n",
    "df_tn = pd.DataFrame(columns=[\"model_name\", \"mode\", \"mission_name\", \"true_negative\"])\n",
    "tn_dict = [{} for i in range(5)]\n",
    "eval_dict = [{} for i in range(5)]\n",
    "\n",
    "save_csv_dir = \"csv_data\"\n",
    "\n",
    "if not os.path.exists(save_csv_dir):\n",
    "    os.makedirs(save_csv_dir)\n",
    "    \n",
    "save_pic_dir = \"pictures\"\n",
    "\n",
    "if not os.path.exists(save_pic_dir):\n",
    "    os.makedirs(save_pic_dir)\n",
    "\n",
    "input_dir = \"./pred\"\n",
    "\n",
    "flag = 1 # 0: internlm 0.95TP 1: llama 0.95TP 2: llama 0.7TP\n",
    "model_names = [\"internlm-7b-8k\", \"llama2-7b-chat-4k\", \"llama2-7b-chat-4k\"]\n",
    "\n",
    "modes0 = [\"internlm-7b-8k_no_g0.1_d10.0\", \"llama2-7b-chat-4k_no_g0.5_d5.0\", \"llama2-7b-chat-4k_no_g0.5_d5.0\"]\n",
    "modes1 = [\"internlm-7b-8k_old_hard_g0.15_d10.0\", \"llama2-7b-chat-4k_old_hard_g0.25_d5.0\", \"llama2-7b-chat-4k_old_hard_g0.75_d5.0\"]\n",
    "modes2 = [\"internlm-7b-8k_old_soft_g0.1_d10.0\", \"llama2-7b-chat-4k_old_soft_g0.1_d10.0\", \"llama2-7b-chat-4k_old_soft_g0.75_d15.0\"]\n",
    "modes3 = [\"internlm-7b-8k_gpt_g0.25_d15.0\", \"llama2-7b-chat-4k_gpt_g0.1_d10.0\", \"llama2-7b-chat-4k_gpt_g0.65_d12.5\"]\n",
    "modes4 = [\"internlm-7b-8k_v2_g0.1_d10.0\", \"llama2-7b-chat-4k_v2_g0.25_d15.0\", \"llama2-7b-chat-4k_v2_g0.75_d15.0\"]\n",
    "\n",
    "all_modes = [modes0, modes1, modes2, modes3, modes4]\n",
    "\n",
    "# All missions\n",
    "dict_mission = {\n",
    "    0: \"konwledge_memorization\",\n",
    "    1: \"konwledge_understanding\",\n",
    "    2: \"longform_qa\",\n",
    "    3: \"finance_qa\",\n",
    "    4: \"hotpotqa\",\n",
    "    5: \"lcc\",\n",
    "    6: \"multi_news\",\n",
    "    7: \"qmsum\",\n",
    "    8: \"alpacafarm\",\n",
    "    9: \"avarage_mode\"\n",
    "}\n",
    "\n",
    "# All missions Categories\n",
    "dict_cate_mission = {\n",
    "    0: \"C1:(Short Q, Short A)\",\n",
    "    1: \"C2:(Short Q, Long A)\",\n",
    "    2: \"C3:(Long Q, Short A)\",\n",
    "    3: \"C4:(Long Q, Long A)\",\n",
    "    4: \"C5:Open-Ended\",\n",
    "    5: \"C6:OverAll\"\n",
    "}\n",
    "  \n",
    "model_name = model_names[flag]\n",
    "modes = [modes_i[flag] for modes_i in all_modes]\n",
    "\n",
    "tn_detect_modes = [\"_\".join(modex.split(\"_\")[1:]) for modex in modes]\n",
    "# print(tn_detect_modes)\n",
    "\n",
    "num_eval = 0\n",
    "num_z = 0\n",
    "p = r\"(?P<model_name>.+)_(?P<mode>old|v2|gpt|new|no)_g(?P<gamma>.+)_d(?P<delta>\\d+(\\.\\d+)?)\"\n",
    "p_z = r\"(?P<mission_name>[a-zA-Z_]+)_(?P<gamma>\\d+(\\.\\d+)?)_(?P<delta>.+)_z\"\n",
    "p_mutual_z = r\"(?P<mission_name>[a-zA-Z_]+)_(?P<threshold>\\d+(\\.\\d+)?)_z\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate TP and Eval score for every group experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all files from input_dir\n",
    "for subfolder in os.listdir(input_dir):\n",
    "    \n",
    "    if \"human\" in subfolder:\n",
    "        continue\n",
    "    matcher = re.match(p, subfolder)\n",
    "    \n",
    "    model_name_now = matcher.group(\"model_name\")\n",
    "    if model_name_now != model_name:\n",
    "        continue\n",
    "    print(\"subfolder is:\", subfolder)\n",
    "    mode = matcher.group(\"mode\")\n",
    "    gamma = matcher.group(\"gamma\")\n",
    "    delta = matcher.group(\"delta\")\n",
    "    \n",
    "    bl_type = \"None\"\n",
    "    bl_type = (subfolder.split(\"_\")[-1]).split(\".\")[0]\n",
    "    \n",
    "    if bl_type != \"hard\":\n",
    "        if \"old\" in subfolder:\n",
    "            bl_type = \"soft\"\n",
    "        else:\n",
    "            bl_type = \"None\"\n",
    "        \n",
    "    if bl_type == \"hard\" or bl_type == \"soft\":\n",
    "        final_mode = model_name + \"_\" + mode + \"_\" + bl_type + \"_\" + \"g\"+gamma + \"_\" + \"d\" + delta  \n",
    "    else:\n",
    "        final_mode = model_name + \"_\" + mode + \"_\" + \"g\"+gamma + \"_\" + \"d\" + delta \n",
    "        \n",
    "    eval_path = os.path.join(input_dir, subfolder, \"eval\")\n",
    "    z_score_path = os.path.join(input_dir, subfolder, \"z_score\")\n",
    "    threshold = 4.0\n",
    "    if os.path.exists(z_score_path):\n",
    "        temp_df = pd.DataFrame(columns=[\"model_name\", \"mode\", \"mission_name\", \"z_score\", \"true_positive\", \"sum\"])\n",
    "        all_z = []\n",
    "        all_sum = []\n",
    "        all_tp = 0\n",
    "        \n",
    "        \n",
    "        files = os.listdir(z_score_path)\n",
    "        for file in files:\n",
    "            tp = 0\n",
    "            matcher1 = re.match(p_z, file)\n",
    "            \n",
    "            if matcher1:\n",
    "                mission_name = matcher1.group(\"mission_name\")\n",
    "            \n",
    "            with open(os.path.join(z_score_path, file), \"r\") as f:\n",
    "                data = json.load(f) \n",
    "                \n",
    "            z_score_list = data[\"z_score_list\"]\n",
    "            _sum = len(data[\"z_score_list\"])   \n",
    "            tp += len([x for x in z_score_list if x > threshold])\n",
    "            avarage_z = data[\"avarage_z\"]\n",
    "            temp = pd.DataFrame({\n",
    "                \"model_name\": [model_name],\n",
    "                \"mode\": [final_mode],\n",
    "                \"mission_name\": [mission_name],\n",
    "                \"z_score\": [avarage_z],\n",
    "                \"true_positive\": [tp/_sum],\n",
    "                \"sum\": [_sum]})\n",
    "            \n",
    "            df_tp = pd.concat([df_tp, temp], ignore_index=True)\n",
    "            all_z.append(avarage_z * _sum)\n",
    "            all_sum.append(_sum)\n",
    "            all_tp += tp\n",
    "            num_z += 1\n",
    "            \n",
    "        temp_df = pd.DataFrame({\n",
    "            \"model_name\": [model_name],\n",
    "            \"mode\": [final_mode],\n",
    "            \"mission_name\": [\"avarage_mode\"],\n",
    "            \"z_score\": [sum(all_z) / sum(all_sum)],\n",
    "            \"true_positive\": [all_tp/sum(all_sum)], \n",
    "            \"sum\": [all_sum]\n",
    "        })\n",
    "        df_tp = pd.concat([df_tp, temp_df], ignore_index=True)   \n",
    "         \n",
    "    if os.path.exists(eval_path):\n",
    "        result_file = os.path.join(eval_path, \"result.json\")\n",
    "        if os.path.exists(result_file):\n",
    "            with open (result_file, \"r\") as f:\n",
    "                data = json.load(f)\n",
    "            keys = data.keys()\n",
    "            temp_df = pd.DataFrame(columns=[\"model_name\", \"mode\", \"mission_name\", \"score\"])\n",
    "            all_scores = []\n",
    "            for key in keys:\n",
    "                eval_score = float((str(data[key])).split(\" \")[-1])\n",
    "                temp_df = pd.DataFrame({\n",
    "                    \"model_name\":[model_name],\n",
    "                    \"mode\":[final_mode],\n",
    "                    \"mission_name\": [key],\n",
    "                    \"score\":[eval_score]\n",
    "                })\n",
    "                \n",
    "                all_scores.append(eval_score)\n",
    "                df_eval = pd.concat([df_eval, temp_df], ignore_index=True)\n",
    "                \n",
    "                num_eval += 1\n",
    "                \n",
    "            temp_df = pd.DataFrame({\n",
    "                    \"model_name\":[model_name],\n",
    "                    \"mode\":[final_mode],\n",
    "                    \"mission_name\": [\"avarage_mode\"],\n",
    "                    \"score\":[sum(all_scores) / len(keys)]\n",
    "                })   \n",
    "            \n",
    "            df_eval = pd.concat([df_eval, temp_df], ignore_index=True) \n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Record the results in the csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tp.to_csv(os.path.join(save_csv_dir, f\"z_score_tp_{model_name}.csv\"))           \n",
    "# print(df_tp)\n",
    "# print(num_z)\n",
    "\n",
    "df_eval = df_eval.sort_values(by=\"mode\", ascending=True)                 \n",
    "df_eval.to_csv(os.path.join(save_csv_dir, f\"eval_{model_name}.csv\"))                \n",
    "# print(\"num is:\", num_eval)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate and Record TN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dir = f\"./detect_human/{model_name}/\"\n",
    "\n",
    "df_human = pd.DataFrame(columns=[\"model_name\", \"mission_name\", \"ref_mode\", \"det_mode\", \"threshold\", \"z_score\", \"true_negative\", \"sum\", \"tn_num\"])\n",
    "\n",
    "num = 0\n",
    "for subfolder in os.listdir(input_dir):\n",
    "    ref_mode = \"_\".join(subfolder.split(\"_\")[1:])\n",
    "    # print(\"ref_mode is\", ref_mode)\n",
    "    if ref_mode in tn_detect_modes:\n",
    "        # print(\"mode_des is\", ref_mode)\n",
    "        \n",
    "        for subsubfolder in os.listdir(os.path.join(input_dir,subfolder)):\n",
    "            det_mode = subsubfolder.split(\"_z\")[0]\n",
    "            print(det_mode)\n",
    "            \n",
    "            z_score_path = os.path.join(input_dir, subfolder, subsubfolder)\n",
    "            if os.path.exists(z_score_path):\n",
    "                files = os.listdir(z_score_path)\n",
    "                \n",
    "                all_z = []\n",
    "                all_sum = []\n",
    "                all_tn = 0\n",
    "                \n",
    "                for file in files:\n",
    "                    tn = 0\n",
    "                    matcher2 = re.match(p_mutual_z, file)\n",
    "                    if matcher2:\n",
    "                        mission_name = matcher2.group(\"mission_name\")\n",
    "                        threshold = float(matcher2.group(\"threshold\"))\n",
    "                        \n",
    "                    with open(os.path.join(z_score_path, file), \"r\") as f:\n",
    "                        data = json.load(f)\n",
    "                        \n",
    "                        \n",
    "                    avarage_z = data[\"avarage_z\"]\n",
    "                    z_score_list = data[\"z_score_list\"]\n",
    "                    _sum = len(data[\"z_score_list\"])\n",
    "                    \n",
    "                    tn += len([x for x in z_score_list if x <= threshold])\n",
    "                    num += 1\n",
    "                    \n",
    "                    temp_df = pd.DataFrame({\n",
    "                        \"model_name\": [model_name],\n",
    "                        \"ref_mode\": [ref_mode],\n",
    "                        \"mission_name\": [mission_name],\n",
    "                        \"det_mode\":[det_mode],\n",
    "                        \"threshold\": [threshold],\n",
    "                        \"z_score\": [avarage_z],\n",
    "                        \"true_negative\": [tn/_sum],\n",
    "                        \"tn_num\": [tn],\n",
    "                        \"sum\": [_sum]\n",
    "                    })\n",
    "                    all_tn += tn\n",
    "                    all_z.append(avarage_z * _sum)\n",
    "                    all_sum.append(_sum)\n",
    "                    \n",
    "                    df_human = pd.concat([df_human, temp_df], ignore_index=True)\n",
    "                temp_df = pd.DataFrame({\n",
    "                        \"model_name\": [model_name],\n",
    "                        \"ref_mode\": [ref_mode],\n",
    "                        \"mission_name\": [\"avarage_mode\"],\n",
    "                        \"det_mode\":[det_mode],\n",
    "                        \"threshold\": [threshold],\n",
    "                        \"z_score\": [sum(all_z) / sum(all_sum)],\n",
    "                        \"true_negative\": [all_tn/sum(all_sum)],\n",
    "                        \"tn_num\": [all_tn],\n",
    "                        \"sum\": [sum(all_sum)]\n",
    "                    })\n",
    "                \n",
    "                df_human = pd.concat([df_human, temp_df], ignore_index=True)\n",
    "                    \n",
    "df_human.to_csv(os.path.join(save_csv_dir, f\"human_z_tn_{model_name}.csv\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Obtain TN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tn = pd.read_csv(f\"{save_csv_dir}/human_z_tn_{model_name}.csv\")\n",
    "\n",
    "# print(df_tn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 3 & Table 5 C1, C2, C3(Latex Format)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_str = model_name\n",
    "if \"llama\" in model_name:\n",
    "    output_str = \"Llama2-7B-chat\"\n",
    "elif \"internlm\" in model_name:\n",
    "    output_str = \"Internlm-7B-8k\"    \n",
    "\n",
    "eval_no = [0 for i in range(6)]\n",
    "\n",
    "for j in range(len(tn_detect_modes)):\n",
    "    \n",
    "    for i in range(0, 6, 2):\n",
    "        # Get tp scores\n",
    "        output_str += \" & \"\n",
    "        \n",
    "        cond1 = (df_tp[\"mode\"].str.contains(tn_detect_modes[j])) & (df_tp[\"mission_name\"] == dict_mission[i])\n",
    "        tp_score1 = df_tp.loc[cond1, \"true_positive\"]\n",
    "        sum1 = df_tp.loc[cond1, \"sum\"]\n",
    "        \n",
    "        cond2 = (df_tp[\"mode\"].str.contains(tn_detect_modes[j])) & (df_tp[\"mission_name\"] == dict_mission[i + 1])\n",
    "        tp_score2 = df_tp.loc[cond2, \"true_positive\"]\n",
    "        sum2 = df_tp.loc[cond2, \"sum\"]\n",
    "        \n",
    "        if tp_score1.empty or tp_score2.empty or j == 0:\n",
    "            output_str += \"--\"\n",
    "        else:\n",
    "            tp_score_i = float(tp_score1.iloc[0])\n",
    "            tp_score_i1 = float(tp_score2.iloc[0])\n",
    "            \n",
    "            tp_score = (tp_score_i * int(sum1.iloc[0]) + tp_score_i1 * int(sum2.iloc[0])) / (int(sum1.iloc[0]) + int(sum2.iloc[0]))\n",
    "            \n",
    "            print(\"tp_score is\", tp_score)\n",
    "            output_str += str(round(float(tp_score) * 100, 1))\n",
    "\n",
    "        # Get tn scores\n",
    "        output_str += \" & \"\n",
    "        cond1 = (df_tn[\"ref_mode\"].str.contains(tn_detect_modes[j])) & (df_tn[\"mission_name\"] == dict_mission[i])\n",
    "        tn_num1 = df_tn.loc[cond1, \"tn_num\"]\n",
    "        sum1 = df_tn.loc[cond1, \"sum\"]\n",
    "        \n",
    "        cond2 = (df_tn[\"ref_mode\"].str.contains(tn_detect_modes[j])) & (df_tn[\"mission_name\"] == dict_mission[i + 1])\n",
    "        tn_num2 = df_tn.loc[cond2, \"tn_num\"]\n",
    "        sum2 = df_tn.loc[cond2, \"sum\"]\n",
    "        \n",
    "        if tn_num1.empty or tn_num2.empty:\n",
    "            output_str += \"--\" \n",
    "        else:\n",
    "            tn_score = (int(tn_num1.iloc[0]) + int(tn_num2.iloc[0])) / (int(sum1.iloc[0]) + int(sum2.iloc[0]))\n",
    "            \n",
    "            tn_dict[j].update({f\"{dict_cate_mission[i // 2]}\":tn_score})\n",
    "            \n",
    "            print(\"tn_score is\", tn_score)\n",
    "            output_str += str(round(float(tn_score) * 100, 1))\n",
    "\n",
    "        # Get eval scores\n",
    "        output_str += \" & \"\n",
    "        cond1 = (df_eval[\"mode\"].str.contains(tn_detect_modes[j])) & (df_eval[\"mission_name\"] == dict_mission[i])\n",
    "        cond2 = (df_eval[\"mode\"].str.contains(tn_detect_modes[j])) & (df_eval[\"mission_name\"] == dict_mission[i + 1])\n",
    "        eval_score1 = df_eval.loc[cond1, \"score\"]\n",
    "        eval_score2 = df_eval.loc[cond2, \"score\"]\n",
    "        if eval_score1.empty or eval_score2.empty:\n",
    "            output_str += \"--\" \n",
    "        else:\n",
    "            eval_score3 = (float(eval_score1.iloc[0]) + float(eval_score2.iloc[0])) / 2\n",
    "            \n",
    "            eval_dict[j].update({f\"{dict_cate_mission[i // 2]}\":eval_score3})\n",
    "            print(\"eval_score is\", eval_score3)\n",
    "            output_str += str(round(eval_score3, 1))\n",
    "        print(\"j =\", j)    \n",
    "        if j == 0:\n",
    "            eval_no[i % 6] = float(eval_score3)\n",
    "            print(\"eval_no is\", eval_no)\n",
    "            \n",
    "        output_str += \" & \"\n",
    "        if j != 0:\n",
    "            print(f\"eval_no[{i} % 6] is\", eval_no[i % 6])\n",
    "            print(\"eval_score3 is\", eval_score3)\n",
    "            down_rate = (eval_no[i % 6] - eval_score3) / eval_no[i % 6]\n",
    "            down_rate = round((down_rate) * 100, 1)\n",
    "            # if j == 1 and i == 0:\n",
    "            output_str += f\"$\\\\downarrow$ {down_rate}\\\\%\"\n",
    "            # else:\n",
    "            #     output_str += f\"{down_rate}\"\n",
    "        else:\n",
    "            output_str += \"--\"    \n",
    "    \n",
    "    output_str += \"\\\\\\\\\\n\"\n",
    "    if j == 0:\n",
    "        output_str += \"+ hard watermark\"\n",
    "    if j == 1:\n",
    "        output_str += \"+ soft watermark\"\n",
    "    if j == 2:\n",
    "        output_str += \"+ gpt watermark\"\n",
    "    if j == 3:\n",
    "        output_str += \"+ v2 watermark\"\n",
    "        \n",
    "print(output_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 4 & Table 5 C4, C5, Overall(Latex Format)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_str = model_name\n",
    "if \"llama\" in model_name:\n",
    "    output_str = \"Llama2-7B-chat\"\n",
    "elif \"internlm\" in model_name:\n",
    "    output_str = \"Internlm-7B-8k\"    \n",
    "\n",
    "# output_str = \"Internlm-7B-8k\"\n",
    "eval_no = [0 for i in range(10)]\n",
    "\n",
    "for j in range(len(tn_detect_modes)):\n",
    "    \n",
    "    i = 6\n",
    "    # Get tp scores\n",
    "    output_str += \" & \"\n",
    "    \n",
    "    cond1 = (df_tp[\"mode\"].str.contains(tn_detect_modes[j])) & (df_tp[\"mission_name\"] == dict_mission[i])\n",
    "    tp_score1 = df_tp.loc[cond1, \"true_positive\"]\n",
    "    sum1 = df_tp.loc[cond1, \"sum\"]\n",
    "    \n",
    "    cond2 = (df_tp[\"mode\"].str.contains(tn_detect_modes[j])) & (df_tp[\"mission_name\"] == dict_mission[i + 1])\n",
    "    tp_score2 = df_tp.loc[cond2, \"true_positive\"]\n",
    "    sum2 = df_tp.loc[cond2, \"sum\"]\n",
    "    \n",
    "    # tp_score_L1 = (tp_score1 * sum1 + tp_score2 *sum2) / (sum1 + sum2)\n",
    "    \n",
    "    if tp_score1.empty or tp_score2.empty or j == 0:\n",
    "        output_str += \"--\"\n",
    "    else:\n",
    "        tp_score_i = float(tp_score1.iloc[0])\n",
    "        tp_score_i1 = float(tp_score2.iloc[0])\n",
    "        \n",
    "        tp_score = (tp_score_i * int(sum1.iloc[0]) + tp_score_i1 * int(sum2.iloc[0])) / (int(sum1.iloc[0]) + int(sum2.iloc[0]))\n",
    "        \n",
    "        print(\"tp_score is\", tp_score)\n",
    "        output_str += str(round(float(tp_score) * 100, 1))\n",
    "\n",
    "    # Get tn scores\n",
    "    output_str += \" & \"\n",
    "    cond1 = (df_tn[\"ref_mode\"].str.contains(tn_detect_modes[j])) & (df_tn[\"mission_name\"] == dict_mission[i])\n",
    "    tn_num1 = df_tn.loc[cond1, \"tn_num\"]\n",
    "    sum1 = df_tn.loc[cond1, \"sum\"]\n",
    "    \n",
    "    cond2 = (df_tn[\"ref_mode\"].str.contains(tn_detect_modes[j])) & (df_tn[\"mission_name\"] == dict_mission[i + 1])\n",
    "    tn_num2 = df_tn.loc[cond2, \"tn_num\"]\n",
    "    sum2 = df_tn.loc[cond2, \"sum\"]\n",
    "    \n",
    "    if tn_num1.empty or tn_num2.empty:\n",
    "        output_str += \"--\" \n",
    "    else:\n",
    "        tn_score = (int(tn_num1.iloc[0]) + int(tn_num2.iloc[0])) / (int(sum1.iloc[0]) + int(sum2.iloc[0]))\n",
    "        \n",
    "        tn_dict[j].update({f\"{dict_cate_mission[i // 2]}\":tn_score})\n",
    "        print(\"tn_score is\", tn_score)\n",
    "        output_str += str(round(float(tn_score) * 100, 1))\n",
    "\n",
    "    # Get eval scores\n",
    "    output_str += \" & \"\n",
    "    cond1 = (df_eval[\"mode\"].str.contains(tn_detect_modes[j])) & (df_eval[\"mission_name\"] == dict_mission[i])\n",
    "    cond2 = (df_eval[\"mode\"].str.contains(tn_detect_modes[j])) & (df_eval[\"mission_name\"] == dict_mission[i + 1])\n",
    "    eval_score1 = df_eval.loc[cond1, \"score\"]\n",
    "    eval_score2 = df_eval.loc[cond2, \"score\"]\n",
    "    if eval_score1.empty or eval_score2.empty:\n",
    "        output_str += \"--\" \n",
    "    else:\n",
    "        eval_score3 = (float(eval_score1.iloc[0]) + float(eval_score2.iloc[0])) / 2\n",
    "        \n",
    "        eval_dict[j].update({f\"{dict_cate_mission[i // 2]}\":eval_score3})\n",
    "        print(\"eval_score is\", eval_score3)\n",
    "        output_str += str(round(eval_score3, 1))\n",
    "    print(\"j =\", j)    \n",
    "    if j == 0:\n",
    "        eval_no[i % 10] = float(eval_score3)\n",
    "        print(\"eval_no is\", eval_no)\n",
    "        \n",
    "    output_str += \" & \"\n",
    "    if j != 0:\n",
    "        print(f\"eval_no[{i} % 10] is\", eval_no[i % 10])\n",
    "        print(\"eval_score3 is\", eval_score3)\n",
    "        down_rate = (eval_no[i % 10] - eval_score3) / eval_no[i % 10]\n",
    "        down_rate = round((down_rate) * 100, 1)\n",
    "        \n",
    "        output_str += f\"$\\\\downarrow$ {down_rate}\\\\%\"\n",
    "        \n",
    "    else:\n",
    "        output_str += \"--\"   \n",
    "    \n",
    "    # C5 & Overall\n",
    "    for i in range(8, 10):\n",
    "        # Get tp scores\n",
    "        output_str += \" & \"\n",
    "        tp_score = df_tp.loc[(df_tp[\"mode\"].str.contains(tn_detect_modes[j])) & (df_tp[\"mission_name\"] == dict_mission[i]), \"true_positive\"]\n",
    "        if tp_score.empty or float(tp_score.iloc[0]) < 0.05:\n",
    "            output_str += \"--\"\n",
    "        else:\n",
    "            tp_score = tp_score.iloc[0]\n",
    "            print(\"tp_score is\", tp_score)\n",
    "            output_str += str(round(float(tp_score) * 100, 1))\n",
    "    \n",
    "        # Get tn scores\n",
    "        output_str += \" & \"\n",
    "        tn_score = df_tn.loc[(df_tn[\"ref_mode\"].str.contains(tn_detect_modes[j])) & (df_tn[\"mission_name\"] == dict_mission[i]), \"true_negative\"]\n",
    "        if tn_score.empty:\n",
    "            output_str += \"--\" \n",
    "        else:\n",
    "            tn_score = tn_score.iloc[0]\n",
    "            if i % 2 == 0:\n",
    "                tn_dict[j].update({f\"{dict_cate_mission[i // 2]}\":tn_score})\n",
    "            else:\n",
    "                tn_dict[j].update({f\"{dict_cate_mission[i // 2 + 1]}\":tn_score})\n",
    "            print(\"tn_score is\", tn_score)\n",
    "            output_str += str(round(float(tn_score) * 100, 1))\n",
    "\n",
    "        # Get eval scores\n",
    "        output_str += \" & \"\n",
    "        eval_score = df_eval.loc[(df_eval[\"mode\"].str.contains(tn_detect_modes[j])) & (df_eval[\"mission_name\"] == dict_mission[i]), \"score\"]\n",
    "        if eval_score.empty:\n",
    "            output_str += \"--\" \n",
    "        else:\n",
    "            eval_score = eval_score.iloc[0]\n",
    "            if i % 2 == 0:\n",
    "                eval_dict[j].update({f\"{dict_cate_mission[i // 2]}\":eval_score})\n",
    "            else:\n",
    "                eval_dict[j].update({f\"{dict_cate_mission[i // 2 + 1]}\":eval_score})\n",
    "            print(\"eval_score is\", eval_score)\n",
    "            output_str += str(round(float(eval_score), 1))  \n",
    "            \n",
    "        if j == 0:\n",
    "            eval_no[i % 10] = float(eval_score)\n",
    "            print(\"eval_no is\", eval_no)\n",
    "            \n",
    "        output_str += \" & \"\n",
    "        if j != 0:\n",
    "            print(f\"eval_no[{i} % 10] is\", eval_no[i % 10])\n",
    "            print(\"eval_score3 is\", eval_score)\n",
    "            down_rate = (eval_no[i % 10] - eval_score) / eval_no[i % 10]\n",
    "            down_rate = round((down_rate) * 100, 1)\n",
    "            output_str += f\"$\\\\downarrow$ {down_rate}\\\\%\"\n",
    "        else:\n",
    "            output_str += \"--\" \n",
    "     \n",
    "    \n",
    "    output_str += \"\\\\\\\\\\n\"\n",
    "    if j == 0:\n",
    "        output_str += \"+ hard watermark\"\n",
    "    if j == 1:\n",
    "        output_str += \"+ soft watermark\"\n",
    "    if j == 2:\n",
    "        output_str += \"+ gpt watermark\"\n",
    "    if j == 3:\n",
    "        output_str += \"+ v2 watermark\"\n",
    "        \n",
    "print(output_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Appendix B Table 7(C1, C2 Full Results + Latex Format) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_str = model_name\n",
    "if \"llama\" in model_name:\n",
    "    output_str = \"Llama2-7B-chat\"\n",
    "elif \"internlm\" in model_name:\n",
    "    output_str = \"Internlm-7B-8k\"    \n",
    "\n",
    "for j in range(len(tn_detect_modes)):\n",
    "    for i in range(4):\n",
    "        \n",
    "        # Get tp scores \n",
    "        output_str += \" & \"\n",
    "        tp_score = df_tp.loc[(df_tp[\"mode\"].str.contains(tn_detect_modes[j])) & (df_tp[\"mission_name\"] == dict_mission[i]), \"true_positive\"]\n",
    "        if tp_score.empty or float(tp_score.iloc[0]) < 0.05 or j == 0:\n",
    "            output_str += \"--\"\n",
    "        else:\n",
    "            tp_score = tp_score.iloc[0]\n",
    "            print(\"tp_score is\", tp_score)\n",
    "            output_str += str(round(float(tp_score) * 100, 1))\n",
    "    \n",
    "        # Get tn scores\n",
    "        output_str += \" & \"\n",
    "        tn_score = df_tn.loc[(df_tn[\"ref_mode\"].str.contains(tn_detect_modes[j])) & (df_tn[\"mission_name\"] == dict_mission[i]), \"true_negative\"]\n",
    "        if tn_score.empty:\n",
    "            output_str += \"--\" \n",
    "        else:\n",
    "            tn_score = tn_score.iloc[0]\n",
    "            print(\"tn_score is\", tn_score)\n",
    "            output_str += str(round(float(tn_score) * 100, 1))\n",
    "    \n",
    "        # Get eval scores\n",
    "        output_str += \" & \"\n",
    "        eval_score = df_eval.loc[(df_eval[\"mode\"].str.contains(tn_detect_modes[j])) & (df_eval[\"mission_name\"] == dict_mission[i]), \"score\"]\n",
    "        if eval_score.empty:\n",
    "            output_str += \"--\" \n",
    "        else:\n",
    "            eval_score = eval_score.iloc[0]\n",
    "            print(\"eval_score is\", eval_score)\n",
    "            output_str += str(round(float(eval_score), 1))\n",
    "    \n",
    "    output_str += \"\\\\\\\\\\n\"\n",
    "    if j == 0:\n",
    "        output_str += \"+ hard watermark\"\n",
    "    if j == 1:\n",
    "        output_str += \"+ soft watermark\"\n",
    "    if j == 2:\n",
    "        output_str += \"+ gpt watermark\"\n",
    "    if j == 3:\n",
    "        output_str += \"+ v2 watermark\"\n",
    "        \n",
    "print(output_str)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Appendix B Table 8(C3, C4 Full Results + Latex Format) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_str = model_name\n",
    "if \"llama\" in model_name:\n",
    "    output_str = \"Llama2-7B-chat\"\n",
    "elif \"internlm\" in model_name:\n",
    "    output_str = \"Internlm-7B-8k\"    \n",
    "\n",
    "\n",
    "for j in range(len(tn_detect_modes)):\n",
    "    for i in range(4, 8):\n",
    "    \n",
    "        # Get tp scores    \n",
    "        output_str += \" & \"\n",
    "        tp_score = df_tp.loc[(df_tp[\"mode\"].str.contains(tn_detect_modes[j])) & (df_tp[\"mission_name\"] == dict_mission[i]), \"true_positive\"]\n",
    "        if tp_score.empty or float(tp_score.iloc[0]) < 0.05 or j == 0:\n",
    "            output_str += \"--\"\n",
    "        else:\n",
    "            tp_score = tp_score.iloc[0]\n",
    "            print(\"tp_score is\", tp_score)\n",
    "            output_str += str(round(float(tp_score) * 100, 1))\n",
    "    \n",
    "        # Get tn scores\n",
    "        output_str += \" & \"\n",
    "        tn_score = df_tn.loc[(df_tn[\"ref_mode\"].str.contains(tn_detect_modes[j])) & (df_tn[\"mission_name\"] == dict_mission[i]), \"true_negative\"]\n",
    "        if tn_score.empty:\n",
    "            output_str += \"--\" \n",
    "        else:\n",
    "            tn_score = tn_score.iloc[0]\n",
    "            print(\"tn_score is\", tn_score)\n",
    "            output_str += str(round(float(tn_score) * 100, 1))\n",
    "    \n",
    "        # Get eval scores\n",
    "        output_str += \" & \"\n",
    "        eval_score = df_eval.loc[(df_eval[\"mode\"].str.contains(tn_detect_modes[j])) & (df_eval[\"mission_name\"] == dict_mission[i]), \"score\"]\n",
    "        if eval_score.empty:\n",
    "            output_str += \"--\" \n",
    "        else:\n",
    "            eval_score = eval_score.iloc[0]\n",
    "            print(\"eval_score is\", eval_score)\n",
    "            output_str += str(round(float(eval_score), 1))\n",
    "    \n",
    "    output_str += \"\\\\\\\\\\n\"\n",
    "    if j == 0:\n",
    "        output_str += \"+ hard watermark\"\n",
    "    if j == 1:\n",
    "        output_str += \"+ soft watermark\"\n",
    "    if j == 2:\n",
    "        output_str += \"+ gpt watermark\"\n",
    "    if j == 3:\n",
    "        output_str += \"+ v2 watermark\"\n",
    "        \n",
    "print(output_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Appendix B Table 9(C5, Overall Full Results + Latex Format) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_str = model_name\n",
    "if \"llama\" in model_name:\n",
    "    output_str = \"Llama2-7B-chat\"\n",
    "elif \"internlm\" in model_name:\n",
    "    output_str = \"Internlm-7B-8k\"    \n",
    "\n",
    "\n",
    "for j in range(len(tn_detect_modes)):\n",
    "    for i in range(8, 10):\n",
    "        \n",
    "        # Get tp scores\n",
    "        output_str += \" & \"\n",
    "        tp_score = df_tp.loc[(df_tp[\"mode\"].str.contains(tn_detect_modes[j])) & (df_tp[\"mission_name\"] == dict_mission[i]), \"true_positive\"]\n",
    "        if tp_score.empty or float(tp_score.iloc[0]) < 0.05 or j == 0:\n",
    "            output_str += \"--\"\n",
    "        else:\n",
    "            tp_score = tp_score.iloc[0]\n",
    "            print(\"tp_score is\", tp_score)\n",
    "            output_str += str(round(float(tp_score) * 100, 1))\n",
    "    \n",
    "        # Get tn scores\n",
    "        output_str += \" & \"\n",
    "        tn_score = df_tn.loc[(df_tn[\"ref_mode\"].str.contains(tn_detect_modes[j])) & (df_tn[\"mission_name\"] == dict_mission[i]), \"true_negative\"]\n",
    "        if tn_score.empty:\n",
    "            output_str += \"--\" \n",
    "        else:\n",
    "            tn_score = tn_score.iloc[0]\n",
    "            print(\"tn_score is\", tn_score)\n",
    "            output_str += str(round(float(tn_score) * 100, 1))\n",
    "    \n",
    "        # Get eval scores\n",
    "        output_str += \" & \"\n",
    "        eval_score = df_eval.loc[(df_eval[\"mode\"].str.contains(tn_detect_modes[j])) & (df_eval[\"mission_name\"] == dict_mission[i]), \"score\"]\n",
    "        if eval_score.empty:\n",
    "            output_str += \"--\" \n",
    "        else:\n",
    "            eval_score = eval_score.iloc[0]\n",
    "            print(\"eval_score is\", eval_score)\n",
    "            output_str += str(round(float(eval_score), 1))\n",
    "    \n",
    "    output_str += \"\\\\\\\\\\n\"\n",
    "    if j == 0:\n",
    "        output_str += \"+ hard\"\n",
    "    if j == 1:\n",
    "        output_str += \"+ soft\"\n",
    "    if j == 2:\n",
    "        output_str += \"+ gpt\"\n",
    "    if j == 3:\n",
    "        output_str += \"+ v2\"\n",
    "        \n",
    "print(output_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyper-parameters Search(Figure 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from turtle import title\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def combine_mode_bl_type (x):\n",
    "  if pd.isnull (x ['bl_type']):\n",
    "    return x ['mode']\n",
    "  else:\n",
    "    return x['bl_type']\n",
    "\n",
    "df_avg = pd.read_csv(f\"{save_csv_dir}/z_score_avg_{model_name}.csv\")\n",
    "sns.set_theme(style=\"whitegrid\")\n",
    "rs = np.random.RandomState(4)\n",
    "df_clean = df_avg.drop(df_avg[(df_avg[\"mode\"] == \"new\") | (df_avg[\"mode\"] == \"no\") | (df_avg[\"gamma\"] == 0.65) | (df_avg[\"gamma\"] == 0.7) | (df_avg[\"gamma\"] == 0.6) | (df_avg[\"delta\"] == 9.0)].index)\n",
    "\n",
    "df_clean['watermark type'] = df_clean.apply(combine_mode_bl_type, axis=1)\n",
    "\n",
    "g = sns.FacetGrid(df_clean, col=\"gamma\", hue=\"watermark type\", palette=\"Set1\", hue_kws={\"marker\": [\"P\", \"s\", \"^\", \"v\"], \"markersize\": [10, 8, 8, 8], \"markeredgewidth\": [0, 0, 0, 0], \"alpha\": [0.8, 0.8, 0.8, 0.8]})\n",
    "\n",
    "g.map(sns.lineplot, \"delta\", \"true_positive\", marker=True, dashes=False)\n",
    "\n",
    "g.set_axis_labels(\"Delta\", \"Watermarking Strength\")\n",
    "g.set_titles(\"Gamma = {col_name}\")\n",
    "\n",
    "g.add_legend(title=\"\", loc=\"upper center\", ncol=4, bbox_to_anchor=(0.4, 1.1), columnspacing=4.0)\n",
    "\n",
    "# plt.savefig(f\"{save_pic_dir}/{model_name}_search.pdf\", dpi=400, bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TN and GM Results(Figure 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as mpatches\n",
    "\n",
    "print(tn_dict)\n",
    "print(eval_dict)\n",
    "\n",
    "\n",
    "scores_mode = [\"no\", \"hard\", \"soft\", \"gpt\", \"v2\"]\n",
    "colors = ['b', 'g', 'r', 'y', 'c', 'm', 'k']\n",
    "\n",
    "new_mode_tn_dict = {}\n",
    "for index, t in enumerate(tn_dict):\n",
    "    if t:\n",
    "        new_mode_tn_dict.update({scores_mode[index]:t})\n",
    "        \n",
    "new_mode_tn_list = list(new_mode_tn_dict.items())\n",
    "\n",
    "plt.style.use('seaborn')\n",
    "\n",
    "def draw(ax, start_point, end_point, para_dict, title):\n",
    "    \n",
    "    new_mode_para_dict = {}\n",
    "    for index, t in enumerate(para_dict[1:]):\n",
    "        new_mode_para_dict.update({scores_mode[index]:t})\n",
    "        \n",
    "    new_mode_para_list = list(new_mode_para_dict.items())\n",
    "    data_length = len(new_mode_para_list[0][1].values())\n",
    "    print(\"data_length is\", data_length)\n",
    "    \n",
    "    angles = np.linspace(0, 2*np.pi, data_length, endpoint=False)\n",
    "    print(new_mode_para_list[0][1].values())\n",
    "    print(new_mode_para_list[0][0])\n",
    "    labels = [key for key in new_mode_para_list[0][1].keys()]\n",
    "    print(\"labels is\", labels)\n",
    "    score = [list(result[1].values()) for result in new_mode_para_list]\n",
    "    \n",
    "    print(\"score is\", score)\n",
    "    \n",
    "    scores = []\n",
    "    patches = []\n",
    "    for i in range(len(score)):\n",
    "        score_i = np.concatenate((score[i], [score[i][0]]))\n",
    "        scores.append(score_i)\n",
    "        \n",
    "    angles = np.concatenate((angles, [angles[0]]))\n",
    "    labels = np.concatenate((labels, [labels[0]]))\n",
    "    \n",
    "    for index, score_i in enumerate(scores):\n",
    "        patch = mpatches.Patch(color=colors[index], label=list(new_mode_tn_dict.keys())[index].split(\"_\")[-1])\n",
    "        patches.append(patch)\n",
    "        ax.plot(angles, score_i, color=colors[index])\n",
    "        ax.fill(angles, score_i, alpha=0.25)\n",
    "    \n",
    "    ax.set_thetagrids(angles*180/np.pi, labels)\n",
    "    \n",
    "    ax.set_theta_zero_location('N')\n",
    "    \n",
    "    # Set the range of polar axis\n",
    "    ax.set_rlim(start_point, end_point)\n",
    "    ax.tick_params(axis='y', labelsize=8, labelcolor='gray')\n",
    "    \n",
    "    ax.set_rlabel_position(0)\n",
    "    ax.set_title(title, fontweight='bold', x=-0.25, y=1, fontsize=16)\n",
    "    \n",
    "    # Adjust Label Positiona\n",
    "    for label in ax.get_xticklabels():\n",
    "        label_text = label.get_text()\n",
    "        if \"C1\" in label_text or \"C4\" in label_text:\n",
    "            label.set_position((label.get_position()[0], -0.05))\n",
    "        else:\n",
    "            label.set_position((label.get_position()[0], -0.2))\n",
    "            \n",
    "    return patches\n",
    "\n",
    "fig, (ax1, ax2) = plt.subplots(1, 2, subplot_kw=dict(polar=True),  figsize=(16, 4))\n",
    "\n",
    "if \"llama\" in model_name:\n",
    "    patches = draw(ax1, 0.8, 1, tn_dict, \"True Negative\") \n",
    "elif \"internlm\" in model_name:\n",
    "    patches = draw(ax1, 0.9, 1, tn_dict, \"True Negative\")    \n",
    "\n",
    "patches = draw(ax2, 0, 20, eval_dict, \"Generation Metric\")  \n",
    "\n",
    "plt.subplots_adjust(wspace=0)\n",
    "    \n",
    "lgd=fig.legend(handles=patches, loc='lower center', ncols=4, bbox_to_anchor=(0.5,-0.05), prop={'size': 12})\n",
    "\n",
    "my_suptitle=fig.suptitle(f'{model_name}', x=0.475, y=1.1, fontweight='bold', fontsize=18)\n",
    "\n",
    "plt.subplots_adjust(wspace=0.25)\n",
    "\n",
    "# plt.savefig(f\"{save_pic_dir}/{model_name}_radar.pdf\", dpi=100, bbox_inches='tight', bbox_extra_artists=(my_suptitle, lgd))\n",
    "\n",
    "# plt.savefig(f\"{save_pic_dir}/{model_name}_radar.png\", dpi=100, bbox_inches='tight', bbox_extra_artists=(my_suptitle, lgd))\n",
    "\n",
    "\n",
    "plt.show()   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare GM scores and WaterMark classes for Figure 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read eval_{model_name}.csv\n",
    "df_eval_llama = pd.read_csv(f\"{save_csv_dir}/eval_llama2-7b-chat-4k.csv\")\n",
    "df_eval_internlm = pd.read_csv(f\"{save_csv_dir}/eval_internlm-7b-8k.csv\") \n",
    "df_eval_all = pd.concat([df_eval_llama.iloc[:, 1:], df_eval_internlm.iloc[:, 1:]], axis=0, ignore_index=True)\n",
    "\n",
    "print(df_eval_all)\n",
    "df_eval_all.to_csv(f\"{save_csv_dir}/eval_all.csv\")\n",
    "\n",
    "id2dataset = {\n",
    "    '1-1': 'konwledge_memorization',\n",
    "    '1-2': 'konwledge_understanding',\n",
    "    '2-1': 'longform_qa',\n",
    "    '2-2': 'finance_qa',\n",
    "    '3-1': 'hotpotqa',\n",
    "    '3-2': 'lcc',\n",
    "    '4-1': 'multi_news',\n",
    "    '4-2': 'qmsum',\n",
    "    '5-1': 'alpacafarm'\n",
    "}\n",
    "\n",
    "# All the conditions under 0.95TP\n",
    "process_modes = {'internlm': modes0[0],\n",
    "                 'internlm_hard': modes1[0],\n",
    "                 'internlm_soft': modes2[0],\n",
    "                 'internlm_gpt': modes3[0],\n",
    "                 'internlm_v2': modes4[0],\n",
    "                 'llama2': modes0[1],\n",
    "                 'llama2_hard': modes1[1],\n",
    "                 'llama2_soft': modes2[1],\n",
    "                 'llama2_gpt': modes3[1],\n",
    "                 'llama2_v2': modes4[1]}\n",
    "all_eval_modes = []\n",
    "for i in range(len(all_modes[0])):\n",
    "    for j in range(len(all_modes)):\n",
    "        all_eval_modes.append(all_modes[j][i])\n",
    "\n",
    "print(all_eval_modes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_matrix = pd.DataFrame(index=id2dataset.keys(), columns=process_modes.keys())\n",
    "for index, row in draw_matrix.iterrows():\n",
    "    for process_mode in process_modes.keys():\n",
    "        score = df_eval_all[(df_eval_all['mission_name'] == \n",
    "                             id2dataset.get(index)) & \n",
    "                            (df_eval_all['mode'] == process_modes.get(process_mode))]['score']\n",
    "        draw_matrix.loc[index, process_mode] = score.values[0]\n",
    "        \n",
    "print(draw_matrix)\n",
    "draw_matrix.to_csv(f\"{save_csv_dir}/heatmap.csv\")\n",
    "\n",
    "\n",
    "# print(draw_matrix.columns)\n",
    "# print(draw_matrix.index)\n",
    "draw_matrix.index.name = \"Unnamed: 0\"\n",
    "draw_matrix1 = pd.read_csv(f\"{save_csv_dir}/heatmap.csv\")\n",
    "\n",
    "# Set the first column as index\n",
    "draw_matrix1.set_index('Unnamed: 0', inplace=True)\n",
    "draw_matrix1.index.name = None\n",
    "draw_matrix1.columns.name = None\n",
    "print(draw_matrix1.columns)\n",
    "print(draw_matrix1.index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Spearman Correlation for GM between each pair of task in WaterBench(Figure 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spr_scores = draw_matrix1.transpose().corr(method='spearman').round(2)\n",
    "fig, ax = plt.subplots(figsize=(20,15))\n",
    "colormap = sns.cubehelix_palette(start=2.8, rot=0.1, gamma=0.2, dark=-0.0, light=0.95, as_cmap=True)\n",
    "hs = sns.heatmap(spr_scores, vmin=-1, vmax=1, annot=True, annot_kws={'size':36}, cmap=colormap, cbar_kws={\"pad\":0.02})\n",
    "cbar = ax.collections[0].colorbar\n",
    "# here set the labelsize by 20\n",
    "cbar.ax.tick_params(labelsize=36)\n",
    "\n",
    "# ax.set_title('Spearman\\'s ρ between datasets with model performance as variables', fontsize=20)\n",
    "ax.tick_params(labelsize=36, rotation=30)\n",
    "\n",
    "y_colors=['blue']*2 + ['green']*2 + ['orange']*2 + ['red']*2 + ['black']*1\n",
    "x_colors=['blue']*5 + ['green']*5\n",
    "[t.set_color(y_colors[i]) for i,t in enumerate(ax.xaxis.get_ticklabels())]\n",
    "[t.set_color(y_colors[i]) for i,t in enumerate(ax.yaxis.get_ticklabels())]\n",
    "plt.tight_layout()\n",
    "# fig.savefig(f'{save_pic_dir}/heat_datasets.pdf')\n",
    "plt.show()\n",
    "# fig.savefig(f'{save_pic_dir}/heat_datasets.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scatter plots for each pair of tasks(Figure 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import (MultipleLocator, AutoMinorLocator)\n",
    "from matplotlib import gridspec\n",
    "import numpy as np\n",
    "import scipy\n",
    "from scipy.interpolate import make_interp_spline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_scatter_multilines(Xs, Ys, names=None, xlabels=None, ylabels=None, xlim=None, ylim=None,\n",
    "                               titles=None, save_name=None, fit='polyfit'):\n",
    "    \n",
    "    titles = titles if titles is not None else [\"\"]*len(Xs)\n",
    "    xlabels = xlabels if xlabels is not None else [\"\"]*len(Xs)\n",
    "    ylabels = ylabels if ylabels is not None else [\"\"]*len(Xs)\n",
    "    cols = 2\n",
    "    rows = len(Xs) // 2\n",
    "    fig, axes = plt.subplots(rows, cols, figsize=(20,15), tight_layout=True)\n",
    "    if cols <= 1:\n",
    "        axes = np.array([axes])\n",
    "        \n",
    "    for i, (X,Y) in enumerate(zip(Xs,Ys)):\n",
    "        r, c = i // rows, i % rows\n",
    "        title = titles[i] if titles is not None else None\n",
    "        axes[r,c].set_title(title, fontsize=30)\n",
    "        axes[r,c].set_xlabel(xlabels[i], fontsize=16)\n",
    "        axes[r,c].set_ylabel(ylabels[i], fontsize=16)\n",
    "        axes[r,c].xaxis.set_minor_locator(MultipleLocator(1))\n",
    "        if xlim is not None: axes[r,c].set_xlim(xlim)\n",
    "        if ylim is not None: axes[r,c].set_ylim(ylim)\n",
    "#         axes[i].scatter(X, Y, s=25, c=\"#9370DB\", alpha=0.5)\n",
    "        axes[r,c].scatter(X, Y, c='gray', marker='o', edgecolors='k', s=40)\n",
    "#         axes[i].set_yticks(np.unique(Y))\n",
    "        # annotate points\n",
    "        if names:\n",
    "            for i in range(len(X)):\n",
    "                plt.annotate(names[i], xy=(X[i],Y[i]), xytext=(X[i]-0.5,Y[i]+0.8), fontsize=6)\n",
    "        axes[r,c].grid(alpha=0.2)\n",
    "        axes[r,c].tick_params(labelsize=36)\n",
    "        \n",
    "        # Fit\n",
    "        if fit != '':\n",
    "            p = np.polyfit(X, Y, 1) # # p[0]=Gradient, p[1]=y-intercept\n",
    "            model = np.poly1d(p)\n",
    "            # Fit the model\n",
    "            y_model = model(X)\n",
    "            \n",
    "            # Number of observations\n",
    "            n = Y.size\n",
    "            # Number of parameters: equal to the degree of the fitted polynomial (ie the\n",
    "            # number of coefficients) plus 1 (ie the number of constants)\n",
    "            m = p.size\n",
    "            # Degrees of freedom (number of observations - number of parameters)\n",
    "            dof = n - m\n",
    "            # Significance level\n",
    "            alpha = 0.05\n",
    "            # We're using a two-sided test\n",
    "            tails = 2\n",
    "            # The percent-point function (aka the quantile function) of the t-distribution\n",
    "            # gives you the critical t-value that must be met in order to get significance\n",
    "            t_critical = scipy.stats.t.ppf(1 - (alpha / tails), dof)\n",
    "            # Mean\n",
    "            y_bar = np.mean(Y)\n",
    "            # Coefficient of determination, R²\n",
    "            R2 = np.sum((y_model - y_bar)**2) / np.sum((Y - y_bar)**2)\n",
    "            \n",
    "            # Calculate the residuals (the error in the data, according to the model)\n",
    "            resid = Y - y_model\n",
    "            # Chi-squared (estimates the error in data)\n",
    "            chi2 = sum((resid / y_model)**2)\n",
    "            # Reduced chi-squared (measures the goodness-of-fit)\n",
    "            chi2_red = chi2 / dof\n",
    "            # Standard deviation of the error\n",
    "            std_err = np.sqrt(sum(resid**2) / dof)\n",
    "\n",
    "            xlim = axes[r,c].get_xlim()\n",
    "            ylim = axes[r,c].get_ylim()\n",
    "            # Line of best fit\n",
    "            axes[r,c].plot(np.array(xlim), p[1] + p[0] * np.array(xlim), label=f'Line of Best Fit, R² = {R2:.2f}')\n",
    "            \n",
    "            # Fit\n",
    "            x_fitted = np.linspace(xlim[0], xlim[1], 300)\n",
    "            y_fitted = np.polyval(p, x_fitted)\n",
    "            # Confidence interval\n",
    "            ci = t_critical * std_err * np.sqrt(1 / n + (x_fitted - np.mean(X))**2 / np.sum((X - np.mean(X))**2))\n",
    "            axes[r,c].fill_between(\n",
    "                x_fitted, y_fitted + ci, y_fitted - ci, facecolor='#b9cfe7', zorder=0,\n",
    "                label=r'95% Confidence Interval'\n",
    "            )\n",
    "            axes[r,c].legend(fontsize=32, loc='upper left')\n",
    "    \n",
    "    # ax = plt.gca()\n",
    "    # ax.set_yticks(Ys[0])\n",
    "    plt.tight_layout()\n",
    "    if save_name:\n",
    "        fig.savefig(f\"{save_pic_dir}/{save_name}\", dpi=400)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_matrix_T = draw_matrix1.transpose()\n",
    "\n",
    "\n",
    "\n",
    "Xs = [draw_matrix_T['1-1'], draw_matrix_T['2-1'], draw_matrix_T['3-1'], draw_matrix_T['4-1']]\n",
    "\n",
    "Ys = [draw_matrix_T['1-2'], draw_matrix_T['2-2'], draw_matrix_T['3-2'], draw_matrix_T['4-2']]\n",
    "\n",
    "# print(len(Ys))\n",
    "# print(Ys)\n",
    "\n",
    "display_scatter_multilines(Xs, Ys, titles=['category-1: Short Q, Short A', 'category-2: Short Q, Long A', 'category-3: Long Q, Short A', 'category-4: Long Q, Long A', 'category-5: Open-End'], save_name='scatters_all.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate ROC Curve And Calculate AUC Values(Figure 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import sklearn.metrics as metrics\n",
    "import numpy as np\n",
    "\n",
    "# plt.rc('font',family='times')\n",
    "plt.clf()\n",
    "plt.figure(constrained_layout=True)\n",
    "# plt.figure(figsize=(5, 4))\n",
    "figure ,axes =plt.subplots(1,2,figsize=(14,6),dpi=400)\n",
    "pred_dir = \"./pred\"\n",
    "\n",
    "# Make colormap\n",
    "viridis = plt.colormaps['viridis'].resampled(len(modes) + 1) \n",
    "cmap = viridis.colors[:(len(modes))][::-1]\n",
    "mutual_dir = f\"mutual_detect/{model_name}\"\n",
    "\n",
    "for flag in [1, 2]:\n",
    "    tp_var = \"0.95TP\" if flag == 1 else \"0.7TP\"\n",
    "    modes = [modes_i[flag] for modes_i in all_modes]\n",
    "    for j in range(1, 5): # mode:0 1 2 3 4\n",
    "        all_z_scores = []\n",
    "        all_human_z_scores = []\n",
    "        all_wm_z_scores = []\n",
    "        \n",
    "        # get as reference to detect human generation z_scores\n",
    "        # print(modes[j])\n",
    "        ref_substr = \"_\".join(modes[j].split(\"_\")[1:])   \n",
    "        # print(ref_substr) \n",
    "        ref_subfolder = \"ref_\" + ref_substr\n",
    "        # print(ref_subfolder)\n",
    "        human_z_path = os.path.join(ref_subfolder, \"human_generation_z\")\n",
    "        human_z_path = os.path.join(mutual_dir, human_z_path)\n",
    "        files = os.listdir(human_z_path)\n",
    "        for file in files:\n",
    "            with open(os.path.join(human_z_path, file), \"r\") as f:\n",
    "                data = json.load(f) \n",
    "                \n",
    "            z_score_list = data[\"z_score_list\"]     \n",
    "            # concatenate all_z_score and z_score_list\n",
    "            all_z_scores.extend(z_score_list)\n",
    "            all_human_z_scores.extend(z_score_list)\n",
    "        wm_type = \"hard\" if \"hard\" in modes[j] else \"soft\"\n",
    "            \n",
    "        if \"old_hard\" in modes[j]:\n",
    "            elem = modes[j].split(\"_\")\n",
    "            # print(elem)\n",
    "            # print(elem[2])\n",
    "            new_elem = elem[:]\n",
    "            new_elem.pop(2)\n",
    "            # print(elem)\n",
    "            new_elem.append(elem[2])\n",
    "            act_subfolder = \"_\".join(new_elem)\n",
    "            \n",
    "        elif \"old_soft\" in modes[j]:\n",
    "            elem = modes[j].split(\"_\")\n",
    "            new_elem = elem\n",
    "            new_elem.pop(2)\n",
    "            act_subfolder = \"_\".join(new_elem)\n",
    "        else:\n",
    "            act_subfolder = modes[j]\n",
    "            wm_type = modes[j].split(\"_\")[1]\n",
    "        \n",
    "        filedir_path = os.path.join(pred_dir, act_subfolder)\n",
    "        z_score_path = os.path.join(filedir_path, \"z_score\")\n",
    "        \n",
    "        # print(\"z_score_path\", z_score_path)\n",
    "        if os.path.exists(z_score_path):\n",
    "            files = os.listdir(z_score_path)\n",
    "            for file in files:\n",
    "                with open(os.path.join(z_score_path, file), \"r\") as f:\n",
    "                    data = json.load(f) \n",
    "                    \n",
    "                z_score_list = data[\"z_score_list\"]     \n",
    "                # concatenate all_z_score and z_score_list\n",
    "                all_z_scores.extend(z_score_list)\n",
    "                all_wm_z_scores.extend(z_score_list)\n",
    "                \n",
    "        baseline_labels = np.zeros_like(all_human_z_scores)\n",
    "        wm_labels = np.ones_like(all_wm_z_scores)\n",
    "        all_labels = np.concatenate([baseline_labels, wm_labels])\n",
    "        \n",
    "        fpr, tpr, thresholds = metrics.roc_curve(all_labels, all_z_scores, pos_label=1)\n",
    "        roc_auc = metrics.auc(fpr, tpr)\n",
    "        model_name0 = modes[j].split(\"-\")[0]\n",
    "        \n",
    "        axes[flag - 1].plot(fpr, tpr, color=cmap[j], label = f\"{model_name0}_{wm_type},AUC:%0.3f\" % roc_auc, linewidth=3)   \n",
    "        \n",
    "    axes[flag - 1].set_xlabel('False Positive Rate', fontsize = 12)\n",
    "    axes[flag - 1].set_ylabel('True Positive Rate', fontsize = 12)\n",
    "    axes[flag - 1].set_title(tp_var, fontsize=30)\n",
    "    axes[flag - 1].plot([0, 1], [0, 1],'r--')\n",
    "    axes[flag - 1].set_xlim([0, 1])\n",
    "    axes[flag - 1].set_ylim([0, 1])\n",
    "    axes[flag - 1].legend(loc = 'lower right', fontsize = 12)\n",
    "    \n",
    "plot_name = (f\"roc_auc\")\n",
    "\n",
    "figure.savefig(f\"{save_pic_dir}/{plot_name}.pdf\", format=\"pdf\")\n",
    "# print(plot_name)\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "wmklmm",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}