noise_defense_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a946ceac-1b7c-4577-b968-45814fdcf3a4",
   "metadata": {},
   "source": [
    "# Noise Defense (Table 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e6a6602-d4b4-42e8-b626-f7c605aeaf3f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29098888-b102-4c7b-8a10-df2cf253c5b7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataset_name = 'ADULT'\n",
    "experiments = {\n",
    "    46: 'TabLeak',\n",
    "    0: 'Inverting Gradients'\n",
    "}\n",
    "batch_size = 32\n",
    "noise_scales = [1e-3, 1e-2, 1e-1]\n",
    "noise_type = 'Gaussian'\n",
    "random_seed = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c392167-6f01-47a0-b62c-93185fa76c3b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "table_dataframe = pd.DataFrame()\n",
    "table_dataframe['Noise Scale'] = noise_scales\n",
    "for experiment_number, experiment_name in experiments.items():\n",
    "    post_selection = 1 if experiment_number == 0 else 30\n",
    "    base_path = f'experiment_data/DP_experiments/{dataset_name}/experiment_{experiment_number}/DP_data_all_{experiment_number}_{dataset_name}_50_{post_selection}_1500_128_{noise_scales[-1]}_0.319_{random_seed}.npy'\n",
    "    if os.path.isfile(base_path):\n",
    "        experiment_data = np.load(base_path)[0, :, :, 0]\n",
    "    else:\n",
    "        print(f'Experiment {experiment_name} not found')\n",
    "        continue\n",
    "    \n",
    "    data_to_add = [(np.around(100 - 100*mean, 1), np.around(100*std, 1)) for mean, std in experiment_data[5, :, :2]]\n",
    "    table_dataframe[experiment_name] = data_to_add\n",
    "\n",
    "# load the trained network accuracies\n",
    "load_path = f'experiment_data/DP_experiments/ADULT/trained_model_accuracies_ADULT_{random_seed}.npy'\n",
    "if os.path.isfile(load_path):\n",
    "    mean_accuracies, std_accuracies = np.around(100*np.mean(np.load(load_path)[1:, :, -1, 0], axis=1), 1), np.around(100*np.std(np.load(load_path)[1:, :, -1, 0], axis=1), 1)\n",
    "    data_to_add = [(m, s) for m, s in zip(mean_accuracies, std_accuracies)]\n",
    "    table_dataframe['Network Accuracy'] = data_to_add\n",
    "else:\n",
    "    print('Trained network data missing, please run the python script train_noise_defended_networks.py')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d6e57f2-bce1-4f39-a233-ae5f8dcf09a1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "table_dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d90ee4f7-e02e-4c0f-9f32-185b71dda599",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}