visualization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a176614b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.decomposition import PCA\n",
    "import pandas as pd\n",
    "from multiprocessing import Pool\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.gridspec as grid_spec\n",
    "from KDEpy import FFTKDE\n",
    "from tqdm.auto import tqdm\n",
    "from math import ceil\n",
    "import itertools\n",
    "import scipy\n",
    "from fast_histogram import histogram2d\n",
    "import colorcet as cc\n",
    "import matplotlib.colors as colors\n",
    "import h5py\n",
    "import io\n",
    "import cloudpickle as cpl\n",
    "from collections import defaultdict\n",
    "import os\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "lmap = lambda func, *iterable: list(map(func, *iterable))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "965450c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_path = \"/data/output/20220226_robustness/dataset.h5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4bb9b628",
   "metadata": {},
   "outputs": [],
   "source": [
    "with h5py.File(dataset_path, \"r\") as f:\n",
    "    dX = f[\"filters\"][:].reshape(-1, 9) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8bfcca50",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_meta = pd.read_hdf(dataset_path, \"meta\")\n",
    "df_meta[\"filter_ids\"] = df_meta[\"filter_ids\"].apply(lambda s: np.arange(int(s.split(\":\")[0]), 1 + int(s.split(\":\")[1])))\n",
    "df_meta.Robust = df_meta.Robust.apply(bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2ee00039",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pairings = pd.read_csv(\"robust_vs_normal_pairings.csv\", index_col=0)\n",
    "all_used_model_names = pd.concat([df_pairings.Robust, df_pairings.Normal]).unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "9dd39504",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_scaled_filters(ids, limit=32):\n",
    "    filters = dX[ids[:limit]]\n",
    "    t = np.abs(filters).max()\n",
    "    if t == 0:\n",
    "        return filters\n",
    "    return filters / t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8fa9dc90",
   "metadata": {},
   "outputs": [],
   "source": [
    "from mpl_toolkits.axes_grid1 import ImageGrid\n",
    "\n",
    "def plot_and_save_filter_grid(name, data):\n",
    "    \n",
    "    from matplotlib.colors import LinearSegmentedColormap\n",
    "    \n",
    "    fig = plt.figure(figsize=(8, 6))\n",
    "    grid = ImageGrid(fig, 111, nrows_ncols=data.shape[:2], axes_pad=0.0)\n",
    "\n",
    "    for i, ax in enumerate(grid):\n",
    "        col_idx = i % data.shape[1]\n",
    "        row_idx = i // data.shape[1]\n",
    "\n",
    "        if col_idx == 0:\n",
    "            ax.set_ylabel(str(row_idx), rotation=0, ha=\"left\", va=\"center_baseline\", labelpad=15)\n",
    "\n",
    "        ax.set_xticks([])\n",
    "        ax.set_yticks([])\n",
    "        ax.imshow(data[row_idx, col_idx].reshape(3, 3), cmap=LinearSegmentedColormap.from_list(\"CyanOrange\", [\"C0\", \"white\", \"C1\"]), vmin=-1, vmax=1)\n",
    "        \n",
    "    plt.savefig(f\"plots/filters/filters_{name}.pdf\", bbox_inches='tight')\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a73cca7",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.shape[1] / 512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "7c12e32a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOsAAADrCAYAAACICmHVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAIhklEQVR4nO3dS2xcZxnG8ffM8Yw9ztiJ60mJbzMjtYtWFJCoBRVBLZUolwVC4rJDYotou0SIZTcskcoSiQVC6gYkJESARVVoqQJRRkVcBCikML4kJcSpO048mds5p4v6WAh5gg/vF6VP9f9tJ3rm2J6/ZyTrfImyLDMA736le30BAI6HWAERxAqIIFZABLECIogVEDFV5B+fum8xO7Oy5n7SUhS5N3Ih/vR0/Y1ta7Wa/osZD/0buVLsnuhsblur6f+6UvfCf2wF+kvh1uaGNRr+ry3kHy5DvKw3Nzbsxs7OkUuFYj2zsmbf/8mL7guarfhfiLlR4v92f/3LT1n7wu/8F9Pd8m/kpufdE+uPf9raF867d24l4X5et4dJkJ1PPfExe+k3v3XvjEP99jCz6bL/g+oTZx+b+BgfgwERxAqIIFZABLECIogVEEGsgAhiBUQQKyCCWAERxAqIIFZABLECIogVEEGsgAhiBUQUup+1VontbGvO/6zDff/GgWRmwT+SJmb9rn8nxEZuPHBPDJLM/v6m/4b4rRs990ZosY1tIX3TP5QEPDDAKu6FOBtPfIx3VkAEsQIiiBUQQayACGIFRBArIIJYARHECoggVkAEsQIiiBUQQayACGIFRBArIIJYARHECoggVkAEsQIiCh3rYuO+2c4l/7NmqX/jQFyK3RuJxdaNTrp35pcW3Ru5NPNvJGlq3d7IvTNT8X+Pc5U40PvDaBDmtdjd9m/kFh/0b4wnHzPDOysgglgBEcQKiCBWQASxAiKIFRBBrIAIYgVEECsgglgBEcQKiCBWQASxAiKIFRBBrIAIYgVEECsgIsqy4x9JUK/Xs0az6X7SAIcgHIoCbGz+83VrrZzxD4W4mFyauCf+urVjC+9bce8kIY6tOFDg5XZHg91/2fJqwz8U8GdWLftP1Oh0Orazs3PkVRU61qXRbNor5y+4LygN9RMzs6mS/7v9+KOPWPun3/NfTBTwg8rgpnti9XPfsGe/+2P3zt5t/9EwuXGg8H/23FfthXO/cu/EAV4/uUeWau6N9Y98dOJjfAwGRBArIIJYARHECoggVkAEsQIiiBUQQayACGIFRBArIIJYARHECoggVkAEsQIiiBUQQayAiEI3n5fSkdV623frWv4/ccW/kfTNblx+d1xLrt91T9weJvbnK/6dwTh1b+Rq04VechOVp0q2fKrq3pmvhrkeM7Pe2H9j/Z0OZuCdFRBBrIAIYgVEECsgglgBEcQKiCBWQASxAiKIFRBBrIAIYgVEECsgglgBEcQKiCBWQASxAiKIFRBBrICIQmda9JKStbvz7icdJuGOCamW4wArkVkU4PdWXPZv5GYX3RPVSmwfXD3p3tkfJO6N3AP314Ls/LEU2X0n/MfoxMO9AFeTj/mvp2Qc6wLII1ZABLECIogVEEGsgAhiBUQQKyCCWAERxAqIIFZABLECIogVEEGsgAhiBUQQKyCCWAERxAqIiLJs8p3p/61er2eNZtP9pKUs3MkDIXQ6HWutLgVYOv738n+K/CdgdLbfsFZjzb2zN3RPHJqbCXOaxt8uv24L96+4d+JSFOBq3pEUaGmS3WtXbL+7e+RFFTrWpdFs2ivnL7gvqDZ+y71xKMBxLOuPnbX2uR/4ryUJ+Kqe8R/Hsv7Zr1j75V+4d365Ge4D2JMPnQ6y88DDH7Knn/+ReyfE0TC5vf7IvfH8s1+a+BgfgwERxAqIIFZABLECIogVEEGsgAhiBUQQKyCCWAERxAqIIFZABLECIogVEEGsgAhiBUQUup81zcwGI/+N47UTi+6NQ+nYPTGKq3Z17gPunXIc7ndfb+j/upJoynbjunvnD9v/cG/k6rUw94/OlGN7eGnevROFu/fcTs36b6yfvsNriHdWQASxAiKIFRBBrIAIYgVEECsgglgBEcQKiCBWQASxAiKIFRBBrIAIYgVEECsgglgBEcQKiCBWQASxAiIKHesyZYktlvb9z3rtkn8jF/uPCbnZH9vLl667d05V/cd6hBTfvGILv/6We+ebZ/xH3hzaqwaZ6Q3H9trmrnune3sU4GresXjC/1rsjycfm8Q7KyCCWAERxAqIIFZABLECIogVEEGsgAhiBUQQKyCCWAERxAqIIFZABLECIogVEEGsgAhiBUQQKyAiyrLs2P+4vjCXtZZPB3jW2L+RK/m3OlevW6ux5r+WdPJd/oXF/lMnOhub1mq1/Ncy2PNvHOhFs0F2/r3dsdbasn8o5GvRjt/SJJ2tK7ZzYzc66rFCx7q0lk9b+4Vvuy/Ipuf8G7mZk+6J9S88Y+1XX/JfS/8t/0buhP+X4vrZJ+3ixYvunejyi+6NXLv8aJCdr33+49b++Q/9Q9Pz/o1clron1p/64sTH+BgMiCBWQASxAiKIFRBBrIAIYgVEECsgglgBEcQKiCBWQASxAiKIFRBBrIAIYgVEECsgotD9rDY9b/bgJ/3PWg7zX9WbmVmp2JdwpDQJcy/qcN+/kasu+DduXbPo1e/4d67+3r9xYH31L2GG0sSs3/XvhPg+5/q3/Bt3uCeWd1ZABLECIogVEEGsgAhiBUQQKyCCWAERxAqIIFZABLECIogVEEGsgAhiBUQQKyCCWAERxAqIIFZABLECIgqdiTLMSrY9nHU/6aDn/+/cc9VygK0sDXMky9ySfyNXqbkn/tSdsda5h9w7n3n/J9wbuc7FQEffxBWzk2v+nel5/0YuGfk3osnvn7yzAiKIFRBBrIAIYgVEECsgglgBEcQKiCBWQASxAiKIFRBBrIAIYgVEECsgglgBEcQKiCBWQASxAiKiLMuO/4+j6LqZbdy9y7lnPmxmr93ri7gL3qtfl9l792trZll2+qgHCsUK4N7hYzAgglgBEcQKiCBWQASxAiKIFRBBrIAIYgVEECsg4m2CKYGo2z9bSgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 32 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib.colors import LinearSegmentedColormap\n",
    "\n",
    "data = np.stack(df_meta[(df_meta.Name == \"robustbench_Addepalli2021Towards_WRN34_cifar10_linf_13\") & (df_meta.index.get_level_values(\"conv_depth_norm\") == 1)].filter_ids.apply(lambda x: get_scaled_filters(x, 16)).values)\n",
    "fig = plt.figure(figsize=(4, 4))\n",
    "grid = ImageGrid(fig, 111, nrows_ncols=(4, 4), axes_pad=0.0)\n",
    "\n",
    "for i, ax in enumerate(grid):\n",
    "    row_idx, col_idx  = divmod(i, data.shape[1])\n",
    "    ax.set_xticks([])\n",
    "    ax.set_yticks([])\n",
    "    ax.imshow(data[row_idx, col_idx].reshape(3, 3), cmap=LinearSegmentedColormap.from_list(\"CyanOrange\", [\"C0\", \"white\", \"C1\"]), vmin=-1, vmax=1)\n",
    "plt.savefig(\"/gavrikov/cvpr22w_RobustnessThroughTheLense/plots/last_filters_robust.pdf\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "9c3bff23",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPEAAAAfCAYAAAA2hSv7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAEqklEQVR4nO2cy09cdRTHv3fuvHgOj1Iewwy3NTaVujJguyiJ2NaoIWkbFzUxXfc/6NqdO5cuNK7cWON7QeojTRtp0iLFR7CNqdRhmBkIdJyhIDCv+3OhiTnn3EImLvRnzmd3vvzO/J6HO+f8LjjGGCiKYi+hf3sAiqL8MzSIFcVyNIgVxXI0iBXFcjSIFcVyws007urpNQPJFNE2d+vEbou6wu/hah7D6TTRGj6tioccR/jlsksYSlG/3WqD2K0xOYVcdglJ5rexUyN2Rzwi/Aq5LPj8dlh/nS3UbyWXxciIR7RKnfoEza2QfQAvNcRU2Y6TWc7DSw4wld8wyBuHzEoRHtsDhOheBd1TZDIZpNMjROPzc0PyWZBfzoo999meB/stwfM8ovGz4oL2DwD3f11Gf3J4T7+gi5jyWkGMk7cL2D4UFu/BO9hBxXqF2pG48MuUGmJ+lbpP7GhYrsv8nTsPjTF9ciRNBvFAMoV3Pv6KaDd+Xif2mNcj/C5ffBnT12aIVt6mQdUWEIxTp07iyvR1ov1U2Ni3v7OnJ/DhFzeIdnVhldgTT8r1eG1qEu9+8jXRFvK0v+efOkjsV1+axM1bt4n2YP13YnfE5dzOT45hbvo9KoblpnPGTr+Cuc/fpqJfZ7Y85GMXLmPu5jWiNWJd1C3glD87Po7rM7eIdn9tk9i97THhd/bMhNjzrQodZ1dLVPhNnTqJ27PfUj/2oEj4JeF35MQZvPXBl0T7bbtKbB7UAPDGpXNinDyoYgFBdW7cw9ybF6hY/IXafUeF39jr32Bulp6XxeIOsdM9rcIvGnaXhPgX+nVaUSxHg1hRLEeDWFEsp6mcuLRdxUfzeaJ9+h2182X6/R74s6DxaIfmNetbFdGO0/ANSiyvyRS3id3dKvOq8k4Nn31fINr7s1nahuXkAFBvGKxv0nHxnLi7jfZXbfhizgssb092tYi+4EaBBC2iwfiyHaexK3OvjkFqx1jBBYBxXFQinURbXKW5rXdA5mLGyEJWrkTn64b2L8gBQKVG53e3tCHa1H2DMtvzH9kePHdE1jN8X+bcvKAYDgeM08hCVnGL9h9UK0B9F1i7S7W1e2xQrFYBoNbwUXhEP/+H5TKxeTF1P/RJrCiWo0GsKJajQawoltNUTpyIR/DiMfqiQX8HvSMcHaJ5FwDMOA5a2EsgB9qoH/85AMQjLkYH6efV2B3ecMCdWmc8ghdG+wNm8DcnDvcK7YrrYCBB72pHhxLE5vlt2HXQx9bgib52Yg8mAu5/61WgTPN0tLI7b1fm+3BjQM9hqvHcy5G/mx34iBmaiw330Lm0RuQe+MaIe9MUW/PBhMz5jQFqDZpLtrP78qeTdG0BIOwY9EbpfI4fouvibK4Iv3gkhKODtBaQZfWT4W55VtyQg0724g9/MSjobMKNAd2HqFZj9aDUceEWca+K88DHdSwghvZCn8SKYjkaxIpiORrEimI5GsSKYjlOM/9jy3GcdQCPfRF7D54BMP8/9bNhjOpnv9/I4/6KqakgVhTlv4d+nVYUy9EgVhTL0SBWFMvRIFYUy9EgVhTL0SBWFMvRIFYUy9EgVhTL0SBWFMv5A3I9h4YJ0MNBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 32 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib.colors import LinearSegmentedColormap\n",
    "\n",
    "data = np.stack(df_meta[(df_meta.Name == \"hso_normal_training_robustbench_Addepalli2021Towards_WRN34_cifar10_13\") & (df_meta.index.get_level_values(\"conv_depth_norm\") == 1)].filter_ids.apply(lambda x: get_scaled_filters(x, 16)).values)\n",
    "fig = plt.figure(figsize=(4, 4))\n",
    "grid = ImageGrid(fig, 111, nrows_ncols=(1, 16), axes_pad=0.0)\n",
    "\n",
    "for i, ax in enumerate(grid):\n",
    "    row_idx, col_idx  = divmod(i, data.shape[1])\n",
    "    ax.set_xticks([])\n",
    "    ax.set_yticks([])\n",
    "    ax.imshow(data[row_idx, col_idx].reshape(3, 3), cmap=LinearSegmentedColormap.from_list(\"CyanOrange\", [\"C0\", \"white\", \"C1\"]), vmin=-1, vmax=1)\n",
    "plt.savefig(\"/gavrikov/cvpr22w_RobustnessThroughTheLense/plots/last_filters_normal.pdf\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7c7817b",
   "metadata": {},
   "outputs": [],
   "source": [
    "todo = [(model_name, np.stack(df_meta[df_meta.Name == model_name].filter_ids.apply(get_scaled_filters))) for model_name in all_used_model_names]\n",
    "Parallel(n_jobs=-1)(delayed(plot_and_save_filter_grid)(name, data) for (name, data) in todo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "a4d1925b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_and_save_rgb_filter_grid(name, data):\n",
    "    fig = plt.figure(figsize=(8, 6))\n",
    "    grid = ImageGrid(fig, 111, nrows_ncols=(data.shape[0] // 16, 16), axes_pad=0.0)\n",
    "    for i, ax in enumerate(grid):\n",
    "        col_idx = i % data.shape[0]\n",
    "        row_idx = i // data.shape[0]\n",
    "\n",
    "        ax.set_xticks([])\n",
    "        ax.set_yticks([])\n",
    "\n",
    "        f = data[i].transpose(1, 2, 0)\n",
    "        #layer = layer - layer.min()\n",
    "        f = 0.5+0.5*f/np.abs(data).max()  \n",
    "        ax.imshow(f)\n",
    "            \n",
    "    plt.savefig(f\"plots/filters/rgb_filters_{name}.pdf\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2f7337d",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_and_save_rgb_filter_grid(, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "755c8078",
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = [(model_name, df_meta.reset_index()[(df_meta.reset_index()[\"conv_depth\"] == 0) & (df_meta.reset_index()[\"Name\"] == model_name)].filter_ids.values) for model_name in all_used_model_names]\n",
    "todo = [(name, dX[idset[0]].reshape(-1, 3, 3, 3)) for (name, idset) in ids if len(idset) >= 1]\n",
    "Parallel(n_jobs=-1)(delayed(plot_and_save_rgb_filter_grid)(name, data) for (name, data) in todo)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}