5.0_DSAE-Hyp_ablations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e33df9c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.decomposition import TruncatedSVD, PCA\n",
    "import sklearn\n",
    "from tqdm import tqdm\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e3ac262",
   "metadata": {},
   "outputs": [],
   "source": [
    "def directed_classifier(embedding, train_pairs, test_pairs):\n",
    "    \n",
    "    embedding.index = embedding.index.astype(str)\n",
    "    train_pairs = train_pairs.astype(str)\n",
    "    test_pairs = test_pairs.astype(str)\n",
    "    \n",
    "    X_train_pos = np.hstack((embedding.loc[train_pairs[:, 0]], embedding.loc[train_pairs[:, 1]]))\n",
    "    X_train_neg = np.hstack((embedding.loc[train_neg_pairs[:, 0]], embedding.loc[train_neg_pairs[:, 1]]))\n",
    "    X_train = np.vstack((X_train_pos, X_train_neg))\n",
    "    y_train = [1]*(X_train_pos.shape[0]) + [0]*(X_train_neg.shape[0])\n",
    "\n",
    "    X_test_pos = np.hstack((embedding.loc[test_pairs[:, 0]], embedding.loc[test_pairs[:, 1]]))\n",
    "    X_test_neg = np.hstack((embedding.loc[test_neg_pairs[:, 0]], embedding.loc[test_neg_pairs[:, 1]]))\n",
    "    X_test = np.vstack((X_test_pos, X_test_neg))\n",
    "    y_test = [1]*(X_test_pos.shape[0]) + [0]*(X_test_neg.shape[0])\n",
    "\n",
    "    clf = sklearn.linear_model.RidgeClassifier(random_state=0)\n",
    "    clf.fit(X_train, y_train, )\n",
    "    y_score_test = clf.decision_function(X_test)\n",
    "\n",
    "    return ((sklearn.metrics.roc_auc_score(y_test, y_score_test), \n",
    "            sklearn.metrics.average_precision_score(y_test, y_score_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e23017df",
   "metadata": {},
   "source": [
    "## Directed Scattering without AE (best q, J)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65676575",
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in ['Texas', 'Cornell', 'omnipath', 'SIGNOR', 'iPTMnet']:\n",
    "    config = np.load(f'results/DS-PM/{dataset}/best_val.npz', allow_pickle=True)['config'][()]\n",
    "    best_q, best_J =  config['q'], config['J']\n",
    "    print (dataset, best_q, best_J)\n",
    "    df = pd.read_csv(f'results/Directed_Scattering/Directed_Scattering_J{best_J}_q{best_q}_{dataset}_train_val_embedding.csv', compression='gzip', index_col=0)\n",
    "    \n",
    "    dim = min(df.shape[1], 128)\n",
    "    df_pc = pd.DataFrame(PCA(n_components=dim).fit_transform(df), index=df.index)\n",
    "    \n",
    "    splits = np.load(f'data/{dataset}_curated_interactions.npz')\n",
    "    train, val, test = splits['train'], splits['val'], splits['test']\n",
    "    \n",
    "    all_data = np.vstack((train, val, test))\n",
    "    \n",
    "    train = np.vstack((train, val))\n",
    "    \n",
    "    train_neg_pairs = []\n",
    "    for pair in tqdm(train):\n",
    "        if len(set(np.where(all_data[:, 0] == pair[1])[0]).intersection(np.where(all_data[:, 1] == pair[0])[0])) == 0:\n",
    "            train_neg_pairs.append([pair[1], pair[0]])\n",
    "\n",
    "    train_neg_pairs = np.array(train_neg_pairs).reshape(-1,2)\n",
    "\n",
    "    test_neg_pairs = []\n",
    "    for pair in tqdm(test):\n",
    "        if len(set(np.where(all_data[:, 0] == pair[1])[0]).intersection(np.where(all_data[:, 1] == pair[0])[0])) == 0:\n",
    "            test_neg_pairs.append([pair[1], pair[0]])\n",
    "\n",
    "    test_neg_pairs = np.array(test_neg_pairs).reshape(-1,2)\n",
    "    \n",
    "    train_neg_pairs = train_neg_pairs.astype(str)\n",
    "    test_neg_pairs = test_neg_pairs.astype(str)\n",
    "    \n",
    "    auroc = directed_classifier(df_pc, train, test)[0]\n",
    "    print (dataset, auroc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46dead0e",
   "metadata": {},
   "source": [
    "## Run DS-PM with varying q and J 5 times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24304f6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_runs = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c5f13b5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for dataset in ['Texas', 'Cornell', 'omnipath', 'SIGNOR', 'iPTMnet']:\n",
    "    config = np.load(f'results/DS-PM/{dataset}/best_val.npz', allow_pickle=True)['config'][()]\n",
    "    for q in [0.0, 0.1, 0.2]:\n",
    "        for J in [5, 10, 15]:\n",
    "            for i in range(num_runs):\n",
    "                print (f\"python test.py --model DS-PM --lr {config['lr']} --c {config['c']} --act linear --weight-decay {config['weight_decay']} --save-as best_test_ablated_q_{q}_J_{J}_{i} --q {q} --dataset {dataset} --J {J} --seed {1234+i}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be51d514",
   "metadata": {},
   "source": [
    "## Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06351295",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "for dataset in ['Texas', 'Cornell', 'SIGNOR', 'iPTMnet', 'omnipath']:\n",
    "    splits = np.load(f'data/{dataset}_curated_interactions.npz')\n",
    "    train, val, test = splits['train'], splits['val'], splits['test']\n",
    "    \n",
    "    all_data = np.vstack((train, val, test))\n",
    "    \n",
    "    train = np.vstack((train, val))\n",
    "    \n",
    "    train_neg_pairs = []\n",
    "    for pair in tqdm(train):\n",
    "        if len(set(np.where(all_data[:, 0] == pair[1])[0]).intersection(np.where(all_data[:, 1] == pair[0])[0])) == 0:\n",
    "            train_neg_pairs.append([pair[1], pair[0]])\n",
    "\n",
    "    train_neg_pairs = np.array(train_neg_pairs).reshape(-1,2)\n",
    "\n",
    "    test_neg_pairs = []\n",
    "    for pair in tqdm(test):\n",
    "        if len(set(np.where(all_data[:, 0] == pair[1])[0]).intersection(np.where(all_data[:, 1] == pair[0])[0])) == 0:\n",
    "            test_neg_pairs.append([pair[1], pair[0]])\n",
    "\n",
    "    test_neg_pairs = np.array(test_neg_pairs).reshape(-1,2)\n",
    "    \n",
    "    train_neg_pairs = train_neg_pairs.astype(str)\n",
    "    test_neg_pairs = test_neg_pairs.astype(str)\n",
    "    \n",
    "    for q in [0.0, 0.1, 0.2]:\n",
    "        for J in [5, 10, 15]:\n",
    "            for i in range(num_runs):\n",
    "                res = np.load(f'results/DS-PM/{dataset}/best_test_ablated_q_{q}_J_{J}_{i}_results.npz', allow_pickle=True)\n",
    "                res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "                auroc = directed_classifier(res, train, test)[0]\n",
    "                \n",
    "                results.append([dataset, q, J, i, auroc])\n",
    "                \n",
    "results = pd.DataFrame(results)\n",
    "results.columns = ['Dataset', 'q', 'J', 'run', 'AUROC']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdd3e6ad",
   "metadata": {},
   "source": [
    "## Get table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f1e300d",
   "metadata": {},
   "outputs": [],
   "source": [
    "auroc_per_dataset = defaultdict(list)\n",
    "for dataset in ['Texas', 'Cornell', 'omnipath', 'SIGNOR', 'iPTMnet']:\n",
    "    \n",
    "    res = results[results['Dataset'] == dataset]\n",
    "    for q in [0.0, 0.1, 0.2]:\n",
    "        for J in [5, 10, 15]:\n",
    "            auroc_per_dataset[dataset].append(res[(res['q'] == q) & (res['J'] == J)]['AUROC'].mean())\n",
    "            \n",
    "auroc_per_dataset = pd.DataFrame(auroc_per_dataset.values())\n",
    "auroc_per_dataset.index = ['Texas', 'Cornell', 'omnipath', 'SIGNOR', 'iPTMnet']\n",
    "auroc_per_dataset = auroc_per_dataset.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f57b5375",
   "metadata": {},
   "outputs": [],
   "source": [
    "print ('Method', '&', ' & '.join(['Texas', 'Cornell', 'omnipath', 'SIGNOR', 'iPTMnet']), '\\\\\\\\')\n",
    "print ('\\hline\\hline')\n",
    "count = 0\n",
    "for q in [0.0, 0.1, 0.2]:\n",
    "    for J in [5, 10, 15]:\n",
    "        print (f'q={q} J={J}', '&', ' & '.join(['%.3f'% x for x in auroc_per_dataset.loc[count].values]), \"\\\\\\\\\")\n",
    "        count += 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}