IM-WCE-MSCOCO.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# General Imports\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from time import strftime, localtime\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sn\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, roc_auc_score, f1_score, cohen_kappa_score, hamming_loss\n",
    "\n",
    "# [TODO] edit the path to the tfslim directoy \"https://github.com/tensorflow/models/tree/master/research/slim\"\n",
    "SLIM_PATH = '/srv/workspace/research/mlml/models/research/slim'\n",
    "os.chdir(SLIM_PATH)\n",
    "\n",
    "# Deep Learning\n",
    "import tensorflow as tf\n",
    "from nets import inception\n",
    "from datasets import dataset_utils\n",
    "from tensorflow.contrib import slim\n",
    "\n",
    "\n",
    "plt.rcParams.update({'font.size': 22})\n",
    "\n",
    "# [TODO] edit the paths to match the machine's directories \n",
    "SOURCE_PATH = \"/srv/workspace/research/mlml/mlml_weightedLoss/\"\n",
    "# path to the formatted dataset\n",
    "IMAGES_PATH = \"/srv/workspace/research/mlml/datasets/mscoco/train_formatted_normalized_npz/\"\n",
    "TEST_IMAGES_PATH = IMAGES_PATH\n",
    "# path to the output direcorty\n",
    "OUTPUT_PATH =\"/srv/workspace/research/mlml/experiments_results_balanced/\"\n",
    "# path to the pretrained model (if not downloaded already, call the download_pretrained_model function below)\n",
    "PRETRAINED_MODEL_DIR = '/srv/workspace/research/mlml/pretrained_models/'\n",
    "\n",
    "INPUT_SHAPE = (224, 224, 3)\n",
    "INPUT_IMAGE_MEAN = [0.485, 0.456, 0.406]\n",
    "INPUT_IMAGE_STD = [0.229, 0.224, 0.225]\n",
    "\n",
    "labels_set = pd.read_csv('/srv/workspace/research/mlml/mlml_weightedLoss/labels_balanced_4labels/missing_labels0.0/train1_0.0.csv')\n",
    "labels_set.drop(\"Unnamed: 0\", axis = 1, inplace = True)\n",
    "\n",
    "LABELS_LIST = labels_set.columns\n",
    "NUM_CLASSES = len(LABELS_LIST)\n",
    "\n",
    "# Training paramaeters\n",
    "BATCH_SIZE = 32\n",
    "TRAINING_STEPS = int(len(labels_set)/BATCH_SIZE)\n",
    "NUM_EPOCHS = 20\n",
    "# compute the weights to balance between the ratio of positive and negative samples per label (not the confidence-based weights) \n",
    "Pos_balance_weights = np.round((1 / (np.sum(labels_set.values, axis = 0)/ len(labels_set))/2)).astype(np.float32)\n",
    "Pos_balance_weights = np.clip(Pos_balance_weights,1,10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "code_folding": [
     1,
     7,
     12,
     18,
     24,
     41,
     48,
     51,
     77,
     96,
     106,
     118,
     125,
     136,
     193,
     218,
     335,
     366,
     389,
     407,
     428,
     454
    ]
   },
   "outputs": [],
   "source": [
    "# Define helper functions\n",
    "\n",
    "def download_pretrained_model():\n",
    "    url = \"http://download.tensorflow.org/models/inception_resnet_v2_2016_08_30.tar.gz\"\n",
    "    if not tf.gfile.Exists(PRETRAINED_MODEL_DIR):\n",
    "        tf.gfile.MakeDirs(PRETRAINED_MODEL_DIR)\n",
    "    dataset_utils.download_and_uncompress_tarball(url, PRETRAINED_MODEL_DIR)\n",
    "\n",
    "# functions for defining the model\n",
    "def get_weights(shape):\n",
    "    w = tf.Variable(tf.truncated_normal(shape, stddev=0.1))\n",
    "    return w\n",
    "\n",
    "def bias_variable(shape):\n",
    "    initial = tf.constant(0.1, shape=shape)\n",
    "    b = tf.Variable(initial)\n",
    "    return b\n",
    "\n",
    "def full_layer(input, size):\n",
    "    in_size = int(input.get_shape()[1])\n",
    "    W = get_weights([in_size, size])\n",
    "    b = bias_variable([size])\n",
    "    return tf.matmul(input, W) + b\n",
    "\n",
    "def dataset_from_csv(csv_path, **kwargs):\n",
    "    \"\"\"\n",
    "        Load dataset from a csv file.\n",
    "        kwargs are forwarded to the pandas.read_csv function.\n",
    "    \"\"\"\n",
    "    df = pd.read_csv(csv_path, **kwargs)\n",
    "\n",
    "    dataset = (\n",
    "        tf.data.Dataset.from_tensor_slices(\n",
    "            {\n",
    "                key:df[key].values\n",
    "                for key in df\n",
    "            }\n",
    "        )\n",
    "    )\n",
    "    return dataset\n",
    "\n",
    "def set_tensor_shape(tensor, tensor_shape):\n",
    "        \"\"\"\n",
    "            set shape for a tensor (not in place, as opposed to tf.set_shape)\n",
    "        \"\"\"\n",
    "        tensor.set_shape(tensor_shape)\n",
    "        return tensor\n",
    "\n",
    "def mscoco_labels_idx_to_names(labels,labelnames = LABELS_LIST):\n",
    "    return labelnames[np.where(labels == 1)]\n",
    "\n",
    "# Dataset piplines [load images]\n",
    "def load_image_npz(*args):\n",
    "    \"\"\"\n",
    "        loads image with error tracking.\n",
    "        args : image ID, path to dataset\n",
    "        return:\n",
    "            Features: numpy ndarray, computed features (if no error occured, otherwise: 0)\n",
    "            Error: boolean, False if no error, True if an error was raised during features computation.\n",
    "    \"\"\"\n",
    "    path = IMAGES_PATH\n",
    "    image_id, dummy_path = args\n",
    "    try:\n",
    "        image = np.load(os.path.join(path, str(image_id) + '.npz'))['image']\n",
    "        return image, False\n",
    "    except Exception as err:\n",
    "        print(\"\\n Error while computing features for \" + str(image_id) + '\\n')\n",
    "        return np.float32(0.0), True\n",
    "\n",
    "def load_imge_tf(sample, identifier_key=\"Unnamed: 0\",\n",
    "                 path=\"/srv/workspace/research/mlml/datasets/mscoco/train_formatted_npz/\", device=\"/cpu:0\",\n",
    "                 features_key=\"features\"):\n",
    "    \"\"\"\n",
    "        wrap load_images into a tensorflow function.\n",
    "    \"\"\"\n",
    "    with tf.device(device):\n",
    "        input_args = [sample[identifier_key], tf.constant(path)]\n",
    "        res = tf.py_func(load_image_npz,\n",
    "                         input_args,\n",
    "                         (tf.float32, tf.bool),\n",
    "                         stateful=False),\n",
    "        image, error = res[0]\n",
    "        image = tf.convert_to_tensor(image, dtype=tf.float32)\n",
    "        res = dict(list(sample.items()) + [(features_key, image), (\"error\", error)])\n",
    "        return res\n",
    "\n",
    "    \n",
    "# Dataset pipelines [load weights]\n",
    "def get_weights_py(image_id):\n",
    "    weights_negative = global_weights_negative[global_weights_negative.iloc[:,0] == image_id]\n",
    "    samples_weights_negative = weights_negative.iloc[:, 1:].values.flatten()\n",
    "    samples_weights_negative = samples_weights_negative.astype(np.float32)\n",
    "    return samples_weights_negative\n",
    "\n",
    "def tf_get_weights_py(sample,device = \"/cpu:0\"):\n",
    "    with tf.device(device):\n",
    "        input_args = [sample[\"Unnamed: 0\"]]\n",
    "        negative_weights = tf.py_func(get_weights_py,\n",
    "            input_args,\n",
    "            [tf.float32],\n",
    "            stateful=False)\n",
    "        res = dict(list(sample.items()) + [(\"negative_weights\", negative_weights)])\n",
    "        return res\n",
    "    \n",
    "\n",
    "# Dataset pipelines [load labels]\n",
    "def get_labels_py(image_id):\n",
    "    labels = global_labels[global_labels.iloc[:,0] == image_id]\n",
    "    labels = labels.iloc[:, 1:].values.flatten()\n",
    "    labels = labels.astype(np.float32)\n",
    "    return labels\n",
    "\n",
    "\n",
    "def tf_get_labels_py(sample, device=\"/cpu:0\"):\n",
    "    with tf.device(device):\n",
    "        input_args = [sample[\"Unnamed: 0\"]]\n",
    "        labels = tf.py_func(get_labels_py,\n",
    "                            input_args,\n",
    "                            [tf.float32],\n",
    "                            stateful=False)\n",
    "        res = dict(list(sample.items()) + [(\"binary_label\", labels)])\n",
    "        return res\n",
    "\n",
    "\n",
    "def get_dataset(input_csv, input_shape=INPUT_SHAPE, batch_size=BATCH_SIZE, shuffle=True,\n",
    "                infinite_generator=True, random_crop=False,\n",
    "                num_parallel_calls=32):\n",
    "    # build dataset from csv file\n",
    "    dataset = dataset_from_csv(input_csv)\n",
    "    # Shuffle data\n",
    "    if shuffle:\n",
    "        dataset = dataset.shuffle(buffer_size=100, seed=0, reshuffle_each_iteration=True)\n",
    "\n",
    "    # load image\n",
    "    dataset = dataset.map(lambda sample: load_imge_tf(sample), num_parallel_calls=1)\n",
    "\n",
    "    # filter out errors\n",
    "    dataset = dataset.filter(lambda sample: tf.logical_not(sample[\"error\"]))\n",
    "\n",
    "    # set features shape\n",
    "    dataset = dataset.map(lambda sample: dict(sample,\n",
    "                                              features=set_tensor_shape(sample[\"features\"], input_shape)))\n",
    "\n",
    "    dataset = dataset.map(lambda sample: tf_get_labels_py(sample), num_parallel_calls=1)\n",
    "\n",
    "    # set output shape\n",
    "    dataset = dataset.map(lambda sample: dict(sample, binary_label=set_tensor_shape(\n",
    "        sample[\"binary_label\"], (NUM_CLASSES))))\n",
    "    \n",
    "    # load weights\n",
    "    dataset = dataset.map(lambda sample: tf_get_weights_py(sample), num_parallel_calls=1)\n",
    "    \n",
    "    dataset = dataset.map(lambda sample: dict(sample, negative_weights=set_tensor_shape(\n",
    "    sample[\"negative_weights\"], (NUM_CLASSES))))\n",
    "    \n",
    "    if infinite_generator:\n",
    "        # Repeat indefinitly\n",
    "        dataset = dataset.repeat(count=-1)\n",
    "\n",
    "    # Make batch\n",
    "    dataset = dataset.batch(batch_size)\n",
    "\n",
    "    # Select only features and annotation\n",
    "    dataset = dataset.map(lambda sample: (sample[\"features\"], sample[\"binary_label\"],\n",
    "                                          sample[\"negative_weights\"]))\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " def create_analysis_report(model_output, groundtruth, output_path, LABELS_LIST, validation_output=None,\n",
    "                           validation_groundtruth=None):\n",
    "    \"\"\"\n",
    "    Create a report of all the different evaluation metrics, including optimizing the threshold with the validation set\n",
    "    if it is passed in the parameters\n",
    "    \"\"\"\n",
    "    # Round the probabilities at 0.5\n",
    "    model_output_rounded = np.round(model_output)\n",
    "    model_output_rounded = np.clip(model_output_rounded, 0, 1)\n",
    "    # Create a dataframe where we keep all the evaluations, starting by prediction accuracy\n",
    "    accuracies_perclass = sum(model_output_rounded == groundtruth) / len(groundtruth)\n",
    "    results_df = pd.DataFrame(columns=LABELS_LIST)\n",
    "    results_df.index.astype(str, copy=False)\n",
    "    percentage_of_positives_perclass = sum(groundtruth) / len(groundtruth)\n",
    "    results_df.loc[0] = percentage_of_positives_perclass\n",
    "    results_df.loc[1] = accuracies_perclass\n",
    "    results_df.index = ['Ratio of positive samples', 'Model accuracy']\n",
    "\n",
    "    # plot the accuracies per class\n",
    "    results_df.T.plot.bar(figsize=(22, 12), fontsize=18)\n",
    "    plt.title('Model accuracy vs the ratio of positive samples per class')\n",
    "    plt.xticks(rotation=45)\n",
    "    plt.savefig(os.path.join(output_path, \"accuracies_vs_positiveRate.pdf\"), format=\"pdf\")\n",
    "    plt.savefig(os.path.join(output_path, \"accuracies_vs_positiveRate.png\"))\n",
    "\n",
    "    # Getting the true positive rate perclass\n",
    "    true_positives_ratio_perclass = sum((model_output_rounded == groundtruth) * (groundtruth == 1)) / sum(groundtruth)\n",
    "    results_df.loc[2] = true_positives_ratio_perclass\n",
    "    # Get true negative ratio\n",
    "    true_negative_ratio_perclass = sum((model_output_rounded == groundtruth)\n",
    "                                       * (groundtruth == 0)) / (len(groundtruth) - sum(groundtruth))\n",
    "    results_df.loc[3] = true_negative_ratio_perclass\n",
    "    # compute additional metrics (AUC,f1,recall,precision)\n",
    "    auc_roc_per_label = roc_auc_score(groundtruth, model_output, average=None)\n",
    "    precision_perlabel = precision_score(groundtruth, model_output_rounded, average=None)\n",
    "    recall_perlabel = recall_score(groundtruth, model_output_rounded, average=None)\n",
    "    f1_perlabel = f1_score(groundtruth, model_output_rounded, average=None)\n",
    "    kappa_perlabel = [cohen_kappa_score(groundtruth[:, x], model_output_rounded[:, x]) for x in range(len(LABELS_LIST))]\n",
    "    results_df = results_df.append(\n",
    "        pd.DataFrame([auc_roc_per_label,recall_perlabel, precision_perlabel, f1_perlabel, kappa_perlabel], columns=LABELS_LIST))\n",
    "    results_df.index = ['Ratio of positive samples', 'Model accuracy', 'True positives ratio',\n",
    "                        'True negatives ratio', \"AUC\", \"Recall\", \"Precision\", \"f1-score\", \"Kappa score\"]\n",
    "\n",
    "\n",
    "    # Adjusting threshold based on validation set\n",
    "    if (validation_groundtruth is not None and validation_output is not None):\n",
    "        np.savetxt(os.path.join(output_path, 'validation_predictions.out'), validation_output, delimiter=',')\n",
    "        np.savetxt(os.path.join(output_path, 'valid_ground_truth_classes.txt'), validation_groundtruth, delimiter=',')\n",
    "        thresholds = np.arange(0, 1, 0.01)\n",
    "        f1_array = np.zeros((len(LABELS_LIST), len(thresholds)))\n",
    "        for idx, label in enumerate(LABELS_LIST):\n",
    "            f1_array[idx, :] = [\n",
    "                f1_score(validation_groundtruth[:, idx], np.clip(np.round(validation_output[:, idx] - threshold + 0.5), 0, 1))\n",
    "                for threshold in thresholds]\n",
    "        threshold_arg = np.argmax(f1_array, axis=1)\n",
    "        threshold_per_class = thresholds[threshold_arg]\n",
    "\n",
    "        # plot the f1 score across thresholds\n",
    "        plt.figure(figsize=(20, 20))\n",
    "        for idx, x in enumerate(LABELS_LIST):\n",
    "            plt.plot(thresholds, f1_array[idx, :], linewidth=5)\n",
    "        plt.legend(LABELS_LIST, loc='best')\n",
    "        plt.title(\"F1 Score vs different prediction threshold values for each class\")\n",
    "        plt.savefig(os.path.join(output_path, \"f1_score_vs_thresholds.pdf\"), format=\"pdf\")\n",
    "        plt.savefig(os.path.join(output_path, \"f1_score_vs_thresholds.png\"))\n",
    "\n",
    "        # Applying thresholds optimized per class\n",
    "        model_output_rounded = np.zeros_like(model_output)\n",
    "        for idx, label in enumerate(LABELS_LIST):\n",
    "            model_output_rounded[:, idx] = np.clip(np.round(model_output[:, idx] - threshold_per_class[idx] + 0.5), 0, 1)\n",
    "\n",
    "        accuracies_perclass = sum(model_output_rounded == groundtruth) / len(groundtruth)\n",
    "        # Getting the true positive rate perclass\n",
    "        true_positives_ratio_perclass = sum((model_output_rounded == groundtruth) * (groundtruth == 1)) / sum(\n",
    "            groundtruth)\n",
    "        # Get true negative ratio\n",
    "        true_negative_ratio_perclass = sum((model_output_rounded == groundtruth)\n",
    "                                           * (groundtruth == 0)) / (len(groundtruth) - sum(groundtruth))\n",
    "        results_df = results_df.append(\n",
    "            pd.DataFrame([accuracies_perclass, true_positives_ratio_perclass,\n",
    "                          true_negative_ratio_perclass], columns=LABELS_LIST))\n",
    "        # compute additional metrics (AUC,f1,recall,precision)\n",
    "        auc_roc_per_label = roc_auc_score(groundtruth, model_output, average=None)\n",
    "        precision_perlabel = precision_score(groundtruth, model_output_rounded, average=None)\n",
    "        recall_perlabel = recall_score(groundtruth, model_output_rounded, average=None)\n",
    "        f1_perlabel = f1_score(groundtruth, model_output_rounded, average=None)\n",
    "        kappa_perlabel = [cohen_kappa_score(groundtruth[:, x], model_output_rounded[:, x]) for x in\n",
    "                          range(len(LABELS_LIST))]\n",
    "        results_df = results_df.append(\n",
    "            pd.DataFrame([auc_roc_per_label,recall_perlabel, precision_perlabel, f1_perlabel,kappa_perlabel],\n",
    "                         columns=LABELS_LIST))\n",
    "        results_df.index = ['Ratio of positive samples', 'Model accuracy', 'True positives ratio',\n",
    "                            'True negatives ratio', \"AUC\", \"Precision\", \"Recall\", \"f1-score\",  \"Kappa score\",\n",
    "                            'Optimized model accuracy', 'Optimized true positives ratio',\n",
    "                            'Optimized true negatives ratio', \"Optimized AUC\",\n",
    "                            \"Optimized precision\", \"Optimized recall\", \"Optimized f1-score\",  \"Optimized Kappa score\"]\n",
    "\n",
    "    results_df['average'] = results_df.mean(numeric_only=True, axis=1)\n",
    "    results_df.T.to_csv(os.path.join(output_path, \"results_report.csv\"), float_format=\"%.2f\")\n",
    "    return results_df\n",
    "\n",
    "\n",
    "def evaluate_model(test_pred_prob, test_classes, saving_path, evaluation_file_path):\n",
    "    \"\"\"\n",
    "    Evaluates a given model using accuracy, area under curve and hamming loss\n",
    "    :param model: model to be evaluated\n",
    "    :param spectrograms: the test set spectrograms as an np.array\n",
    "    :param test_classes: the ground truth labels\n",
    "    :return: accuracy, auc_roc, hamming_error\n",
    "    \"\"\"\n",
    "    test_pred = np.round(test_pred_prob)\n",
    "    # Accuracy\n",
    "    accuracy = 100 * accuracy_score(test_classes, test_pred)\n",
    "    print(\"Exact match accuracy is: \" + str(accuracy) + \"%\")\n",
    "    # Area Under the Receiver Operating Characteristic Curve (ROC AUC)\n",
    "    auc_roc = roc_auc_score(test_classes, test_pred_prob)\n",
    "    print(\"Macro Area Under the Curve (AUC) is: \" + str(auc_roc))\n",
    "    auc_roc_micro = roc_auc_score(test_classes, test_pred_prob, average=\"micro\")\n",
    "    print(\"Micro Area Under the Curve (AUC) is: \" + str(auc_roc_micro))\n",
    "    auc_roc_weighted = roc_auc_score(test_classes, test_pred_prob, average=\"weighted\")\n",
    "    print(\"Weighted Area Under the Curve (AUC) is: \" + str(auc_roc_weighted))\n",
    "    # Hamming loss is the fraction of labels that are incorrectly predicted.\n",
    "    hamming_error = hamming_loss(test_classes, test_pred)\n",
    "    print(\"Hamming Loss (ratio of incorrect tags) is: \" + str(hamming_error))\n",
    "    with open(evaluation_file_path, \"w\") as f:\n",
    "        f.write(\"Exact match accuracy is: \" + str(accuracy) + \"%\\n\" + \"Area Under the Curve (AUC) is: \" + str(auc_roc)\n",
    "                + \"\\nMicro AUC is:\" + str(auc_roc_micro) + \"\\nWeighted AUC is:\" + str(auc_roc_weighted)\n",
    "                + \"\\nHamming Loss (ratio of incorrect tags) is: \" + str(hamming_error))\n",
    "    print(\"saving prediction to disk\")\n",
    "    np.savetxt(os.path.join(saving_path, 'predictions.out'), test_pred_prob, delimiter=',')\n",
    "    np.savetxt(os.path.join(saving_path, 'test_ground_truth_classes.txt'), test_classes, delimiter=',')\n",
    "    return accuracy, auc_roc, hamming_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the proposed weighted loss function\n",
    "def weighted_loss(y_true, y_pred, positive_weights, negative_weights):\n",
    "    # clip to prevent NaN's and Inf's\n",
    "    y_pred = tf.clip_by_value(y_pred, 1e-7, 1-1e-7, name=None)\n",
    "    # compute loss\n",
    "    loss = (-y_true * tf.log(y_pred) * positive_weights) - ((1.0 - y_true) * tf.log(1.0 - y_pred) * negative_weights)\n",
    "    loss = tf.reduce_mean(loss)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## fine-tune pretrained model on the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Experiment: PosLimitited_weighted_CE_balanced_ratio0.0split_1\n",
      "\n",
      "\n",
      "\n",
      "INFO:tensorflow:Summary name Weighted cross entropy is illegal; using Weighted_cross_entropy instead.\n",
      "Execute the following in a terminal:\n",
      "tensorboard --logdir=/srv/workspace/research/mlml/experiments_results_balanced/PosLimitited_weighted_CE_balanced_ratio0.0split_1/19-12_11-22\n",
      "INFO:tensorflow:Restoring parameters from /srv/workspace/research/mlml/pretrained_models/inception_resnet_v2_2016_08_30.ckpt\n",
      "Epoch #1 Loss: 0.7097 accuracy: 0.8958\n",
      "Epoch #2 Loss: 0.6630 accuracy: 0.9387\n",
      "Epoch #3 Loss: 0.6503 accuracy: 0.9390\n",
      "Epoch #4 Loss: 0.6306 accuracy: 0.9394\n",
      "Epoch #5 Loss: 0.6036 accuracy: 0.9376\n",
      "Epoch #6 Loss: 0.5732 accuracy: 0.9305\n",
      "Epoch #7 Loss: 0.5465 accuracy: 0.9221\n",
      "Epoch #8 Loss: 0.5270 accuracy: 0.9152\n",
      "Epoch #9 Loss: 0.5123 accuracy: 0.9113\n",
      "Epoch #10 Loss: 0.5010 accuracy: 0.9090\n",
      "Epoch #11 Loss: 0.4912 accuracy: 0.9075\n",
      "Epoch #12 Loss: 0.4832 accuracy: 0.9071\n",
      "Epoch #13 Loss: 0.4760 accuracy: 0.9067\n",
      "Epoch #14 Loss: 0.4704 accuracy: 0.9066\n",
      "Epoch #15 Loss: 0.4643 accuracy: 0.9068\n",
      "Epoch #16 Loss: 0.4601 accuracy: 0.9067\n",
      "Epoch #17 Loss: 0.4550 accuracy: 0.9071\n",
      "Epoch #18 Loss: 0.4513 accuracy: 0.9072\n",
      "Epoch #19 Loss: 0.4481 accuracy: 0.9074\n",
      "Epoch #20 Loss: 0.4433 accuracy: 0.9077\n",
      "Exact match accuracy is: 0.12017786323759164%\n",
      "Macro Area Under the Curve (AUC) is: 0.8742265160652121\n",
      "Micro Area Under the Curve (AUC) is: 0.9145818579367677\n",
      "Weighted Area Under the Curve (AUC) is: 0.8534077261557507\n",
      "Hamming Loss (ratio of incorrect tags) is: 0.09977917317630093\n",
      "saving prediction to disk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Experiment: PosLimitited_weighted_CE_balanced_ratio0.2split_1\n",
      "\n",
      "\n",
      "\n",
      "INFO:tensorflow:Summary name Weighted cross entropy is illegal; using Weighted_cross_entropy instead.\n",
      "Execute the following in a terminal:\n",
      "tensorboard --logdir=/srv/workspace/research/mlml/experiments_results_balanced/PosLimitited_weighted_CE_balanced_ratio0.2split_1/19-12_12-17\n",
      "INFO:tensorflow:Restoring parameters from /srv/workspace/research/mlml/pretrained_models/inception_resnet_v2_2016_08_30.ckpt\n",
      "Epoch #1 Loss: 0.6065 accuracy: 0.9076\n",
      "Epoch #2 Loss: 0.5557 accuracy: 0.9517\n",
      "Epoch #3 Loss: 0.5471 accuracy: 0.9517\n",
      "Epoch #4 Loss: 0.5349 accuracy: 0.9517\n",
      "Epoch #5 Loss: 0.5190 accuracy: 0.9518\n",
      "Epoch #6 Loss: 0.4998 accuracy: 0.9512\n",
      "Epoch #7 Loss: 0.4797 accuracy: 0.9485\n",
      "Epoch #8 Loss: 0.4620 accuracy: 0.9431\n",
      "Epoch #9 Loss: 0.4470 accuracy: 0.9371\n",
      "Epoch #10 Loss: 0.4358 accuracy: 0.9320\n",
      "Epoch #11 Loss: 0.4265 accuracy: 0.9279\n",
      "Epoch #12 Loss: 0.4187 accuracy: 0.9256\n",
      "Epoch #13 Loss: 0.4130 accuracy: 0.9239\n",
      "Epoch #14 Loss: 0.4074 accuracy: 0.9232\n",
      "Epoch #15 Loss: 0.4026 accuracy: 0.9224\n",
      "Epoch #16 Loss: 0.3987 accuracy: 0.9222\n",
      "Epoch #17 Loss: 0.3944 accuracy: 0.9218\n",
      "Epoch #18 Loss: 0.3910 accuracy: 0.9217\n",
      "Epoch #19 Loss: 0.3881 accuracy: 0.9217\n",
      "Epoch #20 Loss: 0.3839 accuracy: 0.9219\n",
      "Exact match accuracy is: 0.10816007691383248%\n",
      "Macro Area Under the Curve (AUC) is: 0.8689410530863417\n",
      "Micro Area Under the Curve (AUC) is: 0.9106828477665257\n",
      "Weighted Area Under the Curve (AUC) is: 0.846392445003509\n",
      "Hamming Loss (ratio of incorrect tags) is: 0.08597975003004446\n",
      "saving prediction to disk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Experiment: PosLimitited_weighted_CE_balanced_ratio0.5split_1\n",
      "\n",
      "\n",
      "\n",
      "INFO:tensorflow:Summary name Weighted cross entropy is illegal; using Weighted_cross_entropy instead.\n",
      "Execute the following in a terminal:\n",
      "tensorboard --logdir=/srv/workspace/research/mlml/experiments_results_balanced/PosLimitited_weighted_CE_balanced_ratio0.5split_1/19-12_13-28\n",
      "INFO:tensorflow:Restoring parameters from /srv/workspace/research/mlml/pretrained_models/inception_resnet_v2_2016_08_30.ckpt\n",
      "Epoch #1 Loss: 0.4864 accuracy: 0.9376\n",
      "Epoch #2 Loss: 0.4421 accuracy: 0.9627\n",
      "Epoch #3 Loss: 0.4382 accuracy: 0.9627\n",
      "Epoch #4 Loss: 0.4329 accuracy: 0.9629\n",
      "Epoch #5 Loss: 0.4259 accuracy: 0.9632\n",
      "Epoch #6 Loss: 0.4172 accuracy: 0.9636\n",
      "Epoch #7 Loss: 0.4065 accuracy: 0.9640\n",
      "Epoch #8 Loss: 0.3943 accuracy: 0.9639\n",
      "Epoch #9 Loss: 0.3818 accuracy: 0.9628\n",
      "Epoch #10 Loss: 0.3705 accuracy: 0.9603\n",
      "Epoch #11 Loss: 0.3604 accuracy: 0.9567\n",
      "Epoch #12 Loss: 0.3524 accuracy: 0.9535\n",
      "Epoch #13 Loss: 0.3461 accuracy: 0.9507\n",
      "Epoch #14 Loss: 0.3404 accuracy: 0.9486\n",
      "Epoch #15 Loss: 0.3358 accuracy: 0.9469\n",
      "Epoch #16 Loss: 0.3319 accuracy: 0.9455\n",
      "Epoch #17 Loss: 0.3282 accuracy: 0.9445\n",
      "Epoch #18 Loss: 0.3250 accuracy: 0.9440\n",
      "Epoch #19 Loss: 0.3223 accuracy: 0.9433\n",
      "Epoch #20 Loss: 0.3190 accuracy: 0.9428\n",
      "Exact match accuracy is: 0.6729960341305132%\n",
      "Macro Area Under the Curve (AUC) is: 0.8538588646155809\n",
      "Micro Area Under the Curve (AUC) is: 0.9062317294565412\n",
      "Weighted Area Under the Curve (AUC) is: 0.8400738213563925\n",
      "Hamming Loss (ratio of incorrect tags) is: 0.0694958538637183\n",
      "saving prediction to disk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Experiment: PosLimitited_weighted_CE_balanced_ratio0.8split_1\n",
      "\n",
      "\n",
      "\n",
      "INFO:tensorflow:Summary name Weighted cross entropy is illegal; using Weighted_cross_entropy instead.\n",
      "Execute the following in a terminal:\n",
      "tensorboard --logdir=/srv/workspace/research/mlml/experiments_results_balanced/PosLimitited_weighted_CE_balanced_ratio0.8split_1/19-12_14-39\n",
      "INFO:tensorflow:Restoring parameters from /srv/workspace/research/mlml/pretrained_models/inception_resnet_v2_2016_08_30.ckpt\n",
      "Epoch #1 Loss: 0.3354 accuracy: 0.9480\n",
      "Epoch #2 Loss: 0.2734 accuracy: 0.9832\n",
      "Epoch #3 Loss: 0.2721 accuracy: 0.9833\n",
      "Epoch #4 Loss: 0.2711 accuracy: 0.9832\n",
      "Epoch #5 Loss: 0.2697 accuracy: 0.9832\n",
      "Epoch #6 Loss: 0.2683 accuracy: 0.9832\n",
      "Epoch #7 Loss: 0.2663 accuracy: 0.9831\n",
      "Epoch #8 Loss: 0.2641 accuracy: 0.9828\n",
      "Epoch #9 Loss: 0.2615 accuracy: 0.9824\n",
      "Epoch #10 Loss: 0.2582 accuracy: 0.9819\n",
      "Epoch #11 Loss: 0.2544 accuracy: 0.9815\n",
      "Epoch #12 Loss: 0.2503 accuracy: 0.9811\n",
      "Epoch #13 Loss: 0.2461 accuracy: 0.9808\n",
      "Epoch #14 Loss: 0.2414 accuracy: 0.9805\n",
      "Epoch #15 Loss: 0.2372 accuracy: 0.9802\n",
      "Epoch #16 Loss: 0.2332 accuracy: 0.9797\n",
      "Epoch #17 Loss: 0.2293 accuracy: 0.9790\n",
      "Epoch #18 Loss: 0.2260 accuracy: 0.9783\n",
      "Epoch #19 Loss: 0.2233 accuracy: 0.9776\n",
      "Epoch #20 Loss: 0.2203 accuracy: 0.9769\n",
      "Exact match accuracy is: 0.4566758803028482%\n",
      "Macro Area Under the Curve (AUC) is: 0.8159205103591735\n",
      "Micro Area Under the Curve (AUC) is: 0.8913493774194737\n",
      "Weighted Area Under the Curve (AUC) is: 0.8208432388448303\n",
      "Hamming Loss (ratio of incorrect tags) is: 0.0582336858550655\n",
      "saving prediction to disk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/opt/conda/lib/python3.6/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  max_open_warning, RuntimeWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Experiment: PosLimitited_weighted_CE_balanced_ratio0.0split_2\n",
      "\n",
      "\n",
      "\n",
      "INFO:tensorflow:Summary name Weighted cross entropy is illegal; using Weighted_cross_entropy instead.\n",
      "Execute the following in a terminal:\n",
      "tensorboard --logdir=/srv/workspace/research/mlml/experiments_results_balanced/PosLimitited_weighted_CE_balanced_ratio0.0split_2/19-12_15-52\n",
      "INFO:tensorflow:Restoring parameters from /srv/workspace/research/mlml/pretrained_models/inception_resnet_v2_2016_08_30.ckpt\n",
      "Epoch #1 Loss: 0.7039 accuracy: 0.9082\n",
      "Epoch #2 Loss: 0.6627 accuracy: 0.9384\n",
      "Epoch #3 Loss: 0.6487 accuracy: 0.9389\n",
      "Epoch #4 Loss: 0.6287 accuracy: 0.9397\n",
      "Epoch #5 Loss: 0.6025 accuracy: 0.9381\n",
      "Epoch #6 Loss: 0.5727 accuracy: 0.9326\n",
      "Epoch #7 Loss: 0.5457 accuracy: 0.9246\n",
      "Epoch #8 Loss: 0.5251 accuracy: 0.9171\n",
      "Epoch #9 Loss: 0.5101 accuracy: 0.9115\n",
      "Epoch #10 Loss: 0.4990 accuracy: 0.9085\n",
      "Epoch #11 Loss: 0.4896 accuracy: 0.9067\n",
      "Epoch #12 Loss: 0.4819 accuracy: 0.9059\n",
      "Epoch #13 Loss: 0.4753 accuracy: 0.9056\n",
      "Epoch #14 Loss: 0.4697 accuracy: 0.9053\n",
      "Epoch #15 Loss: 0.4651 accuracy: 0.9050\n",
      "Epoch #16 Loss: 0.4597 accuracy: 0.9055\n",
      "Epoch #17 Loss: 0.4562 accuracy: 0.9054\n",
      "Epoch #18 Loss: 0.4513 accuracy: 0.9057\n",
      "Epoch #19 Loss: 0.4482 accuracy: 0.9059\n",
      "Epoch #20 Loss: 0.4444 accuracy: 0.9064\n",
      "Exact match accuracy is: 0.290065264684554%\n",
      "Macro Area Under the Curve (AUC) is: 0.8704371666926051\n",
      "Micro Area Under the Curve (AUC) is: 0.9135413475870993\n",
      "Weighted Area Under the Curve (AUC) is: 0.8479410707956015\n",
      "Hamming Loss (ratio of incorrect tags) is: 0.10054689388445734\n",
      "saving prediction to disk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  max_open_warning, RuntimeWarning)\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/opt/conda/lib/python3.6/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  max_open_warning, RuntimeWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Experiment: PosLimitited_weighted_CE_balanced_ratio0.2split_2\n",
      "\n",
      "\n",
      "\n",
      "INFO:tensorflow:Summary name Weighted cross entropy is illegal; using Weighted_cross_entropy instead.\n",
      "Execute the following in a terminal:\n",
      "tensorboard --logdir=/srv/workspace/research/mlml/experiments_results_balanced/PosLimitited_weighted_CE_balanced_ratio0.2split_2/19-12_17-11\n",
      "INFO:tensorflow:Restoring parameters from /srv/workspace/research/mlml/pretrained_models/inception_resnet_v2_2016_08_30.ckpt\n",
      "Epoch #1 Loss: 0.5962 accuracy: 0.9254\n",
      "Epoch #2 Loss: 0.5577 accuracy: 0.9515\n",
      "Epoch #3 Loss: 0.5489 accuracy: 0.9516\n",
      "Epoch #4 Loss: 0.5371 accuracy: 0.9516\n",
      "Epoch #5 Loss: 0.5214 accuracy: 0.9518\n",
      "Epoch #6 Loss: 0.5020 accuracy: 0.9507\n",
      "Epoch #7 Loss: 0.4814 accuracy: 0.9477\n",
      "Epoch #8 Loss: 0.4625 accuracy: 0.9423\n",
      "Epoch #9 Loss: 0.4476 accuracy: 0.9364\n",
      "Epoch #10 Loss: 0.4358 accuracy: 0.9311\n",
      "Epoch #11 Loss: 0.4263 accuracy: 0.9273\n",
      "Epoch #12 Loss: 0.4189 accuracy: 0.9249\n",
      "Epoch #13 Loss: 0.4128 accuracy: 0.9234\n",
      "Epoch #14 Loss: 0.4072 accuracy: 0.9223\n",
      "Epoch #15 Loss: 0.4028 accuracy: 0.9216\n",
      "Epoch #16 Loss: 0.3983 accuracy: 0.9213\n"
     ]
    }
   ],
   "source": [
    "# Main experiment loop\n",
    "# for each split and each ratio, fine-tune the pretrained model and evaluate the results\n",
    "for split in np.arange(1,5):\n",
    "    for ratio in np.arange(0,1,0.25): \n",
    "        tf.reset_default_graph()\n",
    "        EXPERIMENTNAME = \"PosLimitited_weighted_CE_balanced_ratio\" + str(round(ratio, 1)) + \"split_\" + str(split)\n",
    "        print(\"Current Experiment: \" + EXPERIMENTNAME + \"\\n\\n\\n\")\n",
    "        global_labels = pd.read_csv('/srv/workspace/research/mlml/mlml_weightedLoss/labels_balanced_4labels/missing_labels'+\n",
    "                                    str(round(ratio, 1))+'/train' + str(split) +\"_\" + str(round(ratio, 1)) + '.csv')\n",
    "        # Loading datasets\n",
    "        training_dataset = get_dataset('/srv/workspace/research/mlml/mlml_weightedLoss/labels_balanced_4labels/missing_labels'+\n",
    "                                    str(round(ratio, 1))+'/train' + str(split) +\"_\" + str(round(ratio, 1)) + '.csv')\n",
    "        \n",
    "        global_weights_negative = pd.read_csv('/srv/workspace/research/mlml/mlml_weightedLoss/labels_balanced_4labels/missing_labels'+\n",
    "                            str(round(ratio, 1))+'/negative_weights' + str(split) +\"_\"  + str(round(ratio, 1)) + '.csv')\n",
    "\n",
    "        with tf.Graph().as_default():\n",
    "            # Setting up training generator\n",
    "            training_iterator = training_dataset.make_one_shot_iterator()\n",
    "            training_next_element = training_iterator.get_next()\n",
    "\n",
    "            # Setting up variables\n",
    "            input_labels = tf.placeholder(tf.float32, [None, NUM_CLASSES], name=\"true_labels\")\n",
    "            negative_weights = tf.placeholder(tf.float32, [None, NUM_CLASSES], name=\"negative_weights\")\n",
    "            input_images = tf.placeholder(tf.float32, [None, 224, 224, 3], name=\"input\")\n",
    "            train_phase = tf.placeholder(tf.bool, name=\"is_training\")\n",
    "\n",
    "            # Create the model, use the default arg scope to configure the batch norm parameters.\n",
    "            with slim.arg_scope(inception.inception_resnet_v2_arg_scope()):\n",
    "                ignored_logits, end_points = inception.inception_resnet_v2(input_images, num_classes=NUM_CLASSES, \n",
    "                                                           is_training=train_phase)\n",
    "            \n",
    "            featured_extracted = end_points['Conv2d_7b_1x1']\n",
    "            \n",
    "            with tf.name_scope('trainable/Fully_connected_1'):\n",
    "                flattened = tf.reshape(featured_extracted, [-1, 5*5* 1536])\n",
    "                fully1 = tf.nn.sigmoid(full_layer(flattened, 2048))\n",
    "                \n",
    "            with tf.name_scope('trainable/Fully_connected_2'):\n",
    "                fully2 = tf.nn.sigmoid(full_layer(fully1, 1024))\n",
    "\n",
    "            with tf.name_scope('trainable/Fully_connected_3'):\n",
    "                fully3 = tf.nn.sigmoid(full_layer(fully2, 256))\n",
    "                            \n",
    "            with tf.name_scope('trainable/Fully_connected_4'):\n",
    "                #dropped = tf.nn.dropout(fully1, keep_prob=current_keep_prob)\n",
    "                output_logits = full_layer(fully3, NUM_CLASSES)\n",
    "\n",
    "            trainable_layers = [var for var in tf.global_variables() if (\"trainable\" in var.op.name)]\n",
    "            # Define loss\n",
    "            positive_imbalance_weights = tf.constant(Pos_balance_weights)\n",
    "            my_weights_loss = weighted_loss(y_true= input_labels, y_pred= probabilities,\n",
    "                  positive_weights= positive_imbalance_weights, negative_weights= negative_weights)\n",
    "                \n",
    "            # define optimizer, with decaying learning rate\n",
    "            probabilities = tf.nn.sigmoid(output_logits)          \n",
    "            global_step = tf.Variable(0, trainable=False)\n",
    "            learning_rate = tf.train.exponential_decay(learning_rate=0.1, global_step=global_step, decay_steps=1000,\n",
    "                                                      decay_rate=0.95,staircase=True)\n",
    "            train_step = tf.train.AdadeltaOptimizer(learning_rate).minimize(my_weights_loss,\n",
    "                                                                            var_list=trainable_layers)            #my_weights_loss = weighted_loss(y_true= y, y_pred= model_output,\n",
    "\n",
    "            # define accuracy\n",
    "            correct_prediction = tf.equal(tf.round(probabilities), input_labels)\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "            # Adding tensorboard summaries\n",
    "            tf.summary.scalar('Weighted cross entropy',  my_weights_loss)\n",
    "            tf.summary.scalar('Accuracy', accuracy)\n",
    "            # Merge all the summaries\n",
    "            merged = tf.summary.merge_all()\n",
    "\n",
    "            # restoring pretrained model weights\n",
    "            checkpoint_exclude_scopes = [\"InceptionResnetV2/Logits\", \"InceptionResnetV2/AuxLogits\"]\n",
    "            exclusions = [scope.strip() for scope in checkpoint_exclude_scopes]\n",
    "            variables_to_restore = []\n",
    "            for var in slim.get_model_variables():\n",
    "                for exclusion in exclusions:\n",
    "                    if var.op.name.startswith(exclusion):\n",
    "                        break\n",
    "                else:\n",
    "                    variables_to_restore.append(var)\n",
    "            init_fn = slim.assign_from_checkpoint_fn(\n",
    "                os.path.join(PRETRAINED_MODEL_DIR, 'inception_resnet_v2_2016_08_30.ckpt'),\n",
    "                variables_to_restore)\n",
    "\n",
    "            # Setting up saving directory\n",
    "            experiment_name = strftime(\"%d-%m_%H-%M\", localtime())\n",
    "            exp_dir = os.path.join(OUTPUT_PATH, EXPERIMENTNAME, experiment_name)\n",
    "\n",
    "            # Add ops to save and restore all the variables.\n",
    "            saver = tf.train.Saver()\n",
    "\n",
    "            with tf.Session() as sess:\n",
    "                # Write summaries to LOG_DIR -- used by TensorBoard\n",
    "                train_writer = tf.summary.FileWriter(exp_dir + '/tensorboard/train', graph=tf.get_default_graph())\n",
    "                test_writer = tf.summary.FileWriter(exp_dir + '/tensorboard/test', graph=tf.get_default_graph())\n",
    "                print(\"Execute the following in a terminal:\\n\" + \"tensorboard --logdir=\" + exp_dir)\n",
    "                sess.run(tf.global_variables_initializer())\n",
    "                init_fn(sess)\n",
    "                for epoch in range(NUM_EPOCHS):\n",
    "                    batch_loss, batch_accuracy = np.zeros([TRAINING_STEPS, 1]), np.zeros([TRAINING_STEPS, 1])\n",
    "                    for batch_counter in range(TRAINING_STEPS):\n",
    "                        batch = sess.run(training_next_element)\n",
    "                        batch_images = batch[0]\n",
    "                        batch_labels = np.squeeze(batch[1])\n",
    "                        batch_negative_weights = np.squeeze(batch[2])\n",
    "                        summary, batch_loss[batch_counter], batch_accuracy[batch_counter], _ \\\n",
    "                        = sess.run([merged, my_weights_loss, accuracy, train_step],\n",
    "                                   feed_dict={input_images:batch_images,\n",
    "                                              input_labels:batch_labels,\n",
    "                                              negative_weights: batch_negative_weights,\n",
    "                                              train_phase:True})\n",
    "                    print(\"Epoch #{}\".format(epoch+1), \"Loss: {:.4f}\".format(np.mean(batch_loss)),\n",
    "                          \"accuracy: {:.4f}\".format(np.mean(batch_accuracy)))\n",
    "                    # Add to summaries\n",
    "                    train_writer.add_summary(summary, epoch)\n",
    "                    \n",
    "                os.makedirs(os.path.join(PRETRAINED_MODEL_DIR,EXPERIMENTNAME,experiment_name))\n",
    "                save_path = saver.save(sess, os.path.join(PRETRAINED_MODEL_DIR,EXPERIMENTNAME, \n",
    "                                              experiment_name, \"last_epoch.ckpt\"))\n",
    "                \n",
    "                # Testing the model\n",
    "                test_split = '/srv/workspace/research/mlml/mlml_weightedLoss/labels_balanced_4labels/missing_labels'+ \\\n",
    "                                    str(round(ratio, 1))+'/test' + str(split) + \"_\" + str(round(ratio, 1)) + '.csv'\n",
    "                global_labels = pd.read_csv(test_split)\n",
    "                test_dataset = get_dataset(test_split)\n",
    "                test_classes = np.zeros_like(global_labels.iloc[:,1:].values, dtype=float)\n",
    "\n",
    "                TEST_NUM_STEPS = int(np.floor((len(global_labels)/BATCH_SIZE)))\n",
    "                test_pred_prob = np.zeros_like(test_classes, dtype=float)\n",
    "                test_iterator = test_dataset.make_one_shot_iterator()\n",
    "                test_next_element = test_iterator.get_next()\n",
    "                \n",
    "                for test_batch_counter in range(TEST_NUM_STEPS):\n",
    "                    start_idx = (test_batch_counter * BATCH_SIZE)\n",
    "                    end_idx = (test_batch_counter * BATCH_SIZE) + BATCH_SIZE\n",
    "                    test_batch = sess.run(test_next_element)\n",
    "                    test_batch_images = test_batch[0]\n",
    "                    test_batch_labels = np.squeeze(test_batch[1])\n",
    "                    test_classes[start_idx:end_idx,:] = test_batch_labels\n",
    "                    test_pred_prob[start_idx:end_idx,:] = sess.run(probabilities,\n",
    "                                                             feed_dict={input_images:test_batch_images,train_phase:False})\n",
    "\n",
    "                accuracy_out, auc_roc, hamming_error = evaluate_model(test_pred_prob, test_classes,\n",
    "                                                                  saving_path=exp_dir,\n",
    "                                                                  evaluation_file_path= \\\n",
    "                                                                  os.path.join(exp_dir,\"evaluation_results.txt\"))           \n",
    "                results = create_analysis_report(test_pred_prob, test_classes, exp_dir, LABELS_LIST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}