NN_sourcecode.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copia di mars.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMZImddhqtSIpFf25KQpL9N",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/elia-orsini/mars-machine-learning/blob/main/mars-final-copy.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ny1BmcUlng-z"
      },
      "source": [
        "-------------------------------------------------\n",
        "MODEL SUMMARY\n",
        "1. IMAGE AUGMENTATION (RANDOM FLIP HORIZONTAL, 0.2 ROTATION)\n",
        "2. EFFICIENT NET B7\n",
        "3. GLOBAL AVERAGE POOLING 2D\n",
        "4. DROPOUT (0.2)\n",
        "5. PREDICTION LAYER (SOFTMAX)\n",
        "\n",
        "- LEARNING RATE: 0.0001\n",
        "- LOSS: CATEGORICAL CROSSENTROPY\n",
        "- METRICS: ACCURACY\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MoFleCV6FIri",
        "outputId": "9329d7ad-fb73-4965-ddbd-dca356a69958",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Connect to the drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sGO2y5foDzXg"
      },
      "source": [
        "import cv2\n",
        "import tensorflow as tf\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WIsGhzdAnNM3"
      },
      "source": [
        "# Spefications of the dataset (256x193 images) (25 different categories)\n",
        "IMG_WIDTH = 256\n",
        "IMG_HEIGHT = 193\n",
        "NUM_CATEGORIES = 25"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rqi95tIJVzbH"
      },
      "source": [
        "# Function to read labels from a text file and to convert images into their \n",
        "# pixel values using the cv2 library\n",
        "\n",
        "def read_data(type):\n",
        "\n",
        "\tdataset = f\"drive/My Drive/msl-images/{type}-calibrated-shuffled.txt\"\n",
        "\timgs = []\n",
        "\tlabels = []\n",
        "\n",
        "\twith open(dataset, 'r') as f:\n",
        "\t\tfor line in f.readlines():\n",
        "\t\t\tline = line.replace('\\n', '')\n",
        "\t\t\tline = line.split(' ')\n",
        "\t\t\tpath = f\"./drive/My Drive/msl-images/{line[0]}\"\n",
        "\n",
        "\t\t\timg = cv2.imread(path)\n",
        "\t\t\timg = cv2.resize(img, (IMG_WIDTH, IMG_HEIGHT))\n",
        "\t\t\timgs.append(img)\n",
        "\t\t\tlabels.append(line[1])\n",
        "\n",
        "\treturn (imgs, labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xu9phdNlWD4D",
        "outputId": "86e9adf5-053f-423d-beb5-319f6412d90b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# Data augmentation slightly changes the images to prevent overfitting the \n",
        "# model on the train dataset\n",
        "data_augmentation = tf.keras.Sequential([\n",
        " \t\ttf.keras.layers.experimental.preprocessing.RandomFlip('horizontal'),\n",
        "\t\ttf.keras.layers.experimental.preprocessing.RandomRotation(0.2),\n",
        "])\n",
        "\n",
        "# Using the pre-trained model EfficientNetB7\n",
        "standard_model = tf.keras.applications.EfficientNetB7(\n",
        "\tinclude_top=False, weights='imagenet', input_shape=(IMG_HEIGHT, IMG_WIDTH, 3)\n",
        ")\n",
        "# Keep the model non-trainable at first\n",
        "standard_model.trainable = False\n",
        "\n",
        "global_average_layer = tf.keras.layers.GlobalAveragePooling2D()\n",
        "prediction_layer = tf.keras.layers.Dense(NUM_CATEGORIES, activation=\"softmax\")\n",
        "\n",
        "# Create model (Added a dropout of 0.2 to prevent overfitting)\n",
        "inputs = tf.keras.Input(shape=(IMG_HEIGHT, IMG_WIDTH, 3))\n",
        "x = data_augmentation(inputs)\n",
        "x = standard_model(inputs)\n",
        "x = global_average_layer(x)\n",
        "x = tf.keras.layers.Dropout(0.2)(x)\n",
        "outputs = prediction_layer(x)\n",
        "\n",
        "model = tf.keras.Model(inputs, outputs)\n",
        "\n",
        "model.compile(\n",
        "\toptimizer=tf.keras.optimizers.Adam(lr=0.0001),\n",
        "\tloss=\"categorical_crossentropy\",\n",
        "\tmetrics=['accuracy'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/keras-applications/efficientnetb7_notop.h5\n",
            "258080768/258076736 [==============================] - 7s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VvGlbvirWyos"
      },
      "source": [
        "# Train, evaluate and test datasets\n",
        "train_x, train_y = read_data('train')\n",
        "val_x, val_y = read_data('val')\n",
        "test_x, test_y = read_data('test')\n",
        "\n",
        "# Convert the y values for each of the datasets to categorical\n",
        "train_y = tf.keras.utils.to_categorical(train_y)\n",
        "val_y = tf.keras.utils.to_categorical(val_y)\n",
        "test_y = tf.keras.utils.to_categorical(test_y)\n",
        "\n",
        "# Convert each dataset to a numpy array\n",
        "train_x = np.array(train_x)\n",
        "train_y = np.array(train_y)\n",
        "val_x = np.array(val_x)\n",
        "val_y = np.array(val_y)\n",
        "test_x = np.array(test_x)\n",
        "test_y = np.array(test_y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5eN0JCo6ncp-"
      },
      "source": [
        "-------------------------------------------------------------\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fYbp6aqPlXmr",
        "outputId": "1fe7fe99-d7c5-465c-aefb-5d82e3d49441",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# Evaluate model before training just for the sake of comparison\n",
        "model.evaluate(val_x,  val_y, verbose=2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "52/52 - 11s - loss: 3.2241 - accuracy: 0.0591\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[3.22412109375, 0.05914634093642235]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5ONqF41Klwg-",
        "outputId": "a5f44d96-f003-483a-a46d-4e369d1dd7b6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 748
        }
      },
      "source": [
        "# Train model for 20 epochs (the EfficientNetB7 is non-trainable)\n",
        "history = model.fit(train_x, train_y, epochs=20)\n",
        "print(\"#EVALUATION RESULTS:\")\n",
        "model.evaluate(val_x,  val_y, verbose=2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "118/118 [==============================] - 26s 224ms/step - loss: 2.0446 - accuracy: 0.6351\n",
            "Epoch 2/20\n",
            "118/118 [==============================] - 26s 222ms/step - loss: 0.9539 - accuracy: 0.7977\n",
            "Epoch 3/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.6622 - accuracy: 0.8409\n",
            "Epoch 4/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.5059 - accuracy: 0.8825\n",
            "Epoch 5/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.4084 - accuracy: 0.9026\n",
            "Epoch 6/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.3407 - accuracy: 0.9231\n",
            "Epoch 7/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.2912 - accuracy: 0.9335\n",
            "Epoch 8/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.2576 - accuracy: 0.9391\n",
            "Epoch 9/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.2244 - accuracy: 0.9517\n",
            "Epoch 10/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.1981 - accuracy: 0.9616\n",
            "Epoch 11/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.1792 - accuracy: 0.9661\n",
            "Epoch 12/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.1637 - accuracy: 0.9704\n",
            "Epoch 13/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.1528 - accuracy: 0.9701\n",
            "Epoch 14/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.1375 - accuracy: 0.9725\n",
            "Epoch 15/20\n",
            "118/118 [==============================] - 26s 222ms/step - loss: 0.1276 - accuracy: 0.9760\n",
            "Epoch 16/20\n",
            "118/118 [==============================] - 26s 222ms/step - loss: 0.1190 - accuracy: 0.9760\n",
            "Epoch 17/20\n",
            "118/118 [==============================] - 26s 222ms/step - loss: 0.1135 - accuracy: 0.9776\n",
            "Epoch 18/20\n",
            "118/118 [==============================] - 26s 221ms/step - loss: 0.1081 - accuracy: 0.9784\n",
            "Epoch 19/20\n",
            "118/118 [==============================] - 26s 222ms/step - loss: 0.0995 - accuracy: 0.9800\n",
            "Epoch 20/20\n",
            "118/118 [==============================] - 26s 222ms/step - loss: 0.0942 - accuracy: 0.9802\n",
            "#EVALUATION RESULTS:\n",
            "52/52 - 11s - loss: 1.0476 - accuracy: 0.6927\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1.0476473569869995, 0.6926829218864441]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XPtddgAqpBnT"
      },
      "source": [
        "# First convert all paramethers of the model as trainable\n",
        "standard_model.trainable = True\n",
        "\n",
        "# Then select first 500 paramethers and convert them into non-trainable\n",
        "# (The last paramethers are the most specific of the model so we want to train\n",
        "# them again specifically for the dataset)\n",
        "fine_tune_at = 500\n",
        "for layer in standard_model.layers[:fine_tune_at]:\n",
        "  layer.trainable =  False\n",
        "\n",
        "# Compile again\n",
        "model.compile(\n",
        "\toptimizer=tf.keras.optimizers.Adam(lr=0.0001/10),\n",
        "\tloss=\"categorical_crossentropy\",\n",
        "\tmetrics=['accuracy'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ExFHF9sp4NJ8",
        "outputId": "513b88f8-ab74-4b03-f1f7-cad0530c12ff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442
        }
      },
      "source": [
        "# Train model from the epoch 20 to the epoch 30 and evaluate.\n",
        "history_fine = model.fit(train_x, train_y, epochs=30, initial_epoch=history.epoch[-1])\n",
        "print(\"#EVALUATION RESULTS:\")\n",
        "model.evaluate(val_x,  val_y, verbose=2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 20/30\n",
            "118/118 [==============================] - 46s 392ms/step - loss: 1.0791 - accuracy: 0.8177\n",
            "Epoch 21/30\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.2588 - accuracy: 0.9672\n",
            "Epoch 22/30\n",
            "118/118 [==============================] - 46s 390ms/step - loss: 0.1208 - accuracy: 0.9816\n",
            "Epoch 23/30\n",
            "118/118 [==============================] - 46s 390ms/step - loss: 0.0807 - accuracy: 0.9880\n",
            "Epoch 24/30\n",
            "118/118 [==============================] - 46s 390ms/step - loss: 0.0587 - accuracy: 0.9883\n",
            "Epoch 25/30\n",
            "118/118 [==============================] - 46s 390ms/step - loss: 0.0468 - accuracy: 0.9896\n",
            "Epoch 26/30\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0417 - accuracy: 0.9925\n",
            "Epoch 27/30\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0384 - accuracy: 0.9904\n",
            "Epoch 28/30\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0295 - accuracy: 0.9939\n",
            "Epoch 29/30\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0369 - accuracy: 0.9920\n",
            "Epoch 30/30\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0210 - accuracy: 0.9960\n",
            "#EVALUATION RESULTS:\n",
            "52/52 - 11s - loss: 1.0658 - accuracy: 0.7409\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1.0657694339752197, 0.7408536672592163]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OpSykBeZJnU1",
        "outputId": "0a2050b7-32cd-48d7-8ce2-b2658de8317d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 782
        }
      },
      "source": [
        "# Train model from the epoch 30 to the epoch 50 and evaluate\n",
        "# In this case the accuracy of the evaluation model decreases (74% vs 73.84%)\n",
        "history_fine2 = model.fit(train_x, train_y, epochs=50, initial_epoch=history_fine.epoch[-1])\n",
        "print(\"#EVALUATION RESULTS:\")\n",
        "model.evaluate(val_x,  val_y, verbose=2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 30/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0171 - accuracy: 0.9979\n",
            "Epoch 31/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0154 - accuracy: 0.9971\n",
            "Epoch 32/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0153 - accuracy: 0.9968\n",
            "Epoch 33/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0135 - accuracy: 0.9971\n",
            "Epoch 34/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0121 - accuracy: 0.9973\n",
            "Epoch 35/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0089 - accuracy: 0.9981\n",
            "Epoch 36/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0089 - accuracy: 0.9984\n",
            "Epoch 37/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0114 - accuracy: 0.9960\n",
            "Epoch 38/50\n",
            "118/118 [==============================] - 46s 392ms/step - loss: 0.0080 - accuracy: 0.9981\n",
            "Epoch 39/50\n",
            "118/118 [==============================] - 46s 392ms/step - loss: 0.0069 - accuracy: 0.9992\n",
            "Epoch 40/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0074 - accuracy: 0.9987\n",
            "Epoch 41/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0077 - accuracy: 0.9984\n",
            "Epoch 42/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0128 - accuracy: 0.9979\n",
            "Epoch 43/50\n",
            "118/118 [==============================] - 46s 392ms/step - loss: 0.0072 - accuracy: 0.9989\n",
            "Epoch 44/50\n",
            "118/118 [==============================] - 46s 392ms/step - loss: 0.0060 - accuracy: 0.9989\n",
            "Epoch 45/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0075 - accuracy: 0.9981\n",
            "Epoch 46/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0071 - accuracy: 0.9984\n",
            "Epoch 47/50\n",
            "118/118 [==============================] - 46s 391ms/step - loss: 0.0044 - accuracy: 0.9989\n",
            "Epoch 48/50\n",
            "118/118 [==============================] - 46s 392ms/step - loss: 0.0058 - accuracy: 0.9989\n",
            "Epoch 49/50\n",
            "118/118 [==============================] - 46s 392ms/step - loss: 0.0062 - accuracy: 0.9979\n",
            "Epoch 50/50\n",
            "118/118 [==============================] - 46s 392ms/step - loss: 0.0068 - accuracy: 0.9981\n",
            "#EVALUATION RESULTS:\n",
            "52/52 - 11s - loss: 1.2483 - accuracy: 0.7384\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1.2482805252075195, 0.7384146451950073]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GFEGINhrwyEb",
        "outputId": "8f305b3a-ba22-4cfe-cd40-ff87d1c80c2d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "# Test the model on the test dataset. Accuracy of 78.77%\n",
        "print(\"#TEST RESULTS:\")\n",
        "model.evaluate(test_x,  test_y, verbose=2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "#TEST RESULTS:\n",
            "41/41 - 9s - loss: 0.7963 - accuracy: 0.7877\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.7963438034057617, 0.7877394556999207]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YIK_NTQTrqrR",
        "outputId": "0553e442-13b8-4776-eca1-9d2b16ad86f1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        }
      },
      "source": [
        "# The following code is just to compare the test results and calculate the most\n",
        "# frequent errors. In this case the model confused category 21 (scoop) \n",
        "# for category 7 (drt side) 42 times.\n",
        "errors = 0\n",
        "errors_list = []\n",
        "plot = []\n",
        "plot2 = []\n",
        "values = []\n",
        "all = []\n",
        "for i in range(len(test_x)):\n",
        "  aa = test_x[i]\n",
        "  predictions = model.predict_on_batch(aa).flatten()\n",
        "\n",
        "  predictions = tf.nn.softmax(predictions)\n",
        "  predictions = str(tf.argmax(predictions))\n",
        "  predictions = predictions.replace('(', ' ')\n",
        "  predictions = predictions.replace(',', ' ')\n",
        "  predictions = predictions.split(' ')\n",
        "  prediction = predictions[1]\n",
        "\n",
        "  actual = str(tf.argmax(test_y[i]))\n",
        "  actual = actual.replace('(', ' ')\n",
        "  actual = actual.replace(',', ' ')\n",
        "  actual = actual.split(' ')\n",
        "  actual = actual[1]\n",
        "\n",
        "  all.append(actual)\n",
        "  if prediction != actual:\n",
        "    errors += 1\n",
        "    errors_list.append((prediction, actual))\n",
        "\n",
        "print(len(test_x))\n",
        "print(errors)\n",
        "\n",
        "errors_ordered = []\n",
        "for i in set(errors_list):\n",
        "  a = [i, errors_list.count(i)]\n",
        "  errors_ordered.append(a)\n",
        "\n",
        "print(sorted(errors_ordered,key=lambda e:e[1], reverse=True))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1305\n",
            "277\n",
            "[[('8', '10'), 3], [('20', '24'), 3], [('18', '3'), 1], [('8', '9'), 31], [('8', '14'), 1], [('8', '12'), 4], [('2', '4'), 2], [('0', '11'), 1], [('17', '6'), 1], [('17', '21'), 1], [('17', '14'), 1], [('10', '6'), 10], [('23', '13'), 1], [('21', '7'), 42], [('9', '14'), 1], [('3', '19'), 1], [('9', '20'), 1], [('9', '13'), 2], [('21', '13'), 2], [('0', '6'), 20], [('19', '6'), 3], [('14', '10'), 1], [('21', '6'), 6], [('17', '7'), 2], [('3', '6'), 3], [('20', '10'), 4], [('11', '6'), 1], [('20', '14'), 5], [('20', '12'), 4], [('23', '17'), 2], [('20', '13'), 5], [('15', '19'), 1], [('10', '3'), 40], [('3', '11'), 1], [('0', '16'), 5], [('24', '19'), 2], [('0', '24'), 7], [('8', '19'), 8], [('18', '12'), 2], [('7', '24'), 1], [('21', '24'), 1], [('24', '10'), 7], [('24', '11'), 2], [('12', '3'), 1], [('9', '8'), 15], [('24', '6'), 10], [('4', '7'), 3], [('8', '3'), 1], [('16', '11'), 1], [('18', '6'), 1], [('16', '6'), 1], [('23', '4'), 2]]\n",
            "[[('21', '7'), 42], [('10', '3'), 40], [('8', '9'), 31], [('0', '6'), 20], [('9', '8'), 15], [('10', '6'), 10], [('24', '6'), 10], [('8', '19'), 8], [('0', '24'), 7], [('24', '10'), 7], [('21', '6'), 6], [('20', '14'), 5], [('20', '13'), 5], [('0', '16'), 5], [('8', '12'), 4], [('20', '10'), 4], [('20', '12'), 4], [('8', '10'), 3], [('20', '24'), 3], [('19', '6'), 3], [('3', '6'), 3], [('4', '7'), 3], [('2', '4'), 2], [('9', '13'), 2], [('21', '13'), 2], [('17', '7'), 2], [('23', '17'), 2], [('24', '19'), 2], [('18', '12'), 2], [('24', '11'), 2], [('23', '4'), 2], [('18', '3'), 1], [('8', '14'), 1], [('0', '11'), 1], [('17', '6'), 1], [('17', '21'), 1], [('17', '14'), 1], [('23', '13'), 1], [('9', '14'), 1], [('3', '19'), 1], [('9', '20'), 1], [('14', '10'), 1], [('11', '6'), 1], [('15', '19'), 1], [('3', '11'), 1], [('7', '24'), 1], [('21', '24'), 1], [('12', '3'), 1], [('8', '3'), 1], [('16', '11'), 1], [('18', '6'), 1], [('16', '6'), 1]]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}