Test_image_Pre_processing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Pre-processing-State-Driver.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Yumlembam/Kaggle-Distracted-Driver/blob/master/Test_image_Pre_processing.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "wXF5DyTNUEd4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# upload the credential of kaggle account\n",
        "from google.colab import files\n",
        "files.upload()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "SWXI1UjtIgfI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#before importing the dataset we want to use this code\n",
        "# The Kaggle API client expects this file to be in ~/.kaggle,\n",
        "!mkdir -p ~/.kaggle\n",
        "!cp kaggle.json ~/.kaggle/\n",
        "\n",
        "# This permissions change avoids a warning on Kaggle tool startup.\n",
        "!chmod 600 ~/.kaggle/kaggle.json"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-cG-RN69Uvgm",
        "colab_type": "code",
        "outputId": "d808d0f5-3dcd-4cd8-f226-48b522221ecc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "cell_type": "code",
      "source": [
        "#import the dataset we want to use for our project\n",
        "!kaggle competitions download -c state-farm-distracted-driver-detection"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "sample_submission.csv.zip: Skipping, found more recently modified local copy (use --force to force download)\n",
            "imgs.zip: Skipping, found more recently modified local copy (use --force to force download)\n",
            "driver_imgs_list.csv.zip: Skipping, found more recently modified local copy (use --force to force download)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "FdeH0hcvz6V9",
        "colab_type": "code",
        "outputId": "3a5dc03c-7dce-4af2-da9e-8ff6b956cb79",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " cnn_utils.py\t\t    imgs.zip\t       sample_data\n",
            " combine_train\t\t   'kaggle (1).json'   sample_submission.csv.zip\n",
            " driver_imgs_list.csv.zip   kaggle.json        test\n",
            " Happyhouse-master.zip\t    __pycache__        train\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "_YNufhSdVGoB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!unzip -q imgs.zip # skip this if dataset have already been extracted"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jwJ24UzYDPns",
        "colab_type": "code",
        "outputId": "cb1674db-4c3c-4c37-d673-0129c7263e65",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "path, dirs, files = next(os.walk(\"/content/test\"))\n",
        "file_count = len(files)\n",
        "print(file_count)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "79726\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "lv6cMN_EELe0",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#import pandas as pd\n",
        "#test_df = pd.DataFrame(columns=['test_img'])\n",
        "#test_df['test_img']= files\n",
        "#print(test_df.shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "NUI3nFvPEuCO",
        "colab_type": "code",
        "outputId": "1903cb15-e70d-4fdd-d527-0f8ea9f7cdac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "!unzip sample_submission.csv.zip"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  sample_submission.csv.zip\n",
            "  inflating: sample_submission.csv   \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "R3hoJHH8FFDT",
        "colab_type": "code",
        "outputId": "a8bc3377-1e3c-48a6-d0f6-a35584fb64db",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1122
        }
      },
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "test_df = pd.read_csv('sample_submission.csv', header=0, sep=',', quotechar='\"')\n",
        "print(test_df.shape)\n",
        "print(test_df.head)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(79726, 11)\n",
            "<bound method NDFrame.head of                   img   c0   c1   c2   c3   c4   c5   c6   c7   c8   c9\n",
            "0           img_1.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "1          img_10.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "2         img_100.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "3        img_1000.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "4      img_100000.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "5      img_100001.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "6      img_100002.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "7      img_100003.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "8      img_100004.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "9      img_100005.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "10     img_100007.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "11     img_100008.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "12     img_100009.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "13      img_10001.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "14     img_100010.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "15     img_100011.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "16     img_100012.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "17     img_100013.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "18     img_100014.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "19     img_100016.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "20     img_100017.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "21     img_100018.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "22     img_100019.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "23      img_10002.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "24     img_100020.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "25     img_100022.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "26     img_100023.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "27     img_100024.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "28     img_100025.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "29     img_100028.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "...               ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...\n",
            "79696   img_99960.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79697   img_99961.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79698   img_99962.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79699   img_99966.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79700   img_99967.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79701   img_99968.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79702   img_99969.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79703    img_9997.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79704   img_99970.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79705   img_99972.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79706   img_99973.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79707   img_99974.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79708   img_99975.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79709   img_99981.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79710   img_99983.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79711   img_99984.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79712   img_99985.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79713   img_99986.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79714   img_99987.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79715   img_99988.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79716   img_99989.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79717    img_9999.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79718   img_99990.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79719   img_99991.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79720   img_99993.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79721   img_99994.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79722   img_99995.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79723   img_99996.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79724   img_99998.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "79725   img_99999.jpg  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
            "\n",
            "[79726 rows x 11 columns]>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "-wVAygmjGhtw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "from skimage import io\n",
        "from skimage.transform import resize\n",
        "def convert_to_array(image):\n",
        "  \n",
        "  image_file = io.imread(\"/content/test/\"+str(image))\n",
        "  image_resized = resize(image_file, (128, 128))\n",
        "  image_data = image_resized/ 255\n",
        "  return image_data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "irodwADjJWZH",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Since 70726 image is too big to load at once we will divide the dataset into chunks of 8000 image each, example:1_test_array.h5 will contain images from 0-7999,2_test_array.h5 will contain images from 8000-15999 and so on.So you should modify the code accordingly."
      ]
    },
    {
      "metadata": {
        "id": "_iRX6nOdF0Gz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "i=0\n",
        "X_test = np.ndarray(shape = (7726, 128, 128, 3), dtype = np.float32)\n",
        "for fname in test_df[\"img\"].iloc[72000:79726]:\n",
        "  image_array = convert_to_array(fname)\n",
        "  X_test[i,:,:,:] = image_array\n",
        "  i += 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "CpcZ3UOZc4FX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "print(X_test.shape)\n",
        "print(X_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "t2fPhfG-yp2r",
        "colab_type": "code",
        "outputId": "7c11c881-c857-4ea7-ddd8-6d08052a186e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /gdrive; to attempt to forcibly remount, call drive.mount(\"/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "z8ZvBAen1WoR",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "##File name and dataset name\n",
        "\n",
        "\n",
        "\n",
        "1. file name: 1_test_array.h5, Dataset name:test_dataset_1(image range: 0-8000)\n",
        "2.  \n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "HtivH62CqtRA",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import h5py\n",
        "with h5py.File('/gdrive/My Drive/10_test_array.h5', 'w') as hf:\n",
        "    hf.create_dataset(\"test_dataset_10\",  data=X_test)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "VrPYeSHYz31y",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "del X_test"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}