create_movie_metadata_using_moviedb.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XS-1QqvSDq8G"
      },
      "source": [
        "# Get latest movie data using moviedb api\n",
        "this guide will give you a starting point on how to fetch data from moviedb, process that data and save as Pandas DataFrame\n",
        "\n",
        "<br>\n",
        "\n",
        "## Steps :\n",
        "1. follow this [guide](https://developers.themoviedb.org/3) and get your own MovieDB api.\n",
        "2. in cell 2, configure with the api key you got and language and number of movies for this language\n",
        "3. run all the cells, you will have dataframes saved as csv.\n",
        "\n",
        "<br>\n",
        "\n",
        "## Notes :\n",
        "- currently there is some biased towards popular movies but for learning purposes it should be ok.\n",
        "- read the MoviDB [docs](https://developers.themoviedb.org/3) and fetch extra datas you need."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "GG3arr-cNIRp"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "import pandas as pd\n",
        "import sys\n",
        "import threading, logging"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QRibDL9TF1he"
      },
      "source": [
        "## language_count object :\n",
        "1. find ISO-639-1  [codes](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) for required languages\n",
        "2. int the key is the laguage and value is the number of movies of the same laguage to be downloaded\n",
        "\n",
        "``` \n",
        "language_count = {\n",
        "  ...,\n",
        "  'language' :number_of_movies,\n",
        "  ...\n",
        "}\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "5wyKZlmANP46"
      },
      "outputs": [],
      "source": [
        "api_key = '<moviedb_apikey>'\n",
        "language_count = {\n",
        "    'en':7000,\n",
        "    'hi':2000,\n",
        "    'bn':1000,\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "hjCOiGY9NpiC"
      },
      "outputs": [],
      "source": [
        "def get_movies(lang, freq):\n",
        "  url = 'https://api.themoviedb.org/3/movie/popular?api_key={api_key}&with_original_language={lang}'.format(api_key=api_key,lang=lang)\n",
        "  # print(url)\n",
        "  movies = []\n",
        "  page = 1\n",
        "  progress = 0\n",
        "  while movies.__len__()<freq:\n",
        "    try:\n",
        "        res = requests.get(url+\"&page=\"+str(page))\n",
        "    except:\n",
        "        raise ('not connected to internet or movidb issue')\n",
        "\n",
        "    if res.status_code != 200:\n",
        "        print ('error')\n",
        "        return []\n",
        "\n",
        "    res = res.json()\n",
        "    \n",
        "    if 'errors' in res.keys():\n",
        "      print('api error !!!')\n",
        "      return movies\n",
        "\n",
        "    movies = movies + res['results']\n",
        "\n",
        "    if progress != round(len(movies)/freq*100):\n",
        "      progress = round(len(movies)/freq*100)\n",
        "      if progress%5==0:\n",
        "        print( progress, end=\"%, \")\n",
        "        \n",
        "    page = page + 1\n",
        "    # break\n",
        "    # print(res)\n",
        "  return movies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O7cJB6QjOzct",
        "outputId": "d43fb317-d219-4af3-f260-9ddfdfb9c47b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading  en : 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, Total movies found :  7000\n",
            "Downloading  hi : 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, Total movies found :  2000\n",
            "Downloading  bn : 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, Total movies found :  1000\n"
          ]
        }
      ],
      "source": [
        "all_movies = []\n",
        "\n",
        "for key in language_count:\n",
        "  # print(key,language_count[key])\n",
        "  print(\"Downloading \", key, end=\" : \")\n",
        "  movies = get_movies(key,language_count[key])\n",
        "  all_movies = all_movies + movies\n",
        "  print('Total movies found : ', movies.__len__())\n",
        "  # break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "WS-3__4fVBI9",
        "outputId": "e6204ff4-429e-4eb9-9213-824779e7c920"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>genre_ids</th>\n",
              "      <th>id</th>\n",
              "      <th>original_language</th>\n",
              "      <th>overview</th>\n",
              "      <th>popularity</th>\n",
              "      <th>release_date</th>\n",
              "      <th>title</th>\n",
              "      <th>vote_average</th>\n",
              "      <th>vote_count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[878, 28, 12]</td>\n",
              "      <td>580489</td>\n",
              "      <td>en</td>\n",
              "      <td>After finding a host body in investigative rep...</td>\n",
              "      <td>6972.003</td>\n",
              "      <td>2021-09-30</td>\n",
              "      <td>Venom: Let There Be Carnage</td>\n",
              "      <td>7.2</td>\n",
              "      <td>4500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[28, 35, 80, 53]</td>\n",
              "      <td>512195</td>\n",
              "      <td>en</td>\n",
              "      <td>An Interpol-issued Red Notice is a global aler...</td>\n",
              "      <td>4496.760</td>\n",
              "      <td>2021-11-04</td>\n",
              "      <td>Red Notice</td>\n",
              "      <td>6.8</td>\n",
              "      <td>2016</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[28, 12, 878, 14]</td>\n",
              "      <td>634649</td>\n",
              "      <td>en</td>\n",
              "      <td>Peter Parker is unmasked and no longer able to...</td>\n",
              "      <td>4062.446</td>\n",
              "      <td>2021-12-15</td>\n",
              "      <td>Spider-Man: No Way Home</td>\n",
              "      <td>8.4</td>\n",
              "      <td>84</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[16, 35, 10751]</td>\n",
              "      <td>585245</td>\n",
              "      <td>en</td>\n",
              "      <td>As Emily struggles to fit in at home and at sc...</td>\n",
              "      <td>2509.512</td>\n",
              "      <td>2021-11-10</td>\n",
              "      <td>Clifford the Big Red Dog</td>\n",
              "      <td>7.6</td>\n",
              "      <td>568</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[28, 12, 14]</td>\n",
              "      <td>566525</td>\n",
              "      <td>en</td>\n",
              "      <td>Shang-Chi must confront the past he thought he...</td>\n",
              "      <td>2453.799</td>\n",
              "      <td>2021-09-01</td>\n",
              "      <td>Shang-Chi and the Legend of the Ten Rings</td>\n",
              "      <td>7.8</td>\n",
              "      <td>4244</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9995</th>\n",
              "      <td>[]</td>\n",
              "      <td>658466</td>\n",
              "      <td>bn</td>\n",
              "      <td></td>\n",
              "      <td>0.600</td>\n",
              "      <td>1992-01-01</td>\n",
              "      <td>Bedeneer Prem</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9996</th>\n",
              "      <td>[18]</td>\n",
              "      <td>658463</td>\n",
              "      <td>bn</td>\n",
              "      <td></td>\n",
              "      <td>0.600</td>\n",
              "      <td>1996-01-01</td>\n",
              "      <td>Bhai Amar Bhai</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9997</th>\n",
              "      <td>[]</td>\n",
              "      <td>658443</td>\n",
              "      <td>bn</td>\n",
              "      <td></td>\n",
              "      <td>0.600</td>\n",
              "      <td>1977-01-01</td>\n",
              "      <td>Barbadhu</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9998</th>\n",
              "      <td>[10749]</td>\n",
              "      <td>658383</td>\n",
              "      <td>bn</td>\n",
              "      <td></td>\n",
              "      <td>0.600</td>\n",
              "      <td>2003-01-01</td>\n",
              "      <td>Andha Prem</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9999</th>\n",
              "      <td>[18]</td>\n",
              "      <td>658235</td>\n",
              "      <td>bn</td>\n",
              "      <td></td>\n",
              "      <td>0.600</td>\n",
              "      <td>2010-01-01</td>\n",
              "      <td>Agni Yuddha</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10000 rows × 9 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "              genre_ids      id  ... vote_average vote_count\n",
              "0         [878, 28, 12]  580489  ...          7.2       4500\n",
              "1      [28, 35, 80, 53]  512195  ...          6.8       2016\n",
              "2     [28, 12, 878, 14]  634649  ...          8.4         84\n",
              "3       [16, 35, 10751]  585245  ...          7.6        568\n",
              "4          [28, 12, 14]  566525  ...          7.8       4244\n",
              "...                 ...     ...  ...          ...        ...\n",
              "9995                 []  658466  ...          0.0          0\n",
              "9996               [18]  658463  ...          0.0          0\n",
              "9997                 []  658443  ...          0.0          0\n",
              "9998            [10749]  658383  ...          0.0          0\n",
              "9999               [18]  658235  ...          0.0          0\n",
              "\n",
              "[10000 rows x 9 columns]"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df = pd.DataFrame(all_movies, columns=['genre_ids', 'id', 'original_language',\n",
        "       'overview', 'popularity', 'release_date', 'title', 'vote_average', 'vote_count'])\n",
        "df.to_csv('movies_dataset.csv', index=False)\n",
        "df.to_pickle('movies_dataset.pk',)\n",
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "LKW9z80RGzC4"
      },
      "outputs": [],
      "source": [
        "ids = df['id'].tolist()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "d8lZTd38kKfd"
      },
      "outputs": [],
      "source": [
        "def get_credits(ids):\n",
        "  total_len = len(ids)\n",
        "  progress = 0\n",
        "  credits = []\n",
        "\n",
        "  def get_credit(id):\n",
        "    url = 'https://api.themoviedb.org/3/movie/{movie_id}/credits?api_key={api_key}'.format(api_key=api_key,movie_id=id)\n",
        "    try:\n",
        "        res = requests.get(url)\n",
        "    except:\n",
        "        raise ('not connected to internet or movidb issue')\n",
        "\n",
        "    if res.status_code != 200:\n",
        "        print ('error')\n",
        "        return []\n",
        "\n",
        "    res = res.json()\n",
        "    \n",
        "    if 'errors' in res.keys():\n",
        "      print('api error !!!')\n",
        "      return credits\n",
        "    # print(res)\n",
        "    credits.append(res)\n",
        "\n",
        "  threads = list()\n",
        "  for id in ids:\n",
        "    x = threading.Thread(target=get_credit,args=(id,))\n",
        "    x.start()\n",
        "  for index, thread in enumerate(threads):\n",
        "      # logging.info(\"Main    : before joining thread %d.\", index)\n",
        "      thread.join()    \n",
        "      progress = progress + 1\n",
        "      if round(len(progress)/total_len*100)%5 == 0:\n",
        "        print( progress, end=\", \")\n",
        "\n",
        "  return credits\n",
        "\n",
        "movie_credits = get_credits(ids)\n",
        "\n",
        "new_movie_credits = {'cast':[],'crew':[]}\n",
        "for movie_credit in movie_credits:\n",
        "  \n",
        "  new_movie_credits['cast'].append( {'id' : movie_credit['id'], 'cast' :[]} )\n",
        "  for credit in movie_credit['cast']:\n",
        "    new_movie_credits['cast'][ -1 ][ 'cast' ].append({\n",
        "        'cast_id': credit['id'],\n",
        "        'name': credit['name'],\n",
        "        'character': credit['character'],\n",
        "    })\n",
        "    # break\n",
        "  \n",
        "  new_movie_credits['crew'].append( {'id' : movie_credit['id'], 'crew' :[]} )\n",
        "  for crew in movie_credit['crew']:\n",
        "    new_movie_credits['crew'][ -1 ][ 'crew' ].append({\n",
        "        'crew_id': crew['id'],\n",
        "        'name': crew['name'],\n",
        "        'department': crew['department'],\n",
        "    })\n",
        "  # break\n",
        "\n",
        "cast_df = pd.DataFrame(new_movie_credits['cast'])\n",
        "crew_df = pd.DataFrame(new_movie_credits['crew'])\n",
        "\n",
        "cast_df.to_csv('cast_dataset.csv', index=False)\n",
        "crew_df.to_csv('crew_dataset.csv', index=False)\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "LTuJfIQgEq6T",
        "outputId": "7b52b0f9-2024-46f1-ba1a-3afe5542c352"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>cast</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>580489</td>\n",
              "      <td>[{'cast_id': 2524, 'name': 'Tom Hardy', 'chara...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>617653</td>\n",
              "      <td>[{'cast_id': 1892, 'name': 'Matt Damon', 'char...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>566525</td>\n",
              "      <td>[{'cast_id': 1489211, 'name': 'Simu Liu', 'cha...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>568124</td>\n",
              "      <td>[{'cast_id': 968367, 'name': 'Stephanie Beatri...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>774741</td>\n",
              "      <td>[{'cast_id': 2239159, 'name': 'Brady Noon', 'c...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9995</th>\n",
              "      <td>658496</td>\n",
              "      <td>[{'cast_id': 2491761, 'name': 'Bimal Chakrabor...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9996</th>\n",
              "      <td>658443</td>\n",
              "      <td>[{'cast_id': 114853, 'name': 'Samit Bhanja', '...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9997</th>\n",
              "      <td>658235</td>\n",
              "      <td>[{'cast_id': 585404, 'name': 'Mukesh Rishi', '...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9998</th>\n",
              "      <td>658383</td>\n",
              "      <td>[{'cast_id': 150408, 'name': 'Prosenjit Chatte...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9999</th>\n",
              "      <td>658616</td>\n",
              "      <td>[{'cast_id': 78951, 'name': 'Victor Banerjee',...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10000 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          id                                               cast\n",
              "0     580489  [{'cast_id': 2524, 'name': 'Tom Hardy', 'chara...\n",
              "1     617653  [{'cast_id': 1892, 'name': 'Matt Damon', 'char...\n",
              "2     566525  [{'cast_id': 1489211, 'name': 'Simu Liu', 'cha...\n",
              "3     568124  [{'cast_id': 968367, 'name': 'Stephanie Beatri...\n",
              "4     774741  [{'cast_id': 2239159, 'name': 'Brady Noon', 'c...\n",
              "...      ...                                                ...\n",
              "9995  658496  [{'cast_id': 2491761, 'name': 'Bimal Chakrabor...\n",
              "9996  658443  [{'cast_id': 114853, 'name': 'Samit Bhanja', '...\n",
              "9997  658235  [{'cast_id': 585404, 'name': 'Mukesh Rishi', '...\n",
              "9998  658383  [{'cast_id': 150408, 'name': 'Prosenjit Chatte...\n",
              "9999  658616  [{'cast_id': 78951, 'name': 'Victor Banerjee',...\n",
              "\n",
              "[10000 rows x 2 columns]"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cast_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "FVt5ZAfbEsMz",
        "outputId": "e170bf20-5ddf-4ea0-e3f1-5f52b01a1ce2"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>crew</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>580489</td>\n",
              "      <td>[{'crew_id': 149, 'name': 'Robert Richardson',...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>617653</td>\n",
              "      <td>[{'crew_id': 120, 'name': 'Dariusz Wolski', 'd...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>566525</td>\n",
              "      <td>[{'crew_id': 7232, 'name': 'Sarah Halley Finn'...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>568124</td>\n",
              "      <td>[{'crew_id': 8159, 'name': 'Shannon Mills', 'd...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>774741</td>\n",
              "      <td>[{'crew_id': 32608, 'name': 'Brad Simpson', 'd...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9995</th>\n",
              "      <td>658496</td>\n",
              "      <td>[{'crew_id': 1658227, 'name': 'Sekhar Das', 'd...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9996</th>\n",
              "      <td>658443</td>\n",
              "      <td>[{'crew_id': 2491583, 'name': 'Bijoy Chatterje...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9997</th>\n",
              "      <td>658235</td>\n",
              "      <td>[{'crew_id': 2491093, 'name': 'Rabindra Pradha...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9998</th>\n",
              "      <td>658383</td>\n",
              "      <td>[{'crew_id': 2488664, 'name': 'Narayan Chatter...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9999</th>\n",
              "      <td>658616</td>\n",
              "      <td>[{'crew_id': 2342446, 'name': 'Swapan Saha', '...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10000 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          id                                               crew\n",
              "0     580489  [{'crew_id': 149, 'name': 'Robert Richardson',...\n",
              "1     617653  [{'crew_id': 120, 'name': 'Dariusz Wolski', 'd...\n",
              "2     566525  [{'crew_id': 7232, 'name': 'Sarah Halley Finn'...\n",
              "3     568124  [{'crew_id': 8159, 'name': 'Shannon Mills', 'd...\n",
              "4     774741  [{'crew_id': 32608, 'name': 'Brad Simpson', 'd...\n",
              "...      ...                                                ...\n",
              "9995  658496  [{'crew_id': 1658227, 'name': 'Sekhar Das', 'd...\n",
              "9996  658443  [{'crew_id': 2491583, 'name': 'Bijoy Chatterje...\n",
              "9997  658235  [{'crew_id': 2491093, 'name': 'Rabindra Pradha...\n",
              "9998  658383  [{'crew_id': 2488664, 'name': 'Narayan Chatter...\n",
              "9999  658616  [{'crew_id': 2342446, 'name': 'Swapan Saha', '...\n",
              "\n",
              "[10000 rows x 2 columns]"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "crew_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "i0XAZ_G65Yd8"
      },
      "outputs": [],
      "source": [
        "def get_keywords(ids):\n",
        "  total_len = len(ids)\n",
        "  progress = 0\n",
        "  keywords = []\n",
        "\n",
        "  def get_keyword(id):\n",
        "    url = 'https://api.themoviedb.org/3/movie/{movie_id}/keywords?api_key={api_key}'.format(api_key=api_key,movie_id=id)\n",
        "    try:\n",
        "        res = requests.get(url)\n",
        "    except:\n",
        "        raise ('not connected to internet or movidb issue')\n",
        "\n",
        "    if res.status_code != 200:\n",
        "        print ('error')\n",
        "        return []\n",
        "\n",
        "    res = res.json()\n",
        "    \n",
        "    if 'errors' in res.keys():\n",
        "      print('api error !!!')\n",
        "      return keywords\n",
        "    # print(res)\n",
        "    keywords.append(res)\n",
        "\n",
        "    # for id in ids:\n",
        "    #   get_cred\n",
        "\n",
        "  threads = list()\n",
        "  for id in ids:\n",
        "    x = threading.Thread(target=get_keyword,args=(id,))\n",
        "    x.start()\n",
        "  for index, thread in enumerate(threads):\n",
        "    logging.info(\"Main    : before joining thread %d.\", index)\n",
        "    thread.join()\n",
        "  \n",
        "    progress = progress + 1\n",
        "    if round(len(progress)/total_len*100)%5 == 0:\n",
        "      print( progress, end=\", \")\n",
        "\n",
        "  return keywords\n",
        "\n",
        "\n",
        "movie_keywords = get_keywords(ids)\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "qSC0jc2X518c",
        "outputId": "1d0008ca-c9e4-4d00-b657-92c1f9c316a8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keywords</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>617653</td>\n",
              "      <td>[{'id': 818, 'name': 'based on novel or book'}...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>580489</td>\n",
              "      <td>[{'id': 1701, 'name': 'hero'}, {'id': 2095, 'n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>568124</td>\n",
              "      <td>[{'id': 4344, 'name': 'musical'}]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>762433</td>\n",
              "      <td>[{'id': 588, 'name': 'rome, italy'}, {'id': 94...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>566525</td>\n",
              "      <td>[{'id': 779, 'name': 'martial arts'}, {'id': 9...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9995</th>\n",
              "      <td>658496</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9996</th>\n",
              "      <td>658383</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9997</th>\n",
              "      <td>658616</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9998</th>\n",
              "      <td>658443</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9999</th>\n",
              "      <td>658235</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10000 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          id                                           keywords\n",
              "0     617653  [{'id': 818, 'name': 'based on novel or book'}...\n",
              "1     580489  [{'id': 1701, 'name': 'hero'}, {'id': 2095, 'n...\n",
              "2     568124                  [{'id': 4344, 'name': 'musical'}]\n",
              "3     762433  [{'id': 588, 'name': 'rome, italy'}, {'id': 94...\n",
              "4     566525  [{'id': 779, 'name': 'martial arts'}, {'id': 9...\n",
              "...      ...                                                ...\n",
              "9995  658496                                                 []\n",
              "9996  658383                                                 []\n",
              "9997  658616                                                 []\n",
              "9998  658443                                                 []\n",
              "9999  658235                                                 []\n",
              "\n",
              "[10000 rows x 2 columns]"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "keywords_df = pd.DataFrame(movie_keywords)\n",
        "keywords_df.to_csv('keywords_dataframe.csv')\n",
        "keywords_df"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "create movie metadata using moviedb.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}