craw_api.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Members\n",
    "- Cao Thanh Khiết   19120544\n",
    "- Ninh Duy Huy      19120533\n",
    "- Lê Minh Hữu       19120525\n",
    "- Nguyễn Tuấn Khoa  19120547\n",
    "- Trần Tuấn Kiệt    19120557\n",
    "# Task of each member\n",
    "- Cao Thanh Khiết\n",
    "    - Use SoundClound API to get data and write down them into csv file\n",
    "- Ninh Duy Huy\n",
    "    - Parse HTML to get data and write down them into csv file\n",
    "- Lê Minh Hữu\n",
    "    - Parse HTML to get data and write down them into csv file\n",
    "- Nguyễn Tuấn Khoa\n",
    "    - Parse HTML to get data and write down them into csv file\n",
    "- Trần Tuấn Kiệt\n",
    "    - Parse HTML to get data and write down them into csv file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cấu trúc thư mục\n",
    "### **Api_data folder**: chứa dữ liệu ở dạng csv thu thập đc sau khi chạy code bằng cách call API\n",
    "### **Crawl_data folder**: chứa dữ liệu sau khi chạy code parse HTML\n",
    "### **docs folder**: chứa file đề bài \n",
    "### **url_entry folder**: chứa các url đầu vào, chương trình đầu tiên để gửi request tới các url này để lấy top 50 tracks, làm nền cho việc gửi các request về sau\n",
    "### **craw_api.ipynb**: file source code chạy bằng cách call API\n",
    "### **parse_html.ipynb**: file source code chạy bằng cách parse HTML\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# More information in README.md file\n",
    "import requests\n",
    "import pandas as pd\n",
    "import concurrent.futures\n",
    "import threading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "api = \"https://api-v2.soundcloud.com\"\n",
    "client_id = \"Ab49vxUKnGiGnHTeZNxE25gBG9ATMnUe\"\n",
    "client_id_query = \"?client_id={}\".format(client_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"top50_url.txt\" contains url of top lists ( system-playlists of top 50 tracks)\n",
    "# This cell Convert url_lists from txt file to List python\n",
    "# url_entrypoint = \"url_entry/top50_url.txt\"\n",
    "url_entrypoint = \"url_entry/top50_url_max.txt\"\n",
    "sys_playlists_url = []\n",
    "\n",
    "with open(url_entrypoint) as file:\n",
    "    lines = [ sys_playlists_url.append(line.rstrip()) for line in file]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sending request for each system-playlists to get track_id of top system-playlists\n",
    "track_ids = []  # holding id of track\n",
    "thread_local = threading.local()\n",
    "\n",
    "def get_session():\n",
    "    if not hasattr(thread_local, \"session\"):\n",
    "        thread_local.session = requests.Session()\n",
    "    return thread_local.session\n",
    "\n",
    "def get_track(sys_url):\n",
    "    try:\n",
    "        resolve_api = api + \"/resolve\" + client_id_query + \"&url={}\".format(sys_url)\n",
    "        session = get_session()\n",
    "        with session.get(resolve_api) as response:\n",
    "            for track in response.json()['tracks']:\n",
    "                track_ids.append(track['id'])\n",
    "    except KeyError:\n",
    "        pass\n",
    "\n",
    "with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:\n",
    "    executor.map(get_track, sys_playlists_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# From that track_id, we send request for each tracks to get user_id of each track\n",
    "user_ids = []   # containing id of user\n",
    "\n",
    "def get_user_id(track):\n",
    "    try:\n",
    "        session = get_session()\n",
    "        track_api = api + \"/tracks/{}\".format(track) + client_id_query\n",
    "        with session.get(track_api) as response:\n",
    "            user_ids.append(response.json()['user']['id'])\n",
    "    except KeyError:\n",
    "        pass\n",
    "\n",
    "with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:\n",
    "    executor.map(get_user_id, track_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "users = []  # containing detail information of each user\n",
    "\n",
    "def get_infor_user(user_id):\n",
    "    try:\n",
    "        session = get_session()\n",
    "        user_api = api + \"/users/{}\".format(user_id) + client_id_query\n",
    "        with session.get(user_api) as response:\n",
    "            users.append(response.json())\n",
    "    except KeyError:\n",
    "        pass\n",
    "\n",
    "with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:\n",
    "    executor.map(get_infor_user, user_ids)\n",
    "\n",
    "users_table = pd.DataFrame(users)\n",
    "users_table.to_csv('Api_data/users.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## This cell parse playlist_id and track_id, writing it into playlists.csv file\n",
    "playlists = [] # Containing List of Dict, Dict includes playlist_id and track_ids property\n",
    "\n",
    "def get_playlists(user_id):\n",
    "    session = get_session()\n",
    "    user_api = api + \"/users/{}/playlists\".format(user_id) + client_id_query\n",
    "    with session.get(user_api) as response:\n",
    "        try:\n",
    "            collection = response.json()['collection']\n",
    "            # loop\n",
    "            for playlist in collection:\n",
    "                playlist_id = playlist['id']\n",
    "                track_id_string = ''\n",
    "                for item in playlist['tracks']:\n",
    "                    track_id_string += str(item['id']) + ','\n",
    "                track_id_string = track_id_string[:-1] if track_id_string != '' else 'None'\n",
    "                playlists.append({\n",
    "                    'playlist_id': playlist_id,\n",
    "                    'track_ids': track_id_string\n",
    "                })\n",
    "        except KeyError:\n",
    "            pass\n",
    "\n",
    "with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:\n",
    "    executor.map(get_playlists, user_ids)\n",
    "\n",
    "playlists_table = pd.DataFrame(playlists)\n",
    "playlists_table.to_csv('Api_data/playlists.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get information of each track from playlists.csv file\n",
    "tracks_detail = []\n",
    "read_playlists = pd.read_csv('Api_data/playlists.csv')\n",
    "# parse string from playlists.csv file to get track_id and send request to retrive detail information of each track\n",
    "\n",
    "def get_track_detail(track):\n",
    "    track_api = api + \"/tracks/{}\".format(track) + client_id_query\n",
    "    session = get_session()\n",
    "    with session.get(track_api) as response:\n",
    "        tracks_detail.append(response.json())\n",
    "\n",
    "with concurrent.futures.ThreadPoolExecutor(max_workers=400) as executor:\n",
    "    for index, row in read_playlists.iterrows():\n",
    "        tracks = row['track_ids'].split(',')\n",
    "        if tracks == 'None':\n",
    "            continue\n",
    "        executor.map(get_track_detail, tracks)\n",
    "# Remove some unnessesary property of each track\n",
    "key_remover = ['artwork_url', 'publisher_metadata', 'media', 'user', 'badges']\n",
    "\n",
    "for key in key_remover:\n",
    "    for dic in tracks_detail:\n",
    "        dic.pop(key, None)\n",
    "\n",
    "# convert infor track into csv file\n",
    "track_df = pd.DataFrame(tracks_detail)\n",
    "track_df.to_csv('Api_data/tracks.csv')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "628dd01b888ab9cdfddc3a0771c5db8a998a3af968dbb11740c2370c8ddf1b03"
  },
  "kernelspec": {
   "display_name": "Python 3.8.8 64-bit ('min_ds-env': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}