parse_html.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dbd9e96c",
   "metadata": {},
   "source": [
    "## Members\n",
    "- Cao Thanh Khiết   19120544\n",
    "- Ninh Duy Huy      19120533\n",
    "- Lê Minh Hữu       19120525\n",
    "- Nguyễn Tuấn Khoa  19120547\n",
    "- Trần Tuấn Kiệt    19120557\n",
    "# Task of each member\n",
    "- Cao Thanh Khiết\n",
    "    - Use SoundClound API to get data and write down them into csv file\n",
    "- Ninh Duy Huy\n",
    "    - Parse HTML to get data and write down them into csv file\n",
    "- Lê Minh Hữu\n",
    "    - Parse HTML to get data and write down them into csv file\n",
    "- Nguyễn Tuấn Khoa\n",
    "    - Parse HTML to get data and write down them into csv file\n",
    "- Trần Tuấn Kiệt\n",
    "    - Parse HTML to get data and write down them into csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c0798153",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'D:\\\\exec-in-cmd\\\\miniconda3\\\\envs\\\\min_ds-env\\\\python.exe'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.executable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ed1a2491",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import time\n",
    "import pandas as pd # Dùng để đọc và hiển thị file csv/tsv\n",
    "from datetime import datetime, timedelta # Dùng để xử lý dữ liệu thời gian\n",
    "import os\n",
    "from selenium import webdriver\n",
    "from concurrent import futures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GetAllUrls(filename):\n",
    "    urls = []\n",
    "    with open(filename, 'r') as f:\n",
    "        while f.tell() != os.fstat(f.fileno()).st_size:\n",
    "            url = f.readline()\n",
    "            if url != '':\n",
    "                url = url.replace('\\n','')\n",
    "                urls.append(url)\n",
    "    f.close()\n",
    "    return urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'url_entry/top50_url.txt'\n",
    "urls = GetAllUrls(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "Users = []\n",
    "Users.append([])\n",
    "Users.append([])\n",
    "\n",
    "def run_get_users(url):\n",
    "    main_url = 'https://soundcloud.com'\n",
    "    UsersLink = []\n",
    "    UsersName = []\n",
    "    \n",
    "    chrome_options = webdriver.ChromeOptions()\n",
    "    prefs = {\n",
    "        \"profile.managed_default_content_settings.images\": 2\n",
    "    }\n",
    "    chrome_options.add_experimental_option(\"prefs\", prefs)\n",
    "    \n",
    "    browser = webdriver.Chrome(executable_path=\"chromedriver.exe\", options=chrome_options)\n",
    "    browser.get(url)\n",
    "    # To make it load more scroll to the bottom of the page (repeat if you want to)\n",
    "    lenOfPage = browser.execute_script(\"window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;\")\n",
    "    match = False\n",
    "    while(match == False):\n",
    "        lastCount = lenOfPage\n",
    "        time.sleep(1)\n",
    "        lenOfPage = browser.execute_script(\"window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;\")\n",
    "        if lastCount == lenOfPage:\n",
    "                match = True\n",
    "\n",
    "    html_source = browser.page_source\n",
    "    browser.quit()\n",
    "    soup = BeautifulSoup(html_source, 'html.parser')\n",
    "    for id in soup.findAll('a', {'class' : 'trackItem__username sc-link-light sc-link-secondary'}):\n",
    "        UsersLink.append(main_url + str(id.get('href')))\n",
    "        UsersName.append(id.text)\n",
    "    \n",
    "    #đoạn này nối chuỗi mới vào Users[0], Users[1], Users[2]\n",
    "    Users[0] += UsersLink \n",
    "    Users[1] += UsersName\n",
    "\n",
    "with futures.ThreadPoolExecutor(max_workers=40) as executor:\n",
    "    executor.map(run_get_users, urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(Users)\n",
    "Users_new = []\n",
    "for index, link in enumerate(Users[0]):\n",
    "    Users_new.append([link, Users[1][index]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "TracksOfAllUsers = []\n",
    "TracksOfAllUsers.append([]) #tên mỗi user\n",
    "TracksOfAllUsers.append([]) #tên mỗi track của user\n",
    "TracksOfAllUsers.append([]) #lấy thời gian post\n",
    "TracksOfAllUsers.append([]) #lấy lượt nghe\n",
    "TracksOfAllUsers.append([]) #lấy lượt thich\n",
    "\n",
    "def run_get_tracks(user):   # user contains [link, user_name]\n",
    "    UserName = []\n",
    "    TrackName = []\n",
    "    PostTime = []\n",
    "    Plays = []\n",
    "    Likes = []\n",
    "    #----------------------------\n",
    "    url = str(user[0]) + '/tracks'\n",
    "    chrome_options = webdriver.ChromeOptions()\n",
    "    prefs = {\n",
    "        \"profile.managed_default_content_settings.images\": 2\n",
    "    }\n",
    "    chrome_options.add_experimental_option(\"prefs\", prefs)\n",
    "    browser = webdriver.Chrome(executable_path=\"chromedriver.exe\", options=chrome_options)\n",
    "    browser.get(url)\n",
    "    # To make it load more scroll to the bottom of the page (repeat if you want to)        \n",
    "    lenOfPage = browser.execute_script(\"window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;\")\n",
    "    match = False\n",
    "    while(match == False):\n",
    "        lastCount = lenOfPage\n",
    "        time.sleep(1)\n",
    "        lenOfPage = browser.execute_script(\"window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;\")\n",
    "        if lastCount == lenOfPage:\n",
    "            match = True\n",
    "    html_source = browser.page_source\n",
    "    browser.quit()\n",
    "    \n",
    "    soup = BeautifulSoup(html_source, 'html.parser')\n",
    "    check = soup.findAll('div', {'class' : 'sound__content'})\n",
    "    if check == []:\n",
    "        UserName.append(user[1])\n",
    "        TrackName.append('None')\n",
    "        PostTime.append('None')\n",
    "        Plays.append('None')\n",
    "        Likes.append('None')\n",
    "        \n",
    "        TracksOfAllUsers[0] += UserName\n",
    "        TracksOfAllUsers[1] += TrackName\n",
    "        TracksOfAllUsers[2] += PostTime\n",
    "        TracksOfAllUsers[3] += Plays\n",
    "        TracksOfAllUsers[4] += Likes\n",
    "    else:\n",
    "        for id in check:\n",
    "            UserName = []\n",
    "            TrackName = []\n",
    "            PostTime = []\n",
    "            Plays = []\n",
    "            Likes = []\n",
    "    \n",
    "            UserName.append(user[1])\n",
    "        \n",
    "            trackname = id.find('a', {'class' : 'sc-link-primary soundTitle__title sc-link-dark sc-text-h4'})\n",
    "            Like = id.find('button', {'class' : 'sc-button-like sc-button-secondary sc-button sc-button-small sc-button-responsive'})\n",
    "            info = id.findAll('span', {'class' : 'sc-visuallyhidden'})\n",
    "        \n",
    "            TrackName.append(trackname.text.replace('\\n', ''))\n",
    "            PostTime.append(info[0].text.replace('\\n', '')) #lay thoi gian post\n",
    "            \n",
    "            if len(info) < 2:\n",
    "                Plays.append('0')\n",
    "                Likes.append('0')\n",
    "            else:\n",
    "                Plays.append(info[1].text.replace('\\n', '')) #lay luot xem\n",
    "                Likes.append(Like.text.replace('\\n', ''))\n",
    "                \n",
    "            TracksOfAllUsers[0] += UserName\n",
    "            TracksOfAllUsers[1] += TrackName\n",
    "            TracksOfAllUsers[2] += PostTime\n",
    "            TracksOfAllUsers[3] += Plays\n",
    "            TracksOfAllUsers[4] += Likes\n",
    "\n",
    "with futures.ThreadPoolExecutor(max_workers=20) as executor:\n",
    "    executor.map(run_get_tracks, Users_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "Users.append([]) #append follower column\n",
    "Users.append([]) #append following column\n",
    "Users.append([]) #append tracks column\n",
    "\n",
    "def run_get_userinfor(user):\n",
    "    url = user[0] #lay duong dan den user    \n",
    "    chrome_options = webdriver.ChromeOptions()\n",
    "    prefs = {\n",
    "        \"profile.managed_default_content_settings.images\": 2\n",
    "    }\n",
    "    chrome_options.add_experimental_option(\"prefs\", prefs)\n",
    "    browser = webdriver.Chrome(executable_path=\"chromedriver.exe\", options=chrome_options)\n",
    "    browser.get(url)\n",
    "    # xóa phần cuộn đến cuối trang\n",
    "    html_source = browser.page_source\n",
    "    browser.quit()\n",
    "\n",
    "    soup = BeautifulSoup(html_source, 'html.parser')\n",
    "    checks = soup.find_all('div', {'class' : 'infoStats__value sc-font-light'})\n",
    "    Users[2].append(checks[0].text)\n",
    "    Users[3].append(checks[1].text)\n",
    "    Users[4].append(checks[2].text)\n",
    "        \n",
    "with futures.ThreadPoolExecutor(max_workers=20) as executor:\n",
    "    executor.map(run_get_userinfor, Users_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef71bc75",
   "metadata": {},
   "outputs": [],
   "source": [
    "PlaylistOfAllUsers = []\n",
    "PlaylistOfAllUsers.append([])\n",
    "PlaylistOfAllUsers.append([])\n",
    "PlaylistOfAllUsers.append([])\n",
    "\n",
    "def run_get_playlists(user):\n",
    "    UserNames = []\n",
    "    PlaylistNames = []\n",
    "    TrackNames = []\n",
    "    \n",
    "    url = str(user[0]) + '/sets' #lay duong dan den playlist\n",
    "    chrome_options = webdriver.ChromeOptions()\n",
    "    prefs = {\n",
    "        \"profile.managed_default_content_settings.images\": 2\n",
    "    }\n",
    "    chrome_options.add_experimental_option(\"prefs\", prefs)\n",
    "    browser = webdriver.Chrome(executable_path=\"chromedriver.exe\", options=chrome_options)\n",
    "    browser.get(url)\n",
    "\n",
    "    # To make it load more scroll to the bottom of the page (repeat if you want to)\n",
    "    lenOfPage = browser.execute_script(\"window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;\")\n",
    "    match = False\n",
    "    while(match == False):\n",
    "        lastCount = lenOfPage\n",
    "        time.sleep(1)\n",
    "        lenOfPage = browser.execute_script(\"window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;\")\n",
    "        if lastCount == lenOfPage:\n",
    "            match = True    \n",
    "    html_source = browser.page_source\n",
    "    browser.quit()\n",
    "    \n",
    "    soup = BeautifulSoup(html_source, 'html.parser')\n",
    "    check = soup.findAll('div', {'class' : 'sound__content'})\n",
    "    if check == []:\n",
    "        UserNames.append(user[1])\n",
    "        PlaylistNames.append('None')\n",
    "        TrackNames.append('None')\n",
    "        \n",
    "        PlaylistOfAllUsers[0] += UserNames\n",
    "        PlaylistOfAllUsers[1] += PlaylistNames\n",
    "        PlaylistOfAllUsers[2] += TrackNames\n",
    "    else:        \n",
    "        for id in check:\n",
    "            Title = id.find('a', {'class' : 'sc-link-primary soundTitle__title sc-link-dark sc-text-h4'})\n",
    "            Tracks = id.findAll('span', {'class' : 'compactTrackListItem__trackTitle sc-text-primary sc-text-h4'})\n",
    "            for Track in Tracks:\n",
    "                UserNames = []\n",
    "                PlaylistNames = []\n",
    "                TrackNames = []\n",
    "                \n",
    "                UserNames.append(user[1])\n",
    "                PlaylistNames.append(Title.text.replace('\\n', ''))\n",
    "                TrackNames.append(Track.text.replace('\\n', ''))\n",
    "                \n",
    "                PlaylistOfAllUsers[0] += UserNames\n",
    "                PlaylistOfAllUsers[1] += PlaylistNames\n",
    "                PlaylistOfAllUsers[2] += TrackNames\n",
    "\n",
    "with futures.ThreadPoolExecutor(max_workers=20) as executor:\n",
    "    executor.map(run_get_playlists, Users_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def writeUsersToCsv(List5D, filename):\n",
    "    df = pd.DataFrame({\n",
    "        'Users Link': pd.Series(link for link in List5D[0]),\n",
    "        'User Name': pd.Series(name for name in List5D[1]),\n",
    "        'Followers': pd.Series(fle for fle in List5D[2]),\n",
    "        'Following': pd.Series(fli for fli in List5D[3]),\n",
    "        'Tracks': pd.Series(track for track in List5D[4]),\n",
    "    })\n",
    "    df.to_csv(filename, index = False)\n",
    "    \n",
    "def writeTracksToCsv(List5D, filename):\n",
    "    df = pd.DataFrame({\n",
    "        'User Name': pd.Series(user for user in List5D[0]),\n",
    "        'Track Name': pd.Series(track for track in List5D[1]),\n",
    "        'Post Time' : pd.Series(t for t in List5D[2]),\n",
    "        'Plays' : pd.Series(p for p in List5D[3]),\n",
    "        'Likes' : pd.Series(L for L in List5D[4]),\n",
    "    })\n",
    "    df.to_csv(filename, index = False)\n",
    "\n",
    "def writePlaylistsToCsv(List3D, filename):\n",
    "    df = pd.DataFrame({\n",
    "        'User Name': pd.Series(user for user in List3D[0]),\n",
    "        'Playlist Name': pd.Series(p for p in List3D[1]),\n",
    "        'Track Name' : pd.Series(t for t in List3D[2]),\n",
    "    })\n",
    "    df.to_csv(filename, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfd72a97",
   "metadata": {},
   "outputs": [],
   "source": [
    "writeUsersToCsv(Users, 'Crawl_data/users.csv')\n",
    "writeTracksToCsv(TracksOfAllUsers, './Crawl_data/tracks.csv')\n",
    "writePlaylistsToCsv(PlaylistOfAllUsers, './Crawl_data/playlists.csv')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "628dd01b888ab9cdfddc3a0771c5db8a998a3af968dbb11740c2370c8ddf1b03"
  },
  "kernelspec": {
   "display_name": "Python 3.8.8 64-bit ('min_ds-env': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}