01RissScraping.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ByungjunKim/DDMKL/blob/main/01RissScraping.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aKPb-TfGi29i"
      },
      "source": [
        "# RISS에서 학위 논문 데이터 수집하기"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KxSbHFz2i29m"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "import math\n",
        "import pickle\n",
        "import time\n",
        "import sys\n",
        "import pandas as pd\n",
        "from tqdm.notebook import tqdm\n",
        "from random import uniform\n",
        "import lxml\n",
        "import lxml.etree as et\n",
        "from bs4 import BeautifulSoup\n",
        "import glob\n",
        "import urllib3\n",
        "urllib3.disable_warnings()\n",
        "from natsort import natsorted\n",
        "import re\n",
        "from concurrent.futures import ThreadPoolExecutor\n",
        "from urllib.parse import urlparse, parse_qs, urlencode, urlunparse"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5rbwZa3ai29n"
      },
      "outputs": [],
      "source": [
        "headers = {\n",
        "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36\",\n",
        "    \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\",\n",
        "    \"Accept-Language\": \"en-US,en;q=0.9\",\n",
        "    \"Accept-Encoding\": \"gzip, deflate, br\",\n",
        "    \"Connection\": \"keep-alive\",\n",
        "    \"DNT\": \"1\", # Do Not Track 요청 헤더 (사용자 추적 거부)\n",
        "    # 필요한 추가 헤더를 여기에 추가\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def request_until_success(url, headers, timeout=7, delay=3, max_retries=5, backoff_factor=2):\n",
        "    \"\"\"\n",
        "    Continuously makes a request to the specified URL with a timeout until the request is successful\n",
        "    or the maximum number of retries is reached.\n",
        "\n",
        "    Args:\n",
        "    - url (str): The URL to request.\n",
        "    - headers (dict): The headers to include in the request.\n",
        "    - timeout (int): The timeout for the request in seconds.\n",
        "    - delay (int): The initial delay between retries in seconds.\n",
        "    - max_retries (int): The maximum number of retries before giving up.\n",
        "    - backoff_factor (int): The factor by which to multiply the delay after each retry.\n",
        "\n",
        "    Returns:\n",
        "    - response: The successful response from the server.\n",
        "\n",
        "    Raises:\n",
        "    - requests.RequestException: If the request fails after max_retries attempts.\n",
        "    \"\"\"\n",
        "    attempt = 1\n",
        "    while attempt <= max_retries:\n",
        "        try:\n",
        "            response = requests.get(url, headers=headers, timeout=timeout)\n",
        "            response.raise_for_status()  # Raises HTTPError for bad responses\n",
        "            return response\n",
        "        except requests.Timeout as e:\n",
        "            print(f\"Request timed out (Attempt {attempt}): {e}\")\n",
        "        except requests.RequestException as e:\n",
        "            print(f\"Request failed (Attempt {attempt}): {e}\")\n",
        "\n",
        "        if attempt == max_retries:\n",
        "            raise requests.RequestException(f\"Max retries reached: Failed to get a successful response from {url}\")\n",
        "\n",
        "        # 지수적 백오프 적용\n",
        "        sleep_time = delay * (backoff_factor ** (attempt - 1))\n",
        "        print(f\"Retrying in {sleep_time} seconds...\")\n",
        "        time.sleep(sleep_time)\n",
        "        attempt += 1"
      ],
      "metadata": {
        "id": "MQIuvPvtelR5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p3tSAUiZi29o"
      },
      "source": [
        "### 검색 조건에 맞는 URL 설정\n",
        "학과정보 : 국어국문 OR 국문  \n",
        "학위수여연도 : 2000~2023  \n",
        "학위유형 : 국내박사  \n",
        "페이지당 출력 : 100개씩  \n",
        "https://www.riss.kr/search/Search.do?isDetailSearch=Y&searchGubun=true&viewYn=OP&queryText=znMajor%2C%EA%B5%AD%EC%96%B4%EA%B5%AD%EB%AC%B8%40op%2COR%40znMajor%2C%EA%B5%AD%EB%AC%B8&strQuery=&exQuery=&exQueryText=&order=%2FDESC&onHanja=false&strSort=RANK&p_year1=2000&p_year2=2023&iStartCount=0&orderBy=&mat_type=&mat_subtype=T2&fulltext_kind=&t_gubun=&learning_type=&ccl_code=&inside_outside=&fric_yn=&db_type=&image_yn=&gubun=&kdc=&ttsUseYn=&l_sub_code=&fsearchMethod=search&sflag=1&isFDetailSearch=N&pageNumber=1&resultKeyword=&fsearchSort=&fsearchOrder=&limiterList=&limiterListText=&facetList=&facetListText=&fsearchDB=&icate=bib_t&colName=bib_t&pageScale=100&isTab=Y&regnm=&dorg_storage=&language=&language_code=&clickKeyword=&relationKeyword=&query="
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "srdiA03Bi29p"
      },
      "outputs": [],
      "source": [
        "# 검색 조건을 넣은 url을 아래에 복사해 넣으세요\n",
        "url = 'https://www.riss.kr/search/Search.do?isDetailSearch=Y&searchGubun=true&viewYn=OP&queryText=znMajor%2C%EA%B5%AD%EC%96%B4%EA%B5%AD%EB%AC%B8%40op%2COR%40znMajor%2C%EA%B5%AD%EB%AC%B8&strQuery=&exQuery=&exQueryText=&order=%2FDESC&onHanja=false&strSort=RANK&p_year1=2000&p_year2=2023&iStartCount=0&orderBy=&mat_type=&mat_subtype=T2&fulltext_kind=&t_gubun=&learning_type=&ccl_code=&inside_outside=&fric_yn=&db_type=&image_yn=&gubun=&kdc=&ttsUseYn=&l_sub_code=&fsearchMethod=search&sflag=1&isFDetailSearch=N&pageNumber=1&resultKeyword=&fsearchSort=&fsearchOrder=&limiterList=&limiterListText=&facetList=&facetListText=&fsearchDB=&icate=bib_t&colName=bib_t&pageScale=100&isTab=Y&regnm=&dorg_storage=&language=&language_code=&clickKeyword=&relationKeyword=&query='"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oe3nXAWEi29p"
      },
      "outputs": [],
      "source": [
        "req = request_until_success(url,headers).text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1d1cQAy5i29p"
      },
      "outputs": [],
      "source": [
        "soup = BeautifulSoup(req, 'lxml')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x9WusU93i29q"
      },
      "outputs": [],
      "source": [
        "cont_ml60_classes = soup.find_all('div', class_='cont ml60')\n",
        "\n",
        "data = [{\n",
        "    'title': cont_ml60.find('p', class_='title').get_text(strip=True) if cont_ml60.find('p', class_='title') else None,\n",
        "    'link': cont_ml60.find('p', class_='title').find('a').get('href') if cont_ml60.find('p', class_='title') and cont_ml60.find('p', class_='title').find('a') else None,\n",
        "    'writer': cont_ml60.find('p', class_='etc').find_all('span')[0].get_text(strip=True) if cont_ml60.find('p', class_='etc') and len(cont_ml60.find('p', class_='etc').find_all('span')) > 0 else None,\n",
        "    'assigned': cont_ml60.find('p', class_='etc').find_all('span')[1].get_text(strip=True) if cont_ml60.find('p', class_='etc') and len(cont_ml60.find('p', class_='etc').find_all('span')) > 1 else None,\n",
        "    'year': cont_ml60.find('p', class_='etc').find_all('span')[2].get_text(strip=True) if cont_ml60.find('p', class_='etc') and len(cont_ml60.find('p', class_='etc').find_all('span')) > 2 else None,\n",
        "    'grad': cont_ml60.find('p', class_='etc').find_all('span')[3].get_text(strip=True) if cont_ml60.find('p', class_='etc') and len(cont_ml60.find('p', class_='etc').find_all('span')) > 3 else None,\n",
        "    'preAbstract': cont_ml60.find('p', class_='preAbstract').get_text(strip=True) if cont_ml60.find('p', class_='preAbstract') else None\n",
        "} for cont_ml60 in cont_ml60_classes]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NGeKfAYmi29q"
      },
      "outputs": [],
      "source": [
        "data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JKkXlcz4i29q"
      },
      "source": [
        "### 기본 정보 자동으로 수집하기"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WZxFFDr1i29r"
      },
      "outputs": [],
      "source": [
        "# 총 논문 개수 확인(total_count)\n",
        "# Assuming 'soup' is your BeautifulSoup object\n",
        "num_span = soup.find('span', class_='num')\n",
        "\n",
        "if num_span is not None:\n",
        "    total_count = int(num_span.get_text().replace(',', ''))\n",
        "else:\n",
        "    total_count = 0\n",
        "\n",
        "total_count"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PzFO8w6Ndou0"
      },
      "outputs": [],
      "source": [
        "# 수집용 URL 세팅\n",
        "# The base URL without the 'iStartCount' parameter\n",
        "# Parse the URL\n",
        "parsed_url = urlparse(url)\n",
        "\n",
        "# Parse the query parameters\n",
        "params = parse_qs(parsed_url.query)\n",
        "\n",
        "# Remove the 'iStartCount' parameter\n",
        "params.pop('iStartCount', None)\n",
        "\n",
        "# Re-encode the query parameters\n",
        "new_query = urlencode(params, doseq=True)\n",
        "\n",
        "# Replace the old query parameters with the new ones\n",
        "new_url = parsed_url._replace(query=new_query)\n",
        "\n",
        "# Unparse the URL\n",
        "base_url = urlunparse(new_url)\n",
        "\n",
        "print(base_url)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "exmS4rC1dou1"
      },
      "source": [
        "##### 실제 수집 코드"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NrhHyFq3p5T3"
      },
      "outputs": [],
      "source": [
        "# The base URL without the 'iStartCount' parameter\n",
        "# base_url = \"https://www.riss.kr/search/Search.do?isDetailSearch=Y&searchGubun=true&viewYn=OP&queryText=znMajor,국어국문@op,OR@znMajor,국문&strQuery=&exQuery=&exQueryText=&order=/DESC&onHanja=false&strSort=RANK&p_year1=2000&p_year2=2023&orderBy=&mat_type=&mat_subtype=T2&fulltext_kind=&t_gubun=&learning_type=&ccl_code=&inside_outside=&fric_yn=&db_type=&image_yn=&gubun=&kdc=&ttsUseYn=&l_sub_code=&fsearchMethod=search&sflag=1&isFDetailSearch=N&pageNumber=1&resultKeyword=&fsearchSort=&fsearchOrder=&limiterList=&limiterListText=&facetList=&facetListText=&fsearchDB=&icate=bib_t&colName=bib_t&pageScale=100&isTab=Y&regnm=&dorg_storage=&language=&language_code=&clickKeyword=&relationKeyword=&query=\"\n",
        "\n",
        "# Calculate the number of iterations\n",
        "iterations = math.ceil(total_count / 100)\n",
        "\n",
        "# Initialize an empty list to store the data\n",
        "data = []\n",
        "\n",
        "# 멀티 쓰레딩 활용 데이터 수집 함수\n",
        "def fetch_data(i):\n",
        "    url = f\"{base_url}&iStartCount={i * 100}\"\n",
        "    response = request_until_success(url, headers)\n",
        "    soup = BeautifulSoup(response.text, 'html.parser')\n",
        "\n",
        "    cont_ml60_classes = soup.find_all('div', class_='cont ml60')\n",
        "\n",
        "    # 필요한 데이터만 한 번에 추출\n",
        "    data = []\n",
        "    for cont_ml60 in cont_ml60_classes:\n",
        "        title = cont_ml60.find('p', class_='title')\n",
        "        etc = cont_ml60.find('p', class_='etc')\n",
        "        spans = etc.find_all('span') if etc else []\n",
        "\n",
        "        data.append({\n",
        "            'title': title.get_text(strip=True) if title else None,\n",
        "            'link': title.find('a').get('href') if title and title.find('a') else None,\n",
        "            'writer': spans[0].get_text(strip=True) if len(spans) > 0 else None,\n",
        "            'assigned': spans[1].get_text(strip=True) if len(spans) > 1 else None,\n",
        "            'year': spans[2].get_text(strip=True) if len(spans) > 2 else None,\n",
        "            'grad': spans[3].get_text(strip=True) if len(spans) > 3 else None,\n",
        "            'preAbstract': cont_ml60.find('p', class_='preAbstract').get_text(strip=True) if cont_ml60.find('p', class_='preAbstract') else None\n",
        "        })\n",
        "    return data\n",
        "\n",
        "# ThreadPoolExecutor를 사용하여 멀티스레딩 구현\n",
        "with ThreadPoolExecutor(max_workers=10) as executor:\n",
        "    data_list = list(tqdm(executor.map(fetch_data, range(iterations)), total=iterations))\n",
        "\n",
        "# 리스트 평탄화\n",
        "data = [item for sublist in data_list for item in sublist]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.DataFrame(data)\n",
        "df"
      ],
      "metadata": {
        "id": "i5AYYHqgfrTA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# csv로 저장\n",
        "df.to_csv('./riss_basic.csv')"
      ],
      "metadata": {
        "id": "2Dty7gj8iWT5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z-gd4GhLi29r"
      },
      "source": [
        "### 논문별 상세정보 수집하기"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ORQvvqOci29r"
      },
      "outputs": [],
      "source": [
        "df['link'] = 'https://www.riss.kr' + df['link']\n",
        "df['link']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9sjyvQZri29s"
      },
      "outputs": [],
      "source": [
        "headers_for_riss = {\n",
        "    \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\",\n",
        "    \"Accept-Encoding\": \"gzip, deflate, br, zstd\",\n",
        "    \"Accept-Language\": \"ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7,ja;q=0.6\",\n",
        "    \"Connection\": \"keep-alive\",\n",
        "    \"Cookie\": \"Your Cookie Here\",\n",
        "    \"Host\": \"www.riss.kr\",\n",
        "    \"Referer\": None,\n",
        "    \"Sec-Fetch-Dest\": \"document\",\n",
        "    \"Sec-Fetch-Mode\": \"navigate\",\n",
        "    \"Sec-Fetch-Site\": \"same-origin\",\n",
        "    \"Upgrade-Insecure-Requests\": \"1\",\n",
        "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36\",\n",
        "    \"sec-ch-ua\": '\"Google Chrome\";v=\"123\", \"Not:A-Brand\";v=\"8\", \"Chromium\";v=\"123\"',\n",
        "    \"sec-ch-ua-mobile\": \"?0\",\n",
        "    \"sec-ch-ua-platform\": \"Windows\"\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def fetch_data(link):\n",
        "    headers_for_riss[\"Referer\"] = link\n",
        "    response = request_until_success(link, headers_for_riss)\n",
        "    soup = BeautifulSoup(response.text, 'lxml')\n",
        "\n",
        "    data = {'link': link}\n",
        "\n",
        "    title_tag = soup.find('h3', class_='title')\n",
        "    data['title'] = title_tag.get_text(strip=True) if title_tag else None\n",
        "\n",
        "    info_detail_div = soup.find('div', class_='infoDetailL')\n",
        "    if info_detail_div:\n",
        "        for li in info_detail_div.find_all('li'):\n",
        "            key_element = li.find('span', {'class': 'strong'})\n",
        "            value_element = li.find('div')\n",
        "            if key_element and value_element:\n",
        "                key = key_element.text.strip()\n",
        "                value = value_element.text.strip()\n",
        "                data[key] = value\n",
        "\n",
        "    try:\n",
        "        additional_info_div = soup.find('div', class_='content additionalInfo')\n",
        "        if additional_info_div:\n",
        "            text_off_divs = additional_info_div.find_all('div', class_='text off')\n",
        "            title_text_dict = {}\n",
        "            for div in text_off_divs:\n",
        "                title = div.find_previous_sibling('p', class_='title')\n",
        "                if title:\n",
        "                    if title.text.strip() == \"참고문헌 (Reference)\":\n",
        "                        title_text_dict[title.text.strip()] = [p.text.strip() for p in div.find_all('p')]\n",
        "                    else:\n",
        "                        title_text_dict[title.text.strip()] = div.text.strip()\n",
        "            data.update(title_text_dict)\n",
        "    except AttributeError:\n",
        "        print(f'참고문헌 등 추가 정보 없음 : {link}')\n",
        "\n",
        "    return data"
      ],
      "metadata": {
        "id": "WEKeaFMQh_zJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 실행 코드\n",
        "links = df['link'].tolist()[:100]  # 100개만 테스트, 다하려면 [:100] 제거\n",
        "with ThreadPoolExecutor(max_workers=10) as executor:\n",
        "    data_list = list(tqdm(executor.map(fetch_data, links), total=len(links)))"
      ],
      "metadata": {
        "id": "jwD-cTtviM0-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mKIEj86ZnAxE"
      },
      "outputs": [],
      "source": [
        "detail = pd.DataFrame(data_list)\n",
        "detail"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sgOcFAJvdou2"
      },
      "outputs": [],
      "source": [
        "# csv로 저장\n",
        "detail.to_csv('riss_detail.csv', index=False)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "python311",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.8"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}