diff --git a/README.rst b/README.rst index ca83a01..128a6ca 100644 --- a/README.rst +++ b/README.rst @@ -240,6 +240,10 @@ Retrieves live market data. Changelog --------- +0.4.9 +~~~~~ +- new retry policy + 0.4.8 ~~~~~ - new ``fetch_crypto_live_data()`` diff --git a/karpet/core.py b/karpet/core.py index 0f312e3..b07af38 100644 --- a/karpet/core.py +++ b/karpet/core.py @@ -5,18 +5,21 @@ import asyncio import time -from datetime import date, datetime, timedelta +from datetime import datetime, timedelta import aiohttp import numpy as np import pandas as pd import requests from bs4 import BeautifulSoup +from requests.adapters import HTTPAdapter, Retry class Karpet: quick_search_data = None + req_retries = 4 + req_backoff_factor = 3 def __init__(self, start=None, end=None): """ @@ -28,6 +31,28 @@ def __init__(self, start=None, end=None): self.start = start self.end = end + self.req_ses = self.get_session() + + def get_session(self): + + # Waits for 1.5s, 3s, 6s, 12s, 24s between requests. + status_forcelist = (500, 502, 503, 504, 429) + + retry = Retry( + total=self.req_retries, + read=self.req_retries, + connect=self.req_retries, + backoff_factor=self.req_backoff_factor, + status_forcelist=status_forcelist, + method_whitelist=False, + ) + adapter = HTTPAdapter(max_retries=retry) + + session = requests.Session() + # session.mount("http://", adapter) + session.mount("https://", adapter) + + return session def get_quick_search_data(self): """ @@ -53,24 +78,12 @@ def get_quick_search_data(self): :rtype: list """ - if self.quick_search_data: - return self.quick_search_data - - url = "https://s2.coinmarketcap.com/generated/search/quick_search.json" - - # Download. - try: - response = requests.get(url) - except: - raise Exception("Couldn't download necessary data from the internet.") - - # Parse. - try: - self.quick_search_data = response.json() + if not self.quick_search_data: + self.quick_search_data = self._get_json( + "https://s2.coinmarketcap.com/generated/search/quick_search.json" + ) - return self.quick_search_data - except: - raise Exception("Couldn't parse downloaded data from the internet.") + return self.quick_search_data def fetch_crypto_historical_data(self, symbol=None, id=None): """ @@ -246,7 +259,12 @@ def fetch_google_trends( stich_overlap = trdays - overlap n_days = (self.end - self.start).days - pytrends = TrendReq(hl=hl, tz=tz) + pytrends = TrendReq( + hl=hl, + tz=tz, + retries=self.req_retries, + backoff_factor=self.req_backoff_factor, + ) # Get the dates for each search. if n_days <= trdays: @@ -402,7 +420,7 @@ def get_top_news(): headers = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0" } - response = requests.get("https://cointelegraph.com/", headers=headers) + response = self.req_ses.get("https://cointelegraph.com/", headers=headers) dom = BeautifulSoup(response.text, "lxml") def parse_news(news_items): @@ -673,10 +691,12 @@ def _get_json(self, url): # Download. try: - response = requests.get(url) + response = self.req_ses.get(url) except: raise Exception("Couldn't download necessary data from the internet.") + response.raise_for_status() + # Parse. try: return response.json()