diff --git a/requirements.txt b/requirements.txt index 9e1438d75..4bf69ee98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,5 +32,5 @@ psycogreen==1.0.2 psycopg2==2.9.6 redis==4.5.5 requests==2.31.0 -safe-eth-py[django]==5.4.3 +safe-eth-py[django]==5.5.0 web3==6.5.0 diff --git a/safe_transaction_service/__init__.py b/safe_transaction_service/__init__.py index b5878ca6a..728595d2f 100644 --- a/safe_transaction_service/__init__.py +++ b/safe_transaction_service/__init__.py @@ -1,4 +1,4 @@ -__version__ = "4.20.2" +__version__ = "4.20.3" __version_info__ = tuple( int(num) if num.isdigit() else num for num in __version__.replace("-", ".", 1).split(".") diff --git a/safe_transaction_service/history/services/collectibles_service.py b/safe_transaction_service/history/services/collectibles_service.py index 63bf3bc60..cdfeae6c8 100644 --- a/safe_transaction_service/history/services/collectibles_service.py +++ b/safe_transaction_service/history/services/collectibles_service.py @@ -209,7 +209,7 @@ def _retrieve_metadata_from_uri(self, uri: str) -> Any: try: logger.debug("Getting metadata for uri=%s", uri) - with requests.get(uri, timeout=15, stream=True) as response: + with requests.get(uri, timeout=10, stream=True) as response: if not response.ok: logger.debug("Cannot get metadata for uri=%s", uri) raise MetadataRetrievalException(uri) diff --git a/safe_transaction_service/tokens/clients/base_client.py b/safe_transaction_service/tokens/clients/base_client.py new file mode 100644 index 000000000..08c1afe58 --- /dev/null +++ b/safe_transaction_service/tokens/clients/base_client.py @@ -0,0 +1,24 @@ +import requests + + +class BaseHTTPClient: + def __init__(self, request_timeout: int = 10): + self.http_session = self._prepare_http_session() + self.request_timeout = request_timeout + + def _prepare_http_session(self) -> requests.Session: + """ + Prepare http session with custom pooling. See: + https://urllib3.readthedocs.io/en/stable/advanced-usage.html + https://docs.python-requests.org/en/v1.2.3/api/#requests.adapters.HTTPAdapter + https://web3py.readthedocs.io/en/stable/providers.html#httpprovider + """ + session = requests.Session() + adapter = requests.adapters.HTTPAdapter( + pool_connections=10, + pool_maxsize=100, # Number of concurrent connections + pool_block=False, + ) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session diff --git a/safe_transaction_service/tokens/clients/binance_client.py b/safe_transaction_service/tokens/clients/binance_client.py index 40e15aa34..7581036e6 100644 --- a/safe_transaction_service/tokens/clients/binance_client.py +++ b/safe_transaction_service/tokens/clients/binance_client.py @@ -1,20 +1,16 @@ import logging -import requests - +from .base_client import BaseHTTPClient from .exceptions import CannotGetPrice logger = logging.getLogger(__name__) -class BinanceClient: # pragma: no cover - def __init__(self): - self.http_session = requests.Session() - +class BinanceClient(BaseHTTPClient): # pragma: no cover def _get_price(self, symbol: str) -> float: url = f"https://api.binance.com/api/v3/avgPrice?symbol={symbol}" try: - response = self.http_session.get(url, timeout=10) + response = self.http_session.get(url, timeout=self.request_timeout) api_json = response.json() if not response.ok: logger.warning("Cannot get price from url=%s", url) diff --git a/safe_transaction_service/tokens/clients/coingecko_client.py b/safe_transaction_service/tokens/clients/coingecko_client.py index 83a063f1a..c9438695d 100644 --- a/safe_transaction_service/tokens/clients/coingecko_client.py +++ b/safe_transaction_service/tokens/clients/coingecko_client.py @@ -3,11 +3,11 @@ from typing import Any, Dict, Optional from urllib.parse import urljoin -import requests from eth_typing import ChecksumAddress from gnosis.eth import EthereumNetwork +from safe_transaction_service.tokens.clients.base_client import BaseHTTPClient from safe_transaction_service.tokens.clients.exceptions import ( CannotGetPrice, Coingecko404, @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) -class CoingeckoClient: +class CoingeckoClient(BaseHTTPClient): ASSET_BY_NETWORK = { EthereumNetwork.ARBITRUM_ONE: "arbitrum-one", EthereumNetwork.AURORA_MAINNET: "aurora", @@ -36,8 +36,10 @@ class CoingeckoClient: } base_url = "https://api.coingecko.com/" - def __init__(self, network: Optional[EthereumNetwork] = None): - self.http_session = requests.Session() + def __init__( + self, network: Optional[EthereumNetwork] = None, request_timeout: int = 10 + ): + super().__init__(request_timeout=request_timeout) self.asset_platform = self.ASSET_BY_NETWORK.get(network, "ethereum") @classmethod @@ -46,7 +48,7 @@ def supports_network(cls, network: EthereumNetwork): def _do_request(self, url: str) -> Dict[str, Any]: try: - response = self.http_session.get(url, timeout=10) + response = self.http_session.get(url, timeout=self.request_timeout) if not response.ok: if response.status_code == 404: raise Coingecko404(url) diff --git a/safe_transaction_service/tokens/clients/coinmarketcap_client.py b/safe_transaction_service/tokens/clients/coinmarketcap_client.py index a7709da13..da322ce8a 100644 --- a/safe_transaction_service/tokens/clients/coinmarketcap_client.py +++ b/safe_transaction_service/tokens/clients/coinmarketcap_client.py @@ -4,10 +4,10 @@ from typing import Any, Dict, List from urllib.parse import urljoin -import requests - from gnosis.eth.utils import fast_to_checksum_address +from .base_client import BaseHTTPClient + logger = logging.getLogger(__name__) @@ -20,22 +20,24 @@ class CoinMarketCapToken: logo_uri: str -class CoinMarketCapClient: +class CoinMarketCapClient(BaseHTTPClient): base_url = "https://pro-api.coinmarketcap.com/" base_logo_uri = "https://s2.coinmarketcap.com/static/img/coins/200x200/" - def __init__(self, api_token: str): + def __init__(self, api_token: str, request_timeout: int = 10): + super().__init__(request_timeout=request_timeout) self.api_token = api_token self.headers = { "Accepts": "application/json", "X-CMC_PRO_API_KEY": api_token, } - self.http_session = requests.Session() def download_file(self, url: str, taget_folder: str, local_filename: str) -> str: if not os.path.exists(taget_folder): os.makedirs(taget_folder) - with self.http_session.get(url, stream=True) as response: + with self.http_session.get( + url, stream=True, timeout=self.request_timeout + ) as response: if not response.ok: logger.warning("Image not found for url %s", url) return None @@ -75,7 +77,12 @@ def get_map(self) -> List[Dict[str, Any]]: try: return ( - self.http_session.get(url, headers=self.headers, params=parameters) + self.http_session.get( + url, + headers=self.headers, + params=parameters, + timeout=self.request_timeout, + ) .json() .get("data", []) ) diff --git a/safe_transaction_service/tokens/clients/kraken_client.py b/safe_transaction_service/tokens/clients/kraken_client.py index e7d477e0c..8e593c954 100644 --- a/safe_transaction_service/tokens/clients/kraken_client.py +++ b/safe_transaction_service/tokens/clients/kraken_client.py @@ -1,20 +1,16 @@ import logging -import requests - +from .base_client import BaseHTTPClient from .exceptions import CannotGetPrice logger = logging.getLogger(__name__) -class KrakenClient: - def __init__(self): - self.http_session = requests.Session() - +class KrakenClient(BaseHTTPClient): def _get_price(self, symbol: str) -> float: url = f"https://api.kraken.com/0/public/Ticker?pair={symbol}" try: - response = self.http_session.get(url, timeout=10) + response = self.http_session.get(url, timeout=self.request_timeout) api_json = response.json() error = api_json.get("error") if not response.ok or error: diff --git a/safe_transaction_service/tokens/clients/kucoin_client.py b/safe_transaction_service/tokens/clients/kucoin_client.py index 2b92e2d59..80fed71c6 100644 --- a/safe_transaction_service/tokens/clients/kucoin_client.py +++ b/safe_transaction_service/tokens/clients/kucoin_client.py @@ -1,21 +1,17 @@ import logging -import requests - +from .base_client import BaseHTTPClient from .exceptions import CannotGetPrice logger = logging.getLogger(__name__) -class KucoinClient: - def __init__(self): - self.http_session = requests.Session() - +class KucoinClient(BaseHTTPClient): def _get_price(self, symbol: str): url = f"https://api.kucoin.com/api/v1/market/orderbook/level1?symbol={symbol}" try: - response = self.http_session.get(url, timeout=10) + response = self.http_session.get(url, timeout=self.request_timeout) result = response.json() return float(result["data"]["price"]) except (ValueError, IOError) as e: