From 658c6f85518f7edaa5bc3db87fc0a2d52595b576 Mon Sep 17 00:00:00 2001 From: Kah Keng Tay Date: Tue, 22 Aug 2023 10:23:46 -0700 Subject: [PATCH 1/4] utils: restore original exception message Will be hidden by default in frontend. --- utils/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/common.py b/utils/common.py index d08988d..fd5fb45 100644 --- a/utils/common.py +++ b/utils/common.py @@ -162,7 +162,7 @@ def wrapped_fn(*args, **kwargs): return str(e) except Exception as e: traceback.print_exc() - return "An error occurred. Please try again." + return f'Got exception evaluating {fn.__name__}(args={args}, kwargs={kwargs}): {e}' @functools.wraps(fn) def wrapped_generator_fn(*args, **kwargs): From 4cadac0802c1bd7b4e56e11ecaba008f7c5d333f Mon Sep 17 00:00:00 2001 From: Kah Keng Tay Date: Tue, 22 Aug 2023 11:03:59 -0700 Subject: [PATCH 2/4] integrations: switch some endpoints to center v2 Note the addresses need to be converted to checksum addresses before passing to OpenSea. --- integrations/center.py | 58 +++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/integrations/center.py b/integrations/center.py index c7f1ad7..c699197 100644 --- a/integrations/center.py +++ b/integrations/center.py @@ -5,6 +5,7 @@ from urllib.parse import urlencode import requests +import web3 import env import utils @@ -31,6 +32,8 @@ MAX_RESULTS = 12 PAGE_LIMIT = 12 +MAX_RESULTS_FOR_SEARCH = 12 + MAX_RESULTS_FOR_TRAITS = 100 PAGE_LIMIT_FOR_TRAITS = 100 @@ -39,7 +42,7 @@ class NFTCollection(ContainerMixin): network: str address: str name: str - num_assets: int + num_assets: Optional[int] preview_image_url: str def container_name(self) -> str: @@ -202,13 +205,16 @@ def container_params(self) -> Dict: ) def fetch_nft_search(search_str: str) -> Generator[Union[NFTCollection, NFTAsset], None, None]: + limit = MAX_RESULTS_FOR_SEARCH + offset = 0 q = urlencode(dict( query=search_str, - type='collection', # too noisy otherwise + limit=limit, + offset=offset, )) count = 0 for network in NETWORKS: - url = f"{API_URL}/{network}/search?{q}" + url = f"{API_V2_URL}/{network}/search?{q}" timing.log('search_begin') response = requests.get(url, headers=HEADERS) try: @@ -218,17 +224,24 @@ def fetch_nft_search(search_str: str) -> Generator[Union[NFTCollection, NFTAsset break timing.log('search_done') obj = response.json() - for r in obj['results']: - if not r.get('previewImageUrl'): + for item in obj['items']: + if 'collection' not in item: + continue + collection = item['collection'] + if 'featuredImageURL' not in collection or not collection['featuredImageURL']: + continue + count += 1 # increment pre-filtered count, to determine filtering cost to speed + # v2 endpoint might return a non-checksum address, convert it for compatibility with elsewhere + address = web3.utils.address.to_checksum_address(collection['address']) + result = NFTCollection( + network=network, + address=address, + name=collection['name'], + num_assets=collection['totalSupply'], + preview_image_url=collection['featuredImageURL'], + ) + if not _is_valid_collection(result): continue - count += 1 - network = r['id'].split('/')[0] - if r['type'].lower() == 'collection': - result = fetch_nft_collection(network, r['address']) - if not _is_valid_collection(result): - continue - else: - result = fetch_nft_asset(network, r['address'], r['tokenId']) yield result timing.log('first_result_done') timing.log('%d_results_done' % count) @@ -236,10 +249,6 @@ def fetch_nft_search(search_str: str) -> Generator[Union[NFTCollection, NFTAsset def _is_valid_collection(collection: NFTCollection) -> bool: """Check if this NFT collection is a valid search result.""" - # there should be traits - collection_traits = fetch_nft_collection_traits(collection.network, collection.address) - if not collection_traits.traits: - return False # should have listed and valid assets if collection.network == "ethereum-mainnet": token_prices = opensea.fetch_contract_listing_prices_with_retries(collection.address) @@ -328,16 +337,25 @@ def fetch_nft_search_collection_by_trait(network: str, address: str, trait_name: def fetch_nft_collection(network: str, address: str) -> NFTCollection: - url = f"{API_URL}/{network}/{address}" + url = f"{API_V2_URL}/{network}/{address}/nft/metadata" response = requests.get(url, headers=HEADERS) response.raise_for_status() obj = response.json() + num_assets = obj['totalSupply'] + if num_assets == 0: # seems to return 0 incorrectly + # use the asset endpoint with dummy token + token_id = 1 + url = f"{API_V2_URL}/{network}/{address}/nft/{token_id}/metadata" + response = requests.get(url, headers=HEADERS) + if response.status_code == 200: + token_obj = response.json() + num_assets = token_obj['collection']['totalSupply'] return NFTCollection( network=network, address=address, name=obj['name'], - num_assets=obj['numAssets'], - preview_image_url=obj['smallPreviewImageUrl'], + num_assets=num_assets, + preview_image_url=obj['featuredImageURL'], ) From 85f17392c0618dfed82aeb0a7b909d442f06ed2c Mon Sep 17 00:00:00 2001 From: Kah Keng Tay Date: Tue, 22 Aug 2023 15:00:59 -0700 Subject: [PATCH 3/4] integrations: limit opensea api calls for nft search --- integrations/center.py | 2 +- integrations/opensea.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/integrations/center.py b/integrations/center.py index c699197..027a7be 100644 --- a/integrations/center.py +++ b/integrations/center.py @@ -251,7 +251,7 @@ def _is_valid_collection(collection: NFTCollection) -> bool: """Check if this NFT collection is a valid search result.""" # should have listed and valid assets if collection.network == "ethereum-mainnet": - token_prices = opensea.fetch_contract_listing_prices_with_retries(collection.address) + token_prices = opensea.fetch_contract_listing_prices_with_retries(collection.address, max_results=1) if not token_prices: return False return True diff --git a/integrations/opensea.py b/integrations/opensea.py index d1bea6a..a1c65ca 100644 --- a/integrations/opensea.py +++ b/integrations/opensea.py @@ -118,18 +118,18 @@ def _exec_request(): return ret -def fetch_all_listings(address: str) -> List[NFTListing]: +def fetch_all_listings(address: str, max_results: Optional[int] = None) -> List[NFTListing]: """Fetch all listings for a collection.""" # NOTE: a given token ID might have more than one listing contract = fetch_contract(address) slug = contract.slug - limit = PAGE_LIMIT next_cursor = None ret = [] # Arbitary limit to optimize for latency, based on hueristics related to observed number of NFTs listed for blue-chip collections. - max_results = 300 - max_queries = 5 + max_results = 300 if max_results is None else max_results + max_queries = 3 queries = 0 + limit = min(PAGE_LIMIT, max_results) while len(ret) < max_results and queries < max_queries: queries += 1 q = urlencode(dict( @@ -170,7 +170,7 @@ def _exec_request(): next_cursor = obj.get("next") if not next_cursor: break - return ret + return ret[:max_results] def fetch_asset_listing_prices_with_retries(address: str, token_id: str) -> Optional[Dict[str, Union[str, int]]]: @@ -183,8 +183,8 @@ def fetch_asset_listing_with_retries(address: str, token_id: str) -> Optional[NF listings = fetch_listings(address, token_id) return listings[0] if len(listings) > 0 else None -def fetch_contract_listing_prices_with_retries(address: str) -> Dict[str, Dict[str, Union[str, int]]]: - listings = fetch_all_listings(address) +def fetch_contract_listing_prices_with_retries(address: str, max_results: Optional[int] = None) -> Dict[str, Dict[str, Union[str, int]]]: + listings = fetch_all_listings(address, max_results=max_results) ret = {} for listing in listings: if listing.token_id not in ret or ret[listing.token_id].price_value > listing.price_value: From acf03a105c081fb37a89ae27dd23ef2565f1300c Mon Sep 17 00:00:00 2001 From: Sagar Shah Date: Fri, 25 Aug 2023 13:08:57 -0500 Subject: [PATCH 4/4] task: update GH workflow file (#244) --- .../workflows/check-update-widget-index.yaml | 31 +++++++------------ .../workflows/check-widget-translation.yaml | 15 ++++++--- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/.github/workflows/check-update-widget-index.yaml b/.github/workflows/check-update-widget-index.yaml index 4925c21..d75cd8b 100644 --- a/.github/workflows/check-update-widget-index.yaml +++ b/.github/workflows/check-update-widget-index.yaml @@ -27,6 +27,13 @@ jobs: with: path: backend + - name: Extract branch name + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + + - name: Show branch name + run: echo "Branch name is ${{ steps.extract_branch.outputs.branch }}" + - id: 'auth' uses: 'google-github-actions/auth@v1' with: @@ -58,25 +65,9 @@ jobs: echo "SERVER_HOST=" >> "$GITHUB_ENV" echo "SERVER_ORIGINS=" >> "$GITHUB_ENV" echo "SERVER_SECRET_KEY=" >> "$GITHUB_ENV" - - if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then - echo "ENV_TAG=prod" >> "$GITHUB_ENV" - else - echo "ENV_TAG=dev" >> "$GITHUB_ENV" - echo "WEAVIATE_URL=${{ secrets.DEV_WEAVIATE_URL }}" >> "$GITHUB_ENV" - echo "WEAVIATE_API_KEY=" >> "$GITHUB_ENV" - echo "CHATDB_URL=${{ secrets.DEV_CHATDB_URL }}" >> "$GITHUB_ENV" - fi - - - name: Set environment for branch - run: | - echo "SERVER_HOST=" >> "$GITHUB_ENV" - echo "SERVER_ORIGINS=" >> "$GITHUB_ENV" - echo "SERVER_SECRET_KEY=" >> "$GITHUB_ENV" - echo "CHATDB_URL=" >> "$GITHUB_ENV" echo "WEAVIATE_URL=${{ secrets.WEAVIATE_URL }}" >> "$GITHUB_ENV" - if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then + if [[ ${{ steps.extract_branch.outputs.branch }} == 'master' ]]; then echo "ENV_TAG=prod" >> "$GITHUB_ENV" echo "WEAVIATE_API_KEY=${{ secrets.PROD_WEAVIATE_API_KEY }}" >> "$GITHUB_ENV" echo "CHATDB_URL=${{ secrets.PROD_CHATDB_URL }}" >> "$GITHUB_ENV" @@ -91,10 +82,10 @@ jobs: - name: Check and update widget index run: | GCP_SSH_CMD="gcloud compute ssh cacti-bastion-server --zone us-east1-b --ssh-key-file /tmp/gcp/google_compute_engine --quiet --tunnel-through-iap --ssh-flag" - if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then - $GCP_SSH_CMD '-vvv -fN -L 8080:${{ secrets.PROD_WEAVIATE_INTERNAL_IP }}' + if [[ ${{ steps.extract_branch.outputs.branch }} == 'master' ]]; then + $GCP_SSH_CMD '-fN -L 8080:${{ secrets.PROD_WEAVIATE_INTERNAL_IP }}' else - $GCP_SSH_CMD '-vvv -fN -L 8080:${{ secrets.DEV_WEAVIATE_INTERNAL_IP }}' + $GCP_SSH_CMD '-fN -L 8080:${{ secrets.DEV_WEAVIATE_INTERNAL_IP }}' fi cd backend diff --git a/.github/workflows/check-widget-translation.yaml b/.github/workflows/check-widget-translation.yaml index 95b3133..cbdbb11 100644 --- a/.github/workflows/check-widget-translation.yaml +++ b/.github/workflows/check-widget-translation.yaml @@ -29,6 +29,13 @@ jobs: with: path: backend + - name: Extract branch name + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + + - name: Show branch name + run: echo "Branch name is ${{ steps.extract_branch.outputs.branch }}" + - id: 'auth' uses: 'google-github-actions/auth@v1' with: @@ -63,7 +70,7 @@ jobs: echo "CHATDB_URL=" >> "$GITHUB_ENV" echo "WEAVIATE_URL=${{ secrets.WEAVIATE_URL }}" >> "$GITHUB_ENV" - if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then + if [[ ${{ steps.extract_branch.outputs.branch }} == 'master' ]]; then echo "ENV_TAG=prod" >> "$GITHUB_ENV" echo "WEAVIATE_API_KEY=${{ secrets.PROD_WEAVIATE_API_KEY }}" >> "$GITHUB_ENV" echo "CHATDB_URL=${{ secrets.PROD_CHATDB_URL }}" >> "$GITHUB_ENV" @@ -76,10 +83,10 @@ jobs: - name: Check widget translation run: | GCP_SSH_CMD="gcloud compute ssh cacti-bastion-server --zone us-east1-b --ssh-key-file /tmp/gcp/google_compute_engine --quiet --tunnel-through-iap --ssh-flag" - if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then - $GCP_SSH_CMD '-vvv -fN -L 8080:${{ secrets.PROD_WEAVIATE_INTERNAL_IP }}' + if [[ ${{ steps.extract_branch.outputs.branch }} == 'master' ]]; then + $GCP_SSH_CMD '-fN -L 8080:${{ secrets.PROD_WEAVIATE_INTERNAL_IP }}' else - $GCP_SSH_CMD '-vvv -fN -L 8080:${{ secrets.DEV_WEAVIATE_INTERNAL_IP }}' + $GCP_SSH_CMD '-fN -L 8080:${{ secrets.DEV_WEAVIATE_INTERNAL_IP }}' fi cd backend