From 1bbf2465df09986ad5d5980ab88c019c5fa8b339 Mon Sep 17 00:00:00 2001 From: Mohammed Date: Sun, 26 Mar 2023 16:06:05 +0700 Subject: [PATCH 1/3] feat: erc-1155 support --- .eslintrc.json | 5 - index.ts | 32 +++- src/eth/ABI.ts | 114 +++++++-------- src/eth/index.ts | 307 ++++++++++++++++++++++++++++++--------- src/eth/models/index.ts | 7 +- src/eth/models/log.ts | 45 +++--- src/models/NFT.ts | 81 ++++++----- src/models/NFTOptions.ts | 20 +-- src/models/constants.ts | 12 +- src/models/index.ts | 10 +- src/services/index.ts | 7 +- src/services/metadata.ts | 165 ++++++++++----------- test.ts | 109 +++++++------- 13 files changed, 536 insertions(+), 378 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 7d007d3..b830451 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -9,10 +9,5 @@ ], "parserOptions": { "ecmaVersion": "latest" - }, - "rules": { - "comma-dangle": ["error", "always"], - "semi": [2, "always"], - "quotes": [2, "single", { "avoidEscape": true }] } } diff --git a/index.ts b/index.ts index 7568b99..d804ec4 100644 --- a/index.ts +++ b/index.ts @@ -1,13 +1,29 @@ import * as eth from './src/eth' -import { NFTOptions } from './src/models'; +import { type NFTOptions, } from './src/models' import { setIpfsHostnames, setArweaveHostnames } from './src/services' async function getERC721(NFTOptions: NFTOptions) { - let NFTs = await eth.getERC721(NFTOptions) - const result = NFTs.map((nft) => nft.toDict()); - return result + const NFTs = await eth.getERC721(NFTOptions,); + const result = NFTs.map((nft) => nft.toDict()); + return result; } -module.exports.setArweaveHostnames = async (hostnames: string[]) => setArweaveHostnames(hostnames); -module.exports.setIpfsHostnames = async (hostnames: string[]) => setIpfsHostnames(hostnames); -module.exports.setEthProvider = async (provider: string) => eth.setProvider(provider); -module.exports.getERC721 = async (options: NFTOptions) => await getERC721(options); \ No newline at end of file + +async function getERC1155(NFTOptions: NFTOptions) { + const NFTs = await eth.getERC1155(NFTOptions,); + const result = NFTs.map((nft) => nft.toDict()); + return result; +} + +async function getEthNFTs(NFTOptions: NFTOptions) { + const NFTs = await eth.getNFTs(NFTOptions,); + const result = NFTs.map((nft) => nft.toDict()); + return result; +} + + +module.exports.setArweaveHostnames = async (hostnames: string[]) => { setArweaveHostnames(hostnames,); } +module.exports.setIpfsHostnames = async (hostnames: string[]) => { setIpfsHostnames(hostnames,); } +module.exports.setEthProvider = async (provider: string) => { eth.setProvider(provider,); } +module.exports.getERC721 = async (options: NFTOptions) => await getERC721(options); +module.exports.getERC1155 = async (options: NFTOptions) => await getERC1155(options); +module.exports.getEthNFTs = async (options: NFTOptions) => await getEthNFTs(options); diff --git a/src/eth/ABI.ts b/src/eth/ABI.ts index 978163c..7b17c24 100644 --- a/src/eth/ABI.ts +++ b/src/eth/ABI.ts @@ -1,62 +1,62 @@ -import { AbiItem } from 'web3-utils' - - -const ERC721Transfer = [{ - type: 'address', - name: 'from', - indexed: true, -}, { - type: 'address', - name: 'to', - indexed: true, -}, { - type: 'uint256', - name: 'tokenId', - indexed: true, -}, -]; - -const ERC721TokenURI: AbiItem= -{ - inputs: [ - { - internalType: 'uint256', - name: 'tokenId', - type: 'uint256' - } - ], +import { type AbiItem } from 'web3-utils' +const ERC721Transfer = [ + { type: 'address', name: 'from', indexed: true }, + { type: 'address', name: 'to', indexed: true }, + { type: 'uint256', name: 'tokenId', indexed: true } +] +const ERC721: AbiItem[] = [ + { + constant: true, + inputs: [{ name: '_tokenId', type: 'uint256' }], name: 'tokenURI', - outputs: [ - { - internalType: 'string', - name: '', - type: 'string' - } - ], + outputs: [{ name: '', type: 'string' }], + payable: false, stateMutability: 'view', type: 'function' -}; + } +] +const ERC1155TransferSingle = [ + { type: 'address', name: 'operator', indexed: true }, + { type: 'address', name: 'from', indexed: true }, + { type: 'address', name: 'to', indexed: true }, + { type: 'uint256', name: 'id' }, + { type: 'uint256', name: 'value' } +] -const ERC721: AbiItem[]= [ - { - "constant": true, - "inputs": [ - { - "name": "_tokenId", - "type": "uint256" - } - ], - "name": "tokenURI", - "outputs": [ - { - "name": "", - "type": "string" - } - ], - "payable": false, - "stateMutability": "view", - "type": "function", - }] - ; +const ERC1155TransferMulti = [ + { type: 'address', name: 'operator', indexed: true }, + { type: 'address', name: 'from', indexed: true }, + { type: 'address', name: 'to', indexed: true }, + { type: 'uint256[]', name: 'ids' }, + { type: 'uint256[]', name: 'values' } +] -export { ERC721TokenURI, ERC721Transfer, ERC721 }; +const ERC1155: AbiItem[] = [ + { + inputs: [{ internalType: 'uint256', name: 'id', type: 'uint256' }], + name: 'uri', + outputs: [{ internalType: 'string', name: '', type: 'string' }], + stateMutability: 'view', + type: 'function' + }, + { + anonymous: false, + inputs: [ + { + indexed: false, + internalType: "string", + name: "value", + type: "string" + }, + { + indexed: true, + internalType: "uint256", + name: "id", + type: "uint256" + } + ], + name: "URI", + type: "event" + } +] +export { ERC721Transfer, ERC721, ERC1155TransferSingle, ERC1155TransferMulti, ERC1155, } diff --git a/src/eth/index.ts b/src/eth/index.ts index b33ff83..8601ed2 100644 --- a/src/eth/index.ts +++ b/src/eth/index.ts @@ -1,97 +1,268 @@ -import Web3 from 'web3'; -const web3 = new Web3(); +import Web3 from 'web3' +import { NFT, type NFTOptions, TX_TYPE } from '../models' +import * as ABIs from './ABI' +import { type Log } from './models' +import { getMetaData } from '../services' +import { Contract } from 'web3-eth-contract' -import { NFT, NFTOptions, TX_TYPE } from '../models'; -import * as ABIs from './ABI'; -import { Log } from './models'; -import { getMetaData} from '../services'; +const web3 = new Web3() +const transfer721 = web3.utils.sha3('Transfer(address,address,uint256)'); +const singleTransfer1155 = web3.utils.sha3('TransferSingle(address,address,address,uint256,uint256)') +const multiTransfer1155 = web3.utils.sha3('TransferBatch(address,address,address,uint256[],uint256[])') -let options721 = { - topics: [ - web3.eth.abi.encodeEventSignature('Transfer(address,address,uint256)') - ] -}; +async function getNFTs(nftOptions: NFTOptions) { + const NFTs: NFT[] = [] + const erc721NFTs = await getERC721(nftOptions); + NFTs.push(...erc721NFTs); + const Single1155NFTs = await getERC1155(nftOptions) + NFTs.push(...Single1155NFTs); + + return NFTs +} async function getERC721(nftOptions: NFTOptions) { - const blockNumber = await validateBlockNumber(nftOptions.blockNumber) - const options = { - fromBlock: blockNumber, - toBlock: blockNumber, - options721, - }; - let logs: Log[] = []; - logs = await web3.eth.getPastLogs(options); - - let NFTs: NFT[] = []; - for (let x = 0; x < logs.length; x++) { - const ERC721NFT = await parseERC721Log(logs[x]); - if (ERC721NFT) { - NFTs.push(ERC721NFT) - } + const blockNumber = await validateBlockNumber(nftOptions.blockNumber) + const options = { + fromBlock: blockNumber, + toBlock: blockNumber, + topics: [transfer721] + } + let logs: Log[] = [] + logs = await web3.eth.getPastLogs(options) + + const NFTs: NFT[] = [] + for (let x = 0; x < logs.length; x++) { + if (logs[x].topics[0] === transfer721 && logs[x].topics.length == 4) { + const ERC721NFT = await parseERC721Log(logs[x]) + if (ERC721NFT != null) { + NFTs.push(ERC721NFT) + } } - return NFTs; + } + console.log('found ', NFTs.length, ' ERC721 NFTs'); + + return NFTs } +async function getERC1155(nftOptions: NFTOptions) { + const blockNumber = await validateBlockNumber(nftOptions.blockNumber) + const NFTs: NFT[] = [] + const Single1155NFTs = await getSingleERC1155Logs(blockNumber) + NFTs.push(...Single1155NFTs); + const Multi1155NFTs = await getMultiERC1155Logs(blockNumber) + NFTs.push(...Multi1155NFTs) -async function parseERC721Log(log: Log) { + return NFTs +} +async function getSingleERC1155Logs(blockNumber: any) { - if (log.topics[0] === options721.topics[0] && log.topics.length == 4) { - - let transaction = web3.eth.abi.decodeLog(ABIs.ERC721Transfer, log.data, [log.topics[1], log.topics[2], log.topics[3]]); - const txType = getTXType(transaction); - console.log(log.address, transaction.tokenId, log.transactionHash); - const erc721Contract = new web3.eth.Contract(ABIs.ERC721, log.address); - const uri = await erc721Contract.methods.tokenURI(transaction.tokenId).call().catch((err: any) => { - console.log('TokenURI not found'); - }); - - const metadata = await getMetaData(uri); - - return new NFT({ - nft_type: 'ERC-721', - tx_type: TX_TYPE[txType], - block_number: log.blockNumber, - transaction_hash: log.transactionHash, - chain: 'ethereum', - from: transaction.from, - to: transaction.to, - token_contract: log.address, - token_id: transaction.tokenId, - token_uri: uri, - metadata: metadata, - }) + const NFTs: NFT[] = [] + const options = { + fromBlock: blockNumber, + toBlock: blockNumber, + topics: [singleTransfer1155] + } + let logs: Log[] = [] + logs = await web3.eth.getPastLogs(options) + console.log('found ', logs.length, ' single 1155 logs'); + + for (let x = 0; x < logs.length; x++) { + if (logs[x].topics[0] === singleTransfer1155 && logs[x].topics.length == 4) { + const ERC1155NFT = await parseSingleERC1155Log(logs[x]) + if (ERC1155NFT != null) { + NFTs.push(ERC1155NFT) + } } + } + return NFTs } +async function getMultiERC1155Logs(blockNumber: any) { + const NFTs: NFT[] = [] + const options = { + fromBlock: blockNumber, + toBlock: blockNumber, + topics: [multiTransfer1155] + } + let logs: Log[] = [] + logs = await web3.eth.getPastLogs(options) + console.log('found ', logs.length, ' multi 1155 logs'); -function getTXType(transaction: any) { - if (transaction.from === '0x0000000000000000000000000000000000000000') { - return TX_TYPE.MINT; - } else if (transaction.to === '0x0000000000000000000000000000000000000000') { - return TX_TYPE.BURN; + for (let x = 0; x < logs.length; x++) { + if (logs[x].topics[0] === multiTransfer1155 && logs[x].topics.length == 4) { + const ERC1155NFT = await parseMultiERC1155Log(logs[x]) + if (ERC1155NFT) { + NFTs.push(...ERC1155NFT) + } } - return TX_TYPE.TRANSFER; + } + return NFTs } +async function parseERC721Log(log: Log) { + const transaction = web3.eth.abi.decodeLog(ABIs.ERC721Transfer, log.data, [ + log.topics[1], + log.topics[2], + log.topics[3] + ]) + const txType = getTXType(transaction) + console.log('ERC721', 'address:', log.address, 'token_id:', transaction.tokenId, 'txHash:', log.transactionHash) + const erc721Contract = new web3.eth.Contract(ABIs.ERC721, log.address) + const uri = await erc721Contract.methods.tokenURI(transaction.tokenId).call().catch((err: any) => { + //console.log('TokenURI not found') + }) -async function validateBlockNumber(blockNumber: any) { - if (isNaN(blockNumber)) { - const block = await web3.eth.getBlock('latest'); - blockNumber = block.number - 100; - } + const metadata = await getMetaData(uri) - return blockNumber; + return new NFT({ + nft_type: 'ERC-721', + tx_type: TX_TYPE[txType], + block_number: log.blockNumber, + transaction_hash: log.transactionHash, + chain: 'Ethereum', + from: transaction.from, + to: transaction.to, + token_contract: log.address, + token_id: transaction.tokenId, + token_uri: uri, + metadata + }) } +async function parseSingleERC1155Log(log: Log) { + const transaction = web3.eth.abi.decodeLog( + ABIs.ERC1155TransferSingle, + log.data, + [log.topics[1], log.topics[2], log.topics[3]] + ) + const txType = getTXType(transaction) + console.log('ERC1155 Single', 'address:', log.address, 'token_id:', transaction.id, 'txHash:', log.transactionHash) + const ercSingle1155Contract = new web3.eth.Contract(ABIs.ERC1155, log.address) + let originalUri = await ercSingle1155Contract.methods.uri(transaction.id).call().catch((err: any) => { + //console.log('TokenURI not found using uri function call') + }) -function setProvider(provider: any) { - web3.setProvider(provider); + if (!originalUri) { + originalUri = await getUriFromEvent(ercSingle1155Contract, transaction.id) + } + + let uri = replaceUriSubstitution(originalUri, transaction.id, 'standard') + let metadata = await getMetaData(uri) + + if (!metadata) { + uri = replaceUriSubstitution(originalUri, transaction.id, 'nonstandard') + metadata = await getMetaData(uri) + } + + return new NFT({ + nft_type: 'ERC-1155', + tx_type: TX_TYPE[txType], + block_number: log.blockNumber, + transaction_hash: log.transactionHash, + chain: 'Ethereum', + from: transaction.from, + to: transaction.to, + token_contract: log.address, + token_id: transaction.id, + token_value: transaction.value, + token_uri: uri, + metadata + }) } +async function parseMultiERC1155Log(log: Log) { + const transaction = web3.eth.abi.decodeLog( + ABIs.ERC1155TransferMulti, + log.data, + [log.topics[1], log.topics[2], log.topics[3]] + ) + let NFTs: NFT[] = []; + for (let i = 0; i < transaction.ids.length; i++) { + const txType = getTXType(transaction) + console.log('ERC1155 Multiple', 'address:', log.address, 'token_id:', transaction.ids[i], 'txHash:', log.transactionHash) + const ercMulti1155Contract = new web3.eth.Contract(ABIs.ERC1155, log.address) + let originalUri = await ercMulti1155Contract.methods.uri(transaction.ids[i]).call().catch((err: any) => { + //console.log('TokenURI not found using uri function call') + }) + if (!originalUri) { + originalUri = await getUriFromEvent(ercMulti1155Contract, transaction.ids[i]) + } + //console.log('original uri', originalUri); + let uri = replaceUriSubstitution(originalUri, transaction.ids[i], 'standard') + let metadata = await getMetaData(uri) + + if (!metadata) { + uri = replaceUriSubstitution(originalUri, transaction.ids[i], 'nonstandard') + metadata = await getMetaData(uri) + } + + NFTs.push(new NFT({ + nft_type: 'ERC-1155', + tx_type: TX_TYPE[txType], + block_number: log.blockNumber, + transaction_hash: log.transactionHash, + chain: 'Ethereum', + from: transaction.from, + to: transaction.to, + token_contract: log.address, + token_id: transaction.ids[i], + token_value: transaction.values[i], + token_uri: uri, + metadata + })) + } + return NFTs; +} -export { getERC721, setProvider }; +async function getUriFromEvent(contract: Contract, id: any) { + let events = []; + try { + events = await contract.getPastEvents('URI', { + filter: { 'id': id }, + fromBlock: 0, + toBlock: 'latest' + }); + if (events.length) { + return events[0].returnValues.value; + } + } catch (error: any) { + + return null; + } + return null; +} +// ref https://forum.openzeppelin.com/t/how-to-erc-1155-id-substitution-for-token-uri/3312/2 +function replaceUriSubstitution(uri: any, token_id: any, operationType: string) { + if (uri) { + if (operationType === 'standard') { + uri = uri.replace('{id}', Web3.utils.numberToHex(token_id).replace('0x', '').padStart(64, '0')); + } else if (operationType === 'nonstandard') { + uri = uri.replace('{id}', token_id); + } + } + return uri +} +function getTXType(transaction: any) { + if (transaction.from === '0x0000000000000000000000000000000000000000') { + return TX_TYPE.MINT + } else if (transaction.to === '0x0000000000000000000000000000000000000000') { + return TX_TYPE.BURN + } + return TX_TYPE.TRANSFER +} +async function validateBlockNumber(blockNumber: any) { + if (isNaN(blockNumber)) { + const block = await web3.eth.getBlock('latest') + blockNumber = block.number - 100 + } + return blockNumber +} +function setProvider(provider: any) { + web3.setProvider(provider) +} +export { getNFTs,getERC721, getERC1155, setProvider } diff --git a/src/eth/models/index.ts b/src/eth/models/index.ts index 95d3de7..730e14f 100644 --- a/src/eth/models/index.ts +++ b/src/eth/models/index.ts @@ -1,5 +1,2 @@ -import Log from "./log"; - -export { - Log, -}; +import Log from './log' +export { Log } diff --git a/src/eth/models/log.ts b/src/eth/models/log.ts index 4d9bf72..e279bc4 100644 --- a/src/eth/models/log.ts +++ b/src/eth/models/log.ts @@ -1,27 +1,24 @@ class Logs { + address: string + data: string + topics: string[] + logIndex: number + transactionIndex: number + transactionHash: string + blockHash: string + blockNumber: number + removed: boolean - address: string; - data: string; - topics: string[]; - logIndex: number; - transactionIndex: number; - transactionHash: string; - blockHash: string; - blockNumber: number; - removed: boolean; - - constructor(nft: any) { - this.address = nft.address; - this.data = nft.data; - this.topics = nft.topics; - this.logIndex = nft.logIndex; - this.transactionIndex = nft.transactionIndex; - this.transactionHash = nft.transactionHash; - this.blockHash = nft.blockHash; - this.blockNumber = nft.blockNumber; - this.removed = nft.removed; - - } - + constructor (nft: any) { + this.address = nft.address + this.data = nft.data + this.topics = nft.topics + this.logIndex = nft.logIndex + this.transactionIndex = nft.transactionIndex + this.transactionHash = nft.transactionHash + this.blockHash = nft.blockHash + this.blockNumber = nft.blockNumber + this.removed = nft.removed + } } -export default Logs; +export default Logs diff --git a/src/models/NFT.ts b/src/models/NFT.ts index 0b3c47d..d22cce9 100644 --- a/src/models/NFT.ts +++ b/src/models/NFT.ts @@ -1,44 +1,47 @@ class NFT { - nft_type: string; - tx_type: string; - block_number: number; - transaction_hash: string; - chain: string; - from: string; - to: string; - token_contract: number; - token_id: number; - token_uri: string; - metadata: any; + nft_type: string + tx_type: string + block_number: number + transaction_hash: string + chain: string + from: string + to: string + token_contract: number + token_id: number + token_value: string + token_uri: string + metadata: any - constructor(nft: any) { - this.nft_type = nft.nft_type; - this.tx_type = nft.tx_type; - this.block_number = nft.block_number; - this.transaction_hash = nft.transaction_hash; - this.chain = nft.chain; - this.from = nft.from; - this.to = nft.to; - this.token_contract = nft.token_contract; - this.token_id = nft.token_id; - this.token_uri = nft.token_uri ? nft.token_uri : null; - this.metadata = nft.metadata ? nft.metadata : null; - } + constructor(nft: any) { + this.nft_type = nft.nft_type + this.tx_type = nft.tx_type + this.block_number = nft.block_number + this.transaction_hash = nft.transaction_hash + this.chain = nft.chain + this.from = nft.from + this.to = nft.to + this.token_contract = nft.token_contract + this.token_id = nft.token_id + this.token_value = nft.token_value + this.token_uri = nft.token_uri ? nft.token_uri : null + this.metadata = nft.metadata ? nft.metadata : null + } - toDict() { - return { - nft_type: this.nft_type, - tx_type: this.tx_type, - block_number: this.block_number, - transaction_hash: this.transaction_hash, - chain: this.chain, - from: this.from, - to: this.to, - token_contract: this.token_contract, - token_id: this.token_id, - token_uri: this.token_uri, - metadata: this.metadata, - }; + toDict() { + return { + nft_type: this.nft_type, + tx_type: this.tx_type, + block_number: this.block_number, + transaction_hash: this.transaction_hash, + chain: this.chain, + from: this.from, + to: this.to, + token_contract: this.token_contract, + token_id: this.token_id, + token_value: this.token_value, + token_uri: this.token_uri, + metadata: this.metadata } + } } -export default NFT; +export default NFT diff --git a/src/models/NFTOptions.ts b/src/models/NFTOptions.ts index 3d5b908..896d0cd 100644 --- a/src/models/NFTOptions.ts +++ b/src/models/NFTOptions.ts @@ -1,14 +1,16 @@ class NFTOptions { - blockNumber: string | number; + blockNumber: string | number - constructor(NFTOptions: any) { - this.blockNumber = NFTOptions.blockNumber ? NFTOptions.blockNumber : 'latest' - } + constructor (NFTOptions: any) { + this.blockNumber = NFTOptions.blockNumber + ? NFTOptions.blockNumber + : 'latest' + } - toDict() { - return { - blockNumber: this.blockNumber, - }; + toDict () { + return { + blockNumber: this.blockNumber } + } } -export default NFTOptions; +export default NFTOptions diff --git a/src/models/constants.ts b/src/models/constants.ts index 6563425..c1ff6b5 100644 --- a/src/models/constants.ts +++ b/src/models/constants.ts @@ -1,9 +1,7 @@ enum TX_TYPE { - MINT, - BURN, - TRANSFER, - } + MINT, + BURN, + TRANSFER, +} - export { - TX_TYPE, - } \ No newline at end of file +export { TX_TYPE } diff --git a/src/models/index.ts b/src/models/index.ts index a9dbc10..e07bf39 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -1,7 +1,5 @@ -import NFT from "./NFT"; -import NFTOptions from "./NFTOptions"; -import { TX_TYPE } from "./constants"; +import NFT from './NFT' +import NFTOptions from './NFTOptions' +import { TX_TYPE } from './constants' -export { - NFT, NFTOptions, TX_TYPE, -}; +export { NFT, NFTOptions, TX_TYPE } diff --git a/src/services/index.ts b/src/services/index.ts index 58d14bf..1482f17 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -1,6 +1,3 @@ +import { getMetaData, setIpfsHostnames, setArweaveHostnames } from './metadata' -import { getMetaData, setIpfsHostnames, setArweaveHostnames } from "./metadata"; - -export { - getMetaData, setIpfsHostnames, setArweaveHostnames -}; +export { getMetaData, setIpfsHostnames, setArweaveHostnames } diff --git a/src/services/metadata.ts b/src/services/metadata.ts index b386eb6..93127eb 100644 --- a/src/services/metadata.ts +++ b/src/services/metadata.ts @@ -1,124 +1,117 @@ +import axios from 'axios' -import axios from 'axios'; - -let ipfsHostnames: any = ['cloudflare-ipfs.com']; -let arweaveHostnames: any = ['arweave.net']; +let ipfsHostnames: any = ['cloudflare-ipfs.com'] +let arweaveHostnames: any = ['arweave.net'] async function getMetaData(uri: string) { - - if (!uri) return null; - - if (uri.startsWith('https://')) { - return await requestMetadata(uri); - } else if (uri.startsWith('http://')) { - return await requestMetadata(uri); - } else if (uri.startsWith('ipfs://')) { - return await getIpfsMetadata(uri); - } else if (uri.startsWith('data:application/json;base64,')) { - console.log('base64'); - return getBase64Metadata(uri); - } else if (uri.startsWith('ar://')) { - console.log('arweave') - return await getArweaveMetadata(uri); - } + if (!uri) return null + + if (uri.startsWith('https://')) { + return await requestMetadata(uri) + } else if (uri.startsWith('http://')) { + return await requestMetadata(uri) + } else if (uri.startsWith('ipfs://')) { + return await getIpfsMetadata(uri) + } else if (uri.startsWith('data:application/json;base64,')) { + return getBase64Metadata(uri) + } else if (uri.startsWith('ar://')) { + return await getArweaveMetadata(uri) + } } async function getIpfsMetadata(uri: any) { - - const cloneIpfsHostnames = [...ipfsHostnames]; - uri = uri.split('ipfs://')[1]; - if (uri.startsWith('ipfs/')) { - uri = uri.split('ipfs/')[1]; - } - let json; - for (let x = 0; x < cloneIpfsHostnames.length; x++) { - json = await requestMetadata('https://' + cloneIpfsHostnames[x] + '/ipfs/' + uri) - - if (json) break; - - pushFailedIpfsHostnameToEnd(); - } - return json; + const cloneIpfsHostnames = [...ipfsHostnames] + uri = uri.split('ipfs://')[1] + if (uri.startsWith('ipfs/')) { + uri = uri.split('ipfs/')[1] + } + let json + for (let x = 0; x < cloneIpfsHostnames.length; x++) { + json = await requestMetadata( + 'https://' + cloneIpfsHostnames[x] + '/ipfs/' + uri + ) + + if (json) break + + pushFailedIpfsHostnameToEnd() + } + return json } - async function getArweaveMetadata(uri: any) { + const cloneArweaveHostnames = [...arweaveHostnames] + uri = uri.split('ar://')[1] - const cloneArweaveHostnames = [...arweaveHostnames]; - uri = uri.split('ar://')[1]; + let json + for (let x = 0; x < cloneArweaveHostnames.length; x++) { + json = await requestMetadata( + 'https://' + cloneArweaveHostnames[x] + '/' + uri + ) - let json; - for (let x = 0; x < cloneArweaveHostnames.length; x++) { - json = await requestMetadata('https://' + cloneArweaveHostnames[x] + '/' + uri) + if (json) break - if (json) break; - - pushFailedArweaveHostnameToEnd(); - } - return json; + pushFailedArweaveHostnameToEnd() + } + return json } -function getBase64Metadata(uri: any){ - uri = uri.split('data:application/json;base64,')[1]; +function getBase64Metadata(uri: any) { + uri = uri.split('data:application/json;base64,')[1] - let json; + let json - try { - const metadataFromBase64 = Buffer.from(uri, 'base64').toString('utf8'); - if(metadataFromBase64) { - json =JSON.parse(metadataFromBase64); - } - } catch (error: any) { - if (error) { - console.log('error: ' + error); - } + try { + const metadataFromBase64 = Buffer.from(uri, 'base64').toString('utf8') + if (metadataFromBase64) { + json = JSON.parse(metadataFromBase64) } + } catch (error: any) { + if (error) { + console.log('error: ' + error) + } + } - return json; - + return json } - async function requestMetadata(uri: string) { - - if (containsEncodedComponents(uri)) { - uri = decodeURIComponent(uri); + if (containsEncodedComponents(uri)) { + uri = decodeURIComponent(uri) + } + + let json + + try { + const response = await axios(uri, { timeout: 30000 }) + json = response.data + } catch (error: any) { + if (error.code === 'ECONNABORTED') { + console.log('metadata error: timeout') } - - console.log('getting metatdata from: ' + uri) - let json; - - try { - const response = await axios(uri, { timeout: 30000 }); - json = response.data; - } catch (error: any) { - if (error.code === 'ECONNABORTED') - console.log('error: timeout'); - if (error.response) { - console.log('error: status code ' + error.response.status); - } + if (error.response) { + console.log('metadata error: status code ' + error.response.status) } - return json; - + } + return json } function containsEncodedComponents(uri: string) { - // ie ?,=,&,/ etc - return (decodeURI(uri) !== decodeURIComponent(uri)); + // ie ?,=,&,/ etc + return decodeURI(uri) !== decodeURIComponent(uri) } function pushFailedIpfsHostnameToEnd() { - ipfsHostnames.push(ipfsHostnames.shift()); + ipfsHostnames.push(ipfsHostnames.shift()) } function setIpfsHostnames(hostnames: string[]) { - ipfsHostnames = hostnames; + ipfsHostnames = hostnames } function pushFailedArweaveHostnameToEnd() { - arweaveHostnames.push(ipfsHostnames.shift()); + arweaveHostnames.push(ipfsHostnames.shift()) } function setArweaveHostnames(hostnames: string[]) { - arweaveHostnames = hostnames; + arweaveHostnames = hostnames } -export { getMetaData, setIpfsHostnames, setArweaveHostnames }; +export { getMetaData, setIpfsHostnames, setArweaveHostnames } diff --git a/test.ts b/test.ts index 99a065e..1d96558 100644 --- a/test.ts +++ b/test.ts @@ -1,68 +1,59 @@ const onChainNFT = require('./index') async function test() { - - onChainNFT.setEthProvider(process.env.PROVIDER_URL); - onChainNFT.setIpfsHostnames([ - 'gateway.pinata.cloud', - 'cloudflare-ipfs.com', - 'ipfs-gateway.cloud', - 'gateway.ipfs.io', - '4everland.io', - 'cf-ipfs.com', - 'ipfs.jpu.jp', - 'dweb.link', - ]); - onChainNFT.setArweaveHostnames([ - 'arweave.net', - ]); - - console.log('# Testing get latest ERC721 with 100 block confirmations') - const defaultOption = await onChainNFT.getERC721({ - blockNumber: 'latest' - }) - console.log(JSON.stringify(defaultOption)); - console.log('found ' + defaultOption.length + ' NFTs using the latest block option'); - - await new Promise(resolve => setTimeout(resolve, 2000)); - - console.log('-------------------------------------------'); - console.log('# Testing get NFTs by block number') - let blockNumber = 16725346 - let blockNumberOption = await onChainNFT.getERC721({ blockNumber: blockNumber }) - console.log(JSON.stringify(blockNumberOption)); - console.log('found ' + blockNumberOption.length + ' NFTs in block ' + blockNumber); - + onChainNFT.setEthProvider(process.env.PROVIDER_URL); + onChainNFT.setIpfsHostnames([ + 'gateway.pinata.cloud', + 'cloudflare-ipfs.com', + 'ipfs-gateway.cloud', + 'gateway.ipfs.io', + '4everland.io', + 'cf-ipfs.com', + 'ipfs.jpu.jp', + 'dweb.link' + ]); + onChainNFT.setArweaveHostnames([ + 'arweave.net' + ]); + + // ########### all NFTs ########### + await getNFTs('latest', 'Testing all NFT types in the latest block with 100 block confirmations', 'all'); + + // ########### 1155 tests ########### + await getNFTs('latest', 'Testing get latest ERC1155 with 100 block confirmations', 'erc-1155'); + await getNFTs(16874683, 'Testing blocks with Transfer batch event 1', 'erc-1155'); + await getNFTs(16874466, 'Testing blocks with Transfer batch event 2', 'erc-1155'); + await getNFTs(16881743, 'Testing blocks with Transfer batch event with {id} substitution that was not done properly', 'erc-1155'); + + + // // ########### 721 tests ########### + await getNFTs('latest', 'Testing latest block ERC721 with 100 block confirmations', 'erc-721'); + await getNFTs(16633190, 'Testing block with TXs that include no tokenURI', 'erc-721'); + await getNFTs(16640530, 'Testing block with TX that has token erc-721 burn', 'erc-721'); + await getNFTs(16740129, 'Testing block with over 300 base64 tokenURIs', 'erc-721'); + + + //await getNFT(16874458, 'Testing get 1155 with opensea uri with {id} substitution', 'erc-1155'); + async function getNFTs(blockNumber: any, message: string, type: string) { await new Promise(resolve => setTimeout(resolve, 2000)); console.log('-------------------------------------------'); - console.log('# Testing get NFTs with TXs that include no tokenURI') - blockNumber = 16633190 - blockNumberOption = await onChainNFT.getERC721({ blockNumber: blockNumber }) - console.log(JSON.stringify(blockNumberOption)); - console.log('found ' + blockNumberOption.length + ' NFTs in block ' + blockNumber); + console.log(type+':' +message) + if (type === 'erc-1155') { + const result = await onChainNFT.getERC1155({ blockNumber: blockNumber }); + console.log(JSON.stringify(result)); + console.log('found ' + result.length + ' NFTs in block:' + blockNumber); + } else if (type === 'erc-721') { + const result = await onChainNFT.getERC721({ blockNumber: blockNumber }); + console.log(JSON.stringify(result)); + console.log('found ' + result.length + ' NFTs in block:' + blockNumber); + } + else if (type === 'all'){ + const result = await onChainNFT.getEthNFTs({ blockNumber: blockNumber }); + console.log(JSON.stringify(result)); + console.log('found ' + result.length + ' NFTs in block:' + blockNumber); + } + } - await new Promise(resolve => setTimeout(resolve, 2000)); - - console.log('-------------------------------------------'); - console.log('# Testing get NFTs with TX that has token burn') - blockNumber = 16640530 - blockNumberOption = await onChainNFT.getERC721({ blockNumber: blockNumber }) - console.log(JSON.stringify(blockNumberOption)); - console.log('found ' + blockNumberOption.length + ' NFTs in block ' + blockNumber); - - console.log('-------------------------------------------'); - console.log('# Testing blocks with over 300 base64 tokenURIs') - blockNumber = 16740129 - blockNumberOption = await onChainNFT.getERC721({ blockNumber: blockNumber }) - console.log(JSON.stringify(blockNumberOption)); - console.log('found ' + blockNumberOption.length + ' NFTs in block ' + blockNumber); - - - // TO-DO - // TEST blocks with IPFS, Arweave and base64/ - // test Set hostname with one hostname for both IPFS and arweave } - test(); - From 4bf810d39bedd1fe9eb08830c08cff616b9b148e Mon Sep 17 00:00:00 2001 From: Mohammed Date: Sun, 26 Mar 2023 18:37:35 +0700 Subject: [PATCH 2/3] chore: update logging --- src/services/metadata.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/metadata.ts b/src/services/metadata.ts index 93127eb..e8f2361 100644 --- a/src/services/metadata.ts +++ b/src/services/metadata.ts @@ -78,7 +78,7 @@ async function requestMetadata(uri: string) { if (containsEncodedComponents(uri)) { uri = decodeURIComponent(uri) } - + console.log('Metadata fetch', uri) let json try { From 2366978dafc2a232780048a65da7c539fc13a205 Mon Sep 17 00:00:00 2001 From: Mohammed Date: Sun, 26 Mar 2023 18:57:19 +0700 Subject: [PATCH 3/3] chore: update readme --- README.md | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9ca9dbd..df01751 100644 --- a/README.md +++ b/README.md @@ -8,18 +8,34 @@ Get on-chain NFT mints/transfers/burns including their metadata. Supported metad ## Dev Clone the repo ``` -git clone git@github.com:mohammed-almujil/on-chain-nft.git -cd on-chain-nft +git clone git@github.com:mohammed-almujil/on-chain-nfts.git +cd on-chain-nfts ``` Install dependencies ``` npm install npm install -g ts-node ``` -Set ETH provider URL in test.ts + +### **Ethereum** + +ETH NFTs ``` +const onChainNFT = require('./index') + onChainNFT.setEthProvider(PROVIDER_URL); + +//All NFTs +const all = await onChainNFT.getEthNFTs({ blockNumber: blockNumber }); + +//ERC-721 NFTs only +const erc721 = await onChainNFT.getERC721({ blockNumber: blockNumber }); + +//ERC-1155 only +const erc1155 = await onChainNFT.getERC1155({ blockNumber: blockNumber }); + ``` + Optionally set IPFS hostnames. The code will try the hostnames one by one in case of failure. More here https://ipfs.github.io/public-gateway-checker/ ``` onChainNFT.setIpfsHostnames(['gateway.pinata.cloud','cloudflare-ipfs.com']); @@ -27,10 +43,11 @@ onChainNFT.setIpfsHostnames(['gateway.pinata.cloud','cloudflare-ipfs.com']); Optionally set Arweave hostnames. The code will try the hostnames one by one in case of failure. ``` -onChainNFT.setIpfsHostnames(['gateway.pinata.cloud','cloudflare-ipfs.com']); +onChainNFT.setArweaveHostnames(['arweave.net']); ``` +### **Tests** -Run tests locally +Run tests locally, make sure PROVIDER_URL ENV variable is set then run ``` ts-node test.ts ``` \ No newline at end of file