From b390e0864dd87d96e8efbbf17ad1933fda90c272 Mon Sep 17 00:00:00 2001 From: Ethan Daya Date: Sat, 3 Jul 2021 15:05:47 +1000 Subject: [PATCH] feat: add mimeType to metadata object & handle for custom IPFS baseURL --- package.json | 9 +++--- src/parser.ts | 24 ++++++++++++---- src/parsers/artblocksMetadataParser.ts | 2 ++ src/parsers/hashmasksMetadataParser.ts | 2 ++ src/parsers/makersplaceMetadataParser.ts | 35 ++++++++++++++++++------ src/parsers/openseaMetadataParser.ts | 27 ++++++++++++------ src/parsers/zoraMetadataParser.ts | 16 +++++++---- src/utils/fetch.ts | 14 ++++++++++ tests/contracts/artblocks.int.test.ts | 16 +++++++---- tests/contracts/bonsai.int.test.ts | 8 ++++-- tests/contracts/boredApe.int.test.ts | 5 ++-- tests/contracts/cryptovoxels.int.test.ts | 5 ++-- tests/contracts/decentraland.int.test.ts | 5 ++-- tests/contracts/hashmasks.int.test.ts | 5 ++-- tests/contracts/makersplace.int.test.ts | 13 +++++---- tests/contracts/rarible.int.test.ts | 5 ++-- tests/contracts/sorare.int.test.ts | 5 ++-- tests/contracts/superYeti.int.test.ts | 5 ++-- tests/contracts/superrare.int.test.ts | 5 ++-- tests/contracts/veeFriends.int.test.ts | 5 ++-- tests/contracts/zora.int.test.ts | 5 ++-- 21 files changed, 150 insertions(+), 66 deletions(-) diff --git a/package.json b/package.json index 85f8c8e..507f8e8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@zoralabs/nft-metadata", - "version": "0.0.1", + "version": "0.0.2", "description": "generic nft metadata parsers", "author": "Zora", "license": "MIT", @@ -11,6 +11,10 @@ "files": [ "dist" ], + "dependencies": { + "cross-fetch": "^3.1.4", + "@zoralabs/core": "^1.0.4" + }, "devDependencies": { "@ethersproject/address": "5.3.0", "@ethersproject/contracts": "5.3.0", @@ -63,8 +67,5 @@ ], "coverageDirectory": "../coverage", "testEnvironment": "node" - }, - "dependencies": { - "@zoralabs/core": "^1.0.4" } } diff --git a/src/parser.ts b/src/parser.ts index 6484bcc..979f865 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -2,6 +2,7 @@ import { JsonRpcProvider } from '@ethersproject/providers' import { getAddress } from '@ethersproject/address' import { fetcherLookup } from './fetchers' import { parserLookup } from './parsers' +import { IPFS_IO_GATEWAY } from './utils/ipfs' export interface NftMetadata { metadata: any @@ -11,15 +12,23 @@ export interface NftMetadata { ownerAddress: string - tokenURI: string - contentURI: string + tokenURL: string + + contentURL: string + contentURLMimeType: string + + previewURL?: string + previewURLMimeType?: string externalLink?: string attributes?: Record[] } export class Parser { - constructor(private readonly provider: JsonRpcProvider) {} + constructor( + private readonly provider: JsonRpcProvider, + private readonly ipfsBaseURL: string = IPFS_IO_GATEWAY, + ) {} public async fetchUnderlyingContractData( contractAddress: string, @@ -35,7 +44,13 @@ export class Parser { tokenURI: string, ) { const parser = parserLookup(contractAddress) - return parser(this.provider, contractAddress, tokenId, tokenURI) + return parser( + this.provider, + this.ipfsBaseURL, + contractAddress, + tokenId, + tokenURI, + ) } public async fetchAndParseTokenMeta( @@ -56,7 +71,6 @@ export class Parser { ) return { - tokenURI, ownerAddress, ...parsedMetadata, } diff --git a/src/parsers/artblocksMetadataParser.ts b/src/parsers/artblocksMetadataParser.ts index 8e33c9f..c7c26ba 100644 --- a/src/parsers/artblocksMetadataParser.ts +++ b/src/parsers/artblocksMetadataParser.ts @@ -3,12 +3,14 @@ import { parseGenericMetadata } from './openseaMetadataParser' export async function parseArtblocksMetadata( provider: JsonRpcProvider, + ipfsBaseURL: string, contractAddress: string, tokenId: string, tokenURI: string, ) { const baseMeta = await parseGenericMetadata( provider, + ipfsBaseURL, contractAddress, tokenId, tokenURI, diff --git a/src/parsers/hashmasksMetadataParser.ts b/src/parsers/hashmasksMetadataParser.ts index a34689e..a5b516e 100644 --- a/src/parsers/hashmasksMetadataParser.ts +++ b/src/parsers/hashmasksMetadataParser.ts @@ -4,12 +4,14 @@ import { parseGenericMetadata } from './openseaMetadataParser' export async function parseHashmasksMetadata( provider: JsonRpcProvider, + ipfsBaseURL: string, contractAddress: string, tokenId: string, tokenURI: string, ) { const baseMeta = await parseGenericMetadata( provider, + ipfsBaseURL, contractAddress, tokenId, tokenURI, diff --git a/src/parsers/makersplaceMetadataParser.ts b/src/parsers/makersplaceMetadataParser.ts index d72c5b9..e7bdbe1 100644 --- a/src/parsers/makersplaceMetadataParser.ts +++ b/src/parsers/makersplaceMetadataParser.ts @@ -1,11 +1,12 @@ import { JsonRpcProvider } from '@ethersproject/providers' import { getIPFSUrl } from '../utils/ipfs' -import { fetchMetadata } from '../utils/fetch' +import { fetchMetadata, fetchMimeType } from '../utils/fetch' export async function parseMakersplaceMetadata( _: JsonRpcProvider, __: string, ___: string, + ____: string, tokenURI: string, ) { const publicTokenURI = getIPFSUrl( @@ -14,9 +15,16 @@ export async function parseMakersplaceMetadata( ) const metadata = await fetchMetadata(tokenURI) - const imageURI = metadata.imageUrl - ? getIPFSUrl(metadata.imageUrl, 'https://ipfsgateway.makersplace.com') - : null + if (!metadata.imageUrl) { + throw new Error( + `Invalid metadata required imageUrl key from metadata missing`, + ) + } + + const imageURI = getIPFSUrl( + metadata.imageUrl, + 'https://ipfsgateway.makersplace.com', + ) const animationURI = metadata?.properties?.preview_media_file2 && @@ -28,14 +36,23 @@ export async function parseMakersplaceMetadata( : null const { name, description, attributes } = metadata - const contentURI = animationURI || imageURI - const previewURI = imageURI && animationURI ? imageURI : undefined + const contentURL = animationURI || imageURI + const previewURL = imageURI && animationURI ? imageURI : undefined + + const contentURLMimeType = animationURI + ? 'video/mp4' + : await fetchMimeType(contentURL) + const previewURLMimeType = previewURL + ? await fetchMimeType(previewURL) + : undefined return { metadata, - tokenURI: publicTokenURI, - contentURI, - ...(previewURI && { previewURI }), + tokenURL: publicTokenURI, + contentURL, + contentURLMimeType, + ...(previewURL && { previewURL }), + ...(previewURLMimeType && { previewURLMimeType }), ...(name && { name }), ...(description && { description }), ...(attributes && { attributes }), diff --git a/src/parsers/openseaMetadataParser.ts b/src/parsers/openseaMetadataParser.ts index 7a995c5..537576f 100644 --- a/src/parsers/openseaMetadataParser.ts +++ b/src/parsers/openseaMetadataParser.ts @@ -1,13 +1,15 @@ import { JsonRpcProvider } from '@ethersproject/providers' import { getIPFSUrl } from '../utils/ipfs' -import { fetchMetadata } from '../utils/fetch' +import { fetchMetadata, fetchMimeType } from '../utils/fetch' +import { NftMetadata } from '../parser' export async function parseGenericMetadata( _: JsonRpcProvider, + ipfsBaseURL: string, __: string, ___: string, tokenURI: string, -) { +): Promise { const publicTokenURI = getIPFSUrl(tokenURI) const metadata = await fetchMetadata(tokenURI) @@ -17,20 +19,27 @@ export async function parseGenericMetadata( ) } - const imageURI = getIPFSUrl(metadata.image) + const imageURI = getIPFSUrl(metadata.image, ipfsBaseURL) const animationURI = metadata?.animation_url - ? getIPFSUrl(metadata.animation_url) + ? getIPFSUrl(metadata.animation_url, ipfsBaseURL) : null const { name, description, attributes, external_url: externalURL } = metadata - const contentURI = animationURI || imageURI - const previewURI = imageURI && animationURI ? imageURI : undefined + const contentURL = animationURI || imageURI + const previewURL = imageURI && animationURI ? imageURI : undefined + + const contentURLMimeType = await fetchMimeType(contentURL) + const previewURLMimeType = previewURL + ? await fetchMimeType(previewURL) + : undefined return { metadata, - tokenURI: publicTokenURI, - contentURI, - ...(previewURI && { previewURI }), + tokenURL: publicTokenURI, + contentURL, + contentURLMimeType, + ...(previewURL && { previewURL }), + ...(previewURLMimeType && { previewURLMimeType }), ...(name && { name }), ...(description && { description }), ...(attributes && { attributes }), diff --git a/src/parsers/zoraMetadataParser.ts b/src/parsers/zoraMetadataParser.ts index 4ea4aef..aaa1e60 100644 --- a/src/parsers/zoraMetadataParser.ts +++ b/src/parsers/zoraMetadataParser.ts @@ -1,26 +1,32 @@ import { JsonRpcProvider } from '@ethersproject/providers' import { getIPFSUrl } from '../utils/ipfs' import { MediaFactory } from '@zoralabs/core/dist/typechain' -import { fetchMetadata } from '../utils/fetch' +import { fetchMetadata, fetchMimeType } from '../utils/fetch' export async function parseZoraMetadata( provider: JsonRpcProvider, + ipfsBaseURL: string, contractAddress: string, tokenId: string, tokenURI: string, ) { - const publicTokenURI = getIPFSUrl(tokenURI) + const publicTokenURI = getIPFSUrl(tokenURI, ipfsBaseURL) const metadata = await fetchMetadata(tokenURI) const ZContract = MediaFactory.connect(contractAddress, provider) const contentURI = await ZContract.tokenURI(tokenId) + const contentURL = getIPFSUrl(contentURI, ipfsBaseURL) - const { name, description, externalURL } = metadata + const { name, description, externalURL, mimeType } = metadata + const contentURLMimeType = mimeType + ? mimeType + : await fetchMimeType(contentURL) return { metadata, - tokenURI: publicTokenURI, - contentURI, + tokenURL: publicTokenURI, + contentURL, + contentURLMimeType, ...(name && { name }), ...(description && { description }), ...(externalURL && { externalURL }), diff --git a/src/utils/fetch.ts b/src/utils/fetch.ts index b86798a..252d881 100644 --- a/src/utils/fetch.ts +++ b/src/utils/fetch.ts @@ -6,3 +6,17 @@ export async function fetchMetadata(uri: string) { const resp = await fetch(cloudflareIPFSURI) return resp.json() } + +export async function fetchMimeType(uri: string) { + try { + const resp = await fetch(uri, { method: 'HEAD' }) + return resp.headers.get('content-type') + } catch (e) { + console.error( + `Failed to fetch mimetype for uri: ${uri} because: ${ + e?.message || 'Unknown Error occured' + }`, + ) + return + } +} diff --git a/tests/contracts/artblocks.int.test.ts b/tests/contracts/artblocks.int.test.ts index 4e1b754..177cfbb 100644 --- a/tests/contracts/artblocks.int.test.ts +++ b/tests/contracts/artblocks.int.test.ts @@ -17,9 +17,11 @@ const ART_BLOCKS_CRITERIA = { metadata: METADATA_STUB, name: METADATA_STUB.name, description: METADATA_STUB.description, - tokenURI: 'https://api.artblocks.io/token/91000216', - contentURI: METADATA_STUB.animation_url, - previewURI: METADATA_STUB.image, + tokenURL: 'https://api.artblocks.io/token/91000216', + contentURL: METADATA_STUB.animation_url, + contentURLMimeType: 'text/html; charset=utf-8', + previewURL: METADATA_STUB.image, + previewURLMimeType: 'image/png', attributes: METADATA_STUB.traits, externalURL: METADATA_STUB.external_url, }, @@ -33,9 +35,11 @@ const ART_BLOCKS_CURATED_CRITERIA = { metadata: CURATED_METADATA_STUB, name: CURATED_METADATA_STUB.name, description: CURATED_METADATA_STUB.description, - tokenURI: 'https://api.artblocks.io/token/100', - contentURI: CURATED_METADATA_STUB.animation_url, - previewURI: CURATED_METADATA_STUB.image, + tokenURL: 'https://api.artblocks.io/token/100', + contentURL: CURATED_METADATA_STUB.animation_url, + contentURLMimeType: 'text/html; charset=utf-8', + previewURL: CURATED_METADATA_STUB.image, + previewURLMimeType: 'image/png', attributes: CURATED_METADATA_STUB.traits, externalURL: CURATED_METADATA_STUB.external_url, }, diff --git a/tests/contracts/bonsai.int.test.ts b/tests/contracts/bonsai.int.test.ts index f935893..5754aa3 100644 --- a/tests/contracts/bonsai.int.test.ts +++ b/tests/contracts/bonsai.int.test.ts @@ -12,10 +12,12 @@ const BONSAI_CRITERIA = { metadata: METADATA_STUB, name: METADATA_STUB.name, description: METADATA_STUB.description, - tokenURI: + tokenURL: 'https://ipfs.io/ipfs/QmXPSFmFqfDTMbLePfGTuYa2Vm9CoqsU11ypiMm1nKL8V9/100', - contentURI: METADATA_STUB.animation_url, - previewURI: METADATA_STUB.image, + contentURL: METADATA_STUB.animation_url, + contentURLMimeType: 'video/mp4', + previewURL: METADATA_STUB.image, + previewURLMimeType: 'image/png', attributes: METADATA_STUB.attributes, }, } diff --git a/tests/contracts/boredApe.int.test.ts b/tests/contracts/boredApe.int.test.ts index d021a90..67f60ff 100644 --- a/tests/contracts/boredApe.int.test.ts +++ b/tests/contracts/boredApe.int.test.ts @@ -10,10 +10,11 @@ const BORED_APE_CRITERIA = { }, output: { metadata: BORED_APE_METADATA_STUB, - tokenURI: + tokenURL: 'https://ipfs.io/ipfs/QmeSjSinHpPnmXmspMjwiXyN6zS4E9zccariGR3jxcaWtq/1', - contentURI: + contentURL: 'https://ipfs.io/ipfs/QmPbxeGcXhYQQNgsC6a36dDyYUcHgMLnGKnF8pVFmGsvqi', + contentURLMimeType: 'image/png', attributes: BORED_APE_METADATA_STUB.attributes, }, } diff --git a/tests/contracts/cryptovoxels.int.test.ts b/tests/contracts/cryptovoxels.int.test.ts index 9bd267c..a90a82c 100644 --- a/tests/contracts/cryptovoxels.int.test.ts +++ b/tests/contracts/cryptovoxels.int.test.ts @@ -12,8 +12,9 @@ const CRYPTOVOXELS_CRITERIA = { metadata: METADATA_STUB, name: METADATA_STUB.name, description: METADATA_STUB.description, - tokenURI: 'https://www.cryptovoxels.com/p/100', - contentURI: METADATA_STUB.image, + tokenURL: 'https://www.cryptovoxels.com/p/100', + contentURL: METADATA_STUB.image, + contentURLMimeType: 'image/png', attributes: METADATA_STUB.attributes, externalURL: METADATA_STUB.external_url, }, diff --git a/tests/contracts/decentraland.int.test.ts b/tests/contracts/decentraland.int.test.ts index fc8f02b..08d86af 100644 --- a/tests/contracts/decentraland.int.test.ts +++ b/tests/contracts/decentraland.int.test.ts @@ -11,9 +11,10 @@ const DECENTRALAND_CRITERIA = { output: { metadata: METADATA_STUB, name: METADATA_STUB.name, - tokenURI: + tokenURL: 'https://api.decentraland.org/v2/contracts/0xf87e31492faf9a91b02ee0deaad50d51d56d5d4d/tokens/100', - contentURI: METADATA_STUB.image, + contentURL: METADATA_STUB.image, + contentURLMimeType: 'image/png', attributes: METADATA_STUB.attributes, externalURL: METADATA_STUB.external_url, }, diff --git a/tests/contracts/hashmasks.int.test.ts b/tests/contracts/hashmasks.int.test.ts index 79fa2e8..ff3f95e 100644 --- a/tests/contracts/hashmasks.int.test.ts +++ b/tests/contracts/hashmasks.int.test.ts @@ -12,8 +12,9 @@ const BORED_APE_CRITERIA = { metadata: METADATA_STUB, name: 'Chocolate', description: METADATA_STUB.description, - tokenURI: 'https://hashmap.azurewebsites.net/getMask/3837', - contentURI: METADATA_STUB.image, + tokenURL: 'https://hashmap.azurewebsites.net/getMask/3837', + contentURL: METADATA_STUB.image, + contentURLMimeType: 'image/jpeg', attributes: METADATA_STUB.attributes, externalURL: METADATA_STUB.external_url, }, diff --git a/tests/contracts/makersplace.int.test.ts b/tests/contracts/makersplace.int.test.ts index 629770e..b0c8b2e 100644 --- a/tests/contracts/makersplace.int.test.ts +++ b/tests/contracts/makersplace.int.test.ts @@ -13,9 +13,10 @@ const IMAGE_CRITERIA = { metadata: IMAGE_METADATA_STUB, name: IMAGE_METADATA_STUB.name, description: IMAGE_METADATA_STUB.description, - tokenURI: + tokenURL: 'https://ipfsgateway.makersplace.com/ipfs/QmQZ66nmWPRFTqfuxVuAMHKb57xwhDZfGkSDGd7MJjxbWm', - contentURI: IMAGE_METADATA_STUB.imageUrl, + contentURL: IMAGE_METADATA_STUB.imageUrl, + contentURLMimeType: 'image/jpeg', attributes: IMAGE_METADATA_STUB.attributes, }, } @@ -28,10 +29,12 @@ const VIDEO_CRITERIA = { metadata: VIDEO_METADATA_STUB, name: VIDEO_METADATA_STUB.name, description: VIDEO_METADATA_STUB.description, - tokenURI: + tokenURL: 'https://ipfsgateway.makersplace.com/ipfs/QmczR8BYHGsfMjiQ988mhrQo4Femozf4RoVsqu8PjxFMNU', - contentURI: VIDEO_METADATA_STUB.properties.preview_media_file2.description, - previewURI: VIDEO_METADATA_STUB.imageUrl, + contentURL: VIDEO_METADATA_STUB.properties.preview_media_file2.description, + contentURLMimeType: 'video/mp4', + previewURL: VIDEO_METADATA_STUB.imageUrl, + previewURLMimeType: 'image/jpeg', attributes: VIDEO_METADATA_STUB.attributes, }, } diff --git a/tests/contracts/rarible.int.test.ts b/tests/contracts/rarible.int.test.ts index 12c3375..d636e3d 100644 --- a/tests/contracts/rarible.int.test.ts +++ b/tests/contracts/rarible.int.test.ts @@ -13,9 +13,10 @@ const RARIBLE_CRITERIA = { metadata: METADATA_STUB, name: METADATA_STUB.name, description: METADATA_STUB.description, - tokenURI: + tokenURL: 'https://ipfs.io/ipfs/QmfHrsEpXXrvi2dTTNake723kkMapDQXeuYDsVmSRdsQNH', - contentURI: METADATA_STUB.image, + contentURL: METADATA_STUB.image, + contentURLMimeType: 'image/jpeg', externalURL: METADATA_STUB.external_url, attributes: METADATA_STUB.attributes, }, diff --git a/tests/contracts/sorare.int.test.ts b/tests/contracts/sorare.int.test.ts index 540d8b7..9098a29 100644 --- a/tests/contracts/sorare.int.test.ts +++ b/tests/contracts/sorare.int.test.ts @@ -13,9 +13,10 @@ const SORARE_CRITERIA = { metadata: METADATA_STUB, name: METADATA_STUB.name, description: METADATA_STUB.description, - tokenURI: + tokenURL: 'https://api.sorare.com/api/v1/cards/100408090256748841933639219919068439822835068705742947267506906666220789418990', - contentURI: METADATA_STUB.image, + contentURL: METADATA_STUB.image, + contentURLMimeType: 'image/png', attributes: METADATA_STUB.attributes, externalURL: METADATA_STUB.external_url, }, diff --git a/tests/contracts/superYeti.int.test.ts b/tests/contracts/superYeti.int.test.ts index 5e29a23..f7b1229 100644 --- a/tests/contracts/superYeti.int.test.ts +++ b/tests/contracts/superYeti.int.test.ts @@ -12,9 +12,10 @@ const SUPER_YET_CRITERIA = { metadata: METADATA_STUB, name: METADATA_STUB.name, description: METADATA_STUB.description, - tokenURI: + tokenURL: 'https://defra.systems/metadata/QmXFtqihiEDP5sJwME5dNB3NnYk6LeiepbD4RPP1XES6Ys/asset/100', - contentURI: METADATA_STUB.image, + contentURL: METADATA_STUB.image, + contentURLMimeType: 'image/jpeg', attributes: METADATA_STUB.attributes, }, } diff --git a/tests/contracts/superrare.int.test.ts b/tests/contracts/superrare.int.test.ts index 29952b5..f804af9 100644 --- a/tests/contracts/superrare.int.test.ts +++ b/tests/contracts/superrare.int.test.ts @@ -12,9 +12,10 @@ const SUPERARE_CRITERIA = { metadata: METADATA_STUB, name: METADATA_STUB.name, description: METADATA_STUB.description, - tokenURI: + tokenURL: 'https://ipfs.pixura.io/ipfs/QmfQs1DPrZgmR7osnWvRuXpDvqP5ihfEfSBKPGAQyeL1WS/metadata.json', - contentURI: METADATA_STUB.image, + contentURL: METADATA_STUB.image, + contentURLMimeType: 'image/gif', }, } diff --git a/tests/contracts/veeFriends.int.test.ts b/tests/contracts/veeFriends.int.test.ts index 90fe93a..473aa9c 100644 --- a/tests/contracts/veeFriends.int.test.ts +++ b/tests/contracts/veeFriends.int.test.ts @@ -12,9 +12,10 @@ const VEE_FRIENDS_CRITERIA = { metadata: METADATA_STUB, name: METADATA_STUB.name, description: METADATA_STUB.description, - tokenURI: + tokenURL: 'https://erc721.veefriends.com/api/metadata/0xa3aee8bce55beea1951ef834b99f3ac60d1abeeb/294', - contentURI: METADATA_STUB.image, + contentURL: METADATA_STUB.image, + contentURLMimeType: 'image/png', attributes: METADATA_STUB.attributes, externalURL: METADATA_STUB.external_url, }, diff --git a/tests/contracts/zora.int.test.ts b/tests/contracts/zora.int.test.ts index 4559f31..93b078c 100644 --- a/tests/contracts/zora.int.test.ts +++ b/tests/contracts/zora.int.test.ts @@ -12,9 +12,10 @@ const ZORA_CRITERIA = { metadata: METADATA_STUB, name: METADATA_STUB.name, description: METADATA_STUB.description, - contentURI: + contentURL: 'https://ipfs.fleek.co/ipfs/bafkreic4zqxiofoxuf6dcefp4n5ykynyjncq3ohpz4mh3edet4qseaifb4', - tokenURI: + contentURLMimeType: 'text/plain', + tokenURL: 'https://ipfs.fleek.co/ipfs/bafkreic364j5bx3tai5esijlnicpssmmudqlo4nwz5reftkpz6w2qrdhym', }, }