Parallelize baby yeah (metaplex-foundation#1025)

stayhungry07212 · Nov 20, 2021 · 3895e83 · 3895e83
1 parent ed75057
commit 3895e83
Show file tree

Hide file tree

Showing 3 changed files with 114 additions and 93 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -36,6 +36,7 @@ incremented for features.
 - If x then y probability for image generation in the Candy Machine CLI
 - Ability to add premade customs to your image generation in the candy machine CLI
 - Ability to use PSD instead of PNGs in the candy machine CLI
+- Parallelize with batchSize command the candy machine CLI uploader
 
 
 ### Fixes

diff --git a/js/packages/cli/src/candy-machine-cli.ts b/js/packages/cli/src/candy-machine-cli.ts
@@ -58,6 +58,7 @@ programCommand('upload')
     },
   )
   .option('-n, --number <number>', 'Number of images to upload')
+  .option('-b, --batchSize <number>', 'Batch size - defaults to 1000')
   .option(
     '-s, --storage <string>',
     'Database to use for storage (arweave, ipfs, aws)',
@@ -94,6 +95,7 @@ programCommand('upload')
       retainAuthority,
       mutable,
       rpcUrl,
+      batchSize,
     } = cmd.opts();
 
     if (storage === 'ipfs' && (!ipfsInfuraProjectId || !ipfsInfuraSecret)) {
@@ -156,6 +158,7 @@ programCommand('upload')
         rpcUrl,
         ipfsCredentials,
         awsS3Bucket,
+        batchSize,
       );
 
       if (successful) {

diff --git a/js/packages/cli/src/commands/upload.ts b/js/packages/cli/src/commands/upload.ts
@@ -27,6 +27,7 @@ export async function upload(
   rpcUrl: string,
   ipfsCredentials: ipfsCreds,
   awsS3Bucket: string,
+  batchSize: number,
 ): Promise<boolean> {
   let uploadSuccessful = true;
 
@@ -72,102 +73,118 @@ export async function upload(
 
   const tick = SIZE / 100; //print every one percent
   let lastPrinted = 0;
-  for (let i = 0; i < SIZE; i++) {
-    const image = images[i];
-    const imageName = path.basename(image);
-    const index = imageName.replace(EXTENSION_PNG, '');
-
-      log.debug(`Processing file: ${i}`);
-
-    let link = cacheContent?.items?.[index]?.link;
-    let imageLink = cacheContent?.items?.[index]?.imageLink;
-    if (!link || !cacheContent.program.uuid) {
-      if (i >= lastPrinted + tick || i === 0) {
-        lastPrinted = i;
-        log.info(`Processing file: ${i}, ${imageName}`);
-      }
-      const manifestPath = image.replace(EXTENSION_PNG, EXTENSION_JSON);
-      const manifestContent = fs
-        .readFileSync(manifestPath)
-        .toString()
-        .replace(imageName, 'image.png')
-        .replace(imageName, 'image.png');
-      const manifest = JSON.parse(manifestContent);
-
-      const manifestBuffer = Buffer.from(JSON.stringify(manifest));
-
-      if (i === 0 && !cacheContent.program.uuid) {
-        // initialize config
-        log.info(`initializing config`);
-        try {
-          const res = await createConfig(anchorProgram, walletKeyPair, {
-            maxNumberOfLines: new BN(totalNFTs),
-            symbol: manifest.symbol,
-            sellerFeeBasisPoints: manifest.seller_fee_basis_points,
-            isMutable: mutable,
-            maxSupply: new BN(0),
-            retainAuthority: retainAuthority,
-            creators: manifest.properties.creators.map(creator => {
-              return {
-                address: new PublicKey(creator.address),
-                verified: true,
-                share: creator.share,
-              };
-            }),
-          });
-          cacheContent.program.uuid = res.uuid;
-          cacheContent.program.config = res.config.toBase58();
-          config = res.config;
-
-          log.info(
-            `initialized config for a candy machine with publickey: ${res.config.toBase58()}`,
-          );
-
-          saveCache(cacheName, env, cacheContent);
-        } catch (exx) {
-          log.error('Error deploying config to Solana network.', exx);
-          throw exx;
-        }
-      }
-
-      if (!link) {
-        try {
-          if (storage === 'arweave') {
-            [link, imageLink] = await arweaveUpload(
-              walletKeyPair,
-              anchorProgram,
-              env,
-              image,
-              manifestBuffer,
-              manifest,
-              index,
-            );
-          } else if (storage === 'ipfs') {
-            [link, imageLink] = await ipfsUpload(ipfsCredentials, image, manifestBuffer);
-          } else if (storage === 'aws') {
-            [link, imageLink] = await awsUpload(awsS3Bucket, image, manifestBuffer);
-          }
+  await Promise.all(
+    chunks(Array.from(Array(SIZE).keys()), batchSize || 1000).map(
+      async allIndexesInSlice => {
+        for (let ind = 0; ind < allIndexesInSlice.length; ind++) {
+          const i = allIndexesInSlice[ind];
+          const image = images[i];
+          const imageName = path.basename(image);
+          const index = imageName.replace(EXTENSION_PNG, '');
+
+          log.debug(`Processing file: ${i}`);
+
+          let link = cacheContent?.items?.[index]?.link;
+          let imageLink = cacheContent?.items?.[index]?.imageLink;
+          if (!link || !cacheContent.program.uuid) {
+            if (i >= lastPrinted + tick || i === 0) {
+              lastPrinted = i;
+              log.info(`Processing file: ${i}, ${imageName}`);
+            }
+            const manifestPath = image.replace(EXTENSION_PNG, EXTENSION_JSON);
+            const manifestContent = fs
+              .readFileSync(manifestPath)
+              .toString()
+              .replace(imageName, 'image.png')
+              .replace(imageName, 'image.png');
+            const manifest = JSON.parse(manifestContent);
+
+            const manifestBuffer = Buffer.from(JSON.stringify(manifest));
+
+            if (i === 0 && !cacheContent.program.uuid) {
+              // initialize config
+              log.info(`initializing config`);
+              try {
+                const res = await createConfig(anchorProgram, walletKeyPair, {
+                  maxNumberOfLines: new BN(totalNFTs),
+                  symbol: manifest.symbol,
+                  sellerFeeBasisPoints: manifest.seller_fee_basis_points,
+                  isMutable: mutable,
+                  maxSupply: new BN(0),
+                  retainAuthority: retainAuthority,
+                  creators: manifest.properties.creators.map(creator => {
+                    return {
+                      address: new PublicKey(creator.address),
+                      verified: true,
+                      share: creator.share,
+                    };
+                  }),
+                });
+                cacheContent.program.uuid = res.uuid;
+                cacheContent.program.config = res.config.toBase58();
+                config = res.config;
+
+                log.info(
+                  `initialized config for a candy machine with publickey: ${res.config.toBase58()}`,
+                );
+
+                saveCache(cacheName, env, cacheContent);
+              } catch (exx) {
+                log.error('Error deploying config to Solana network.', exx);
+                throw exx;
+              }
+            }
 
-          if (link && imageLink) {
-            log.debug('setting cache for ', index);
-            cacheContent.items[index] = {
-              link,
-              imageLink,
-              name: manifest.name,
-              onChain: false,
-            };
-            cacheContent.authority = walletKeyPair.publicKey.toBase58();
-            saveCache(cacheName, env, cacheContent);
+            if (!link) {
+              try {
+                if (storage === 'arweave') {
+                  [link, imageLink] = await arweaveUpload(
+                    walletKeyPair,
+                    anchorProgram,
+                    env,
+                    image,
+                    manifestBuffer,
+                    manifest,
+                    index,
+                  );
+                } else if (storage === 'ipfs') {
+                  [link, imageLink] = await ipfsUpload(
+                    ipfsCredentials,
+                    image,
+                    manifestBuffer,
+                  );
+                } else if (storage === 'aws') {
+                  [link, imageLink] = await awsUpload(
+                    awsS3Bucket,
+                    image,
+                    manifestBuffer,
+                  );
+                }
+
+                if (link && imageLink) {
+                  log.debug('setting cache for ', index);
+                  cacheContent.items[index] = {
+                    link,
+                    imageLink,
+                    name: manifest.name,
+                    onChain: false,
+                  };
+                  cacheContent.authority = walletKeyPair.publicKey.toBase58();
+                  saveCache(cacheName, env, cacheContent);
+                }
+              } catch (er) {
+                uploadSuccessful = false;
+                log.error(`Error uploading file ${index}`, er);
+              }
+            }
+          } else {
+            log.debug(`file ${index} already has a link`);
           }
-        } catch (er) {
-          uploadSuccessful = false;
-          log.error(`Error uploading file ${index}`, er);
         }
-      }
-    } else {
-      log.debug(`file ${index} already has a link`);
-    }
-  }
+      },
+    ),
+  );
+  saveCache(cacheName, env, cacheContent);
 
   const keys = Object.keys(cacheContent.items);
   try {