Utilize ZipStorage (sourcecred#2858)

Description

For now, this class will be used to compress weighted graphs and cred
graphs. these artifacts aren't especially readable on disk and also take
up a 10s of MBs. This will allow larger instances to continue utilizing
Github for the foreseeable future. If further optimization is needed, a
higher-order ShardedStorage class can be implemented and used.

builds towards sourcecred#2768

Test Plan
`yarn build`

in a local instance:
1. `scDev go`
2. `scDev grain`
3. `scDev serve`

open http://localhost:6006 and verify that the frontends load as
expected.

src/api/instance/localInstance.js

@@ -7,11 +7,11 @@ import {
   parser as weightsParser,
   empty as emptyWeights,
 } from "../../core/weights";
-import fs from "fs-extra";
 import {join as pathJoin} from "path";
 import stringify from "json-stable-stringify";
 import {
   type WeightedGraph,
+  type WeightedGraphJSON,
   fromJSON as weightedGraphFromJSON,
 } from "../../core/weightedGraph";
 import {
@@ -33,7 +33,9 @@ import {
   parser as credGraphParser,
 } from "../../core/credrank/credGraph";
 import {DiskStorage} from "../../core/storage/disk";
+import {WritableZipStorage} from "../../core/storage/zip";
 import {encode} from "../../core/storage/textEncoding";
+import * as Combo from "../../util/combo";
 
 const DEPENDENCIES_PATH: $ReadOnlyArray<string> = [
   "config",
@@ -45,9 +47,12 @@ const WEIGHT_OVERRIDES_PATH: $ReadOnlyArray<string> = [
 ];
 const BUDGET_PATH: $ReadOnlyArray<string> = ["config", "pluginBudgets.json"];
 const INSTANCE_CONFIG_PATH: $ReadOnlyArray<string> = ["sourcecred.json"];
-const CREDGRAPH_PATH: $ReadOnlyArray<string> = ["output", "credGraph.json"];
+const CREDGRAPH_PATH: $ReadOnlyArray<string> = [
+  "output",
+  "credGraph.json.gzip",
+];
 const GRAPHS_DIRECTORY: $ReadOnlyArray<string> = ["output", "graphs"];
-const GRAPHS_PATH: $ReadOnlyArray<string> = ["graph.json"];
+const GRAPHS_PATH: $ReadOnlyArray<string> = ["graph.json.gzip"];
 const LEDGER_PATH: $ReadOnlyArray<string> = ["data", "ledger.json"];
 
 /**
@@ -128,7 +133,11 @@ export class LocalInstance implements Instance {
       pluginNames.map(async (name) => {
         const outputDir = this.createPluginDirectory(name);
         const outputPath = pathJoin(outputDir, ...GRAPHS_PATH);
-        const graphJSON = JSON.parse(await fs.readFile(outputPath));
+        const graphJSON = await loadJson(
+          new WritableZipStorage(this._storage),
+          outputPath,
+          ((Combo.raw: any): Combo.Parser<WeightedGraphJSON>)
+        );
         return weightedGraphFromJSON(graphJSON);
       })
     );
@@ -176,7 +185,7 @@ export class LocalInstance implements Instance {
       path = pathJoin(path, pc);
       mkdirx(path);
     }
-    return path;
+    return pathJoin(...pathComponents);
   }
 
   async writeLedger(ledger: Ledger): Promise<void> {
@@ -187,7 +196,8 @@ export class LocalInstance implements Instance {
   async writeCredGraph(credGraph: CredGraph): Promise<void> {
     const cgJson = stringify(credGraph.toJSON());
     const outputPath = pathJoin(...CREDGRAPH_PATH);
-    await this._storage.set(outputPath, encode(cgJson));
+    const zipStorage = new WritableZipStorage(this._storage);
+    await zipStorage.set(outputPath, encode(cgJson));
   }
 
   async writeDependenciesConfig(

src/cli/common.js

@@ -9,6 +9,7 @@ import stringify from "json-stable-stringify";
 
 import {type BonusPolicy} from "../core/bonusMinting";
 import {DiskStorage} from "../core/storage/disk";
+import {ZipStorage} from "../core/storage/zip";
 import {WritableDataStorage} from "../core/storage/index";
 import type {PluginDirectoryContext} from "../api/plugin";
 import {
@@ -191,8 +192,8 @@ export async function loadAndMergePluginWeigtedGraphs(
 }
 
 export async function loadCredGraph(baseDir: string): Promise<CredGraph> {
-  const storage = new DiskStorage(baseDir);
-  const credGraphPath = pathJoin("output", "credGraph.json");
+  const storage = new ZipStorage(new DiskStorage(baseDir));
+  const credGraphPath = pathJoin("output", "credGraph.json.gzip");
   return await loadJson(storage, credGraphPath, credGraphParser);
 }
 

src/cli/graph.js

@@ -33,6 +33,7 @@ import {
 } from "../core/weightedGraph";
 import * as pluginId from "../api/pluginId";
 import {DiskStorage} from "../core/storage/disk";
+import {WritableZipStorage} from "../core/storage/zip";
 import {encode} from "../core/storage/textEncoding";
 import {ensureIdentityExists} from "../core/ledger/identityProposal";
 
@@ -127,8 +128,8 @@ const graphCommand: Command = async (args, std) => {
         stringify(weightedGraphToJSON(weightedGraph))
       );
       const outputDir = makePluginDir(baseDir, graphOutputPrefix, name);
-      const graphStorage = new DiskStorage(outputDir);
-      await graphStorage.set("graph.json", serializedGraph);
+      const graphStorage = new WritableZipStorage(new DiskStorage(outputDir));
+      await graphStorage.set("graph.json.gzip", serializedGraph);
       taskReporter.finish(writeTask);
     }
   }

src/core/storage/zip.js

@@ -30,7 +30,7 @@ export class WritableZipStorage
     this._baseStorage = baseStorage;
   }
 
-  async set(key: string, value: Uint8Array) {
+  async set(key: string, value: Uint8Array): Promise<void> {
     this._baseStorage.set(key, deflate(value));
   }
 }

src/ui/load.js

@@ -14,6 +14,7 @@ import {rawParser as rawInstanceConfigParser} from "../api/rawInstanceConfig";
 import {createLedgerDiskStorage} from "./utils/ledgerDiskStorage";
 import * as Combo from "../util/combo";
 import {NetworkStorage} from "../core/storage/network";
+import {ZipStorage} from "../core/storage/zip";
 import {loadJson, loadJsonWithDefault} from "../util/storage";
 
 export type LoadResult = LoadSuccess | LoadFailure;
@@ -54,8 +55,8 @@ export async function load(): Promise<LoadResult> {
       defaultCurrencyConfig
     ),
     loadJsonWithDefault(
-      networkStorage,
-      "output/credGraph.json",
+      new ZipStorage(networkStorage),
+      "output/credGraph.json.gzip",
       Combo.fmap(credGraphJsonParser, (graphJson) =>
         CredGraph.fromJSON(graphJson)
       ),