Generalize disk utils as storage Utils for use in browser (sourcecred…

…#2840)

The loadJson and loadFile helper functions now accept cross-platform
DataStorage implementations. They should now package for use in
in browser contexts. By moving them into their own file, this is now
achievable since the 'fs' node imports can stay in the disk.js file,
which still contains node-specific helpers.

test plan:
Unit tests have been migrated and flow typechecks successfully.

src/api/instance/localInstance.js

@@ -16,10 +16,10 @@ import {
 } from "../../core/weightedGraph";
 import {
   loadJson,
-  mkdirx,
   loadFileWithDefault,
   loadJsonWithDefault,
-} from "../../util/disk";
+} from "../../util/storage";
+import {mkdirx} from "../../util/disk";
 import {parser as configParser, type InstanceConfig} from "../instanceConfig";
 import {Ledger} from "../../core/ledger/ledger";
 import {

src/cli/common.js

@@ -2,7 +2,8 @@
 
 import {join as pathJoin} from "path";
 import fs from "fs-extra";
-import {loadJson, mkdirx} from "../util/disk";
+import {mkdirx} from "../util/disk";
+import {loadJson} from "../util/storage";
 import deepEqual from "lodash.isequal";
 import stringify from "json-stable-stringify";
 
@@ -28,7 +29,7 @@ import {
   fromJSON as weightedGraphFromJSON,
 } from "../core/weightedGraph";
 import {CredGraph, parser as credGraphParser} from "../core/credrank/credGraph";
-import {loadFileWithDefault, loadJsonWithDefault} from "../util/disk";
+import {loadFileWithDefault, loadJsonWithDefault} from "../util/storage";
 import {parser as pluginBudgetParser} from "../api/pluginBudgetConfig";
 import {applyBudget, type Budget} from "../core/mintBudget";
 import {

src/cli/grain.js

@@ -1,7 +1,7 @@
 // @flow
 
 import {join} from "path";
-import {loadFileWithDefault, loadJson} from "../util/disk";
+import {loadFileWithDefault, loadJson} from "../util/storage";
 import {Ledger} from "../core/ledger/ledger";
 import {applyDistributions2 as applyDistributions} from "../core/ledger/applyDistributions";
 import {computeCredAccounts2 as computeCredAccounts} from "../core/ledger/credAccounts";

src/cli/grain2.js

@@ -1,7 +1,7 @@
 // @flow
 
 import {join} from "path";
-import {loadFileWithDefault, loadJson} from "../util/disk";
+import {loadFileWithDefault, loadJson} from "../util/storage";
 import {Ledger} from "../core/ledger/ledger";
 import {applyDistributions2} from "../core/ledger/applyDistributions";
 import {computeCredAccounts2} from "../core/ledger/credAccounts";

src/cli/score.js

@@ -5,7 +5,8 @@ import stringify from "json-stable-stringify";
 
 import type {Command} from "./command";
 import {loadInstanceConfig, prepareCredData} from "./common";
-import {loadJsonWithDefault, mkdirx} from "../util/disk";
+import {mkdirx} from "../util/disk";
+import {loadJsonWithDefault} from "../util/storage";
 import dedent from "../util/dedent";
 import {LoggingTaskReporter} from "../util/taskReporter";
 import {

src/plugins/discord/plugin.js

@@ -21,7 +21,7 @@ import {
   type PluginId,
   fromString as pluginIdFromString,
 } from "../../api/pluginId";
-import {loadJson} from "../../util/disk";
+import {loadJson} from "../../util/storage";
 import {DiskStorage} from "../../core/storage/disk";
 import {createIdentities} from "./createIdentities";
 import type {IdentityProposal} from "../../core/ledger/identityProposal";

src/plugins/discourse/plugin.js

@@ -19,7 +19,7 @@ import {
   type PluginId,
   fromString as pluginIdFromString,
 } from "../../api/pluginId";
-import {loadJson} from "../../util/disk";
+import {loadJson} from "../../util/storage";
 import {DiskStorage} from "../../core/storage/disk";
 import {createIdentities} from "./createIdentities";
 import type {IdentityProposal} from "../../core/ledger/identityProposal";

src/plugins/ethereum/plugin.js

@@ -12,7 +12,7 @@ import {
   type PluginId,
   fromString as pluginIdFromString,
 } from "../../api/pluginId";
-import {loadJson} from "../../util/disk";
+import {loadJson} from "../../util/storage";
 import {DiskStorage} from "../../core/storage/disk";
 import {
   empty as emptyWeightedGraph,

src/plugins/github/plugin.js

@@ -25,7 +25,7 @@ import {
   type PluginId,
   fromString as pluginIdFromString,
 } from "../../api/pluginId";
-import {loadJson} from "../../util/disk";
+import {loadJson} from "../../util/storage";
 import {createIdentities} from "./createIdentities";
 import {DiskStorage} from "../../core/storage/disk";
 import type {IdentityProposal} from "../../core/ledger/identityProposal";

src/plugins/initiatives/initiativesDirectory.js

@@ -21,7 +21,7 @@ import {
   initiativeFileURL,
   initiativeFileId,
 } from "./initiativeFile";
-import {loadJson} from "../../util/disk";
+import {loadJson} from "../../util/storage";
 import {DiskStorage} from "../../core/storage/disk";
 import {parser as initiativeParser} from "./parseInitiative";
 

src/plugins/initiatives/plugin.js

@@ -12,7 +12,7 @@ import {
   type PluginId,
   fromString as pluginIdFromString,
 } from "../../api/pluginId";
-import {loadJson} from "../../util/disk";
+import {loadJson} from "../../util/storage";
 import {loadDirectory as _loadDirectory} from "./initiativesDirectory";
 import * as Weights from "../../core/weights";
 import type {IdentityProposal} from "../../core/ledger/identityProposal";

src/util/disk.js

@@ -1,8 +1,6 @@
 // @flow
 
 import fs from "fs-extra";
-import {DataStorage} from "../core/storage";
-import * as P from "./combo";
 
 /**
  * Make a directory, if it doesn't already exist.
@@ -17,90 +15,6 @@ export function mkdirx(path: string) {
   }
 }
 
-/**
- * Load and parse a JSON file from disk.
- *
- * If the file cannot be read, then an error is thrown.
- * If parsing fails, an error is thrown.
- */
-export async function loadJson<T>(
-  storage: DataStorage,
-  path: string,
-  parser: P.Parser<T>
-): Promise<T> {
-  const contents = await storage.get(path);
-  return parser.parseOrThrow(JSON.parse(contents.toString()));
-}
-
-/**
- * Load and parse a JSON file from disk, with a default to use if the file is
- * not found.
- *
- * This is intended as a convenience for situations where the user may
- * optionally provide configuration in a json file saved to disk.
- *
- * The default must be provided as a function that returns a default, to
- * accommodate situations where the object may be mutable, or where constructing
- * the default may be expensive.
- *
- * If no file is present at that location, then the default constructor is
- * invoked to create a default value, and that is returned.
- *
- * If attempting to load the file fails for any reason other than ENOENT
- * (e.g. the path actually is a directory), then the error is thrown.
- *
- * If parsing fails, an error is thrown.
- */
-export async function loadJsonWithDefault<T>(
-  storage: DataStorage,
-  path: string,
-  parser: P.Parser<T>,
-  def: () => T
-): Promise<T> {
-  try {
-    const contents = await storage.get(path);
-    return parser.parseOrThrow(JSON.parse(contents.toString()));
-  } catch (e) {
-    if (e.code === "ENOENT") {
-      return def();
-    } else {
-      throw e;
-    }
-  }
-}
-
-/**
- * Read a text file from disk, with a default string value to use if the
- * file is not found. The file is read in the default encoding, UTF-8.
- *
- * This is intended as a convenience for situations where the user may
- * optionally provide configuration in a non-JSON file saved to disk.
- *
- * The default must be provided as a function that returns a default, in
- * case constructing the default may be expensive.
- *
- * If no file is present at that location, then the default constructor is
- * invoked to create a default value, and that is returned.
- *
- * If attempting to load the file fails for any reason other than ENOENT
- * (e.g. the path actually is a directory), then the error is thrown.
- */
-export async function loadFileWithDefault(
-  storage: DataStorage,
-  path: string,
-  def: () => string
-): Promise<string> {
-  try {
-    return (await storage.get(path)).toString();
-  } catch (e) {
-    if (e.code === "ENOENT") {
-      return def();
-    } else {
-      throw e;
-    }
-  }
-}
-
 /**
  * Check if a directory is empty
  *

src/util/disk.test.js

@@ -1,117 +1,11 @@
 // @flow
 
-import {
-  loadFileWithDefault,
-  loadJsonWithDefault,
-  loadJson,
-  mkdirx,
-  isDirEmpty,
-} from "./disk";
+import {mkdirx, isDirEmpty} from "./disk";
 import tmp from "tmp";
 import fs from "fs-extra";
-import * as P from "./combo";
 import {join as pathJoin} from "path";
-import {encode} from "../core/storage/textEncoding";
-import {DiskStorage} from "../core/storage/disk";
 
 describe("util/disk", () => {
-  describe("loadJson / loadJsonWithDefault", () => {
-    function tmpWithContents(contents: mixed, dir: string = basedir.name) {
-      const name = tmp.tmpNameSync({dir});
-      fs.writeFileSync(name, JSON.stringify(contents));
-      const fname = name.split("/").pop();
-      return fname;
-    }
-    const basedir = tmp.dirSync();
-    const storage = new DiskStorage(basedir.name);
-    const badPath = () => pathJoin(basedir.name, "not-a-real-path");
-    const fooParser = P.object({foo: P.number});
-    const fooInstance = Object.freeze({foo: 42});
-    const fooDefault = () => ({foo: 1337});
-    const barInstance = Object.freeze({bar: "1337"});
-    it("loadJson works when valid file is present", async () => {
-      const f = tmpWithContents(fooInstance);
-      await expect(await loadJson(storage, f, fooParser)).toEqual(fooInstance);
-    });
-    it("loadJson errors if the path does not exist", async () => {
-      const fail = async () => await loadJson(storage, badPath(), fooParser);
-      await expect(fail).rejects.toThrow("ENOENT");
-    });
-    it("loadJson errors if the combo parse fails", async () => {
-      const f = tmpWithContents(barInstance);
-      const fail = async () => await loadJson(storage, f, fooParser);
-      await expect(fail).rejects.toThrow("missing key");
-    });
-    it("loadJson errors if JSON.parse fails", async () => {
-      const f = tmpWithContents("");
-      await storage.set(f, encode("zzz"));
-      const fail = async () => await loadJson(storage, f, P.raw);
-      await expect(fail).rejects.toThrow();
-    });
-    it("loadJsonWithDefault works when valid file is present", async () => {
-      const f = tmpWithContents(fooInstance);
-      console.log(storage._basePath, f);
-      await expect(
-        await loadJsonWithDefault(storage, f, fooParser, fooDefault)
-      ).toEqual(fooInstance);
-    });
-    it("loadJsonWithDefault loads default if file not present", async () => {
-      await expect(
-        await loadJsonWithDefault(storage, badPath(), fooParser, fooDefault)
-      ).toEqual(fooDefault());
-    });
-    it("loadJsonWithDefault errors if parse fails", async () => {
-      const f = tmpWithContents(barInstance);
-      const fail = async () =>
-        await loadJsonWithDefault(storage, f, fooParser, fooDefault);
-      await expect(fail).rejects.toThrow("missing key");
-    });
-    it("loadJsonWithDefault errors if JSON.parse fails", async () => {
-      const f = tmpWithContents("");
-      await storage.set(f, encode("zzz"));
-      const fail = async () =>
-        await loadJsonWithDefault(storage, f, P.raw, fooDefault);
-      await expect(fail).rejects.toThrow();
-    });
-    it("loadJsonWithDefault errors if file loading fails for a non-ENOENT reason", async () => {
-      const fail = async () =>
-        await loadJsonWithDefault(storage, "", fooParser, fooDefault);
-      await expect(fail).rejects.toThrow("EISDIR");
-    });
-  });
-
-  describe("loadFileWithDefault", () => {
-    const badPath = () => pathJoin(tmp.dirSync().name, "not-a-real-path");
-    const unreachable = () => {
-      throw new Error("Should not get here");
-    };
-    function tmpWithData(data: string, dir: string = basedir.name) {
-      const name = tmp.tmpNameSync({dir});
-      fs.writeFileSync(name, data);
-      const fname = name.split("/").pop();
-      return fname;
-    }
-    const basedir = tmp.dirSync();
-    const storage = new DiskStorage(basedir.name);
-    it("works when valid file is present", async () => {
-      const f = tmpWithData("hello\n");
-      expect(await loadFileWithDefault(storage, f, unreachable)).toEqual(
-        "hello\n"
-      );
-    });
-    it("loads default if file not present", async () => {
-      expect(
-        await loadFileWithDefault(storage, badPath(), () => "backup")
-      ).toEqual("backup");
-    });
-    it("errors if file loading fails for a non-ENOENT reason", async () => {
-      const directoryPath = "";
-      const fail = async () =>
-        await loadFileWithDefault(storage, directoryPath, unreachable);
-      await expect(fail).rejects.toThrow("EISDIR");
-    });
-  });
-
   describe("mkdirx", () => {
     it("makes the directory if it doesn't exist", () => {
       const name = tmp.tmpNameSync();