refactor(database-client): add new package database-client for databa…

…se implementations

Explanation: Ceramic v2 and related libraries now support ESM modules. However, Jest testing is
extremely finicky when there is a mix of CommonJS and ESM dependencies.
 1) we attempted to
transpile ESM dependencies into CJS when Jest tests are executed; however we ran into many issues
where the modules were simply not transpiled, or Jest was unable to load the transpiled modules,
etc. We tried transpilation with both NextJS 12 default compilation (SWC), as well as Babel - both
methods were unsuccessful.
 2) we attempted to turn on ESM support with Jest; however this is
still unsupported even if we were to use the latest version (jest v28), and Jest docs note that Jest
mocks are difficult to set up when ESM-jest is enabled.
 3) We decided the cleanest course of
action is to isolate any unmockable Ceramic-related implementation into its own module -- in this
case, our custom database interface with Ceramic. In this new package (database-client), we can set
up ESM-jest to allow for integration testing with a real Ceramic node, and also mock database-client
whenever it is imported into other packages (app).

[passportxyz#36]

app/__mocks__/@dpopp/database-client/src/index.js

@@ -0,0 +1,9 @@
+export const createPassportMock = jest.fn();
+export const getPassportMock = jest.fn();
+export const addStampMock = jest.fn();
+
+export class CeramicDatabase {
+  constructor() {
+    return { createPassport: createPassportMock, getPassport: getPassportMock, addStamp: addStampMock };
+  }
+}

app/package.json

@@ -14,11 +14,13 @@
   },
   "dependencies": {
     "@chakra-ui/react": "^1.8.8",
+    "@dpopp/database-client": "^0.0.1",
     "@dpopp/identity": "^0.0.1",
     "@dpopp/types": "^0.0.1",
     "@emotion/react": "^11.9.0",
     "@emotion/styled": "^11.8.1",
     "@ethersproject/providers": "^5.6.4",
+    "@self.id/framework": "^0.3.0",
     "@web3-onboard/core": "^2.2.5",
     "@web3-onboard/injected-wallets": "^2.0.5",
     "@web3-onboard/ledger": "^2.0.3",
@@ -33,6 +35,9 @@
     "react-router-dom": "^6.3.0"
   },
   "devDependencies": {
+    "@ceramicnetwork/common": "^2.0.0",
+    "@glazed/did-datastore-model": "^0.2.0",
+    "@glazed/types": "^0.2.0",
     "@next/eslint-plugin-next": "^12.1.5",
     "@testing-library/jest-dom": "^5.16.4",
     "@testing-library/react": "^12.0.0",
@@ -63,6 +68,7 @@
   },
   "resolutions": {
     "csstype": "3.0.10",
-    "**/@types/react": "17.0.2"
+    "**/@types/react": "17.0.2",
+    "leveldown": "6.1.1"
   }
 }

app/services/databaseStorage.ts

@@ -1,5 +1,8 @@
 import { DID, Passport, Stamp } from "@dpopp/types";
 import { DataStorageBase } from "../types";
+import { CeramicDatabase } from "@dpopp/database-client/src";
+
+const placeholderCeramic = new CeramicDatabase();
 
 export class LocalStorageDatabase implements DataStorageBase {
   passportKey: string;

database-client/.gitignore

@@ -0,0 +1,40 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# build
+/dist
+
+# testing
+/coverage
+
+# production
+/build
+
+# ceramic
+.pinning.store
+
+# misc
+.DS_Store
+.env
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+.eslintcache
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

database-client/integration-tests/ceramicDatabaseTest.ts

@@ -0,0 +1,90 @@
+import { Passport } from "@dpopp/types";
+import { DID } from "dids";
+import { Ed25519Provider } from "key-did-provider-ed25519";
+import { getResolver } from "key-did-resolver";
+
+import { CeramicDatabase } from "../src/ceramicClient";
+
+let testDID: DID;
+let ceramicDatabase: CeramicDatabase;
+
+beforeAll(async () => {
+  const TEST_SEED = new Uint8Array([
+    6, 190, 125, 152, 83, 9, 111, 202, 6, 214, 218, 146, 104, 168, 166, 110, 202, 171, 42, 114, 73, 204, 214, 60, 112,
+    254, 173, 151, 170, 254, 250, 2,
+  ]);
+
+  // Create and authenticate the DID
+  testDID = new DID({
+    provider: new Ed25519Provider(TEST_SEED),
+    resolver: getResolver(),
+  });
+  await testDID.authenticate();
+
+  ceramicDatabase = new CeramicDatabase(testDID);
+});
+
+afterAll(async () => {
+  await ceramicDatabase.store.remove("Passport");
+});
+
+describe("when there is no passport for the given did", () => {
+  beforeEach(async () => {
+    await ceramicDatabase.store.remove("Passport");
+  });
+
+  it("createPassport creates a passport in ceramic", async () => {
+    const actualPassportStreamID = await ceramicDatabase.createPassport();
+
+    expect(actualPassportStreamID).toBeDefined();
+
+    const storedPassport = (await ceramicDatabase.loader.load(actualPassportStreamID)).content;
+    console.log("Stored passport: ", JSON.stringify(storedPassport.content));
+
+    const formattedDate = new Date(storedPassport["issuanceDate"]);
+    const todaysDate = new Date();
+
+    expect(formattedDate.getDay).toEqual(todaysDate.getDay);
+    expect(formattedDate.getMonth).toEqual(todaysDate.getMonth);
+    expect(formattedDate.getFullYear).toEqual(todaysDate.getFullYear);
+    expect(storedPassport["stamps"]).toEqual([]);
+  });
+
+  it("getPassport returns undefined", async () => {
+    const actualPassport = await ceramicDatabase.getPassport();
+
+    expect(actualPassport).toEqual(undefined);
+  });
+
+  it("getPassport returns undefined for invalid stream id", async () => {
+    const actualPassport = await ceramicDatabase.getPassport("bad id");
+
+    expect(actualPassport).toEqual(undefined);
+  });
+});
+
+describe("when there is an existing passport for the given did", () => {
+  const existingPassport: Passport = {
+    issuanceDate: new Date("2022-01-01"),
+    expiryDate: new Date("2022-01-02"),
+    stamps: [],
+  };
+
+  let existingPassportStreamID;
+  beforeEach(async () => {
+    // actualPassportStreamID = await ceramicDatabase.createPassport();
+    const stream = await ceramicDatabase.store.set("Passport", existingPassport);
+    existingPassportStreamID = stream.toUrl();
+  });
+
+  afterEach(async () => {
+    await ceramicDatabase.store.remove("Passport");
+  });
+
+  it("getPassport retrieves the passport from ceramic given the stream id", async () => {
+    const actualPassport = await ceramicDatabase.getPassport(existingPassportStreamID);
+
+    expect(actualPassport).toBeDefined();
+    expect(actualPassport).toEqual(existingPassport);
+  });
+});

database-client/jest.integration.config.js

@@ -0,0 +1,5 @@
+export default {
+  transform: {},
+  testMatch: ["**/integration-tests/**/*.js"],
+  extensionsToTreatAsEsm: [".ts"],
+};

database-client/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@dpopp/database-client",
+  "version": "0.0.1",
+  "license": "MIT",
+  "type": "module",
+  "main": "src/index.js",
+  "directories": {
+    "src": "src",
+    "dist": "dist"
+  },
+  "files": [
+    "src",
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc",
+    "clean": "rimraf dist node_modules",
+    "ceramic": "ceramic daemon",
+    "test:integration": "yarn build && yarn node --experimental-vm-modules $(yarn bin jest) -c jest.integration.config.js"
+  },
+  "dependencies": {
+    "@ceramicnetwork/http-client": "^2.0.0",
+    "@dpopp/schemas": "0.0.1",
+    "@glazed/datamodel": "^0.3.0",
+    "@glazed/did-datastore": "^0.3.0",
+    "@glazed/tile-loader": "^0.2.0",
+    "dids": "^3.0.0",
+    "dotenv": "^16.0.0",
+    "key-did-provider-ed25519": "^2.0.0",
+    "key-did-resolver": "^2.0.0",
+    "uint8arrays": "^3.0.0"
+  },
+  "devDependencies": {
+    "@ceramicnetwork/common": "^2.0.0",
+    "@ceramicnetwork/stream-tile": "^2.0.0",
+    "@glazed/devtools": "^0.1.6",
+    "@glazed/did-datastore-model": "^0.2.0",
+    "@glazed/types": "^0.2.0",
+    "@types/node": "^16.11.6",
+    "jest": "^27.5.1"
+  },
+  "resolutions": {
+    "leveldown": "6.1.1"
+  }
+}

database-client/src/ceramicClient.ts

@@ -0,0 +1,105 @@
+import { DID, Passport, Stamp, VerifiableCredential } from "@dpopp/types";
+
+// -- Ceramic and Glazed
+import type { CeramicApi } from "@ceramicnetwork/common";
+import { CeramicClient } from "@ceramicnetwork/http-client";
+import publishedModel from "@dpopp/schemas/scripts/publish-model.json";
+import { DataModel } from "@glazed/datamodel";
+import { DIDDataStore } from "@glazed/did-datastore";
+import { TileLoader } from "@glazed/tile-loader";
+import type { DID as CeramicDID } from "dids";
+
+import { DataStorageBase } from "./types";
+
+// TODO read this in from env
+const CERAMIC_CLIENT_URL = "http://localhost:7007";
+
+type CeramicStamp = {
+  provider: string;
+  credential: string;
+};
+type CeramicPassport = {
+  issuanceDate: string;
+  expiryDate: string;
+  stamps: CeramicStamp[];
+};
+
+export type ModelTypes = {
+  schemas: {
+    Passport: CeramicPassport;
+    VerifiableCredential: VerifiableCredential;
+  };
+  definitions: {
+    Passport: "Passport";
+    VerifiableCredential: "VerifiableCredential";
+  };
+  tiles: {};
+};
+
+export class CeramicDatabase implements DataStorageBase {
+  loader: TileLoader;
+  ceramicClient: CeramicApi;
+  model: DataModel<ModelTypes>;
+  store: DIDDataStore<ModelTypes>;
+
+  constructor(did?: CeramicDID) {
+    // Create the Ceramic instance and inject the DID
+    const ceramic = new CeramicClient(CERAMIC_CLIENT_URL);
+    // @ts-ignore
+    ceramic.setDID(did);
+    console.log("Current ceramic did: ", ceramic.did?.id);
+
+    // Create the loader, model and store
+    const loader = new TileLoader({ ceramic });
+    const model = new DataModel({ ceramic, aliases: publishedModel });
+    const store = new DIDDataStore({ loader, ceramic, model });
+
+    this.loader = loader;
+    this.ceramicClient = ceramic;
+    this.model = model;
+    this.store = store;
+  }
+
+  async createPassport(): Promise<DID> {
+    const date = new Date();
+    const newPassport: CeramicPassport = {
+      issuanceDate: date.toISOString(),
+      expiryDate: date.toISOString(),
+      stamps: [],
+    };
+    // const passportTile = await this.model.createTile("Passport", newPassport);
+    // console.log("Created passport tile: ", JSON.stringify(passportTile.id.toUrl()));
+    const stream = await this.store.set("Passport", { ...newPassport });
+    console.log("Set Passport: ", JSON.stringify(stream.toUrl()));
+    return stream.toUrl();
+  }
+  async getPassport(did?: DID): Promise<Passport | undefined> {
+    try {
+      const passport = await this.store.get("Passport");
+      console.log("Loaded passport: ", JSON.stringify(passport));
+      // // `stamps` is stored as ceramic URLs - must load actual VC data from URL
+      // const stampsToLoad =
+      //   passport?.stamps.map(async (_stamp) => {
+      //     const { provider, credential } = _stamp;
+      //     const loadedCred = await this.loader.load(credential);
+      //     return {
+      //       provider,
+      //       credential: loadedCred.content,
+      //     } as Stamp;
+      //   }) ?? [];
+      // const loadedStamps = await Promise.all(stampsToLoad);
+
+      return undefined;
+    } catch (e) {
+      console.error(e);
+      return undefined;
+    }
+  }
+  async addStamp(did: DID, stamp: Stamp): Promise<void> {
+    console.log("add stamp ceramic");
+  }
+
+  async deletePassport(): Promise<void> {
+    console.log("remove passport");
+  }
+}

database-client/src/index.ts

@@ -0,0 +1 @@
+export * from "./ceramicClient";

database-client/src/types.ts

@@ -0,0 +1,8 @@
+import { Passport, Stamp, DID } from "@dpopp/types";
+
+// Class used as a base for each DataStorage Type
+export abstract class DataStorageBase {
+  abstract createPassport(): Promise<DID>;
+  abstract getPassport(did: DID): Promise<Passport | undefined>;
+  abstract addStamp(did: DID, stamp: Stamp): Promise<void>;
+}

database-client/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "extends": "../tsconfig.settings.json",
+  "compilerOptions": {
+    "module": "esnext",
+    "esModuleInterop": true,
+    "declaration": true,
+    "allowSyntheticDefaultImports": true,
+    "target": "es5",
+    "moduleResolution": "node",
+    "sourceMap": true,
+    "outDir": "dist/esm",
+    "allowJs": true,
+    "baseUrl": "src",
+    "paths": {
+      "*": ["../node_modules/*", "node_modules/*"]
+    },
+    "skipLibCheck": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src/*", "___test___", "integration-tests"]
+}

lerna.json

@@ -1,5 +1,5 @@
 {
-  "packages": ["app", "schemas", "iam", "identity", "types"],
+  "packages": ["app", "database-client", "iam", "identity", "schemas", "types"],
   "npmClient": "yarn",
   "version": "independent",
   "useWorkspaces": true,