Implement publish command to upload project to IPFS (subquery#486)

* Implement publish command to upload project to IPFS

* Revert jest config change

* Fix eslint issues

* Fix copyright header

* Minify and sort keys for yaml files when publishing

* Rework publish to use manifest as entry point rather than directory

* Explicitly use cid v0

* Update validator and cli  command to support validating projects published to ipfs

* Update publishing to support project manifest spec 0.2.0

* Update README.md

* Clean up

* Fix test

* Update ipfs endpoints for tests

* Increase cli publish test timeout

* Increase timeout for db module tests

* Fix build issues

jest.config.js

@@ -193,7 +193,9 @@ module.exports = {
   //   "\\.pnp\\.[^\\/]+$"
   // ],
 
-  transformIgnorePatterns: ['node_modules/(?!(@polkadot|@babel/runtime/helpers/esm)/)'],
+  "transformIgnorePatterns": [
+    "node_modules/(?!(@polkadot|@babel/runtime/helpers/esm)/)"
+  ],
 
   // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
   // unmockedModulePathPatterns: undefined,

packages/cli/README.md

@@ -8,39 +8,36 @@ cli for polkagraph
 [![License](https://img.shields.io/npm/l/@subql/cli.svg)](https://github.com/packages/cli/blob/master/package.json)
 
 <!-- toc -->
-
-- [@subql/cli](#subqlcli)
-- [Usage](#usage)
-- [Commands](#commands)
+* [@subql/cli](#subqlcli)
+* [Usage](#usage)
+* [Commands](#commands)
 <!-- tocstop -->
 
 # Usage
 
 <!-- usage -->
-
 ```sh-session
 $ npm install -g @subql/cli
 $ subql COMMAND
 running command...
 $ subql (-v|--version|version)
-@subql/cli/0.13.1-0 linux-x64 node-v14.18.1
+@subql/cli/0.13.1-0 darwin-x64 node-v14.15.1
 $ subql --help [COMMAND]
 USAGE
   $ subql COMMAND
 ...
 ```
-
 <!-- usagestop -->
 
 # Commands
 
 <!-- commands -->
-
-- [`subql build`](#subql-build)
-- [`subql codegen`](#subql-codegen)
-- [`subql help [COMMAND]`](#subql-help-command)
-- [`subql init [PROJECTNAME]`](#subql-init-projectname)
-- [`subql validate`](#subql-validate)
+* [`subql build`](#subql-build)
+* [`subql codegen`](#subql-codegen)
+* [`subql help [COMMAND]`](#subql-help-command)
+* [`subql init [PROJECTNAME]`](#subql-init-projectname)
+* [`subql publish`](#subql-publish)
+* [`subql validate`](#subql-validate)
 
 ## `subql build`
 
@@ -112,6 +109,19 @@ OPTIONS
 
 _See code: [lib/commands/init.js](https://github.com/packages/cli/blob/v0.13.1-0/lib/commands/init.js)_
 
+## `subql publish`
+
+Upload this SubQuery project to IPFS
+
+```
+USAGE
+  $ subql publish
+
+OPTIONS
+  -l, --location=location  local folder
+  --ipfs=ipfs              [default: http://localhost:5001/api/v0] IPFS gateway endpoint
+```
+
 ## `subql validate`
 
 check a folder or github repo is a validate subquery project
@@ -121,10 +131,13 @@ USAGE
   $ subql validate
 
 OPTIONS
-  -l, --location=location  local folder or github repo url
+  -l, --location=location  local folder, github repo url or IPFS cid
+
+  --ipfs=ipfs              [default: http://localhost:5001/api/v0] IPFS gateway endpoint, used for validating projects
+                           on IPFS
+
   --silent
 ```
 
 _See code: [lib/commands/validate.js](https://github.com/packages/cli/blob/v0.13.1-0/lib/commands/validate.js)_
-
 <!-- commandsstop -->

packages/cli/package.json

@@ -18,6 +18,7 @@
     "chalk": "^4.1.1",
     "cli-ux": "^5.6.2",
     "ejs": "^3.1.6",
+    "ipfs-http-client": "^52.0.3",
     "rimraf": "^3.0.2",
     "simple-git": "^2.31.0",
     "ts-loader": "^9.2.5",

packages/cli/src/commands/build.ts

@@ -67,7 +67,7 @@ export default class Build extends Command {
 
     // Get the output location from the project package.json main field
     const pjson = JSON.parse(readFileSync(path.join(directory, 'package.json')).toString());
-    const outputPath = path.resolve(pjson.main || 'dist/index.js');
+    const outputPath = path.resolve(directory, pjson.main || 'dist/index.js');
 
     const config = merge(
       getBaseConfig(directory, outputPath, isDev)

packages/cli/src/commands/publish.ts

@@ -0,0 +1,40 @@
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import {lstatSync} from 'fs';
+import path from 'path';
+import {Command, flags} from '@oclif/command';
+import {uploadToIpfs} from '../controller/publish-controller';
+import Build from './build';
+
+export default class Publish extends Command {
+  static description = 'Upload this SubQuery project to IPFS';
+
+  static flags = {
+    location: flags.string({char: 'l', description: 'local folder'}),
+    ipfs: flags.string({description: 'IPFS gateway endpoint', default: 'http://localhost:5001/api/v0'}),
+  };
+
+  async run(): Promise<void> {
+    const {flags} = this.parse(Publish);
+
+    const directory = flags.location ? path.resolve(flags.location) : process.cwd();
+
+    if (!lstatSync(directory).isDirectory()) {
+      this.error('Argument `location` is not a valid directory');
+    }
+
+    // Ensure that the project is built
+    try {
+      await Build.run(['--location', directory]);
+    } catch (e) {
+      this.log(e);
+      this.error('Failed to build project');
+    }
+
+    this.log('Uploading SupQuery project to ipfs');
+    const cid = await uploadToIpfs(flags.ipfs, directory);
+
+    this.log(`SubQuery Project uploaded to IPFS: ${cid}`);
+  }
+}

packages/cli/src/commands/validate.ts

@@ -9,13 +9,17 @@ export default class Validate extends Command {
   static description = 'check a folder or github repo is a validate subquery project';
 
   static flags = {
-    location: flags.string({char: 'l', description: 'local folder or github repo url'}),
+    location: flags.string({char: 'l', description: 'local folder, github repo url or IPFS cid'}),
+    ipfs: flags.string({
+      description: 'IPFS gateway endpoint, used for validating projects on IPFS',
+      default: 'https://ipfs.thechainhub.com/api/v0',
+    }),
     silent: flags.boolean(),
   };
 
   async run(): Promise<void> {
     const {flags} = this.parse(Validate);
-    const v = new Validator(flags.location ?? process.cwd());
+    const v = new Validator(flags.location ?? process.cwd(), {ipfs: flags.ipfs});
     v.addRule(...commonRules);
 
     const reports = await v.getValidateReports();

packages/cli/src/controller/publish-controller.spec.ts

@@ -0,0 +1,77 @@
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import childProcess from 'child_process';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import {promisify} from 'util';
+import rimraf from 'rimraf';
+import Build from '../commands/build';
+import Codegen from '../commands/codegen';
+import Validate from '../commands/validate';
+import {ProjectSpecBase, ProjectSpecV0_0_1, ProjectSpecV0_2_0} from '../types';
+import {createProject} from './init-controller';
+import {uploadToIpfs} from './publish-controller';
+
+const projectSpecV0_0_1: ProjectSpecV0_0_1 = {
+  name: 'mocked_starter',
+  repository: '',
+  endpoint: 'wss://rpc.polkadot.io/public-ws',
+  author: 'jay',
+  description: 'this is test for init controller',
+  version: '',
+  license: '',
+};
+
+const projectSpecV0_2_0: ProjectSpecV0_2_0 = {
+  name: 'mocked_starter',
+  repository: '',
+  genesisHash: '0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3',
+  author: 'jay',
+  description: 'this is test for init controller',
+  version: '',
+  license: '',
+  endpoint: '',
+};
+
+const ipfsEndpoint = 'https://ipfs.thechainhub.com/api/v0';
+
+jest.setTimeout(120000);
+
+async function createTestProject(projectSpec: ProjectSpecBase): Promise<string> {
+  const tmpdir = await fs.promises.mkdtemp(`${os.tmpdir()}${path.sep}`);
+  const projectDir = path.join(tmpdir, projectSpec.name);
+
+  await createProject(tmpdir, projectSpec);
+
+  // Install dependencies
+  childProcess.execSync('npm i', {cwd: projectDir});
+
+  await Codegen.run(['-l', projectDir]);
+  await Build.run(['-l', projectDir]);
+
+  return projectDir;
+}
+
+describe('Cli publish', () => {
+  let projectDir: string;
+
+  afterEach(() => {
+    promisify(rimraf)(projectDir);
+  });
+
+  it('should not allow uploading a v0.0.1 spec version project', async () => {
+    projectDir = await createTestProject(projectSpecV0_0_1);
+
+    await expect(uploadToIpfs(ipfsEndpoint, projectDir)).rejects.toBeDefined();
+  });
+
+  it('should upload appropriate files to IPFS', async () => {
+    projectDir = await createTestProject(projectSpecV0_2_0);
+    const cid = await uploadToIpfs(ipfsEndpoint, projectDir);
+
+    expect(cid).toBeDefined();
+    await expect(Validate.run(['-l', cid, '--ipfs', ipfsEndpoint])).resolves.toBe(undefined);
+  });
+});

packages/cli/src/controller/publish-controller.ts

@@ -0,0 +1,67 @@
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import fs from 'fs';
+import path from 'path';
+import {
+  loadProjectManifest,
+  manifestIsV0_2_0,
+  ProjectManifestV0_0_1Impl,
+  ProjectManifestV0_2_0Impl,
+} from '@subql/common';
+import IPFS from 'ipfs-http-client';
+import yaml from 'js-yaml';
+
+// https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#filecontent
+type FileContent = Uint8Array | string | Iterable<Uint8Array> | Iterable<number> | AsyncIterable<Uint8Array>;
+
+// https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#fileobject
+type FileObject = {
+  path?: string;
+  content?: FileContent;
+  mode?: number | string;
+  mtime?: Date | number[] | {secs: number; nsecs?: number};
+};
+
+export async function uploadToIpfs(ipfsEndpoint: string, projectDir: string): Promise<string> {
+  const ipfs = IPFS.create({url: ipfsEndpoint});
+
+  const projectManifestPath = path.resolve(projectDir, 'project.yaml');
+  const manifest = loadProjectManifest(projectManifestPath).asImpl;
+
+  if (manifestIsV0_2_0(manifest)) {
+    const entryPaths = manifest.dataSources.map((ds) => ds.mapping.file);
+    const schemaPath = manifest.schema.file;
+
+    // Upload referenced files to IPFS
+    const [schema, ...entryPoints] = await Promise.all(
+      [schemaPath, ...entryPaths].map((filePath) =>
+        uploadFile(ipfs, fs.createReadStream(path.resolve(projectDir, filePath))).then((cid) => `ipfs://${cid}`)
+      )
+    );
+
+    // Update referenced file paths to IPFS cids
+    manifest.schema.file = schema;
+
+    entryPoints.forEach((entryPoint, index) => {
+      manifest.dataSources[index].mapping.file = entryPoint;
+    });
+  } else {
+    throw new Error('Unsupported project manifest spec, only 0.2.0 is supported');
+  }
+
+  // Upload schema
+  return uploadFile(ipfs, toMinifiedYaml(manifest));
+}
+
+async function uploadFile(ipfs: IPFS.IPFSHTTPClient, content: FileObject | FileContent): Promise<string> {
+  const result = await ipfs.add(content, {pin: true, cidVersion: 0});
+  return result.cid.toString();
+}
+
+function toMinifiedYaml(manifest: ProjectManifestV0_0_1Impl | ProjectManifestV0_2_0Impl): string {
+  return yaml.dump(manifest, {
+    sortKeys: true,
+    condenseFlow: true,
+  });
+}

packages/common/src/project/versioned/v0_2_0/model.ts

@@ -12,7 +12,7 @@ import {Type} from 'class-transformer';
 import {Equals, IsArray, IsObject, IsOptional, IsString, ValidateNested} from 'class-validator';
 import {CustomDataSourceBase, Mapping, RuntimeDataSourceBase} from '../../models';
 import {ProjectManifestBaseImpl} from '../base';
-import {ProjectManifestV0_2_0, RuntimeDataSourceV0_2_0, SubqlMappingV0_2_0} from './types';
+import {CustomDatasourceV0_2_0, ProjectManifestV0_2_0, RuntimeDataSourceV0_2_0, SubqlMappingV0_2_0} from './types';
 
 export class FileType {
   @IsString()
@@ -77,5 +77,5 @@ export class ProjectManifestV0_2_0Impl extends ProjectManifestBaseImpl implement
     },
     keepDiscriminatorProperty: true,
   })
-  dataSources: (RuntimeDataSourceV0_2_0 | SubqlCustomDatasource)[];
+  dataSources: (RuntimeDataSourceV0_2_0 | CustomDatasourceV0_2_0)[];
 }

packages/common/src/project/versioned/v0_2_0/types.ts

@@ -3,10 +3,12 @@
 
 import {
   SubqlCustomDatasource,
+  SubqlCustomHandler,
   SubqlDatasource,
   SubqlDatasourceKind,
   SubqlHandler,
   SubqlMapping,
+  SubqlNetworkFilter,
   SubqlRuntimeDatasource,
   SubqlRuntimeHandler,
 } from '@subql/types';
@@ -17,6 +19,11 @@ export interface SubqlMappingV0_2_0<T extends SubqlHandler> extends SubqlMapping
 }
 
 export type RuntimeDataSourceV0_2_0 = SubqlRuntimeDatasource<SubqlMappingV0_2_0<SubqlRuntimeHandler>>;
+export type CustomDatasourceV0_2_0 = SubqlCustomDatasource<
+  string,
+  SubqlNetworkFilter,
+  SubqlMappingV0_2_0<SubqlCustomHandler>
+>;
 
 export interface ProjectManifestV0_2_0 extends IProjectManifest {
   name: string;
@@ -33,7 +40,7 @@ export interface ProjectManifestV0_2_0 extends IProjectManifest {
     };
   };
 
-  dataSources: (RuntimeDataSourceV0_2_0 | SubqlCustomDatasource)[];
+  dataSources: (RuntimeDataSourceV0_2_0 | CustomDatasourceV0_2_0)[];
 }
 
 export function isRuntimeDataSourceV0_2_0(dataSource: SubqlDatasource): dataSource is RuntimeDataSourceV0_2_0 {

packages/node/src/db/db.module.test.ts

@@ -31,7 +31,7 @@ describe('DbModule', () => {
     await app.init();
     const sequelize = app.get(Sequelize);
     await expect(sequelize.authenticate()).resolves.not.toThrow();
-  });
+  }, 10000);
 
   it('can load subquery model', async () => {
     const module = await Test.createTestingModule({
@@ -51,5 +51,5 @@ describe('DbModule', () => {
     await app.init();
     const subqueryRepo: SubqueryRepo = app.get('Subquery');
     await expect(subqueryRepo.describe()).resolves.toBeTruthy();
-  });
+  }, 10000);
 });

packages/validator/package.json

@@ -18,6 +18,7 @@
   "dependencies": {
     "@subql/common": "workspace:*",
     "axios": "^0.21.1",
+    "ipfs-http-client": "^52.0.3",
     "js-yaml": "^4.1.0",
     "package-json-type": "^1.0.3"
   },