Introduce Terra Support (subquery#781)

* update * add terra * remove header rule * terra * terra * terra * terra * terra * terra * terra * terra * Update indexerterra.manager.ts * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * add filters support * add filter * add filter * manifest v0.3.0 * dictionary * dictionary * dictionary * terrra * bug fix * refactor * Update README.md * clean up * add newline at end * add header * Update package.json * Update package.json * Update main.ts * Update package.json * Update main.ts * Update load.ts * add types-terra * Update model.ts * @subql/common split * add v0.3.0 support * code reuse * rebasing * Update yarn.lock * debug * rebase * Remove redundant sequalize types package from nodes * Update CI to include new packages * Fix types issue * Update tests to include new packages * Fix most tests * remove substrate specific tests from node-terra * rebase * Remove api from sandbox and allow unsafeApi * Add missing deps from cli * Improve code reuse * Remove unused function * Update copyright date header in ts files * validate terra manifest * refactor * add terra suppory in publish * sync latest node changes to node-terra * rebase * rebase * Fix duplicate import Co-authored-by: Scott Twiname <[email protected]> Co-authored-by: Ian He <[email protected]>
fanligroup · Feb 27, 2022 · 1c694fb · 1c694fb
1 parent d129548
commit 1c694fb
Show file tree

Hide file tree

Showing 171 changed files with 8,970 additions and 497 deletions.
diff --git a/.github/workflows/gh-release.yml b/.github/workflows/gh-release.yml
@@ -30,6 +30,16 @@ jobs:
           package-path: packages/common
           repo-token: ${{ secrets.REPO_TOKEN }}
 
+      - uses: ./.github/actions/create-release
+        with:
+          package-path: packages/common-substrate
+          repo-token: ${{ secrets.REPO_TOKEN }}
+
+      - uses: ./.github/actions/create-release
+        with:
+          package-path: packages/common-terra
+          repo-token: ${{ secrets.REPO_TOKEN }}
+
       - uses: ./.github/actions/create-release
         with:
           package-path: packages/contract-processors
@@ -40,6 +50,11 @@ jobs:
           package-path: packages/node
           repo-token: ${{ secrets.REPO_TOKEN }}
 
+      - uses: ./.github/actions/create-release
+        with:
+          package-path: packages/node-terra
+          repo-token: ${{ secrets.REPO_TOKEN }}
+
       - uses: ./.github/actions/create-release
         with:
           package-path: packages/query
@@ -50,6 +65,11 @@ jobs:
           package-path: packages/types
           repo-token: ${{ secrets.REPO_TOKEN }}
 
+      - uses: ./.github/actions/create-release
+        with:
+          package-path: packages/types-terra
+          repo-token: ${{ secrets.REPO_TOKEN }}
+
       - uses: ./.github/actions/create-release
         with:
           package-path: packages/validator

diff --git a/.github/workflows/node-terra-docker.yml b/.github/workflows/node-terra-docker.yml
@@ -0,0 +1,60 @@
+name: "Node-to-docker"
+on:
+  workflow_dispatch:
+    inputs:
+      isLatest:
+        description: 'Add latest tag'
+        default: 'true'
+        require: true
+
+jobs:
+  node-build-push-docker:
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 100
+          token: ${{ secrets.REPO_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: onfinality
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      ## node
+      - name: Get updated node version
+        id: get-node-version
+        run: | #TODO
+          sh .github/workflows/scripts/nodeTerraVersion.sh
+
+      - name: Build and push
+        if: github.event.inputs.isLatest == 'false'
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          platforms: arm64,amd64
+          file: ./packages/node-terra/Dockerfile
+          tags: onfinality/subql-node-terra:v${{ steps.get-node-version.outputs.NODE_VERSION }}
+          build-args: RELEASE_VERSION=${{ steps.get-node-version.outputs.NODE_VERSION }}
+
+      - name: Build and push
+        if: github.event.inputs.isLatest == 'true'
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          platforms: arm64,amd64
+          file: ./packages/node-terra/Dockerfile
+          tags: onfinality/subql-node-terra:v${{ steps.get-node-version.outputs.NODE_VERSION }},onfinality/subql-node-terra:latest
+          build-args: RELEASE_VERSION=${{ steps.get-node-version.outputs.NODE_VERSION }}
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}
+
diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml
@@ -40,11 +40,26 @@ jobs:
         with:
           paths: packages/types
 
+      - uses: marceloprado/has-changed-path@v1
+        id: changed-types-terra
+        with:
+          paths: packages/types-terra
+
       - uses: marceloprado/has-changed-path@v1
         id: changed-common
         with:
           paths: packages/common
 
+      - uses: marceloprado/has-changed-path@v1
+        id: changed-common-substrate
+        with:
+          paths: packages/common-substrate
+
+      - uses: marceloprado/has-changed-path@v1
+        id: changed-common-terra
+        with:
+          paths: packages/common-terra
+
       - uses: marceloprado/has-changed-path@v1
         id: changed-contract-processors
         with:
@@ -60,6 +75,11 @@ jobs:
         with:
           paths: packages/node
 
+      - uses: marceloprado/has-changed-path@v1
+        id: changed-node-terra
+        with:
+          paths: packages/node-terra
+
       - uses: marceloprado/has-changed-path@v1
         id: changed-query
         with:
@@ -83,13 +103,34 @@ jobs:
           package-path: packages/types
           npm-token: ${{ secrets.NPM_TOKEN }}
 
+      - name: Bump types terra & deploy
+        if: steps.changed-types-terra.outputs.changed == 'true'
+        uses: ./.github/actions/create-prerelease
+        with:
+          package-path: packages/types-terra
+          npm-token: ${{ secrets.NPM_TOKEN }}
+
       - name: Bump common & deploy
         if: steps.changed-common.outputs.changed == 'true'
         uses: ./.github/actions/create-prerelease
         with:
           package-path: packages/common
           npm-token: ${{ secrets.NPM_TOKEN }}
 
+      - name: Bump common substrate & deploy
+        if: steps.changed-common-substrate.outputs.changed == 'true'
+        uses: ./.github/actions/create-prerelease
+        with:
+          package-path: packages/common-substrate
+          npm-token: ${{ secrets.NPM_TOKEN }}
+
+      - name: Bump common terra & deploy
+        if: steps.changed-common-terra.outputs.changed == 'true'
+        uses: ./.github/actions/create-prerelease
+        with:
+          package-path: packages/common-terra
+          npm-token: ${{ secrets.NPM_TOKEN }}
+
       - name: Bump contract-processors & deploy
         if: steps.changed-contract-processors.outputs.changed == 'true'
         uses: ./.github/actions/create-prerelease
@@ -104,6 +145,13 @@ jobs:
           package-path: packages/node
           npm-token: ${{ secrets.NPM_TOKEN }}
 
+      - name: Bump node terra & deploy
+        if: steps.changed-node-terra.outputs.changed == 'true'
+        uses: ./.github/actions/create-prerelease
+        with:
+          package-path: packages/node-terra
+          npm-token: ${{ secrets.NPM_TOKEN }}
+
       - name: Bump query & deploy
         if: steps.changed-query.outputs.changed == 'true'
         uses: ./.github/actions/create-prerelease

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -41,11 +41,26 @@ jobs:
         with:
           paths: packages/types
 
+      - uses: marceloprado/has-changed-path@v1
+        id: changed-types-terra
+        with:
+          paths: packages/types-terra
+
       - uses: marceloprado/has-changed-path@v1
         id: changed-common
         with:
           paths: packages/common
 
+      - uses: marceloprado/has-changed-path@v1
+        id: changed-common-substrate
+        with:
+          paths: packages/common-substrate
+
+      - uses: marceloprado/has-changed-path@v1
+        id: changed-common-terra
+        with:
+          paths: packages/common-terra
+
       - uses: marceloprado/has-changed-path@v1
         id: changed-contract-processors
         with:
@@ -61,6 +76,11 @@ jobs:
         with:
           paths: packages/node
 
+      - uses: marceloprado/has-changed-path@v1
+        id: changed-node-terra
+        with:
+          paths: packages/node-terra
+
       - uses: marceloprado/has-changed-path@v1
         id: changed-query
         with:
@@ -85,6 +105,14 @@ jobs:
           repo-token: ${{ secrets.REPO_TOKEN }}
           npm-token: ${{ secrets.NPM_TOKEN }}
 
+      - name: Publish Types Terra
+        if: steps.changed-types-terra.outputs.changed == 'true'
+        uses: ./.github/actions/create-release
+        with:
+          package-path: packages/types-terra
+          repo-token: ${{ secrets.REPO_TOKEN }}
+          npm-token: ${{ secrets.NPM_TOKEN }}
+
       - name: Publish Common
         if: steps.changed-common.outputs.changed == 'true'
         uses: ./.github/actions/create-release
@@ -93,6 +121,22 @@ jobs:
           repo-token: ${{ secrets.REPO_TOKEN }}
           npm-token: ${{ secrets.NPM_TOKEN }}
 
+      - name: Publish Common Substrate
+        if: steps.changed-common-substrate.outputs.changed == 'true'
+        uses: ./.github/actions/create-release
+        with:
+          package-path: packages/common-substrate
+          repo-token: ${{ secrets.REPO_TOKEN }}
+          npm-token: ${{ secrets.NPM_TOKEN }}
+
+      - name: Publish Common Terra
+        if: steps.changed-common-terra.outputs.changed == 'true'
+        uses: ./.github/actions/create-release
+        with:
+          package-path: packages/common-terra
+          repo-token: ${{ secrets.REPO_TOKEN }}
+          npm-token: ${{ secrets.NPM_TOKEN }}
+
       - name: Publish Contract Processors
         if: steps.changed-contract-processors.outputs.changed == 'true'
         uses: ./.github/actions/create-release
@@ -109,6 +153,14 @@ jobs:
           repo-token: ${{ secrets.REPO_TOKEN }}
           npm-token: ${{ secrets.NPM_TOKEN }}
 
+      - name: Publish Node Terra
+        if: steps.changed-node-terra.outputs.changed == 'true'
+        uses: ./.github/actions/create-release
+        with:
+          package-path: packages/node-terra
+          repo-token: ${{ secrets.REPO_TOKEN }}
+          npm-token: ${{ secrets.NPM_TOKEN }}
+
       - name: Publish Query
         if: steps.changed-query.outputs.changed == 'true'
         uses: ./.github/actions/create-release

diff --git a/.github/workflows/scripts/nodeTerraVersion.sh b/.github/workflows/scripts/nodeTerraVersion.sh
@@ -0,0 +1,9 @@
+PACKAGE_VERSION=$(cat ./packages/node-terra/package.json \
+  | grep version \
+  | head -1 \
+  | awk -F: '{ print $2 }' \
+  | sed 's/[",]//g' \
+  | tr -d '[[:space:]]')
+
+
+echo "::set-output name=NODE_VERSION::$PACKAGE_VERSION"
diff --git a/.prettierignore b/.prettierignore
@@ -7,3 +7,4 @@
 **/.DS_Store
 *.ejs
 *.tgz
+*.cmd
diff --git a/jest.config.js b/jest.config.js
@@ -23,8 +23,11 @@ module.exports = {
   collectCoverageFrom: [
     'packages/cli/src/**/*.ts',
     'packages/common/src/**/*.ts',
+    'packages/common-substrate/src/**/*.ts',
+    'packages/common-terra/src/**/*.ts',
     'packages/contract-processors/src/**/*.ts',
     'packages/node/src/**/*.ts',
+    'packages/node-terra/src/**/*.ts',
     'packages/validator/src/**/*.ts',
   ],
 
@@ -88,6 +91,10 @@ module.exports = {
   // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
   // moduleNameMapper: {},
   moduleNameMapper: {
+    '^@subql/common-substrate': '<rootDir>/packages/common-substrate/src',
+    '^@subql/common-substrate/(.*)$': '<rootDir>/packages/common-substrate/src/$1',
+    '^@subql/common-terra': '<rootDir>/packages/common-terra/src',
+    '^@subql/common-terra/(.*)$': '<rootDir>/packages/common-terra/src/$1',
     '^@subql/common': '<rootDir>/packages/common/src',
     '^@subql/common/(.*)$': '<rootDir>/packages/common/src/$1',
   },

diff --git a/package.json b/package.json
@@ -32,7 +32,8 @@
   },
   "resolutions": {
     "@polkadot/api": "7.9.1",
-    "@polkadot/util": "8.4.1"
+    "@polkadot/util": "8.4.1",
+    "@terra-money/terra.js": "^3.0.1"
   },
   "scripts": {
     "build": "yarn workspaces foreach -ptA run build",

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -13,6 +13,8 @@
     "@oclif/core": "^1.3.3",
     "@oclif/plugin-help": "^3.2.3",
     "@subql/common": "workspace:*",
+    "@subql/common-substrate": "workspace:*",
+    "@subql/common-terra": "workspace:*",
     "@subql/validator": "workspace:*",
     "@types/ejs": "^3.1.0",
     "@types/inquirer": "^8.2.0",

diff --git a/packages/cli/src/commands/migrate.ts b/packages/cli/src/commands/migrate.ts
@@ -3,7 +3,7 @@
 
 import path from 'path';
 import {Command, Flags} from '@oclif/core';
-import {loadProjectManifest, ProjectManifestVersioned} from '@subql/common';
+import {loadSubstrateProjectManifest, SubstrateProjectManifestVersioned} from '@subql/common-substrate';
 import {migrate, prepare} from '../controller/migrate-controller';
 
 export default class Migrate extends Command {
@@ -18,9 +18,9 @@ export default class Migrate extends Command {
   async run(): Promise<void> {
     const {flags} = await this.parse(Migrate);
     const location = flags.location ? path.resolve(flags.location) : process.cwd();
-    let manifest: ProjectManifestVersioned;
+    let manifest: SubstrateProjectManifestVersioned;
     try {
-      manifest = loadProjectManifest(location);
+      manifest = loadSubstrateProjectManifest(location);
     } catch (e) {
       this.error(`Please validate project manifest before migrate. \n ${e}`);
     }
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,3 +7,4 @@ @@
     **/.DS_Store
     *.ejs
     *.tgz
+    *.cmd