generate sourcemap for projects (subquery#1569)

* improve error handling * use inline-source-map * revert api error handling * beautify minified output from webpack * code cleanup * move terser plugin dependency to cli * log warnings for unresolved stack trace * get entry file from datasource * alter convertStack prototype * add ES2020 to lib compiler option
fanligroup · Mar 26, 2023 · a06e9be · a06e9be
1 parent c37093e
commit a06e9be
Show file tree

Hide file tree

Showing 6 changed files with 188 additions and 4 deletions.
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -36,6 +36,7 @@
     "oclif": "^2.4.4",
     "rimraf": "^3.0.2",
     "simple-git": "^3.12.0",
+    "terser-webpack-plugin": "^5.3.7",
     "ts-loader": "^9.2.6",
     "tslib": "^2.3.1",
     "typechain": "8.1.1",

diff --git a/packages/cli/src/controller/build-controller.ts b/packages/cli/src/controller/build-controller.ts
@@ -1,7 +1,7 @@
 // Copyright 2020-2022 OnFinality Limited authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
-import childProcess from 'child_process';
+import TerserPlugin from 'terser-webpack-plugin';
 import webpack, {Configuration} from 'webpack';
 import {merge} from 'webpack-merge';
 
@@ -15,7 +15,20 @@ const getBaseConfig = (
   mode: development ? 'development' : 'production',
   context: projectDir,
   entry: buildEntries,
-  devtool: development && 'inline-source-map',
+  devtool: 'inline-source-map',
+  optimization: {
+    minimize: true,
+    minimizer: [
+      new TerserPlugin({
+        terserOptions: {
+          sourceMap: true,
+          format: {
+            beautify: true,
+          },
+        },
+      }),
+    ],
+  },
   module: {
     rules: [
       {

diff --git a/packages/node-core/package.json b/packages/node-core/package.json
@@ -33,6 +33,7 @@
     "lodash": "^4.17.21",
     "prom-client": "^14.0.1",
     "sequelize": "6.28.0",
+    "source-map": "^0.7.4",
     "vm2": "^3.9.9",
     "yargs": "^16.2.0"
   },

diff --git a/packages/node-core/src/indexer/sandbox.ts b/packages/node-core/src/indexer/sandbox.ts
@@ -1,10 +1,12 @@
 // Copyright 2020-2022 OnFinality Limited authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
+import {existsSync, readFileSync} from 'fs';
 import path from 'path';
 import {Store} from '@subql/types';
 import {levelFilter} from '@subql/utils';
-import {merge} from 'lodash';
+import {last, merge} from 'lodash';
+import {SourceMapConsumer, NullableMappedPosition} from 'source-map';
 import {NodeVM, NodeVMOptions, VMScript} from 'vm2';
 import {NodeConfig} from '../configure/NodeConfig';
 import {getLogger} from '../logger';
@@ -35,6 +37,10 @@ const DEFAULT_OPTION = (unsafe = false): NodeVMOptions => {
 const logger = getLogger('sandbox');
 
 export class Sandbox extends NodeVM {
+  private root: string;
+  private entry: string;
+  private sourceMap: any | undefined;
+
   constructor(option: SandboxOption, protected readonly script: VMScript, protected config: NodeConfig) {
     super(
       merge(DEFAULT_OPTION(config.unsafe), {
@@ -46,11 +52,66 @@ export class Sandbox extends NodeVM {
         },
       })
     );
+    this.root = config.subquery.startsWith('ipfs://') ? '' : option.root;
+    this.entry = option.entry;
+
+    const sourceMapFile = path.join(this.root, this.entry);
+
+    if (existsSync(sourceMapFile)) {
+      this.sourceMap = this.decodeSourceMap(sourceMapFile);
+    }
   }
 
   async runTimeout<T = unknown>(duration: number): Promise<T> {
     return timeout(this.run(this.script), duration);
   }
+
+  protected async convertStack(stackTrace: string): Promise<string> {
+    if (!this.sourceMap) {
+      logger.warn('Unable to find a source map. Rebuild your project with latest @subql/cli to generate a source map.');
+      logger.warn('Logging unresolved stack trace.');
+      return stackTrace;
+    }
+
+    const entryFile = last(this.entry.split('/'));
+    const regex = new RegExp(`${entryFile.split('.')[0]}.${entryFile.split('.')[1]}:([0-9]+):([0-9]+)`, 'gi');
+    const matches = [...stackTrace.matchAll(regex)];
+
+    for (const match of matches) {
+      const lineNumber = Number.parseInt(match[1]);
+      const columnNumber = Number.parseInt(match[2]);
+      const lineInfo = await this.findLineInfo(this.sourceMap, lineNumber, columnNumber);
+      const newLineTrace = `${lineInfo.source}:${lineInfo.line}:${lineInfo.column}`;
+      stackTrace = stackTrace.replace(`${path.join(this.root, this.entry)}:${match[1]}:${match[2]}`, newLineTrace);
+    }
+
+    return stackTrace;
+  }
+
+  decodeSourceMap(sourceMapPath: string) {
+    const source = readFileSync(sourceMapPath).toString();
+    const sourceMapBase64 = source.split(`//# sourceMappingURL=data:application/json;charset=utf-8;base64,`)[1];
+    if (!sourceMapBase64) {
+      logger.warn('Unable to find a source map for project');
+      logger.warn('Build your project with latest @subql/cli to generate a source map');
+      return;
+    }
+    const sourceMap = Buffer.from(sourceMapBase64, 'base64').toString();
+    const json = JSON.parse(sourceMap);
+
+    return json;
+  }
+
+  async findLineInfo(
+    sourcemap: any,
+    compiledLineNumber: number,
+    compiledColumnNumber: number
+  ): Promise<NullableMappedPosition> {
+    const consumer = await new SourceMapConsumer(sourcemap);
+    const lineInfo = consumer.originalPositionFor({line: compiledLineNumber, column: compiledColumnNumber});
+
+    return lineInfo;
+  }
 }
 
 export class IndexerSandbox extends Sandbox {
@@ -74,6 +135,8 @@ export class IndexerSandbox extends Sandbox {
     try {
       await this.runTimeout(this.config.timeout);
     } catch (e) {
+      const newStack = await this.convertStack((e as Error).stack);
+      e.stack = newStack;
       e.handler = funcName;
       if (this.config.logLevel && levelFilter('debug', this.config.logLevel)) {
         e.handlerArgs = JSON.stringify(args);

diff --git a/tsconfig.json b/tsconfig.json
@@ -11,7 +11,7 @@
     "experimentalDecorators": true,
     "esModuleInterop": true,
     "allowSyntheticDefaultImports": true,
-    "lib": ["ES2017"],
+    "lib": ["ES2017", "ES2020"],
     "emitDecoratorMetadata": true,
     "declaration": true,
     "sourceMap": true,

diff --git a/yarn.lock b/yarn.lock
@@ -2499,6 +2499,24 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@jridgewell/gen-mapping@npm:^0.3.0":
+  version: 0.3.2
+  resolution: "@jridgewell/gen-mapping@npm:0.3.2"
+  dependencies:
+    "@jridgewell/set-array": ^1.0.1
+    "@jridgewell/sourcemap-codec": ^1.4.10
+    "@jridgewell/trace-mapping": ^0.3.9
+  checksum: 1832707a1c476afebe4d0fbbd4b9434fdb51a4c3e009ab1e9938648e21b7a97049fa6009393bdf05cab7504108413441df26d8a3c12193996e65493a4efb6882
+  languageName: node
+  linkType: hard
+
+"@jridgewell/resolve-uri@npm:3.1.0":
+  version: 3.1.0
+  resolution: "@jridgewell/resolve-uri@npm:3.1.0"
+  checksum: b5ceaaf9a110fcb2780d1d8f8d4a0bfd216702f31c988d8042e5f8fbe353c55d9b0f55a1733afdc64806f8e79c485d2464680ac48a0d9fcadb9548ee6b81d267
+  languageName: node
+  linkType: hard
+
 "@jridgewell/resolve-uri@npm:^3.0.3":
   version: 3.0.7
   resolution: "@jridgewell/resolve-uri@npm:3.0.7"
@@ -2513,13 +2531,47 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@jridgewell/set-array@npm:^1.0.1":
+  version: 1.1.2
+  resolution: "@jridgewell/set-array@npm:1.1.2"
+  checksum: 69a84d5980385f396ff60a175f7177af0b8da4ddb81824cb7016a9ef914eee9806c72b6b65942003c63f7983d4f39a5c6c27185bbca88eb4690b62075602e28e
+  languageName: node
+  linkType: hard
+
+"@jridgewell/source-map@npm:^0.3.2":
+  version: 0.3.2
+  resolution: "@jridgewell/source-map@npm:0.3.2"
+  dependencies:
+    "@jridgewell/gen-mapping": ^0.3.0
+    "@jridgewell/trace-mapping": ^0.3.9
+  checksum: 1b83f0eb944e77b70559a394d5d3b3f98a81fcc186946aceb3ef42d036762b52ef71493c6c0a3b7c1d2f08785f53ba2df1277fe629a06e6109588ff4cdcf7482
+  languageName: node
+  linkType: hard
+
+"@jridgewell/sourcemap-codec@npm:1.4.14":
+  version: 1.4.14
+  resolution: "@jridgewell/sourcemap-codec@npm:1.4.14"
+  checksum: 61100637b6d173d3ba786a5dff019e1a74b1f394f323c1fee337ff390239f053b87266c7a948777f4b1ee68c01a8ad0ab61e5ff4abb5a012a0b091bec391ab97
+  languageName: node
+  linkType: hard
+
 "@jridgewell/sourcemap-codec@npm:^1.4.10":
   version: 1.4.13
   resolution: "@jridgewell/sourcemap-codec@npm:1.4.13"
   checksum: f14449096f60a5f921262322fef65ce0bbbfb778080b3b20212080bcefdeba621c43a58c27065bd536ecb4cc767b18eb9c45f15b6b98a4970139572b60603a1c
   languageName: node
   linkType: hard
 
+"@jridgewell/trace-mapping@npm:^0.3.17":
+  version: 0.3.17
+  resolution: "@jridgewell/trace-mapping@npm:0.3.17"
+  dependencies:
+    "@jridgewell/resolve-uri": 3.1.0
+    "@jridgewell/sourcemap-codec": 1.4.14
+  checksum: 9d703b859cff5cd83b7308fd457a431387db5db96bd781a63bf48e183418dd9d3d44e76b9e4ae13237f6abeeb25d739ec9215c1d5bfdd08f66f750a50074a339
+  languageName: node
+  linkType: hard
+
 "@jridgewell/trace-mapping@npm:^0.3.9":
   version: 0.3.10
   resolution: "@jridgewell/trace-mapping@npm:0.3.10"
@@ -4142,6 +4194,8 @@ __metadata:
     lodash: ^4.17.21
     prom-client: ^14.0.1
     sequelize: 6.28.0
+    source-map: ^0.7.4
+    terser-webpack-plugin: ^5.3.7
     vm2: ^3.9.9
     yargs: ^16.2.0
   languageName: unknown
@@ -16052,6 +16106,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"serialize-javascript@npm:^6.0.1":
+  version: 6.0.1
+  resolution: "serialize-javascript@npm:6.0.1"
+  dependencies:
+    randombytes: ^2.1.0
+  checksum: 3c4f4cb61d0893b988415bdb67243637333f3f574e9e9cc9a006a2ced0b390b0b3b44aef8d51c951272a9002ec50885eefdc0298891bc27eb2fe7510ea87dc4f
+  languageName: node
+  linkType: hard
+
 "serve-static@npm:1.14.2":
   version: 1.14.2
   resolution: "serve-static@npm:1.14.2"
@@ -16306,6 +16369,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"source-map@npm:^0.7.4":
+  version: 0.7.4
+  resolution: "source-map@npm:0.7.4"
+  checksum: 01cc5a74b1f0e1d626a58d36ad6898ea820567e87f18dfc9d24a9843a351aaa2ec09b87422589906d6ff1deed29693e176194dc88bcae7c9a852dc74b311dbf5
+  languageName: node
+  linkType: hard
+
 "source-map@npm:~0.8.0-beta.0":
   version: 0.8.0-beta.0
   resolution: "source-map@npm:0.8.0-beta.0"
@@ -16976,6 +17046,42 @@ __metadata:
   languageName: node
   linkType: hard
 
+"terser-webpack-plugin@npm:^5.3.7":
+  version: 5.3.7
+  resolution: "terser-webpack-plugin@npm:5.3.7"
+  dependencies:
+    "@jridgewell/trace-mapping": ^0.3.17
+    jest-worker: ^27.4.5
+    schema-utils: ^3.1.1
+    serialize-javascript: ^6.0.1
+    terser: ^5.16.5
+  peerDependencies:
+    webpack: ^5.1.0
+  peerDependenciesMeta:
+    "@swc/core":
+      optional: true
+    esbuild:
+      optional: true
+    uglify-js:
+      optional: true
+  checksum: 095e699fdeeb553cdf2c6f75f983949271b396d9c201d7ae9fc633c45c1c1ad14c7257ef9d51ccc62213dd3e97f875870ba31550f6d4f1b6674f2615562da7f7
+  languageName: node
+  linkType: hard
+
+"terser@npm:^5.16.5":
+  version: 5.16.6
+  resolution: "terser@npm:5.16.6"
+  dependencies:
+    "@jridgewell/source-map": ^0.3.2
+    acorn: ^8.5.0
+    commander: ^2.20.0
+    source-map-support: ~0.5.20
+  bin:
+    terser: bin/terser
+  checksum: f763a7bcc7b98cb2bfc41434f7b92bfe8a701a12c92ea6049377736c8e6de328240d654a20dfe15ce170fd783491b9873fad9f4cd8fee4f6c6fb8ca407859dee
+  languageName: node
+  linkType: hard
+
 "terser@npm:^5.7.2":
   version: 5.13.1
   resolution: "terser@npm:5.13.1"