From 48a21e998b7e44fec069d57519a6694283ce73fb Mon Sep 17 00:00:00 2001
From: Devon Govett <devongovett@gmail.com>
Date: Sat, 5 Aug 2017 09:59:37 -0700
Subject: [PATCH] Initial commit

---
 .gitignore                                    |  3 +
 packages/core/parcel-bundler/package.json     | 14 +++++
 packages/core/parcel-bundler/src/Bundle.js    | 60 +++++++++++++++++++
 packages/core/parcel-bundler/src/Module.js    | 32 ++++++++++
 packages/core/parcel-bundler/src/Parser.js    | 46 ++++++++++++++
 packages/core/parcel-bundler/src/Resolver.js  | 41 +++++++++++++
 packages/core/parcel-bundler/src/_empty.js    |  0
 packages/core/parcel-bundler/src/builtins.js  | 38 ++++++++++++
 .../core/parcel-bundler/src/parsers/es6.js    | 27 +++++++++
 .../core/parcel-bundler/src/parsers/json.js   |  5 ++
 packages/core/parcel-bundler/src/utils/fs.js  |  5 ++
 .../parcel-bundler/src/utils/promisify.js     | 15 +++++
 .../core/parcel-bundler/src/utils/queue.js    | 52 ++++++++++++++++
 .../src/visitors/dependencies.js              | 32 ++++++++++
 packages/core/parcel-bundler/src/worker.js    | 23 +++++++
 test.js                                       | 23 +++++++
 16 files changed, 416 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 packages/core/parcel-bundler/package.json
 create mode 100644 packages/core/parcel-bundler/src/Bundle.js
 create mode 100644 packages/core/parcel-bundler/src/Module.js
 create mode 100644 packages/core/parcel-bundler/src/Parser.js
 create mode 100644 packages/core/parcel-bundler/src/Resolver.js
 create mode 100644 packages/core/parcel-bundler/src/_empty.js
 create mode 100644 packages/core/parcel-bundler/src/builtins.js
 create mode 100644 packages/core/parcel-bundler/src/parsers/es6.js
 create mode 100644 packages/core/parcel-bundler/src/parsers/json.js
 create mode 100644 packages/core/parcel-bundler/src/utils/fs.js
 create mode 100644 packages/core/parcel-bundler/src/utils/promisify.js
 create mode 100644 packages/core/parcel-bundler/src/utils/queue.js
 create mode 100644 packages/core/parcel-bundler/src/visitors/dependencies.js
 create mode 100644 packages/core/parcel-bundler/src/worker.js
 create mode 100644 test.js

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000000..beccf0fbd7a
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+node_modules/
+.DS_Store
+yarn.lock
diff --git a/packages/core/parcel-bundler/package.json b/packages/core/parcel-bundler/package.json
new file mode 100644
index 00000000000..a207b049533
--- /dev/null
+++ b/packages/core/parcel-bundler/package.json
@@ -0,0 +1,14 @@
+{
+  "name": "bundler1",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "dependencies": {
+    "babel-core": "^6.25.0",
+    "babylon": "^6.17.4",
+    "babylon-walk": "^1.0.2",
+    "browser-resolve": "^1.11.2",
+    "node-libs-browser": "^2.0.0",
+    "worker-farm": "^1.4.1"
+  }
+}
diff --git a/packages/core/parcel-bundler/src/Bundle.js b/packages/core/parcel-bundler/src/Bundle.js
new file mode 100644
index 00000000000..746b87c2710
--- /dev/null
+++ b/packages/core/parcel-bundler/src/Bundle.js
@@ -0,0 +1,60 @@
+const fs = require('./utils/fs');
+const Module = require('./Module');
+const Resolver = require('./Resolver');
+const Parser = require('./Parser');
+const workerFarm = require('worker-farm');
+const promisify = require('./utils/promisify');
+
+class Bundle {
+  constructor(main, options) {
+    this.mainFile = main;
+    this.options = options;
+    this.resolver = new Resolver(options);
+    // this.parser = new Parser(options);
+
+    this.loadedModules = new Map;
+    this.loading = new Set;
+    this.farm = workerFarm({autoStart: true},require.resolve('./worker.js'));
+    this.runFarm = promisify(this.farm);
+  }
+
+  async collectDependencies() {
+    let main = await this.resolveModule(this.mainFile);
+    await this.loadModule(main);
+    console.log('done')
+    workerFarm.end(this.farm);
+    return main;
+  }
+
+  async resolveModule(name, parent) {
+    let path = await this.resolver.resolve(name, parent);
+    if (this.loadedModules.has(path)) {
+      return this.loadedModules.get(path);
+    }
+
+    let module = new Module(path, this.options);
+    this.loadedModules.set(path, module);
+    return module;
+  }
+
+  async loadModule(module) {
+    if (this.loading.has(module)) {
+      return;
+    }
+
+    this.loading.add(module);
+
+    let deps = await this.runFarm(module.name, this.options);
+
+    module.dependencies = deps;
+    // module.ast = ast;
+
+    await Promise.all(deps.map(async dep => {
+      let mod = await this.resolveModule(dep, module.name);
+      module.modules.set(dep, mod);
+      await this.loadModule(mod);
+    }));
+  }
+}
+
+module.exports = Bundle;
diff --git a/packages/core/parcel-bundler/src/Module.js b/packages/core/parcel-bundler/src/Module.js
new file mode 100644
index 00000000000..2a9dc0ade64
--- /dev/null
+++ b/packages/core/parcel-bundler/src/Module.js
@@ -0,0 +1,32 @@
+const traverse = require('babel-traverse').default;
+const types = require('babel-types');
+const path = require('path');
+const collectDependencies = require('./visitors/dependencies');
+const walk = require('babylon-walk');
+
+class Module {
+  constructor(name, options = {}) {
+    this.name = name;
+    this.basename = path.basename(this.name, path.extname(this.name));
+    this.code = null;
+    this.ast = null;
+    this.options = options;
+
+    this.dependencies = new Set;
+    this.modules = new Map;
+  }
+
+  async load() {
+
+  }
+
+  traverse(visitor) {
+    return walk.simple(this.ast, visitor, this);
+  }
+
+  collectDependencies() {
+    this.traverse(collectDependencies);
+  }
+}
+
+module.exports = Module;
diff --git a/packages/core/parcel-bundler/src/Parser.js b/packages/core/parcel-bundler/src/Parser.js
new file mode 100644
index 00000000000..26f2c16a2b9
--- /dev/null
+++ b/packages/core/parcel-bundler/src/Parser.js
@@ -0,0 +1,46 @@
+const path = require('path');
+const es6 = require('./parsers/es6');
+const json = require('./parsers/json');
+
+class Parser {
+  constructor(options = {}) {
+    this.extensions = {};
+
+    let extensions = options.extensions || {};
+    for (let ext in extensions) {
+      this.registerExtension(ext, extensions[ext]);
+    }
+
+    this.registerExtension('.js', es6);
+    this.registerExtension('.jsx', es6);
+    this.registerExtension('.es6', es6);
+    this.registerExtension('.json', json);
+  }
+
+  registerExtension(ext, parser) {
+    if (typeof parser === 'string') {
+      parser = require(parser);
+    }
+
+    this.extensions[ext] = parser;
+  }
+
+  findParser(filename) {
+    let extension = path.extname(filename);
+    let parser = this.extensions[extension];
+    if (!parser) {
+      throw new Error('Could not find parser for extension ' + extension);
+    }
+
+    return parser;
+  }
+
+  parse(filename, code) {
+    let parser = this.findParser(filename);
+    let options = Object.assign({filename: filename}, this.options);
+    // console.log('parsing', filename)
+    return parser(code, options);
+  }
+}
+
+module.exports = Parser;
diff --git a/packages/core/parcel-bundler/src/Resolver.js b/packages/core/parcel-bundler/src/Resolver.js
new file mode 100644
index 00000000000..85b52f91d54
--- /dev/null
+++ b/packages/core/parcel-bundler/src/Resolver.js
@@ -0,0 +1,41 @@
+const promisify = require('./utils/promisify');
+// const builtins from './builtins';
+// import _resolve from 'browser-resolve';
+const resolve = promisify(require('browser-resolve'));
+const builtins = require('node-libs-browser');
+const path = require('path');
+
+for (let key in builtins) {
+  if (builtins[key] == null) {
+    builtins[key] = require.resolve('./_empty.js');
+  }
+}
+
+class Resolver {
+  constructor(options = {}) {
+    this.options = options;
+    this.cache = new Map;
+  }
+
+  async resolve(filename, parent) {
+    let key = (parent ? path.dirname(parent) : '') + ':' + filename;
+    if (this.cache.has(key)) {
+      // console.log('cached!', key)
+      return this.cache.get(key);
+    }
+
+    var res = await resolve(filename, {
+      filename: parent,
+      paths: this.options.paths,
+      modules: builtins
+    });
+
+    if (Array.isArray(res))
+      res = res[0];
+
+    this.cache.set(key, res);
+    return res;
+  }
+}
+
+module.exports = Resolver;
diff --git a/packages/core/parcel-bundler/src/_empty.js b/packages/core/parcel-bundler/src/_empty.js
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/packages/core/parcel-bundler/src/builtins.js b/packages/core/parcel-bundler/src/builtins.js
new file mode 100644
index 00000000000..26c8904eca6
--- /dev/null
+++ b/packages/core/parcel-bundler/src/builtins.js
@@ -0,0 +1,38 @@
+exports.assert = require.resolve('assert/');
+exports.buffer = require.resolve('buffer/');
+exports.child_process = require.resolve('./_empty.js');
+exports.cluster = require.resolve('./_empty.js');
+exports.console = require.resolve('console-browserify');
+exports.constants = require.resolve('constants-browserify');
+exports.crypto = require.resolve('crypto-browserify');
+exports.dgram = require.resolve('./_empty.js');
+exports.dns = require.resolve('./_empty.js');
+exports.domain = require.resolve('domain-browser');
+exports.events = require.resolve('events/');
+exports.fs = require.resolve('./_empty.js');
+exports.http = require.resolve('stream-http');
+exports.https = require.resolve('https-browserify');
+exports.module = require.resolve('./_empty.js');
+exports.net = require.resolve('./_empty.js');
+exports.os = require.resolve('os-browserify/browser.js');
+exports.path = require.resolve('path-browserify');
+exports.punycode = require.resolve('punycode/');
+exports.querystring = require.resolve('querystring-es3/');
+exports.readline = require.resolve('./_empty.js');
+exports.repl = require.resolve('./_empty.js');
+exports.stream = require.resolve('stream-browserify');
+exports._stream_duplex = require.resolve('readable-stream/duplex.js');
+exports._stream_passthrough = require.resolve('readable-stream/passthrough.js');
+exports._stream_readable = require.resolve('readable-stream/readable.js');
+exports._stream_transform = require.resolve('readable-stream/transform.js');
+exports._stream_writable = require.resolve('readable-stream/writable.js');
+exports.string_decoder = require.resolve('string_decoder/');
+exports.sys = require.resolve('util/util.js');
+exports.timers = require.resolve('timers-browserify');
+exports.tls = require.resolve('./_empty.js');
+exports.tty = require.resolve('tty-browserify');
+exports.url = require.resolve('url/');
+exports.util = require.resolve('util/util.js');
+exports.vm = require.resolve('vm-browserify');
+exports.zlib = require.resolve('browserify-zlib');
+exports._process = require.resolve('process/browser');
diff --git a/packages/core/parcel-bundler/src/parsers/es6.js b/packages/core/parcel-bundler/src/parsers/es6.js
new file mode 100644
index 00000000000..8cb0d6b8ca7
--- /dev/null
+++ b/packages/core/parcel-bundler/src/parsers/es6.js
@@ -0,0 +1,27 @@
+const babylon = require('babylon');
+
+module.exports = function (code, opts) {
+  const options = {
+    filename: opts.filename,
+    allowImportExportEverywhere: opts.loose,
+    allowReturnOutsideFunction: true,
+    allowHashBang: true,
+    ecmaVersion: Infinity,
+    strictMode: false,
+    sourceType: 'module',
+    locations: true,
+    features: opts.features || {},
+    plugins: opts.plugins || [
+      'asyncFunctions',
+      'asyncGenerators',
+      'classConstructorCall',
+      'classProperties',
+      'decorators',
+      'exportExtensions',
+      'jsx',
+      'flow'
+    ]
+  };
+
+  return babylon.parse(code, options);
+};
diff --git a/packages/core/parcel-bundler/src/parsers/json.js b/packages/core/parcel-bundler/src/parsers/json.js
new file mode 100644
index 00000000000..5619a76c037
--- /dev/null
+++ b/packages/core/parcel-bundler/src/parsers/json.js
@@ -0,0 +1,5 @@
+const es6 = require('./es6');
+
+module.exports = function (code, opts) {
+  return es6('module.exports = ' + code + ';', opts);
+};
diff --git a/packages/core/parcel-bundler/src/utils/fs.js b/packages/core/parcel-bundler/src/utils/fs.js
new file mode 100644
index 00000000000..e03de8b3f04
--- /dev/null
+++ b/packages/core/parcel-bundler/src/utils/fs.js
@@ -0,0 +1,5 @@
+const promisify = require('./promisify');
+const fs = require('fs');
+
+exports.readFile = promisify(fs.readFile);
+exports.writeFile = promisify(fs.writeFile);
diff --git a/packages/core/parcel-bundler/src/utils/promisify.js b/packages/core/parcel-bundler/src/utils/promisify.js
new file mode 100644
index 00000000000..23f0d9e3354
--- /dev/null
+++ b/packages/core/parcel-bundler/src/utils/promisify.js
@@ -0,0 +1,15 @@
+module.exports = function(fn) {
+  return function(...args) {
+    return new Promise(function(resolve, reject) {
+      fn(...args, function(err, ...res) {
+        if (err)
+          return reject(err);
+
+        if (res.length === 1)
+          return resolve(res[0]);
+
+        resolve(res);
+      });
+    });
+  };
+}
diff --git a/packages/core/parcel-bundler/src/utils/queue.js b/packages/core/parcel-bundler/src/utils/queue.js
new file mode 100644
index 00000000000..75694ed3ace
--- /dev/null
+++ b/packages/core/parcel-bundler/src/utils/queue.js
@@ -0,0 +1,52 @@
+class Queue {
+  constructor() {
+    this.q = new Map;
+    this.promises = new Map;
+    this.running = 0;
+    this.concurrency = 50;
+  }
+
+  add(key, fn) {
+    if (!this.q.has(key)) {
+      let promise = new Promise((resolve, reject) => {
+        this.q.set(key, {fn, resolve, reject});
+        this.run();
+      });
+
+      this.promises.set(key, promise);
+      return promise;
+    } else {
+      return this.promises.get(key);
+    }
+  }
+
+  run() {
+    while (this.running < this.concurrency && this.q.size > 0) {
+      this.processNext();
+    }
+  }
+
+  async processNext() {
+    if (this.q.size === 0) {
+      return;
+    }
+
+    let [key, {fn, resolve, reject}] = this.q.entries().next().value;
+    this.q.delete(key);
+
+    this.running++;
+
+    try {
+      let res = await fn();
+      resolve(res);
+    } catch (err) {
+      reject(err);
+    } finally {
+      this.running--;
+      this.promises.delete(key);
+      this.run();
+    }
+  }
+}
+
+module.exports = Queue;
\ No newline at end of file
diff --git a/packages/core/parcel-bundler/src/visitors/dependencies.js b/packages/core/parcel-bundler/src/visitors/dependencies.js
new file mode 100644
index 00000000000..3c45cddaaaa
--- /dev/null
+++ b/packages/core/parcel-bundler/src/visitors/dependencies.js
@@ -0,0 +1,32 @@
+const types = require('babel-types');
+
+module.exports = {
+  ImportDeclaration(node, module) {
+    module.dependencies.add(node.source.value);
+  },
+
+  ExportNamedDeclaration(node, module) {
+    if (node.source) {
+      module.dependencies.add(node.source.value);
+    }
+  },
+
+  ExportAllDeclaration(node, module) {
+    module.dependencies.add(node.source.value);
+  },
+
+  CallExpression(node, module) {
+    let {callee, arguments: args} = node;
+
+    let isRequire = types.isIdentifier(callee)
+                 && callee.name === 'require'
+                 && args.length === 1
+                 && types.isStringLiteral(args[0]);
+
+    if (!isRequire) {
+      return;
+    }
+
+    module.dependencies.add(args[0].value);
+  }
+};
diff --git a/packages/core/parcel-bundler/src/worker.js b/packages/core/parcel-bundler/src/worker.js
new file mode 100644
index 00000000000..ce5fb6748b6
--- /dev/null
+++ b/packages/core/parcel-bundler/src/worker.js
@@ -0,0 +1,23 @@
+const fs = require('./utils/fs');
+const Module = require('./Module');
+const Parser = require('./Parser');
+const babel = require('babel-core');
+
+process.on('unhandledRejection', console.error)
+
+let parser;
+
+module.exports = async function (path, options, callback) {
+  if (!parser) {
+    parser = new Parser(options || {});
+  }
+
+  let mod = new Module(path, options);
+  mod.code = await fs.readFile(path, 'utf8');
+  mod.ast = parser.parse(path, mod.code);
+  mod.collectDependencies();
+
+  // let res = babel.transformFromAst(mod.ast);
+  // console.log(res.code)
+  callback(null, Array.from(mod.dependencies));
+};
diff --git a/test.js b/test.js
new file mode 100644
index 00000000000..aa1619a43b5
--- /dev/null
+++ b/test.js
@@ -0,0 +1,23 @@
+const Bundle = require('./src/Bundle');
+// const babel = require('babel-core');
+
+process.on('unhandledRejection', console.error)
+
+async function run() {
+  let bundle = new Bundle('/Users/devongovett/projects/Storify/liveblog-editor/src/Editor.js');
+  let module = await bundle.collectDependencies();
+  printDeps(module);
+}
+
+function printDeps(module, indent = '', deps = new Set) {
+  for (let [file, mod] of module.modules) {
+    console.log(indent + file);
+    if (!deps.has(mod.name)) {
+      deps.add(mod.name);
+      printDeps(mod, indent + '  ', deps);
+      // babel.transformFromAst(mod.ast);
+    }
+  }
+}
+
+run().then(console.log, console.error);