can run a standalone that includes the parser, and the filesystem

Makefile

@@ -168,15 +168,15 @@ $(PHASE3)/main-wrapper.js: src/scripts/main-wrapper.js
 	cp $< $@
 
 $(PHASE1)/$(JS)/%-parser.js: src/$(JSBASE)/%-grammar.bnf src/$(JSBASE)/%-tokenizer.js $(wildcard lib/jglr/*.js)
-	$(NODE) lib/jglr/parser-generator.js src/$(JSBASE)/$*-grammar.bnf $(PHASE1)/$(JS)/$*-grammar.js
+	$(NODE) lib/jglr/parser-generator.js src/$(JSBASE)/$*-grammar.bnf $(PHASE1)/$(JS)/$*-grammar.js "../../../lib/jglr/jglr"
 	$(NODE) $(PHASE1)/$(JS)/$*-grammar.js $(PHASE1)/$(JS)/$*-parser.js
 
 $(PHASE2)/$(JS)/%-parser.js: src/$(JSBASE)/%-grammar.bnf src/$(JSBASE)/%-tokenizer.js $(wildcard lib/jglr/*.js)
-	$(NODE) lib/jglr/parser-generator.js src/$(JSBASE)/$*-grammar.bnf $(PHASE2)/$(JS)/$*-grammar.js
+	$(NODE) lib/jglr/parser-generator.js src/$(JSBASE)/$*-grammar.bnf $(PHASE2)/$(JS)/$*-grammar.js  "../../../lib/jglr/jglr"
 	$(NODE) $(PHASE2)/$(JS)/$*-grammar.js $(PHASE2)/$(JS)/$*-parser.js
 
 $(PHASE3)/$(JS)/%-parser.js: src/$(JSBASE)/%-grammar.bnf src/$(JSBASE)/%-tokenizer.js $(wildcard lib/jglr/*.js)
-	$(NODE) lib/jglr/parser-generator.js src/$(JSBASE)/$*-grammar.bnf $(PHASE3)/$(JS)/$*-grammar.js
+	$(NODE) lib/jglr/parser-generator.js src/$(JSBASE)/$*-grammar.bnf $(PHASE3)/$(JS)/$*-grammar.js  "../../../lib/jglr/jglr"
 	$(NODE) $(PHASE3)/$(JS)/$*-grammar.js $(PHASE3)/$(JS)/$*-parser.js
 
 $(PHASE1)/$(JS)/%.js : src/$(JSBASE)/%.js

build/phaseA/runtime.js

@@ -4156,7 +4156,7 @@ function isMethod(obj) { return obj instanceof PMethod; }
       else {
         var uri = toLoad[0];
         var mod = staticMods[uri];
-        console.log(uri, mod);
+        // console.log(uri, mod);
 
         var reqs = mod.requires;
         if(depMap[uri] === undefined) {
@@ -4172,30 +4172,42 @@ function isMethod(obj) { return obj instanceof PMethod; }
           }
           return getExported(thisRuntime.modules[duri]);
         });
+
         return thisRuntime.safeCall(function() {
-          return mod.theModule.apply(null, [thisRuntime, thisRuntime.namespace, uri].concat(reqInstantiated));
-        }, function(r) {
-          console.log("Result from module: ", r);
-          thisRuntime.modules[uri] = r;
-          function continu() {
-            return thisRuntime.safeTail(function() {
-              return runStandalone(staticMods, depMap, toLoad.slice(1), postLoadHooks);
+            thisRuntime.pauseStack(function(restarter) {
+              console.log("About to load: ", mod.nativeRequires);
+              require(mod.nativeRequires, function(/* varargs */) {
+                var nativeInstantiated = Array.prototype.slice.call(arguments);
+                console.log("Loaded: ", nativeInstantiated);
+                restarter.resume(nativeInstantiated);
+              });
             });
-          }
-          if(uri in postLoadHooks) {
+          }, function(natives) {
             return thisRuntime.safeCall(function() {
-              return postLoadHooks[uri](r);
-            }, function(_) {
-              return continu();
-            });
+              return mod.theModule.apply(null, [thisRuntime, thisRuntime.namespace, uri].concat(reqInstantiated).concat(natives));
+            }, 
+            function(r) {
+//              console.log("Result from module: ", r);
+              thisRuntime.modules[uri] = r;
+              function continu() {
+                return thisRuntime.safeTail(function() {
+                  return runStandalone(staticMods, depMap, toLoad.slice(1), postLoadHooks);
+                });
+              }
+              if(uri in postLoadHooks) {
+                return thisRuntime.safeCall(function() {
+                  return postLoadHooks[uri](r);
+                }, function(_) {
+                  return continu();
+                });
 
-          }
-          else {
-            return continu();
-          }
-        });
+              }
+              else {
+                return continu();
+              }
+            });
+          });
       }
-
     }
 
     function JSModuleReturn(jsmod) {

lib/jglr/parser-generator.js

@@ -1,10 +1,6 @@
 const R = require('requirejs');
 
-R.config({
-  baseUrl: "./"
-});
-
-R(['lib/jglr/jglr', 'fs'], function(E, fs) {
+R(['jglr', 'fs'], function(E, fs) {
   const Grammar = E.Grammar
   const Nonterm = E.Nonterm
   const Token = E.Token
@@ -300,11 +296,12 @@ R(['lib/jglr/jglr', 'fs'], function(E, fs) {
     // console.log(parses[0].toString());
     var bnfJS = generateGrammar(parses[0]);
     var filename = process.argv[3];
+    var jglrPath = process.argv[4];
     var out = fs.createWriteStream(filename);
     out.write("const R = require('requirejs');\n\n");
 //    out.write("R.config({ baseUrl: __dirname });\n");
 //    out.write("console.log(__dirname);\n");
-    out.write("R(['fs', '../../../lib/jglr/jglr'], function(fs, E) {\n");
+    out.write("R(['fs', '"  + jglrPath + "'], function(fs, E) {\n");
     out.write("  const Grammar = E.Grammar\n");
     out.write("  const Nonterm = E.Nonterm\n");
     out.write("  const Token = E.Token\n");
@@ -327,7 +324,7 @@ R(['lib/jglr/jglr', 'fs'], function(E, fs) {
     out.write("  var filename = process.argv[2];\n");
     out.write("  var out = fs.createWriteStream(filename);\n");
 
-    out.write("  out.write(\"define(['../../../lib/jglr/jglr'],\\n\");\n");
+    out.write("  out.write(\"define(['" + jglrPath + "'],\\n\");\n");
     out.write("  out.write(\"/** @param {{Grammar : {fromSerializable : !Function}, Nonterm : !Object, Token : !Object, Rule : !Object}} E */\\n\");\n");
     out.write("  out.write(\"function(E) {\\n\");\n");
     out.write("  out.write(\"  const Grammar = E.Grammar;\\n\");\n");

module-design.txt

@@ -17,11 +17,15 @@
     + done by assigning into modules via URI
   - do any currying massaging necessary to emulate defineModule for pure JS
     + not doing it, just changing the shape of module funs
+
 4.5.
   - find nativeRequires using requirejs
+  - Add nativeRequires visibility on locators to kick off pure JS imports
+    + Make sure to check existing locators to add get-native-modules!
   - change all js/trove modules to have new object literal shape (including
     nativeRequires)
   - nuke initialize-trove?
+
 [x] 5. "Manually" (e.g. with cat) put 1-3 together with some modules to test
 [x] 6. New entrypoint to anf-loop-compiler that generates object literals
        - Added as the compile-module flag in compile options
@@ -41,7 +45,7 @@
       save-modules (List<Loadable> -> Nothing)
     }
 
-11. Figure out how to checkpoint the dependency list on ffi.  Doing
+[x] 11. Figure out how to checkpoint the dependency list on ffi.  Doing
 
     "import ffi as _"
 
@@ -60,7 +64,6 @@ rather than Pyret-shaped returns, for e.g. ffi, image-lib, and other JS-to-JS
 modules.  This avoids them being import-able from Pyret, which doesn't make
 much sense.
 
-
 [x] 14. Wrangle legacy-path and maybe builtin includes?
     - Change arr/trove modules to have explicit includes for list, etc
     - Ditch legacy path locator, and change all compiler-to-compiler

src/arr/compiler/cli-module-loader.arr

@@ -161,12 +161,16 @@ fun build-standalone(path, options):
       end))
   end)
 
+  natives = j-list(true, for fold(natives from [clist:], w from wl):
+    C.map_list(lam(r): j-str(r.path) end, w.locator.get-native-modules()) + natives
+  end)
+
   to-load = j-list(false, for C.map_list(w from wl):
     j-str(w.locator.uri())
   end)
 
   prog = j-block([clist:
-    j-app(define-name, [clist: j-list(true, [clist:]), j-fun([clist:],
+    j-app(define-name, [clist: natives, j-fun([clist:],
         j-block([clist:
           j-return(j-obj([clist:
             j-field("staticModules", static-modules),

src/arr/compiler/compile-lib.arr

@@ -57,6 +57,9 @@ type Locator = {
   # Pre-compile (had better be known with no other help)
   get-dependencies :: ( -> List<CS.Dependency>),
 
+  # Pre-compile (to find list of requirejs dependencies to include)
+  get-native-modules :: ( -> List<CS.NativeModule>),
+
   # Pre-compile
   get-extra-imports :: ( -> CS.ExtraImports),
 

src/arr/compiler/compile-structs.arr

@@ -48,6 +48,10 @@ data Dependency:
     key(self): "builtin(" + self.modname + ")" end
 end
 
+data NativeModule:
+  | requirejs(path :: String)
+end
+
 data NameResolution:
   | resolved(
       ast :: A.Program,

src/arr/compiler/locators/builtin.arr

@@ -64,6 +64,10 @@ fun make-builtin-js-locator(basedir, builtin-name):
       deps = raw.get-raw-dependencies()
       raw-array-to-list(deps).map(make-dep)
     end,
+    get-native-modules(_):
+      natives = raw.get-raw-native-modules()
+      raw-array-to-list(natives).map(CM.requirejs)
+    end,
     get-globals(_):
       raise("Should never get compile-env for builtin module " + builtin-name)
     end,
@@ -107,6 +111,9 @@ fun make-builtin-arr-locator(basedir, builtin-name):
     get-dependencies(self):
       CL.get-dependencies(self.get-module(), self.uri())
     end,
+    get-native-modules(self):
+      [list:]
+    end,
     get-extra-imports(self):
       CM.minimal-imports
     end,

src/arr/compiler/locators/file.arr

@@ -28,6 +28,9 @@ fun mockable-file-locator(file-ops):
     get-dependencies(self):
       CL.get-standard-dependencies(self.get-module(), self.uri())
     end,
+    get-native-modules(self):
+      [list:]
+    end,
     get-extra-imports(self):
       CS.standard-imports
     end,

src/js/trove/builtin-modules.js

@@ -41,6 +41,14 @@ define(["fs", "js/secure-loader", "js/runtime-util", "js/type-util"], function(f
                   return [];
                 }
               }),
+            "get-raw-native-modules":
+              F(function() {
+                if(Array.isArray(m.nativeRequires)) {
+                  return m.nativeRequires.map(RUNTIME.makeString);
+                } else {
+                  return [];
+                }
+              }),
             "get-raw-datatype-provides":
               F(function() {
                 if(m.provides && m.provides.datatypes) {

src/js/troveA/base.js

@@ -1,4 +1,4 @@
-{
+({
   requires: [
     { "import-type": "builtin", name: "option" },
     { "import-type": "builtin", name: "either" },
@@ -15,7 +15,7 @@
     { "import-type": "builtin", name: "ffi" }
   ],
   provides: {},
-  nativeRequires: {},
+  nativeRequires: [],
   theModule: function(runtime, namespace, uri /* intentionally blank */) {
     return runtime.makeObject({
       "provide-plus-types": runtime.makeObject({
@@ -24,4 +24,4 @@
       })
     });
   }
-}
+})