Prototype next gen i18n framework (kubernetes#1239)

* Next gen i18n framework

Works on serve, serve:prod and build. Message extraction works too.

* Remove commented-out code

build/conf.js

@@ -205,6 +205,7 @@ export default {
     materialIcons: path.join(basePath, 'bower_components/material-design-icons/iconfont'),
     nodeModules: path.join(basePath, 'node_modules'),
     partials: path.join(basePath, '.tmp/partials'),
+    messagesForExtraction: path.join(basePath, '.tmp/messages_for_extraction'),
     prodTmp: path.join(basePath, '.tmp/prod'),
     protractorConf: path.join(basePath, 'build/protractor.conf.js'),
     robotoFonts: path.join(basePath, 'bower_components/roboto-fontface/fonts'),

build/i18n.js

@@ -19,24 +19,27 @@ import childProcess from 'child_process';
 import fileExists from 'file-exists';
 import gulp from 'gulp';
 import gulpUtil from 'gulp-util';
+import jsesc from 'jsesc';
 import path from 'path';
 import q from 'q';
+import regexpClone from 'regexp-clone';
 
 import conf from './conf';
 
 /**
  * Extracts the translatable text messages for the given language key from the pre-compiled
- * files under conf.paths.serve.
+ * files under conf.paths.{serve|messagesForExtraction}.
  * @param  {string} langKey - the locale key
  * @return {!Promise} A promise object.
  */
 function extractForLanguage(langKey) {
   let deferred = q.defer();
 
   let translationBundle = path.join(conf.paths.base, `i18n/messages-${langKey}.xtb`);
-  let codeSource = path.join(conf.paths.serve, '*.js');
+  let codeSource = path.join(conf.paths.serve, '**.js');
+  let messagesSource = path.join(conf.paths.messagesForExtraction, '**.js');
   let command = `java -jar ${conf.paths.xtbgenerator} --lang ${langKey}` +
-      ` --xtb_output_file ${translationBundle} --js ${codeSource}`;
+      ` --xtb_output_file ${translationBundle} --js ${codeSource} --js ${messagesSource}`;
   if (fileExists(translationBundle)) {
     command = `${command} --translations_file ${translationBundle}`;
   }
@@ -45,7 +48,6 @@ function extractForLanguage(langKey) {
     if (err) {
       gulpUtil.log(stdout);
       gulpUtil.log(stderr);
-      deferred.reject();
       deferred.reject(new Error(err));
     }
     return deferred.resolve();
@@ -57,7 +59,76 @@ function extractForLanguage(langKey) {
 /**
  * Extracts all translation messages into XTB bundles.
  */
-gulp.task('extract-translations', ['scripts'], function() {
+gulp.task('extract-translations', ['scripts', 'angular-templates'], function() {
   let promises = conf.translations.map((translation) => extractForLanguage(translation.key));
   return q.all(promises);
 });
+
+
+// Regex to match [[Foo | Bar]] or [[Foo]] i18n placeholders.
+// Technical details:
+// * First capturing group is lazy math for any string not-containing |. This is to make
+//   both [[ message | desription ]] and [[ message ]] work.
+// * Second is non-capturing and optional. It has a capturing group inside. This is to
+//   extract description that is optional.
+const I18N_REGEX = /\[\[([^|]*?)(?:\|(.*?))?\]\]/mg;
+
+export function processI18nMessages(file, minifiedHtml) {
+  let pureHtmlContent = minifiedHtml;
+  let content = jsesc(minifiedHtml);
+  let filePath = path.relative(file.base, file.path);
+  let messageVarPrefix = filePath.toUpperCase().split('/').join('_').replace('.HTML', '');
+
+  /**
+   * Finds all i18n messages inside a template and returns its text, description and original
+   * string.
+   * @param {string} htmlContent
+   * @return {!Array<{text: string, desc: string, original: string}>}
+   */
+  function findI18nMessages(htmlContent) {
+    let matches = htmlContent.match(I18N_REGEX);
+    if (matches) {
+      return matches.map((match) => {
+        let exec = regexpClone(I18N_REGEX).exec(match);
+        // Default to no description when it is not provided.
+        let desc = (exec[2] || '(no description provided)').trim();
+        return {text: exec[1], desc: desc, original: match};
+      });
+    }
+    return [];
+  }
+
+  let i18nMessages = findI18nMessages(content);
+
+  /**
+   * @param {number} index
+   * @return {string}
+   */
+  function createMessageVarName(index) {
+    return `MSG_${messageVarPrefix}_${index}`;
+  }
+
+  i18nMessages.forEach((message, index) => {
+    let messageVarName = createMessageVarName(index);
+    // Replace i18n messages with english messages for testing and MSG_ vars invocations
+    // for compiler passses.
+    content = content.replace(message.original, `' + ${messageVarName} + '`);
+    pureHtmlContent = pureHtmlContent.replace(message.original, message.text);
+  });
+
+  let messageVariables = i18nMessages.map((message, index) => {
+    let messageVarName = createMessageVarName(index);
+    return `/** @desc ${message.desc} */\n` +
+        `var ${messageVarName} = goog.getMsg('${message.text}');\n`;
+  });
+
+  file.messages = messageVariables.join('\n');
+  file.pureHtmlContent = pureHtmlContent;
+  file.moduleContent = `` +
+      `import module from 'index_module';\n\n${file.messages}\n` +
+      `module.run(['$templateCache', ($templateCache) => {\n` +
+      `    $templateCache.put('${filePath}', '${content}');\n` +
+      `}]);\n`;
+
+  return minifiedHtml;
+}

build/script.js

@@ -17,13 +17,15 @@
  */
 import async from 'async';
 import gulp from 'gulp';
-import gulpAngularTemplatecache from 'gulp-angular-templatecache';
 import gulpClosureCompiler from 'gulp-closure-compiler';
 import gulpHtmlmin from 'gulp-htmlmin';
+import gulpModify from 'gulp-modify';
+import gulpRename from 'gulp-rename';
 import path from 'path';
 import webpackStream from 'webpack-stream';
 
 import conf from './conf';
+import {processI18nMessages} from './i18n';
 
 /**
  * Returns function creating a stream that compiles frontend JavaScript files into development bundle located in
@@ -197,7 +199,30 @@ gulp.task('scripts:prod', ['angular-templates', 'extract-translations'], functio
 });
 
 /**
- * Compiles Angular HTML template files into one JS file that serves them through $templateCache.
+ * Compiles frontend JavaScript files into production bundle located in {conf.paths.prodTmp}
+ * directory. A separated bundle is created for each i18n locale.
+ */
+gulp.task('scripts:prod', ['angular-templates', 'extract-translations'], function(doneFn) {
+  // add a compilation step to stream for each translation file
+  let streams = conf.translations.map((translation) => {
+    return createCompileTask(translation);
+  });
+
+  // add a default compilation task (no localization)
+  streams = streams.concat(createCompileTask());
+
+  // TODO (taimir) : do not run the tasks sequentially once
+  // gulp-closure-compiler can be run in parallel
+  async.series(streams, doneFn);
+});
+
+/**
+ * Compiles each Angular HTML template file (path/foo.html) into three processed forms:
+ *   * serve/path/foo.html - minified html with i18n messages stripped out into english form
+ *   * partials/path/foo.html.js - JS module file with template added to $templateCache, ready to be
+ *         compiled by closure compiler
+ *   * messages_for_extraction/path/foo.html.js - file with only MSG_FOO i18n message
+ *         definitions - used to extract messages
  */
 gulp.task('angular-templates', function() {
   return gulp.src(path.join(conf.paths.frontendSrc, '**/!(index).html'))
@@ -206,8 +231,26 @@ gulp.task('angular-templates', function() {
         collapseWhitespace: true,
         conservativeCollapse: true,
       }))
-      .pipe(gulpAngularTemplatecache('angular-templates.js', {
-        module: conf.frontend.rootModuleName,
+      .pipe(gulpModify({fileModifier: processI18nMessages}))
+      .pipe(gulpModify({
+        fileModifier: function(file) {
+          return file.pureHtmlContent;
+        },
+      }))
+      .pipe(gulp.dest(conf.paths.serve))
+      .pipe(gulpModify({
+        fileModifier: function(file) {
+          return file.moduleContent;
+        },
+      }))
+      .pipe(gulpRename(function(path) {
+        path.extname = '.html.js';
+      }))
+      .pipe(gulp.dest(conf.paths.partials))
+      .pipe(gulpModify({
+        fileModifier: function(file) {
+          return file.messages;
+        },
       }))
-      .pipe(gulp.dest(conf.paths.partials));
+      .pipe(gulp.dest(conf.paths.messagesForExtraction));
 });

build/serve.js

@@ -108,8 +108,7 @@ function serveDevelopmentMode() {
   browserSyncInit(
       [
         conf.paths.serve,
-        conf.paths.frontendSrc,  // For angular templates to work.
-        conf.paths.app,          // For assets to work.
+        conf.paths.app,  // For assets to work.
       ],
       true);
 }
@@ -189,7 +188,7 @@ gulp.task('kill-backend', function(doneFn) {
 /**
  * Watches for changes in source files and runs Gulp tasks to rebuild them.
  */
-gulp.task('watch', ['index'], function() {
+gulp.task('watch', ['index', 'angular-templates'], function() {
   gulp.watch([path.join(conf.paths.frontendSrc, 'index.html'), 'bower.json'], ['index']);
 
   gulp.watch(
@@ -207,5 +206,6 @@ gulp.task('watch', ['index'], function() {
       });
 
   gulp.watch(path.join(conf.paths.frontendSrc, '**/*.js'), ['scripts-watch']);
+  gulp.watch(path.join(conf.paths.frontendSrc, '**/*.html'), ['angular-templates']);
   gulp.watch(path.join(conf.paths.backendSrc, '**/*.go'), ['spawn-backend']);
 });

package.json

@@ -25,7 +25,6 @@
     "file-exists": "2.0.0",
     "google-closure-compiler": "~20160911.0.0",
     "gulp": "~3.9.1",
-    "gulp-angular-templatecache": "~2.0.0",
     "gulp-autoprefixer": "~3.1.0",
     "gulp-browserify": "~0.5.1",
     "gulp-clang-format": "1.0.23",
@@ -39,6 +38,7 @@
     "gulp-if": "~2.0.1",
     "gulp-inject": "~4.1.0",
     "gulp-minify-css": "~1.2.4",
+    "gulp-modify": "~0.1.1",
     "gulp-protractor": "~2.6.0",
     "gulp-rename": "~1.2.2",
     "gulp-replace": "~0.5.4",
@@ -53,6 +53,7 @@
     "gulp-watch": "~4.3.8",
     "html-minifier": "~3.1.0",
     "isparta": "~4.0.0",
+    "jsesc": "~2.2.0",
     "karma": "1.3.0",
     "karma-browserify": "~5.1.0",
     "karma-chrome-launcher": "~2.0.0",
@@ -69,6 +70,7 @@
     "npm-check-updates": "~2.8.0",
     "proxy-middleware": "~0.15.0",
     "q": "~1.4.1",
+    "regexp-clone": "~0.0.1",
     "semver": "~5.3.0",
     "through2": "~2.0.1",
     "uglify-save-license": "~0.4.1",