Initial i18n implementation for frontend and backend

.babelrc

@@ -11,6 +11,16 @@
   "env": {
     "development": {
       "presets": []
+    },
+    "extract": {
+      "plugins": [
+        ["c-3po", {
+          "extract": {
+            "output": "locales/metabase-frontend.pot"
+          },
+          "discover": ["t"]
+        }]
+      ]
     }
   }
 }

.eslintrc

@@ -42,7 +42,8 @@
         "flowtype/use-flow-type": 1
     },
     "globals": {
-        "pending": false
+        "pending": false,
+        "t": false
     },
     "env": {
         "browser": true,

.gitignore

@@ -45,3 +45,6 @@ bin/release/aws-eb/metabase-aws-eb.zip
 /build.xml
 /test-report-*
 /crate-*
+*.po~
+/resources/locales.clj
+/locales/*.pot

bin/i18n/build-translation-frontend-resource

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+
+// This program compiles a ".po" translations file to a JSON version suitable for use on the frontend
+// It removes strings that aren't used on the frontend, and other extraneous information like comments
+
+const fs = require("fs");
+const _ = require("underscore");
+const gParser = require("gettext-parser");
+
+if (process.argv.length !== 4) {
+  console.log("USAGE: build-translation-frontend-resource input.po output.json");
+  process.exit(1);
+}
+
+const inputFile = process.argv[2];
+const outputFile = process.argv[3];
+
+const translationObject = gParser.po.parse(fs.readFileSync(inputFile, "utf-8"));
+
+// NOTE: unsure why the headers are duplicated in a translation for "", but we don't need it
+delete translationObject.translations[""][""]
+
+for (const id in translationObject.translations[""]) {
+  const translation = translationObject.translations[""][id];
+  if (!translation.comments.reference || _.any(translation.comments.reference.split("\n"), reference => reference.startsWith("frontend/"))) {
+    // remove comments:
+    delete translation.comments;
+    // NOTE: would be nice if we could remove the message id since it's redundant:
+    // delete translation.msgid;
+  } else {
+    // remove strings that aren't in the frontend
+    delete translationObject.translations[""][id];
+  }
+}
+
+fs.writeFileSync(outputFile, JSON.stringify(translationObject, null, 2), "utf-8");

bin/i18n/build-translation-resources

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+set -eu
+
+# gettext installed via homebrew is "keg-only", add it to the PATH
+if [ -d "/usr/local/opt/gettext/bin" ]; then
+  export PATH="/usr/local/opt/gettext/bin:$PATH"
+fi
+
+POT_NAME="locales/metabase.pot"
+LOCALES=$(find locales -type f -name "*.po" -exec basename {} .po \;)
+
+FRONTEND_LANG_DIR="resources/frontend_client/app/locales"
+
+# backend
+cat << EOF > "resources/locales.clj"
+{
+  :locales  #{"de_DE" "en"}
+  :packages ["metabase"]
+  :bundle   "metabase.Messages"
+}
+EOF
+
+mkdir -p "$FRONTEND_LANG_DIR"
+
+for LOCALE in $LOCALES; do
+  LOCALE_FILE="locales/$LOCALE.po"
+  # frontend
+  # NOTE: just copy these for now, but eventially precompile from .po to .json
+  ./bin/i18n/build-translation-frontend-resource \
+    "$LOCALE_FILE" \
+    "$FRONTEND_LANG_DIR/$LOCALE.json"
+
+  # backend
+  msgfmt                            \
+    --java2                         \
+    -d "resources"                  \
+    -r "metabase.Messages" \
+    -l "$LOCALE"                    \
+    "$LOCALE_FILE"
+done

bin/i18n/update-translation

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -eu
+
+# gettext installed via homebrew is "keg-only", add it to the PATH
+if [ -d "/usr/local/opt/gettext/bin" ]; then
+  export PATH="/usr/local/opt/gettext/bin:$PATH"
+fi
+
+POT_NAME="locales/metabase.pot"
+PO_NAME="locales/$1.po"
+
+if [ $# -lt 1 ]; then
+  echo "USAGE: update-translation en_US"
+  exit 1
+fi
+
+if [ -f "$PO_NAME" ]; then
+  exec msgmerge -U "$PO_NAME" "$POT_NAME"
+else
+  exec msginit -i "$POT_NAME" -o "$PO_NAME" -l "$1"
+fi

bin/i18n/update-translation-template

@@ -0,0 +1,48 @@
+#!/bin/sh
+
+set -eu
+
+# gettext installed via homebrew is "keg-only", add it to the PATH
+if [ -d "/usr/local/opt/gettext/bin" ]; then
+  export PATH="/usr/local/opt/gettext/bin:$PATH"
+fi
+
+POT_NAME="locales/metabase.pot"
+POT_BACKEND_NAME="locales/metabase-backend.pot"
+POT_FRONTEND_NAME="locales/metabase-frontend.pot"
+
+mkdir -p "locales"
+
+# update frontend pot
+
+# NOTE: about twice as fast to call babel directly rather than a full webpack build
+BABEL_ENV=extract ./node_modules/.bin/babel -q -x .js,.jsx -o /dev/null frontend/src
+# BABEL_ENV=extract BABEL_DISABLE_CACHE=1 yarn run build
+
+# update backend pot
+
+# xgettext before 0.19 does not understand --add-location=file. Even CentOS
+# 7 ships with an older gettext. We will therefore generate full location
+# info on those systems, and only file names where xgettext supports it
+LOC_OPT=$(xgettext --add-location=file -f - </dev/null >/dev/null 2>&1 && echo --add-location=file || echo --add-location)
+
+find src -name "*.clj" | xgettext                   \
+  --from-code=UTF-8                                 \
+  --language=lisp                                   \
+  --copyright-holder='Metabase <[email protected]>' \
+  --package-name="metabase"                         \
+  --msgid-bugs-address="[email protected]"          \
+  -k                                                \
+  -kmark:1 -ki18n/mark:1                            \
+  -ktrs:1 -ki18n/trs:1                              \
+  -ktru:1 -ki18n/tru:1                              \
+  -ktrun:1,2 -ki18n/trun:1,2                        \
+  -ktrsn:1,2 -ki18n/trsn:1,2                        \
+  $LOC_OPT                                          \
+  --add-comments --sort-by-file                     \
+  -o $POT_BACKEND_NAME -f -
+
+sed -i "" -e 's/charset=CHARSET/charset=UTF-8/' "$POT_BACKEND_NAME"
+
+# merge frontend and backend pots
+msgcat "$POT_FRONTEND_NAME" "$POT_BACKEND_NAME" > "$POT_NAME"

frontend/src/metabase/app.js

@@ -3,6 +3,10 @@
 import 'babel-polyfill';
 import 'number-to-locale-string';
 
+// make the i18n function "t" global so we don't have to import it in basically every file
+import { t } from "c-3po";
+global.t = t;
+
 import React from 'react'
 import ReactDOM from 'react-dom'
 import { Provider } from 'react-redux'
@@ -20,6 +24,8 @@ import { Router, useRouterHistory } from "react-router";
 import { createHistory } from 'history'
 import { syncHistoryWithStore } from 'react-router-redux';
 
+import { loadLocale } from "metabase/lib/i18n";
+
 // remove trailing slash
 const BASENAME = window.MetabaseRoot.replace(/\/+$/, "");
 
@@ -56,6 +62,9 @@ function _init(reducers, getRoutes, callback) {
         window['ga-disable-' + MetabaseSettings.get('ga_code')] = MetabaseSettings.isTrackingEnabled() ? null : true;
     });
 
+    // TODO: detect user's prefered locale
+    loadLocale("en");
+
     if (callback) {
         callback(store);
     }

frontend/src/metabase/lib/i18n.js

@@ -0,0 +1,13 @@
+import MetabaseSettings from "metabase/lib/settings";
+
+import { addLocale, useLocale } from "c-3po";
+import { I18NApi } from "metabase/services";
+
+export async function loadLocale(locale) {
+    // load and parse the locale
+    const translationsObject = await I18NApi.locale({ locale });
+
+    // add and set locale with C-3PO
+    addLocale(locale, translationsObject);
+    useLocale(locale);
+}

frontend/src/metabase/services.js

@@ -241,4 +241,8 @@ export const GeoJSONApi = {
     get:                         GET("/api/geojson/:id"),
 };
 
+export const I18NApi = {
+    locale:                      GET("/app/locales/:locale.json"),
+}
+
 global.services = exports;

package.json

@@ -12,6 +12,7 @@
   "dependencies": {
     "ace-builds": "^1.2.2",
     "babel-polyfill": "^6.6.1",
+    "c-3po": "^0.5.8",
     "chevrotain": "0.21.0",
     "classnames": "^2.1.3",
     "color": "^1.0.3",
@@ -81,6 +82,7 @@
     "babel-eslint": "^7.1.1",
     "babel-loader": "^6.2.4",
     "babel-plugin-add-react-displayname": "^0.0.4",
+    "babel-plugin-c-3po": "^0.5.8",
     "babel-plugin-transform-builtin-extend": "^1.1.2",
     "babel-plugin-transform-decorators-legacy": "^1.3.4",
     "babel-plugin-transform-flow-strip-types": "^6.8.0",
@@ -179,4 +181,4 @@
       "git add"
     ]
   }
-}
+}

project.clj

@@ -58,6 +58,7 @@
                  [environ "1.1.0"]                                    ; easy environment management
                  [hiccup "1.0.5"]                                     ; HTML templating
                  [honeysql "0.8.2"]                                   ; Transform Clojure data structures to SQL
+                 [io.crate/crate-jdbc "2.1.6"]                        ; Crate JDBC driver
                  [log4j/log4j "1.2.17"                                ; logging framework
                   :exclusions [javax.mail/mail
                                javax.jms/jms
@@ -75,7 +76,7 @@
                  [org.yaml/snakeyaml "1.18"]                          ; YAML parser (required by liquibase)
                  [org.xerial/sqlite-jdbc "3.16.1"]                    ; SQLite driver
                  [postgresql "9.3-1102.jdbc41"]                       ; Postgres driver
-                 [io.crate/crate-jdbc "2.1.6"]                        ; Crate JDBC driver
+                 [puppetlabs/i18n "0.8.0"]                            ; Internationalization library
                  [prismatic/schema "1.1.5"]                           ; Data schema declaration and validation library
                  [ring/ring-core "1.6.0"]
                  [ring/ring-jetty-adapter "1.6.0"]                    ; Ring adapter using Jetty webserver (used to run a Ring server for unit tests)
@@ -86,7 +87,8 @@
   :repositories [["bintray" "https://dl.bintray.com/crate/crate"]]    ; Repo for Crate JDBC driver
   :plugins [[lein-environ "1.1.0"]                                    ; easy access to environment variables
             [lein-ring "0.11.0"                                       ; start the HTTP server with 'lein ring server'
-             :exclusions [org.clojure/clojure]]]                      ; TODO - should this be a dev dependency ?
+             :exclusions [org.clojure/clojure]]                       ; TODO - should this be a dev dependency ?
+            [puppetlabs/i18n "0.8.0"]]                                ; i18n helpers
   :main ^:skip-aot metabase.core
   :manifest {"Liquibase-Package" "liquibase.change,liquibase.changelog,liquibase.database,liquibase.parser,liquibase.precondition,liquibase.datatype,liquibase.serializer,liquibase.sqlgenerator,liquibase.executor,liquibase.snapshot,liquibase.logging,liquibase.diff,liquibase.structure,liquibase.structurecompare,liquibase.lockservice,liquibase.sdk,liquibase.ext"}
   :target-path "target/%s"

webpack.config.js

@@ -41,7 +41,7 @@ if (IS_WATCHING) {
 
 // Babel:
 var BABEL_CONFIG = {
-    cacheDirectory: ".babel_cache"
+    cacheDirectory: process.env.BABEL_DISABLE_CACHE ? null : ".babel_cache"
 };
 
 // Build mapping of CSS variables