Bug 1353680, create test to prevent bad content in localizable string…

…s, r=ahal,flod

Differential Revision: https://phabricator.services.mozilla.com/D20465

--HG--
extra : moz-landing-system : lando

taskcluster/ci/source-test/mozlint.yml

@@ -174,3 +174,18 @@ shellcheck:
     when:
         files-changed:
             - '**/*.sh'
+
+
+localization:
+    description: l10n tests for strings with errors and conflicts with cross-channel
+    platform: lint/opt
+    treeherder:
+        symbol: l1nt
+    run:
+        mach: lint -l l10n -f treeherder
+    when:
+        files-changed:
+            - '**/locales/en-US/**'
+            - '**/l10n.toml'
+            - 'third_party/python/compare-locales/**'
+            - 'third_party/python/fluent/**'

tools/lint/docs/index.rst

@@ -28,6 +28,7 @@ like mach, mozreview and taskcluster.
   create
   linters/eslint
   linters/flake8
+  linters/l10n
 
 Indices and tables
 ==================

tools/lint/docs/linters/l10n.rst

@@ -0,0 +1,39 @@
+L10n
+====
+
+The l10n linter checks for mistakes and problems in the localizable files.
+Most of the code lives inside the
+`compare-locales <https://pypi.org/project/compare-locales/>`_
+package, and is shipping as the ``moz-l10n-lint`` command.
+
+The linter checks for fundamental issues like parsing errors, but it also
+finds more subtle mistakes like duplicated messages. It also warns if you're
+trying to change a string without changing the ID, or to add a string that's
+still in use in a stable channel with a different value.
+
+The warnings on string ID changes get reported on phabricator, but they're
+not making the build fail. To find out when to change IDs and when not to,
+read the :doc:`Lifecycle & Workflow </intl/localization>` section in the
+localization documentation.
+
+Run Locally
+-----------
+
+The can be run using mach:
+
+.. parsed-literal::
+
+    $ mach lint --linter l10n <file paths>
+
+Alternatively, omit the ``--linter l10n`` and run all configured linters, which
+will include the l10n linter.
+
+
+Updating the Reference
+----------------------
+
+The linter checks out the cross-channel localization files into your
+``.mozbuild`` state directory. By default this is updated automatically after
+48 hours. There might be new strings anyway, if you want to ensure an
+updated clone, remove the marker file in
+``~/.mozbuild/gecko-strings/.hg/l10n_pull_marker``.

tools/lint/l10n.yml

@@ -0,0 +1,41 @@
+---
+l10n:
+    description: Localization linter
+    # list of include directories of both
+    # browser and mobile/android l10n.tomls
+    include:
+        - browser/branding/official/locales/en-US
+        - browser/extensions/formautofill/locales/en-US
+        - browser/extensions/fxmonitor/locales/en-US
+        - browser/extensions/report-site-issue/locales/en-US
+        - browser/locales/en-US
+        - devtools/client/locales/en-US
+        - devtools/shared/locales/en-US
+        - devtools/startup/locales/en-US
+        - dom/locales/en-US
+        - mobile/android/base/locales/en-US
+        - mobile/android/locales/en-US
+        - mobile/locales/en-US
+        - netwerk/locales/en-US
+        - security/manager/locales/en-US
+        - services/sync/locales/en-US
+        - toolkit/locales/en-US
+        - tools/lint/l10n.yml
+    # files not supported by compare-locales,
+    # and also not relevant to this linter
+    exclude:
+        - browser/locales/en-US/firefox-l10n.js
+        - mobile/android/locales/en-US/mobile-l10n.js
+        - toolkit/locales/en-US/chrome/global/intl.css
+    l10n_configs:
+        - browser/locales/l10n.toml
+        - mobile/android/locales/l10n.toml
+    type: external
+    payload: python.l10n_lint:lint
+    setup: python.l10n_lint:gecko_strings_setup
+    support-files:
+        - '**/l10n.toml'
+        - 'third_party/python/compare-locales/**'
+        - 'third_party/python/fluent/**'
+        - 'tools/lint/python/l10n_lint.py'
+        - 'tools/lint/l10n.yml'

tools/lint/python/l10n_lint.py

@@ -0,0 +1,157 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+
+from datetime import datetime, timedelta
+import os
+
+from mozboot import util as mb_util
+from mozlint import result, pathutils
+from mozpack import path as mozpath
+import mozversioncontrol.repoupdate
+
+from compare_locales.lint.linter import L10nLinter
+from compare_locales.lint.util import l10n_base_reference_and_tests
+from compare_locales import parser
+from compare_locales.paths import TOMLParser, ProjectFiles
+
+
+LOCALE = 'gecko-strings'
+
+
+PULL_AFTER = timedelta(days=2)
+
+
+def lint(paths, lintconfig, **lintargs):
+    l10n_base = mb_util.get_state_dir()
+    root = lintargs['root']
+    exclude = lintconfig.get('exclude')
+    extensions = lintconfig.get('extensions')
+
+    # Load l10n.toml configs
+    l10nconfigs = load_configs(lintconfig, root, l10n_base)
+
+    # Check include paths in l10n.yml if it's in our given paths
+    # Only the l10n.yml will show up here, but if the l10n.toml files
+    # change, we also get the l10n.yml as the toml files are listed as
+    # support files.
+    if lintconfig['path'] in paths:
+        results = validate_linter_includes(lintconfig, l10nconfigs, lintargs)
+        paths.remove(lintconfig['path'])
+    else:
+        results = []
+
+    all_files = []
+    for p in paths:
+        fp = pathutils.FilterPath(p)
+        if fp.isdir:
+            for _, fileobj in fp.finder:
+                all_files.append(fileobj.path)
+        if fp.isfile:
+            all_files.append(p)
+    # Filter again, our directories might have picked up files the
+    # explicitly excluded in the l10n.yml configuration.
+    # `browser/locales/en-US/firefox-l10n.js` is a good example.
+    all_files, _ = pathutils.filterpaths(
+        lintargs['root'], all_files, lintconfig['include'],
+        exclude=exclude, extensions=extensions
+    )
+    # These should be excluded in l10n.yml
+    skips = {p for p in all_files if not parser.hasParser(p)}
+    results.extend(
+        result.from_config(
+            lintconfig,
+            level='warning',
+            path=path,
+            message="file format not supported in compare-locales"
+            )
+        for path in skips
+    )
+    all_files = [p for p in all_files if p not in skips]
+    files = ProjectFiles(LOCALE, l10nconfigs)
+
+    get_reference_and_tests = l10n_base_reference_and_tests(files)
+
+    linter = MozL10nLinter(lintconfig)
+    results += linter.lint(all_files, get_reference_and_tests)
+    return results
+
+
+def gecko_strings_setup(**lint_args):
+    gs = mozpath.join(mb_util.get_state_dir(), LOCALE)
+    marker = mozpath.join(gs, '.hg', 'l10n_pull_marker')
+    try:
+        last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime)
+        skip_clone = datetime.now() < last_pull + PULL_AFTER
+    except OSError:
+        skip_clone = False
+    if skip_clone:
+        return
+    hg = mozversioncontrol.get_tool_path('hg')
+    mozversioncontrol.repoupdate.update_mercurial_repo(
+        hg,
+        'https://hg.mozilla.org/l10n/gecko-strings',
+        gs
+    )
+    with open(marker, 'w') as fh:
+        fh.flush()
+
+
+def load_configs(lintconfig, root, l10n_base):
+    '''Load l10n configuration files specified in the linter configuration.'''
+    configs = []
+    env = {
+        'l10n_base': l10n_base
+    }
+    for toml in lintconfig['l10n_configs']:
+        cfg = TOMLParser().parse(
+            mozpath.join(root, toml),
+            env=env,
+            ignore_missing_includes=True
+        )
+        cfg.set_locales([LOCALE], deep=True)
+        configs.append(cfg)
+    return configs
+
+
+def validate_linter_includes(lintconfig, l10nconfigs, lintargs):
+    '''Check l10n.yml config against l10n.toml configs.'''
+    reference_paths = set(
+        mozpath.relpath(p['reference'].prefix, lintargs['root'])
+        for project in l10nconfigs
+        for config in project.configs
+        for p in config.paths
+    )
+    # Just check for directories
+    reference_dirs = sorted(p for p in reference_paths if os.path.isdir(p))
+    missing_in_yml = [
+        refd for refd in reference_dirs if refd not in lintconfig['include']
+    ]
+    # These might be subdirectories in the config, though
+    missing_in_yml = [
+        d for d in missing_in_yml
+        if not any(d.startswith(parent + '/') for parent in lintconfig['include'])
+    ]
+    if missing_in_yml:
+        dirs = ', '.join(missing_in_yml)
+        return [result.from_config(
+            lintconfig, path=lintconfig['path'],
+            message='l10n.yml out of sync with l10n.toml, add: ' + dirs
+        )]
+    return []
+
+
+class MozL10nLinter(L10nLinter):
+    '''Subclass linter to generate the right result type.'''
+    def __init__(self, lintconfig):
+        super(MozL10nLinter, self).__init__()
+        self.lintconfig = lintconfig
+
+    def lint(self, files, get_reference_and_tests):
+        return [
+            result.from_config(self.lintconfig, **result_data)
+            for result_data in super(MozL10nLinter, self).lint(
+                files, get_reference_and_tests
+            )
+        ]