Build setup.py and requirements files using moban

coala-mobans provides a set of metadata common to
all repositories, with local overrides.

The bear parser now creates bear-requirements.yaml
which is loaded by moban to generate the
bear-requirements.txt, Gemfile and package.json.

Continues coala/coala#4897

.ci/generate_bear_requirements.py

@@ -20,7 +20,7 @@
 import sys
 from collections import OrderedDict
 
-from jinja2 import Environment, FileSystemLoader
+from yaml import dump
 from pyprint.NullPrinter import NullPrinter
 
 from coalib.bears.BEAR_KIND import BEAR_KIND
@@ -30,9 +30,7 @@
 from dependency_management.requirements.NpmRequirement import NpmRequirement
 from dependency_management.requirements.PipRequirement import PipRequirement
 
-NPM_REQUIREMENTS_TEMPLATE_FILE = "package.json.jinja2"
-
-GEM_REQUIREMENTS_TEMPLATE_FILE = "Gemfile.jinja2"
+BEAR_REQUIREMENTS_YAML = "bear-requirements.yaml"
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 
@@ -45,18 +43,14 @@
    'lxml',
 )
 
-env = Environment(loader=FileSystemLoader(THIS_DIR))
-env.filters['jsonify'] = json.dumps
-
-
 def get_args():
     parser = argparse.ArgumentParser(
-        description='This program generates a pip requirement file for '
+        description='This program generates a yaml requirement file for '
                     'installation of linters that are used by the bears.')
     parser.add_argument('--output', '-o',
                         help='name of file to generate, or - for stdout',
                         default=os.path.join(PROJECT_DIR,
-                                             'bear-requirements.txt'))
+                                             BEAR_REQUIREMENTS_YAML))
     parser.add_argument('--bear-dirs', '-d', nargs='+', metavar='DIR',
                         help='additional directories which may contain bears')
 
@@ -118,65 +112,60 @@ def get_all_requirements(bears):
         )
 
 
-def write_gem_requirements(requirements):
-    gem_dependencies = []
-    template = env.get_template(GEM_REQUIREMENTS_TEMPLATE_FILE)
+def _to_entry(obj, version=None):
+    entry = {}
 
-    for requirement in requirements:
-        gem_dependencies.append(
-            {'name': requirement.package,
-             'version': requirement.version,
-             }
-        )
+    if version:
+        entry['version'] = version
+    elif obj.version:
+        entry['version'] = obj.version
+    else:
+        return True
+
+    return entry
 
-    gemfile_string = template.render(
-        gems=gem_dependencies
-        )
 
-    with open(os.path.join(PROJECT_DIR, "Gemfile"), 'w') as file:
-        file.write(gemfile_string)
+def get_gem_requirements(requirements):
+    gem_dependencies = {}
+
+    for requirement in requirements:
+        gem_dependencies[requirement.package] = _to_entry(requirement)
 
+    return gem_dependencies
 
-def write_npm_requirements(requirements):
+
+def get_npm_requirements(requirements):
     npm_dependencies = {}
-    template = env.get_template(NPM_REQUIREMENTS_TEMPLATE_FILE)
 
     for requirement in requirements:
-        req_version = requirement.version
         package_name = requirement.package
+        req_version = requirement.version
         if req_version:
-            if req_version[0] in ('<', '>', '~', '='):
-                npm_dependencies[package_name] = req_version
-            else:
-                npm_dependencies[package_name] = "~" + req_version
-        else:
-            npm_dependencies[package_name] = "*"
+            if req_version[0] not in ('<', '>', '~', '='):
+                req_version = "~" + req_version
+        npm_dependencies[package_name] = _to_entry(requirement, req_version)
 
-    package_json_string = template.render(
-        dependencies=npm_dependencies, version="0.8.0")
-    pretty_json = json.dumps(
-        json.loads(package_json_string, object_pairs_hook=OrderedDict),
-        indent=2)
+    return npm_dependencies
 
-    with open(os.path.join(PROJECT_DIR, "package.json"), 'w') as file:
-        file.write(pretty_json)
-        file.write('\n')
 
-
-def write_pip_requirements(requirements, output):
+def get_pip_requirements(requirements):
     inherited_requirements = get_inherited_requirements()
 
+    pip_requirements = {}
+
     for requirement in requirements:
         if requirement.package in inherited_requirements:
             continue
 
+        package_name = requirement.package
+        req_version = None
         if requirement.version:
             marker = '==' if requirement.package in PINNED_PACKAGES else '~='
-            output.write('{0}{1}{2}\n'.format(requirement.package,
-                                              marker,
-                                              requirement.version))
-        else:
-            output.write(requirement.package + '\n')
+            req_version = '{0}{1}'.format(marker, requirement.version)
+
+        pip_requirements[package_name] = _to_entry(requirement, req_version)
+
+    return pip_requirements
 
 
 if __name__ == '__main__':
@@ -190,9 +179,11 @@ def write_pip_requirements(requirements, output):
     pip_reqs, npm_reqs, gem_reqs = (
         get_all_requirements(get_all_bears(bear_dirs)))
 
-    write_gem_requirements(gem_reqs)
-
-    write_npm_requirements(npm_reqs)
+    requirements = {}
+    requirements['overrides'] = 'coala-build.yaml'
+    requirements['pip_requirements'] = get_pip_requirements(pip_reqs)
+    requirements['npm_requirements'] = get_npm_requirements(npm_reqs)
+    requirements['gem_requirements'] = get_gem_requirements(gem_reqs)
 
     output = None
 
@@ -201,5 +192,5 @@ def write_pip_requirements(requirements, output):
     else:
         output = open(args.output, 'w')
 
-    write_pip_requirements(pip_reqs, output)
+    dump(requirements, output, default_flow_style=False)
     output.close()

.coafile

@@ -69,11 +69,17 @@ ignore = *.py
 
 [all.yml]
 bears = YAMLLintBear
-files = *.yml, .ci/*.yml, tests/**/*.yml
+files = *.yml, *.yaml, .ci/*.yml, tests/**/*.yml
 
 [bash]
 bears = ShellCheckBear
 files = .ci/*.sh
 ignore = .ci/deps.pip.sh
 shell = bash
 shellcheck_ignore = SC2046, SC2154, SC2155
+
+[jinja2]
+bears = Jinja2Bear
+files = **.jj2
+# Bug in handling of ".. list | sort"
+ignore = .moban.dt/Gemfile.jj2, .moban.dt/bear-requirements.txt.jj2, .moban.dt/package.json.jj2

.moban.dt/Gemfile.jj2

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+ruby '2.2.2'
+
+{% for name, data in gem_requirements.items() | sort %}
+{% if data is mapping %}
+gem "{{ name + '", "' + data.version + '", require: false' }}
+{% else %}
+gem "{{ name + '", require: false' }}
+{% endif %}{# if data is mapping #}
+{% endfor %}{# for name, data in gem_requirements.items() #}

.moban.dt/bear-requirements.txt.jj2

@@ -0,0 +1,3 @@
+{% for package, data in pip_requirements.items() | sort %}
+{{ package }}{{ data.version }}
+{% endfor %}{# for package, data in pip_requirements.items() #}

.moban.dt/bears-setup.py.jj2

@@ -0,0 +1,19 @@
+{% extends 'coala-setup.py.jj2' %}
+
+{% block custom_requirements %}
+with open('bear-requirements.txt') as requirements:
+    bear_required = requirements.read().splitlines()
+
+with open('ignore.txt') as ignore:
+    ignore_requirements = ignore.read().splitlines()
+
+extras_require = {
+    'alldeps': bear_required,
+}
+
+# For the average user we leave out some of the more complicated requirements,
+# e.g. language-check (needs java).
+required += [req for req in bear_required
+             if not any(req.startswith(ignore)
+                        for ignore in ignore_requirements)]
+{% endblock %}

.moban.dt/package.json.jj2

@@ -0,0 +1,13 @@
+{
+  "name": "coala-bears",
+  "version": "0.8.0",
+  "dependencies": {
+{% for package, data in npm_requirements.items() | sort %}
+{% if not loop.last %}
+    "{{ package }}": "{{ data.version }}",
+{% else %}
+    "{{ package }}": "{{ data.version }}"
+{% endif %}{# if not loop.last #}
+{% endfor %}{# for package, data in npm_requirements.items() #}
+  }
+}

.moban.dt/requirements.txt.jj2

@@ -0,0 +1,8 @@
+# NOTE: This file is parsed by .ci/generate_bear_requirements.py
+# Edit coala_dependency in coala-build.yaml
+coala{{ coala_dependency }}
+# Dependencies inherited from coala
+# coala_utils
+# dependency_management
+# requests
+-r bear-requirements.txt

.moban.yaml

@@ -0,0 +1,30 @@
+overrides: bear-requirements.yaml
+
+name: coala-bears
+description: "Bears for coala (Code Analysis Application)"
+build_version: 0.12.0.dev99999999999999
+current_version: 0.12.0.dev
+version: 0.12.0.dev
+release: 0.12.0.dev
+branch: master
+package_module: bears
+docs_output_dir: docs/API
+entry_points:
+  coalabears:
+    - coala_official_bears = bears
+
+configuration:
+  template_dir:
+    - .moban.dt/
+    - .ci/
+    - ../coala-mobans/templates/
+    - ../setupmobans/templates/
+  configuration: .moban.yaml
+  configuration_dir: ../coala-mobans/
+targets:
+  - setup.py: bears-setup.py.jj2
+  - requirements.txt: requirements.txt.jj2
+  - bears/VERSION: VERSION.jj2
+  - Gemfile: Gemfile.jj2
+  - package.json: package.json.jj2
+  - bear-requirements.txt: bear-requirements.txt.jj2

.travis.yml

@@ -116,7 +116,10 @@ script:
   - bash .ci/tests.sh
   # Ensure bear requirements are in sync with the bear PipRequirement
   - .ci/generate_bear_requirements.py
-  - git diff --exit-code bear-requirements.txt Gemfile package.json
+  - git clone https://github.com/moremoban/setupmobans ../setupmobans
+  - git clone https://gitlab.com/coala/mobans ../coala-mobans
+  - moban
+  - git diff --exit-code
   # https://github.com/coala/coala-bears/issues/1037
   - >
     if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then