Re-implement plugin manifest code generator using Python + jinja2 ste…

…emit#1157

Dockerfile

@@ -24,6 +24,7 @@ RUN \
         pkg-config \
         python3 \
         python3-dev \
+        python3-jinja2 \
         python3-pip \
         nginx \
         fcgiwrap \

doc/building.md

@@ -45,7 +45,9 @@ will build out of the box without further effort:
         libssl-dev \
         libtool \
         make \
-        pkg-config
+        pkg-config \
+        python3 \
+        python3-jinja2
 
     # Boost packages (also required)
     sudo apt-get install -y \
@@ -102,7 +104,9 @@ Here are the required packages:
         libreadline-dev \
         libbz2-dev \
         python-dev \
-        perl
+        perl \
+        python3 \
+        python3-jinja2
 
 The Boost provided in the Ubuntu 14.04 package manager (Boost 1.55) is too old.
 Steem requires Boost 1.58 (as in Ubuntu 16.04) and works with versions up to 1.60 (including).
@@ -168,7 +172,8 @@ Install Homebrew by following the instructions here: http://brew.sh/
         homebrew/versions/boost160 \
         libtool \
         openssl \
-        python3
+        python3 \
+        python3-jinja2
 
 Note: brew recently updated to boost 1.61.0, which is not yet supported by
 steem. Until then, this will allow you to install boost 1.60.0.

libraries/manifest/CMakeLists.txt

@@ -1,55 +1,110 @@
 file(GLOB HEADERS "include/steemit/manifest/*.hpp")
 
-################# external plugins ###################
+# helpful hints from
+# https://blog.kangz.net/posts/2016/05/26/integrating-a-code-generator-with-cmake/
+# https://stackoverflow.com/questions/26074450/how-to-include-generated-files-for-compilation-with-cmake
 
-# external_plugins target depends on all plugins
-add_library( steemit_external_plugins
-   external_plugins.cpp
-   )
+##############################################################################
+# First, we generate plugins.json by reading all the plugins.json files.
+# We run the Python script once at configure time with execute_process()
+# to figure out which files will be read (using the --print-dependencies option),
+# then we run it again at build time with add_custom_command() to
+# actually read the files.
+#
+# This way it will be properly regenerated.
+#
 
-# generate file based on comments
-file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/inc/mf_external_plugins.inc" "// this file is autogenerated\n#pragma once\n#define STEEMIT_EXTERNAL_PLUGIN_LIST \\\n   ")
-
-foreach(pin $ENV{STEEMIT_EXTERNAL_PLUGINS})
-   file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/inc/mf_external_plugins.inc" "( ${pin} )")
-   target_link_libraries( steemit_external_plugins "${pin}" )
-endforeach()
+execute_process(
+   COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}
+   python3 -m steem_manifest.plugins
+   ${CMAKE_CURRENT_SOURCE_DIR}/../plugins
+   -o ${CMAKE_CURRENT_BINARY_DIR}/template_context/plugins.json
+   --print-dependencies
+   OUTPUT_VARIABLE PLUGINS_DEPS
+   RESULT_VARIABLE RETURN_VALUE
+)
 
-file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/inc/mf_external_plugins.inc" "\n")
+if( NOT RETURN_VALUE EQUAL 0 )
+   message(FATAL_ERROR "Could not get plugin dependencies")
+endif()
+message( STATUS "PLUGINS_DEPS: ${PLUGINS_DEPS}" )
 
-################# internal plugins ###################
+add_custom_command(
+   COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}
+   python3 -m steem_manifest.plugins
+   ${CMAKE_CURRENT_SOURCE_DIR}/../plugins
+   -o ${CMAKE_CURRENT_BINARY_DIR}/template_context/plugins.json
+   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/template_context/plugins.json
+   DEPENDS ${PLUGINS_DEPS} ${CMAKE_CURRENT_SOURCE_DIR}/steem_manifest/plugins.py
+)
 
-add_library( steemit_internal_plugins
-   internal_plugins.cpp
-   )
+##############################################################################
+# We instantiate jinja2 templates with the values in plugins.json.
+# Just like generating plugins.json, we use execute_process() to do dry runs
+# at configure time to figure out the dependencies and outputs.
+# Then we use add_custom_command() to execute at build time.
 
-file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/inc/mf_internal_plugins.inc" "// this file is autogenerated\n#pragma once\n#define STEEMIT_INTERNAL_PLUGIN_LIST \\\n   ")
+execute_process(
+   COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}
+   python3 -m steem_manifest.build
+   -j ${CMAKE_CURRENT_BINARY_DIR}/template_context
+   -t ${CMAKE_CURRENT_SOURCE_DIR}/templates
+   -o ${CMAKE_CURRENT_BINARY_DIR}/gen
+   --print-dependencies
+   OUTPUT_VARIABLE MANIFEST_DEPS
+   RESULT_VARIABLE RETURN_VALUE
+)
 
-foreach(pin $ENV{STEEMIT_INTERNAL_PLUGINS})
-   file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/inc/mf_internal_plugins.inc" "( ${pin} )")
-   target_link_libraries( steemit_internal_plugins "steemit_${pin}" )
-endforeach()
+if( NOT RETURN_VALUE EQUAL 0 )
+   message(FATAL_ERROR "Could not get manifest dependencies")
+endif()
+message( STATUS "MANIFEST_DEPS: ${MANIFEST_DEPS}" )
 
-file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/inc/mf_internal_plugins.inc" "\n")
+execute_process(
+   COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}
+   python3 -m steem_manifest.build
+   -j ${CMAKE_CURRENT_BINARY_DIR}/template_context
+   -t ${CMAKE_CURRENT_SOURCE_DIR}/templates
+   -o ${CMAKE_CURRENT_BINARY_DIR}/gen
+   --print-outputs
+   OUTPUT_VARIABLE MANIFEST_OUTPUTS
+   RESULT_VARIABLE RETURN_VALUE
+)
 
-################# common ###################
+if( NOT RETURN_VALUE EQUAL 0 )
+   message(FATAL_ERROR "Could not get manifest outputs")
+endif()
+message( STATUS "MANIFEST_OUTPUTS: ${MANIFEST_OUTPUTS}" )
 
-add_library( steemit_mf_plugins
-   mf_plugins.cpp
-   ${HEADERS}
-   ${CMAKE_CURRENT_BINARY_DIR}/inc/mf_internal_plugins.inc
-   ${CMAKE_CURRENT_BINARY_DIR}/inc/mf_external_plugins.inc
-   )
-target_link_libraries( steemit_mf_plugins fc )
+add_custom_command(
+   COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}
+   python3 -m steem_manifest.build
+   -j ${CMAKE_CURRENT_BINARY_DIR}/template_context
+   -t ${CMAKE_CURRENT_SOURCE_DIR}/templates
+   -o ${CMAKE_CURRENT_BINARY_DIR}/gen
+   OUTPUT ${MANIFEST_OUTPUTS}
+   DEPENDS ${MANIFEST_DEPS} ${CMAKE_CURRENT_SOURCE_DIR}/steem_manifest/build.py
+)
 
-target_include_directories( steemit_mf_plugins
-   PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include"
-   PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/inc" )
+##############################################################################
+# Now we need to repeat the two above add_custom_command() as execute_process()
+# because CMakeLists.txt is one of the generated files, and it must be
+# created at configure time!
+#
 
-INSTALL( TARGETS
-   steemit_mf_plugins
+execute_process(
+   COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}
+   python3 -m steem_manifest.plugins
+   ${CMAKE_CURRENT_SOURCE_DIR}/../plugins
+   -o ${CMAKE_CURRENT_BINARY_DIR}/template_context/plugins.json
+)
 
-   RUNTIME DESTINATION bin
-   LIBRARY DESTINATION lib
-   ARCHIVE DESTINATION lib
+execute_process(
+   COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}
+   python3 -m steem_manifest.build
+   -j ${CMAKE_CURRENT_BINARY_DIR}/template_context
+   -t ${CMAKE_CURRENT_SOURCE_DIR}/templates
+   -o ${CMAKE_CURRENT_BINARY_DIR}/gen
 )
+
+add_subdirectory( ${CMAKE_CURRENT_BINARY_DIR}/gen/plugins )

libraries/manifest/steem_manifest/__init__.py

@@ -0,0 +1,3 @@
+
+from . import build
+from . import plugins

libraries/manifest/steem_manifest/build.py

@@ -0,0 +1,105 @@
+
+import jinja2
+
+import argparse
+import json
+import os
+import sys
+
+def needs_overwrite( target_path, text ):
+    if not os.path.exists( target_path ):
+        return True
+    with open(target_path, "r") as f:
+        current_text = f.read()
+    if current_text == text:
+        return False
+    return True
+
+def overwrite_if_different( target_path, text ):
+    if not needs_overwrite( target_path, text ):
+        return
+    with open(target_path, "w") as f:
+        f.write(text)
+    return
+
+def build(template_dir="templates", output_dir=None, ctx=None, do_write=True, deps=None, outputs=None):
+    os.makedirs(output_dir, exist_ok=True)
+    for root, dirs, files in os.walk(template_dir):
+        dirs_to_remove = [d for d in dirs if d.startswith(".")]
+        for d in dirs_to_remove:
+            dirs.remove(d)
+        for f in files:
+            fpath = os.path.join(root, f)
+            rpath = os.path.relpath(fpath, template_dir)
+            target_path = os.path.join(output_dir, rpath)
+            target_dir = os.path.dirname(target_path)
+            os.makedirs(target_dir, exist_ok=True)
+            with open(fpath, "r") as infile:
+                infile_text = infile.read()
+            if f.endswith(".j2"):
+                template = jinja2.Template( infile_text )
+                outfile_text = template.render( **ctx )
+                target_path = target_path[:-3]
+            else:
+                outfile_text = infile_text
+            if do_write:
+                overwrite_if_different( target_path, outfile_text )
+            if deps is not None:
+                deps.append(fpath)
+            if outputs is not None:
+                outputs.append(target_path)
+    return
+
+def load_context(json_dir, ctx=None, do_read=True, deps=None):
+    if ctx is None:
+        ctx = {}
+    for root, dirs, files in os.walk(json_dir):
+        dirs_to_remove = [d for d in dirs if d.startswith(".")]
+        for d in dirs_to_remove:
+            dirs.remove(d)
+        dirs.sort()
+        for f in sorted(files):
+            if not f.endswith(".json"):
+                continue
+            fpath = os.path.join(root, f)
+            if do_read:
+                with open(fpath, "r") as infile:
+                    obj = json.load(infile)
+                ctx.update(obj)
+            if deps is not None:
+                deps.append(fpath)
+    return ctx
+
+def main(argv):
+
+    parser = argparse.ArgumentParser( description="Build the manifest library" )
+    parser.add_argument( "--json-dir", "-j", type=str, required=True, help="Location of JSON template context files")
+    parser.add_argument( "--template-dir", "-t", type=str, required=True, help="Location of template files" )
+    parser.add_argument( "--output-dir", "-o", type=str, required=True, help="Output location" )
+    parser.add_argument( "--print-dependencies", action="store_true", help="Print dependencies and exit")
+    parser.add_argument( "--print-outputs", action="store_true", help="Print dependencies and exit")
+    args = parser.parse_args()
+
+    deps = []
+    outputs = []
+
+    do_write = not (args.print_dependencies or args.print_outputs)
+
+    ctx = load_context(args.json_dir, do_read=do_write, deps=deps)
+
+    build(
+        template_dir=args.template_dir,
+        output_dir=args.output_dir,
+        ctx=ctx,
+        do_write=do_write,
+        deps=deps,
+        outputs=outputs,
+        )
+
+    if args.print_dependencies:
+        print(";".join(deps))
+    if args.print_outputs:
+        print(";".join(outputs))
+
+if __name__ == "__main__":
+    main(sys.argv)

libraries/manifest/steem_manifest/plugins.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import sys
+
+def find_plugin_filenames(basedir):
+    for root, dirs, files in os.walk(basedir):
+        dirs_to_remove = [d for d in dirs if d.startswith(".")]
+        for d in dirs_to_remove:
+            dirs.remove(d)
+        if "plugin.json" in files:
+            yield os.path.join(root, "plugin.json")
+
+def find_plugins(basedir):
+    for plugin_json in find_plugin_filenames(basedir):
+        with open(plugin_json, "r") as f:
+            yield json.load(f)
+
+def process_plugin(ctx, plugin):
+    ctx["plugins"].append(plugin)
+    for iext in plugin.get("index_extensions", []):
+        ctx["index_extensions"].append(iext)
+    return
+
+def main(argv):
+
+    parser = argparse.ArgumentParser( description="Find plugins" )
+
+    parser.add_argument("plugin_dir", metavar="DIR", type=str, nargs="+",
+                        help="Dir(s) to search for templates")
+    parser.add_argument("-o", "--output", metavar="FILE", type=str, default="-",
+                        help="Output file")
+    parser.add_argument( "--print-dependencies", action="store_true", help="Print dependencies and exit")
+
+    args = parser.parse_args()
+
+    if args.print_dependencies:
+        deps = []
+        for d in args.plugin_dir:
+            for plugin in find_plugin_filenames(d):
+                deps.append(plugin)
+        print(";".join(deps))
+        return
+
+    if args.output == "-":
+        outfile = sys.stdout
+    else:
+        os.makedirs(os.path.dirname(args.output), exist_ok=True)
+        outfile = open(args.output, "w")
+
+    ctx = {
+        "plugins" : [],
+        "index_extensions" : [],
+        }
+    for d in args.plugin_dir:
+        for plugin in find_plugins(d):
+            process_plugin(ctx, plugin)
+    outfile.write(json.dumps(ctx))
+    outfile.write("\n")
+    outfile.close()
+
+if __name__ == "__main__":
+    main(sys.argv)