utils: add functionality to generate bindings

This currently exists in two places
1) Bindtool (longevity TBD) which calls blocktool to parse the public
header file in the include directory
2) Modtool - binding of headers added to add and bind.  rm, update,
info, etc still TODO

Author: mormj

gr-utils/CMakeLists.txt

@@ -84,7 +84,10 @@ add_subdirectory(blocktool)
 endif(ENABLE_GR_BLOCKTOOL)
 
 if(ENABLE_GR_MODTOOL)
+add_subdirectory(bindtool)
 add_subdirectory(modtool)
 endif(ENABLE_GR_MODTOOL)
 
+
+
 endif(ENABLE_GR_UTILS)

gr-utils/bindtool/CMakeLists.txt

@@ -0,0 +1,32 @@
+# Copyright 2019 Free Software Foundation, Inc.
+#
+# This file is part of GNU Radio
+#
+# GNU Radio is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# GNU Radio is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Radio; see the file COPYING.  If not, write to
+# the Free Software Foundation, Inc., 51 Franklin Street,
+# Boston, MA 02110-1301, USA.
+
+include(GrPython)
+
+GR_PYTHON_INSTALL(FILES
+    __init__.py
+    DESTINATION ${GR_PYTHON_DIR}/gnuradio/bindtool
+)
+
+########################################################################
+# Add subdirectories
+########################################################################
+add_subdirectory(core)
+add_subdirectory(templates)
+

gr-utils/bindtool/README.md

@@ -0,0 +1,3 @@
+Tool to generate python bindings from headers
+
+May eventually be absorbed into modtool or blocktool

gr-utils/bindtool/__init__.py

@@ -0,0 +1 @@
+from .core.generator import BindingGenerator

gr-utils/bindtool/core/CMakeLists.txt

@@ -0,0 +1,9 @@
+include(GrPython)
+
+GR_PYTHON_INSTALL(FILES
+    __init__.py
+    base.py
+    generator.py
+    # types.py
+    DESTINATION ${GR_PYTHON_DIR}/gnuradio/bindtool/core
+)

gr-utils/bindtool/core/__init__.py

@@ -0,0 +1,9 @@
+
+
+
+class BindTool(object):
+    
+    target_bindings = 'pybind11'
+    
+    def __init__(self):
+        pass

gr-utils/bindtool/core/base.py

@@ -0,0 +1,268 @@
+#
+# Copyright 2020 Free Software Foundation, Inc.
+#
+# This file is part of GNU Radio
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+#
+
+from .base import BindTool
+from gnuradio.blocktool import BlockHeaderParser, GenericHeaderParser
+
+import os
+import pathlib
+import json
+from mako.template import Template
+from datetime import datetime
+
+
+class BindingGenerator:
+
+    def __init__(self, prefix, namespace, prefix_include_root, output_dir="", addl_includes="", match_include_structure=False):
+        """Initialize BindingGenerator
+        prefix -- path to installed gnuradio prefix (use gr.prefix() if unsure)
+        namespace -- desired namespace to parse e.g. ['gr','module_name']
+            module_name is stored as the last element of namespace
+        prefix_include_root -- relative path to module headers, e.g. "gnuradio/modulename"
+
+        Keyword arguments:
+        output_dir -- path where bindings will be placed
+        addl_includes -- comma separated list of additional include directories (default "")
+        match_include_structure -- 
+            If set to False, a bindings/ dir will be placed directly under the specified output_dir
+            If set to True, the directory structure under include/ will be mirrored
+        """
+
+        self.header_extensions = ['.h', '.hh', '.hpp']
+        self.addl_include = addl_includes
+        self.prefix = prefix
+        self.namespace = namespace
+        self.module_name = namespace[-1]
+        self.prefix_include_root = prefix_include_root
+        self.output_dir = output_dir
+        self.match_include_structure = match_include_structure
+
+        pass
+
+    def gen_pydoc_h(self, header_info, base_name):
+        current_path = os.path.dirname(pathlib.Path(__file__).absolute())
+        tpl = Template(filename=os.path.join(
+            current_path, '..', 'templates', 'license.mako'))
+        license = tpl.render(year=datetime.now().year)
+
+        tpl = Template(filename=os.path.join(current_path, '..',
+                                             'templates', 'pydoc_h.mako'))
+        return tpl.render(
+            license=license,
+            header_info=header_info,
+            basename=base_name,
+            prefix_include_root=self.prefix_include_root,
+        )
+
+
+    def gen_pybind_cc(self, header_info, base_name):
+        current_path = os.path.dirname(pathlib.Path(__file__).absolute())
+        tpl = Template(filename=os.path.join(
+            current_path, '..', 'templates', 'license.mako'))
+        license = tpl.render(year=datetime.now().year)
+
+        tpl = Template(filename=os.path.join(current_path, '..',
+                                             'templates', 'generic_python_cc.mako'))
+        return tpl.render(
+            license=license,
+            header_info=header_info,
+            basename=base_name,
+            prefix_include_root=self.prefix_include_root,
+        )
+
+    def write_pydoc_h(self, header_info, base_name, output_dir):
+
+        doc_pathname = os.path.join(
+            output_dir, 'docstrings', '{}_pydoc_template.h'.format(base_name))
+
+        try:
+            pybind_code = self.gen_pydoc_h(
+                header_info, base_name)
+            with open(doc_pathname, 'w+') as outfile:
+                print("Writing binding code to {}".format(doc_pathname))
+                outfile.write(pybind_code)
+            return doc_pathname
+        except Exception as e:
+            print(e)
+            return None
+
+
+    def write_pybind_cc(self, header_info, base_name, output_dir):
+
+        binding_pathname_cc = os.path.join(
+            output_dir, '{}_python.cc'.format(base_name))
+
+        try:
+            pybind_code = self.gen_pybind_cc(
+                header_info, base_name)
+            with open(binding_pathname_cc, 'w+') as outfile:
+                print("Writing binding code to {}".format(binding_pathname_cc))
+                outfile.write(pybind_code)
+            return binding_pathname_cc
+        except Exception as e:
+            print(e)
+            return None
+
+    def write_json(self, header_info, base_name, output_dir):
+        json_pathname = os.path.join(output_dir, '{}.json'.format(base_name))
+        with open(json_pathname, 'w') as outfile:
+            json.dump(header_info, outfile)
+
+    def read_json(self, pathname):
+        with open(pathname, 'r') as fp:
+            header_info = json.load(fp)
+        return header_info
+
+    def gen_file_binding(self, file_to_process):
+        """Produce the blockname_python.cc python bindings"""
+        output_dir = self.get_output_dir(file_to_process)
+        binding_pathname = None
+        base_name = os.path.splitext(os.path.basename(file_to_process))[0]
+        module_include_path = os.path.abspath(os.path.dirname(file_to_process))
+        top_include_path = os.path.join(
+            module_include_path.split('include'+os.path.sep)[0], 'include')
+
+        include_paths = ','.join(
+            (module_include_path, top_include_path))
+        if self.prefix:
+            prefix_include_path = os.path.abspath(
+                os.path.join(self.prefix, 'include'))
+            include_paths = ','.join(
+                (include_paths, prefix_include_path)
+            )
+        if self.addl_include:
+            include_paths = ','.join((include_paths, self.addl_include))
+
+        parser = GenericHeaderParser(
+            include_paths=include_paths, file_path=file_to_process)
+        try:
+            header_info = parser.get_header_info(self.namespace)
+            # TODO: Scrape the docstrings
+            self.write_json(header_info, base_name, output_dir)
+            self.write_pybind_cc(header_info, base_name, output_dir)
+            self.write_pydoc_h(header_info, base_name, output_dir)
+
+        except Exception as e:
+            print(e)
+            failure_pathname = os.path.join(
+                output_dir, 'failed_conversions.txt')
+            with open(failure_pathname, 'a+') as outfile:
+                outfile.write(file_to_process)
+                outfile.write(str(e))
+                outfile.write('\n')
+
+        return binding_pathname
+
+    def get_output_dir(self, filename):
+        """Get the output directory for a given file"""
+        output_dir = self.output_dir
+        rel_path_after_include = ""
+        if self.match_include_structure:
+            if 'include'+os.path.sep in filename:
+                rel_path_after_include = os.path.split(
+                    filename.split('include'+os.path.sep)[-1])[0]
+
+        output_dir = os.path.join(
+            self.output_dir, rel_path_after_include, 'bindings')
+        doc_dir = os.path.join(output_dir,'docstrings')
+        if output_dir and not os.path.exists(output_dir) and not os.path.exists(doc_dir):
+            output_dir = os.path.abspath(output_dir)
+            print('creating output directory {}'.format(output_dir))
+            os.makedirs(output_dir)
+
+        if doc_dir and not os.path.exists(doc_dir):
+            doc_dir = os.path.abspath(doc_dir)
+            print('creating docstrings directory {}'.format(doc_dir))
+            os.makedirs(doc_dir)
+
+        return output_dir
+
+    def gen_top_level_cpp(self, file_list):
+        """Produce the python_bindings.cc for the bindings"""
+        current_path = os.path.dirname(pathlib.Path(__file__).absolute())
+        file = file_list[0]
+        output_dir = self.get_output_dir(file)
+
+        tpl = Template(filename=os.path.join(
+            current_path, '..', 'templates', 'license.mako'))
+        license = tpl.render(year=datetime.now().year)
+
+        binding_pathname = os.path.join(output_dir, 'python_bindings.cc')
+        file_list = [os.path.split(f)[-1] for f in file_list]
+        tpl = Template(filename=os.path.join(current_path, '..',
+                                             'templates', 'python_bindings_cc.mako'))
+        pybind_code = tpl.render(
+            license=license,
+            files=file_list,
+            module_name=self.module_name
+        )
+
+        # print(pybind_code)
+        try:
+            with open(binding_pathname, 'w+') as outfile:
+                outfile.write(pybind_code)
+            return binding_pathname
+        except:
+            return None
+
+    def gen_cmake_lists(self, file_list):
+        """Produce the CMakeLists.txt for the bindings"""
+        current_path = os.path.dirname(pathlib.Path(__file__).absolute())
+        file = file_list[0]
+        output_dir = self.get_output_dir(file)
+
+        tpl = Template(filename=os.path.join(
+            current_path, '..', 'templates', 'license.mako'))
+        license = tpl.render(year=datetime.now().year)
+
+        binding_pathname = os.path.join(output_dir, 'CMakeLists.txt')
+        file_list = [os.path.split(f)[-1] for f in file_list]
+        tpl = Template(filename=os.path.join(current_path, '..',
+                                             'templates', 'CMakeLists.txt.mako'))
+        pybind_code = tpl.render(
+            license=license,
+            files=file_list,
+            module_name=self.module_name
+        )
+
+        # print(pybind_code)
+        try:
+            with open(binding_pathname, 'w+') as outfile:
+                outfile.write(pybind_code)
+            return binding_pathname
+        except:
+            return None
+
+    def get_file_list(self, include_path):
+        """Recursively get sorted list of files in path"""
+        file_list = []
+        for root, _, files in os.walk(include_path):
+            for file in files:
+                _, file_extension = os.path.splitext(file)
+                if (file_extension in self.header_extensions):
+                    pathname = os.path.abspath(os.path.join(root, file))
+                    file_list.append(pathname)
+        return sorted(file_list)
+
+    def gen_bindings(self, module_dir):
+        """Generate bindings for an entire GR module
+
+        Produces CMakeLists.txt, python_bindings.cc, and blockname_python.cc 
+            for each block in the module
+
+        module_dir -- path to the include directory where the public headers live
+        """
+        file_list = self.get_file_list(module_dir)
+        api_pathnames = [s for s in file_list if 'api.h' in s]
+        for f in api_pathnames:
+            file_list.remove(f)
+        self.gen_top_level_cpp(file_list)
+        self.gen_cmake_lists(file_list)
+        for fn in file_list:
+            self.gen_file_binding(fn)

gr-utils/bindtool/core/generator.py

@@ -0,0 +1,25 @@
+from gnuradio.bindtool import BindingGenerator
+
+prefix = '/share/gnuradio/grnext'
+output_dir = '/share/tmp/test_pybind'
+namespace = ['gr']
+module_dir = '/share/gnuradio/grnext/src/gnuradio/gnuradio-runtime/include/gnuradio'
+prefix_include_root = 'gnuradio'  #pmt, gnuradio/digital, etc.
+
+prefix = '/share/gnuradio/grnext'
+output_dir = '/share/tmp/test_pybind'
+namespace = ['gr','digital']
+module_dir = '/share/gnuradio/grnext/src/gnuradio/gr-digital/include'
+prefix_include_root = 'gnuradio/digital'  #pmt, gnuradio/digital, etc.
+
+prefix = '/share/gnuradio/grnext'
+output_dir = '/share/tmp/test_pybind'
+namespace = ['gr','fft']
+module_dir = '/share/gnuradio/grnext/src/gnuradio/gr-fft/include'
+prefix_include_root = 'gnuradio/fft'  #pmt, gnuradio/digital, etc.
+
+import warnings
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore", category=DeprecationWarning)
+    bg = BindingGenerator(prefix, namespace, prefix_include_root, output_dir)
+    bg.gen_bindings(module_dir)

gr-utils/bindtool/scripts/batch_gen_gr_modules.py

@@ -0,0 +1,15 @@
+import argparse
+import os
+from gnuradio.bindtool import BindingGenerator
+import pathlib
+
+parser = argparse.ArgumentParser(description='Bind a GR header file from the generated json')
+parser.add_argument('pathnames', type=str, nargs='+',
+                    help='Json files to bind')
+
+args = parser.parse_args()
+
+bg = BindingGenerator()
+
+for p in args.pathnames:
+    bg.bind_from_json(p)