[F] Added local python handling

-added local python handle for pipeline processing
-added build requirements
-updated build scripts
-added custom spec for PyInstaller to include standard library
-changed common module versions for 32 and 64-bit version
-fixed vcruntime error for 64-bit PyHook version
-updated ai_test pipeline and pipeline_template
-updated readme

.gitignore

@@ -14,7 +14,6 @@ dist/
 
 # PyInstaller
 *.manifest
-*.spec
 
 # VScode
 .vscode/

PyHook.spec

@@ -0,0 +1,65 @@
+# -*- mode: python ; coding: utf-8 -*-
+from PyInstaller.utils.hooks import collect_all
+from stdlib_list import stdlib_list
+import os
+import sys
+
+with open('PyHook\\_version.py') as ver_file:
+    exec(ver_file.read())
+
+is_64_bit = os.getenv('CONDA_FORCE_32BIT', '0') == '0'
+name = f'PyHook-{__version__}-{"win_amd64" if is_64_bit else "win32"}'
+
+datas = []
+binaries = []
+hiddenimports = stdlib_list("3.9") # Update to 3.10 when possible
+tmp_ret = collect_all('pyinjector')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+
+# Pack DLLs from conda environment
+conda_env_path = os.path.dirname(sys.executable)
+binaries += [(f"{conda_env_path}\\python3.dll", ".")]
+if is_64_bit:
+    binaries += [(f"{conda_env_path}\\vcruntime140_1.dll", ".")]
+
+block_cipher = None
+
+a = Analysis(
+    ['PyHook\\pyhook.py'],
+    pathex=[],
+    binaries=binaries,
+    datas=datas,
+    hiddenimports=hiddenimports,
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher,
+    noarchive=False,
+)
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    [],
+    name=name,
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    runtime_tmpdir=None,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+    version='VERSION.txt',
+)

PyHook/PyHook.py

@@ -5,8 +5,6 @@
 :copyright: (c) 2022 by Dominik Wojtasik.
 :license: MIT, see LICENSE for more details.
 """
-__version__ = "0.0.1"
-
 import logging
 import os
 import sys
@@ -15,6 +13,7 @@
 
 # pylint: disable=unused-import
 import utils  # To be available in PyInstaller frozen bundle
+from _version import __version__
 from dll_utils import (
     AddonHandler,
     AddonNotFoundException,
@@ -141,6 +140,7 @@ def _main():
                     displayed_ms_error = True
                 memory_manager.unlock()
                 continue
+            # Process pipelines changes
             active_pipelines, to_unload, to_load, changes = memory_manager.read_pipelines()
             for unload_pipeline in to_unload:
                 pipelines[unload_pipeline].unload()
@@ -150,7 +150,7 @@ def _main():
                 is_active = update_pipeline in active_pipelines
                 for key, value in settings.items():
                     pipelines[update_pipeline].change_settings(is_active, key, value)
-            # Skip if user didn't select any pipeline
+            # Skip frame processing if user didn't select any pipeline
             if len(active_pipelines) == 0:
                 memory_manager.unlock()
                 continue

PyHook/_version.py

@@ -0,0 +1,2 @@
+# PyHook version
+__version__ = "0.0.1"

PyHook/pipelines/ai_test.py

@@ -1,11 +1,13 @@
-import cv2
 import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from torchvision import transforms
 
-from utils import build_variable, read_value, resolve_path
+from utils import *
+
+with use_local_python():
+    import cv2
+    import torch
+    import torch.nn as nn
+    import torch.nn.functional as F
+    from torchvision import transforms
 
 name = "AI Fast style transfer"
 version = "0.0.1"

PyHook/pipelines/pipeline_template

@@ -1,10 +1,19 @@
 # Optional PyHook utils import
+# Has to be used to import external (OS) modules
 # Should be used to build/read settings values and for resolving resources paths
 # Check docstrings for more informations
-from utils import build_variable, read_value, resolve_path
-# Optional numpy import
+from utils import *
+
+# Optional numpy import from PyHook frozen bundle
 import numpy as np
 
+# External (OS) modules have to be imported in 'with use_local_python() block'.
+# Example:
+# with use_local_python():
+#     import module
+#     import otherModule
+#     from module import xyz
+
 name = "Pipeline name up to 64 characters"
 # OPTIONAL
 # Version string up to 12 characters.
@@ -19,10 +28,10 @@ settings = {                     # Util method to build variable.
                                  # Needs values in order: initial, min, max, step (for slider).
                                  # Supports bool, int and float.
                                  # For bool set None value for min, max and step [see below 'Bool example'].
-    'Label up to 32 characters': build_variable(1, 0, 10, 1, "Variable tooltip up to 128 characters."),
-    'Bool example': build_variable(False, None, None, None, "Test bool variable."),
-    'Int example': build_variable(1, 0, 10, 1, "Test int variable."),
-    'Float example': build_variable(1.0, 0.0, 10.0, 1.0, "Test float variable.")
+    "Label up to 32 characters": build_variable(1, 0, 10, 1, "Variable tooltip up to 128 characters."),
+    "Bool example": build_variable(False, None, None, None, "Test bool variable."),
+    "Int example": build_variable(1, 0, 10, 1, "Test int variable."),
+    "Float example": build_variable(1.0, 0.0, 10.0, 1.0, "Test float variable."),
 }
 
 # OPTIONAL
@@ -33,6 +42,7 @@ def before_change_settings(key: str, value: float) -> None:
     # Should reinit all objects connected to given 'key'.
     pass
 
+
 # OPTIONAL
 def after_change_settings(key: str, value: float) -> None:
     # Called right after the settings change.
@@ -41,19 +51,22 @@ def after_change_settings(key: str, value: float) -> None:
     # Should reinit all objects connected to given 'key'.
     pass
 
+
 # OPTIONAL
 def on_load() -> None:
     # Should initialize all necessary objects for processing.
     pass
 
+
 def on_frame_process(frame: np.array, width: int, height: int, frame_num: int) -> np.array:
     # Should process frame.
     # Frame array shape has to remain unchanged after processing.
     # Array has to be 3-D with height, width, channels as dimensions.
     # Array has to contains uint8 values.
     return frame
 
+
 # OPTIONAL
 def on_unload() -> None:
     # Should destroy all necessary objects for processing
-    pass
+    pass

PyHook/utils.py

@@ -6,10 +6,143 @@
 :license: MIT, see LICENSE for more details.
 """
 
-from os.path import abspath, dirname
+import ctypes
+import os
+import sys
+from subprocess import check_output
 from typing import Any, Dict, List, Union
 
-_DIR = dirname(abspath(__file__))
+# Runtime info
+_LOCAL_PYTHON_EXE = None
+_RUNTIME_HANDLE = None
+
+_DIR = os.getcwd()
+_IS_64_BIT = sys.maxsize > 2**32
+_LOCAL_PYTHON_ENV = "LOCAL_PYTHON"
+_RUNTIME_DLL = "vcruntime140_1.dll"
+_MEIPASS = "_MEIPASS"
+
+
+class _LocalPython:
+    """Allows to use local Python setup.
+
+    Go back to bundled only env by calling close() or using it in a with statement.
+
+    _sys_path (List[str]): Frozen sys.path list from bundled Python.
+    _added_paths (List[os._AddedDllDirectory]): List of added paths to sys.path and DLL search path.
+        All DLL directory handles will be closed by calling close() on _LocalPython object.
+    """
+
+    def __init__(self):
+        local_paths = (
+            check_output(f"{_LOCAL_PYTHON_EXE} -c \"import sys;print(';'.join(sys.path),end='')\"")
+            .decode("utf-8")
+            .split(";")
+        )
+        self._sys_path = [p for p in sys.path]
+        self._added_paths = [self._add_path(p) for p in local_paths if self._is_valid_path(p)]
+
+    def _is_valid_path(self, path: str) -> bool:
+        """Checks if path is valid to be used in bundled sys.path.
+
+        Args:
+            path (str): The path to be checked.
+
+        Returns:
+            bool: True if path is valid to use.
+        """
+        return len(path) > 0 and not path.endswith(".zip")
+
+    def _add_path(self, path: str) -> os._AddedDllDirectory:
+        """Adds path to bundled sys.path and DLL search path.
+
+        Args:
+            path (str): The path to be added.
+
+        Returns:
+            os._AddedDllDirectory: DLL directory handle to be closed after usage.
+        """
+        sys.path.append(path)
+        return os.add_dll_directory(path)
+
+    def close(self):
+        """Restores bundled sys.path and closes all DLL directory handles."""
+        for path in self._added_paths:
+            path.close()
+        sys.path.clear()
+        sys.path.extend(self._sys_path)
+        self._sys_path = None
+        self._added_paths = None
+
+    def __enter__(self) -> "_LocalPython":
+        """Called at the start of with block .
+
+        Returns:
+            _LocalPython: Local Python handle.
+        """
+        return self
+
+    def __exit__(self, *args) -> None:
+        """Called at the end of with block.
+
+        Closes local Python handle.
+        """
+        self.close()
+
+
+def _set_local_python() -> None:
+    """Reads and stores local Python executable path.
+
+    Firstly checks user defined env "LOCAL_PYTHON".
+    If not set it will try to read executable path from python3 binary that is set in path.
+    """
+    # pylint: disable=global-statement
+    global _LOCAL_PYTHON_EXE
+    path_from_env = os.getenv(_LOCAL_PYTHON_ENV, None)
+    if path_from_env is None:
+        try:
+            _LOCAL_PYTHON_EXE = check_output("python3 -c \"import sys;print(sys.executable,end='')\"").decode("utf-8")
+        except FileNotFoundError as ex:
+            raise ValueError(
+                "Local Python3 executable not found. Please update system path or set LOCAL_PYTHON env."
+            ) from ex
+    else:
+        try:
+            check_output(f'{path_from_env} -c "1"')
+            _LOCAL_PYTHON_EXE = path_from_env
+        except FileNotFoundError as ex:
+            raise ValueError("LOCAL_PYTHON is pointing to invalid Python3 executable.") from ex
+
+
+def _is_frozen_bundle() -> bool:
+    """Checks if app is running in PyInstaller frozen bundle.
+
+    Returns:
+        bool: True if app is running in PyInstaller frozen bundle.
+    """
+    return getattr(sys, "frozen", False) and hasattr(sys, _MEIPASS)
+
+
+def use_local_python() -> _LocalPython:
+    """Allows to use local Python setup in pipelines.
+
+    Use it with when statement for imports, e.g.
+    with use_local_python():
+        import moduleA
+        from moduleB import X
+        ...
+
+    Returns:
+        _LocalPython: Local Python handle. When not closed it allows to load modules from local setup.
+    """
+    # pylint: disable=global-statement
+    global _RUNTIME_HANDLE
+    # For 64-bit vcruntime needs additional library to be loaded
+    if _IS_64_BIT and _RUNTIME_HANDLE is None and _is_frozen_bundle():
+        _RUNTIME_HANDLE = ctypes.cdll[f"{getattr(sys, _MEIPASS)}\\{_RUNTIME_DLL}"]
+    if _LOCAL_PYTHON_EXE is None:
+        _set_local_python()
+    return _LocalPython()
 
 
 def resolve_path(file_path: str) -> str: