add readme, make service, gui and kernels configurable

desktop/manjaro/python/README.md

@@ -0,0 +1,39 @@
+# Scientific Python using the Jupyter ecosystem
+
+## notes
+
+additional packages (currently not installed)
+
+### jupyter related
+
++ itables - Pandas DataFrames and Series as interactive datatables
++ finos/perspective - interactive analytics and data visualization component, which is especially well-suited for large and/or streaming datasets.
++ elyra-pipeline-editor-extension
+
+### sklearn related
++ auto-sklearn
+
+### torch related
++ optuna            # hyperparameter optimization framework to automate hyperparameter search
++ pytorchvideo      # a deeplearning library with a focus on video understanding work
++ detectron2        # Facebook AI Research's next generation library that provides state-of-the-art detection and segmentation algorithms
++ torch-geometric   # PyTorch Geometric (PyG) is a geometric deep learning extension library for PyTorch
++ monai             # Medical Open Network for AI for pytorch
++ flair             # A very simple framework for state-of-the-art NLP. Developed by Humboldt University of Berlin and friends
++ allennlp          # A natural language processing platform for building state-of-the-art models
++ vissl             # A computer VIsion library for state-of-the-art Self-Supervised Learning research with PyTorch
++ albumentations    # a library for image augmentation to create new training samples from the existing data
++ pfrl              # a deep reinforcement learning library that implements various state-of-the-art deep reinforcement algorithms
++ parlai            # a framework for sharing, training and testing dialogue models, from open-domain chitchat, to task-oriented dialogue, to visual question answering
++ KevinMusgrave/pytorch-metric-learning
+
+### inference related
++ https://github.com/Tencent/ncnn
+    + ncnn is a high-performance neural network inference framework optimized for the mobile platform
+    + Supports GPU acceleration via the next-generation low-overhead vulkan api
+    + Extensible model design, supports 8bit quantization and half-precision floating point storage
+    + can import caffe/pytorch/mxnet/onnx/darknet/keras/tensorflow(mlir) models
++ https://github.com/Tencent/TNN
+    + A high-performance, lightweight neural network inference framework open sourced by Tencent Youtu Lab.
+    + It also has many outstanding advantages such as cross-platform, high performance, model compression, and code tailoring.
+#}

desktop/manjaro/python/defaults.jinja

@@ -0,0 +1,40 @@
+{% load_yaml as defaults %}
+default:
+  packages:
+    ## jupterlab extensions not available in arch
+    # jupyterlab-system-monitor - display system information (memory and cpu usage)
+    - jupyterlab-system-monitor
+    # jupyterlab_execute_time - display cell timings in Jupyter Lab
+    - jupyterlab_execute_time
+    # jupyternotify - provides %%notify via a browser push to notify the user upon completion of a cell
+    - jupyternotify
+    # hugo_jupyter - publish Jupyter notebooks with Hugo
+    - hugo_jupyter
+    # nbdev - Create delightful python projects/libraries/tests using Jupyter Notebooks
+    - nbdev
+
+    ## jupyterlab themes
+    - jupyterlab_miami_nights
+    - jupyterlab_darkside_ui
+    - theme-darcula
+    - jupyterlab_materialdarker
+user:
+  default_packages: true
+  packages: []
+kernel:
+  lab:
+    system_packages: true
+    default_packages: true
+    packages: []
+  lab_user:
+    system_packages: false
+    default_packages: true
+    rebuild: false
+    packages: []
+service:
+  lab:
+    notebook_dir: ~/lab
+{% endload %}
+
+{% set settings=salt['grains.filter_by']({'default': defaults},
+    grain='default', default= 'default', merge= salt['pillar.get']('jupyter', {})) %}

desktop/manjaro/python/init.sls

@@ -1,4 +1,43 @@
+{% from 'python/lib.sls' import pipx_install %}
+{% from 'desktop/user/lib.sls' import user with context %}
+{% from 'desktop/manjaro/python/lib.sls' import jupyter_user_service, jupyter_user_kernel %}
+{% from "desktop/manjaro/python/defaults.jinja" import settings with context %}
+
 include:
   - desktop.manjaro.python.development
   - desktop.manjaro.python.scientific
   - desktop.manjaro.python.machinelearning
+
+desktop_manjaro_python_init:
+  test.nop:
+    - require:
+      - sls: desktop.manjaro.python.development
+      - sls: desktop.manjaro.python.scientific
+      - sls: desktop.manjaro.python.machinelearning
+
+# install user packages for jupyter if configured
+{% if settings.user.default_packages or settings.user.packages %}
+user_python_env_jupyter_packages:
+  pip.installed:
+    - user: {{ user }}
+    - pkgs: {{ settings.default.packages + settings.user.packages  }}
+    - use_vt: true
+{% endif %}
+
+# install all configured kernels for jupyter
+{% for key, value in settings.kernel.items() %}
+{{ jupyter_user_kernel(user=user, name=key, pkgs=value.packages,
+    system_packages=value.system_packages|d(true),
+    rebuild=value.rebuild|d(false),
+    require='test: desktop_manjaro_python_init') }}
+{% endfor %}
+
+{% for key, value in settings.service.items() %}
+# create a systemd user service for starting jupyter lab in background without gui
+# and a desktop entry chromium app to start the gui app of jupyterlab
+{{ jupyter_user_service(user=user, notebook_dir=value.notebook_dir,
+    require='test: desktop_manjaro_python_init') }}
+{% endfor %}
+
+# euporie - jupyter Text-User-Interface
+{{ pipx_install('euporie', user=user) }}

desktop/manjaro/python/lib.sls

@@ -0,0 +1,132 @@
+{% macro jupyter_user_kernel(user, name, pkgs=[], system_packages=True, rebuild=False) %}
+  {% set BASE_DIR= salt['user.info'](user)['home'] ~ '/.local/lib/ipykernel' %}
+  {% set VIRTUAL_ENV= BASE_DIR ~ '/' ~ name %}
+
+create_jupyter_python_env_{{ name }}:
+  file.directory:
+    - name: {{ BASE_DIR }}
+    - user: {{ user }}
+    - group: {{ user }}
+    - makedirs: true
+  cmd.run:
+    - name: python -m venv {{ '--system-site-packages' if system_packages }} {{ VIRTUAL_ENV }}
+    - unless: test -f {{ VIRTUAL_ENV }}/bin/activate
+    - user: {{ user }}
+    - group: {{ user }}
+  {%- if 'require' in kwargs %}
+    - require:
+    {%- set data = kwargs['require'] %}
+    {%- if data is sequence and data is not string %}
+      {%- for value in data %}
+      - {{ value }}
+      {%- endfor %}
+    {%- else %}
+      - {{ data }}
+    {%- endif %}
+  {%- endif %}
+
+install_ipykernel_and_packages_{{ name }}:
+  pip.installed:
+    - user: {{ user }}
+    - group: {{ user }}
+    - pkgs: {{ ['ipykernel'] + pkgs }}
+    - bin_env: {{ VIRTUAL_ENV }}
+    - use_vt: true
+    - require:
+      - cmd: create_jupyter_python_env_{{ name }}
+
+register_ipykernel_{{ name }}:
+  cmd.run:
+    - name: python -m ipykernel install --user --name={{ name }}
+    - user: {{ user }}
+    - group: {{ user }}
+    - require:
+      - pip: install_ipykernel_and_packages_{{ name }}
+
+installed_jupyter_kernel_{{ name }}:
+  test.nop:
+    - require:
+      - cmd: register_ipykernel_{{ name}}
+
+{% endmacro %}
+
+
+{% macro jupyter_user_service(user, notebook_dir) %}
+
+  {% from 'desktop/user/lib.sls' import user_desktop %}
+  {% set basename= salt['file.basename'](notebook_dir) %}
+  {% set home= salt['user.info'](user)['home'] %}
+  {% set WMID= 'jupterlab_' ~ salt['cmd.run_stdout'](
+    'python -c "import binascii; print(\'{:x}\'.format(binascii.crc_hqx(b\'' ~
+    notebook_dir ~ '\', 0)))"' ) %}
+  {% set ice_profile= home ~ '.local/share/ice/profiles/' ~ WMID %}
+  {% set WMClass= 'WebApp-' ~ WMID %}
+  {% set token= '' %}
+  {% set port= '8888' %}
+
+# create a systemd user service for starting jupyter lab in background without gui
+jupyter_user_service_{{ WMID }}:
+  file.managed:
+    - name: {{ home }}/.config/systemd/user/{{ WMID }}.service
+    - contents: |
+        [Unit]
+        Description=Jupyter notebook server ({{ notebook_dir }})
+
+        [Service]
+        Type=simple
+        WorkingDirectory={{ notebook_dir }}
+        Restart=on-failure
+        ExecStart=jupyter lab \
+          --notebook-dir={{ notebook_dir }} \
+          --ip=localhost \
+          --port={{ port }} \
+          --pylab=True \
+          --NotebookApp.token='{{ token }}' \
+          --no-browser
+
+        # --autoreload \
+        # --collaborative \
+
+        [Install]
+        WantedBy=default.target
+
+  {%- if 'require' in kwargs %}
+    - require:
+    {%- set data = kwargs['require'] %}
+    {%- if data is sequence and data is not string %}
+      {%- for value in data %}
+      - {{ value }}
+      {%- endfor %}
+    {%- else %}
+      - {{ data }}
+    {%- endif %}
+  {%- endif %}
+
+
+{% load_yaml as desktop_config %}
+Type: Application
+Name: Jupyterlab-{{ basename }}
+Comment: Jupyter Lab Web-App ({{ notebook_dir }})
+Icon: jupyter
+Categories: Development;Science;GTK;Network;
+Exec: chromium --app=http://localhost:{{ port }}/lab?token={{ token }} --class={{ WMClass }} --user-data-dir={{ ice_profile }}
+Path: {{ notebook_dir }}
+StartupWMClass: {{ WMClass }}
+StartupNotify: true
+X-MultipleArgs: false
+X-WebApp-Browser: Chromium
+X-WebApp-URL: http://localhost:{{ port }}/lab?token={{ token }}
+X-WebApp-Isolated: true
+{% endload %}
+
+# create a jupyterlab desktop entry to start the gui-browser as "app"
+{{ user_desktop(user, desktop_config, require='file: jupyter_user_service_'~ WMID) }}
+
+
+installed_jupyter_user_service_{{ WMID }}:
+  test.nop:
+    - require:
+      - file: jupyter_user_service_{{ WMID }}
+      - file: {{ desktop_config.Name }}.desktop
+
+{% endmacro %}

desktop/manjaro/python/machinelearning.sls

@@ -4,27 +4,25 @@
 include:
   - desktop.manjaro.python.scientific
   - hardware.amd.rocm.pytorch
-  # - hardware.amd.rocm.tensorflow
-
+  - hardware.amd.rocm.tensorflow
 
 ml_tools:
   pkg.installed:
     - pkgs:
-      ## tensorboard - visualization and tooling needed for machine learning experimentation
+      # tensorboard - visualization and tooling needed for machine learning experimentation
       - tensorboard
     - require:
       - test: scientific_python
 {% load_yaml as pkgs %}
-      ## tensorboardX - Tensorboard for PyTorch
+      # tensorboardX - Tensorboard for PyTorch
       - python-tensorboardx
-      ## spacy - library for Natural Language Processing in Python
+      # spacy - library for Natural Language Processing in Python
       - python-spacy
-      ## transformers - pretrained models to perform text, vision, and audio tasks for Jax, pytorch and tensorflow
+      # transformers - pretrained models to perform text, vision, and audio tasks for Jax, pytorch and tensorflow
       - transformers
 {% endload %}
 {{ pamac_install('ml_tools_aur', pkgs, require='pkg: ml_tools') }}
 
-
 ml_sklearn:
   pkg.installed:
     - pkgs:
@@ -35,18 +33,15 @@ ml_sklearn:
 {% load_yaml as pkgs %}
       ## sklearn
       - python-sklearn-pandas
-      # auto-sklearn
 {% endload %}
 {{ pamac_install('ml_sklearn_aur', pkgs, require='pkg: ml_sklearn') }}
 
-
 ml_tensorflow:
   test.nop:
     - require:
       - test: scientific_python
       - test: ml_tools_aur
-      # - sls: hardware.amd.rocm.tensorflow
-
+      - sls: hardware.amd.rocm.tensorflow
 
 ml_pytorch:
   test.nop:
@@ -56,54 +51,26 @@ ml_pytorch:
       - sls: hardware.amd.rocm.pytorch
 
 {% load_yaml as pkgs %}
-      ## torchtext - data processing utilities and popular datasets for natural language
+      # torchtext - data processing utilities and popular datasets for natural language
       - python-torchtext
-      ## torchdata - modular data loading primitives for easily constructing flexible and performant data pipelines
+      # torchdata - modular data loading primitives for easily constructing flexible and performant data pipelines
       - python-torchdata
-      ## functorch - JAX-like composable function transforms for PyTorch
+      # functorch - JAX-like composable function transforms for PyTorch
       - python-functorch
-      ## pytorch-lightning - lightweight PyTorch wrapper for high-performance AI research
+      # pytorch-lightning - lightweight PyTorch wrapper for high-performance AI research
       - python-pytorch-lightning
-      ## kornia - classical computer vision integrated into deep learning models
+      # kornia - classical computer vision integrated into deep learning models
       - python-kornia
-      ## skorch - scikit-learn compatible neural network library that wraps PyTorch
+      # skorch - scikit-learn compatible neural network library that wraps PyTorch
       - python-skorch
 {% endload %}
 {{ pamac_install('ml_pytorch_extra_aur', pkgs,
     require=['test: ml_pytorch', 'test: ml_tools_aur']) }}
 
-## fastai - simplifies training fast and accurate neural nets using modern best practices
+# fastai - simplifies training fast and accurate neural nets using modern best practices
 {{ pamac_patch_install_dir('python-fastcore',
     'salt://desktop/manjaro/python/python-fastcore',
     require= 'test: ml_pytorch_extra_aur') }}
 {{ pamac_patch_install_dir('python-fastai2',
     'salt://desktop/manjaro/python/python-fastai2',
     require= 'test: python-fastcore') }}
-
-
-{#
-
-# torch related
-optuna            # hyperparameter optimization framework to automate hyperparameter search
-pytorchvideo      # a deeplearning library with a focus on video understanding work
-detectron2        # Facebook AI Research's next generation library that provides state-of-the-art detection and segmentation algorithms
-torch-geometric   # PyTorch Geometric (PyG) is a geometric deep learning extension library for PyTorch
-monai             # Medical Open Network for AI for pytorch
-flair             # A very simple framework for state-of-the-art NLP. Developed by Humboldt University of Berlin and friends
-allennlp          # A natural language processing platform for building state-of-the-art models
-vissl             # A computer VIsion library for state-of-the-art Self-Supervised Learning research with PyTorch
-albumentations    # a library for image augmentation to create new training samples from the existing data
-pfrl              # a deep reinforcement learning library that implements various state-of-the-art deep reinforcement algorithms
-parlai            # a framework for sharing, training and testing dialogue models, from open-domain chitchat, to task-oriented dialogue, to visual question answering
-KevinMusgrave/pytorch-metric-learning
-
-# inference related
-+ https://github.com/Tencent/ncnn
-    + ncnn is a high-performance neural network inference framework optimized for the mobile platform
-    + Supports GPU acceleration via the next-generation low-overhead vulkan api
-    + Extensible model design, supports 8bit quantization and half-precision floating point storage
-    + can import caffe/pytorch/mxnet/onnx/darknet/keras/tensorflow(mlir) models
-+ https://github.com/Tencent/TNN
-    + A high-performance, lightweight neural network inference framework open sourced by Tencent Youtu Lab.
-    + It also has many outstanding advantages such as cross-platform, high performance, model compression, and code tailoring.
-#}