Merge branch 'qkitgroup:master' into ADwinProII-driver

.gitignore

@@ -21,3 +21,8 @@ qkit/logs
 qkit/data/*
 data/*
 src/qkit/data/*
+!data/README.md
+
+# Notebook folders
+notebooks/
+!notebooks/README.md

README.md

@@ -15,16 +15,6 @@
   The qkit framework has been tested under windows and with limits under macos x and linux. 
   The gui requires h5py, qt and pyqtgraph, which work fine on these platforms. 
   The core of the framework should run with python 2.7.x/3.4+
-
-## Installation:
-If you intend to actively work on the code, look at the section for developing this code.
-To install this package, run:
-```bash
-pip install 'qkit[jupyter,analysis] @ git+https://github.com/qkitgroup/qkit.git@master'
-```
-If you are simply using qkit, changes to files in `src/` **should not** be necessary. If you find things you need to change there, this might be a bug. Please report this.
-
-If you intend to use the option to click on .h5 files in order to open them, don't install qkit in a virtual environment.
 
 ## Configuration
 There are three sources of configuration, which are used to set up your environment
@@ -37,7 +27,7 @@ On import, qkit will look for a file called `qkit_local_config.py` or `local.py`
 
 If you can't put such a configuration file into a suitable location, you can set the environment variable `QKIT_LOCAL_CONFIG` with a path pointing to such a configuration file. This variable circumvents the search, and the file is directly loaded.
 
-## Developing
+## Developing and Installation
 Clone this repository to wherever is convenient.
 ```bash
 git clone https://github.com/qkitgroup/qkit
@@ -51,6 +41,8 @@ For more, look [here](https://docs.python.org/3/library/venv.html).
 python -m venv .venv
 ```
 
+**NOTE: WE DO NOT GUARANTEE COMPATIBILITY WITH ANACONDA/MINICONDA/CONDA! USE AT YOUR OWN RISK!**
+
 Now, you can install qkit as an editable package. This means, that you can change files in the cloned repository. Changes will affect your python setup.
 
 The brackets contain optional dependencies. This pulls the libraries needed for jupyter lab.
@@ -99,6 +91,17 @@ notebook_dir = r'C:\notebooks' # Change this line
 
 Also, you will need to migrate your local config to a parent directory of your notebooks, or point to it using the environment variable `QKIT_LOCAL_CONFIG`. It is possible to set this environment variable in `jupyter_lab_config.py`, as environment variables are inherited to process children.
 
+
+## Long Term Goal: Pip Installation (NOT YET WORKING)
+If you intend to actively work on the code, look at the section for developing this code.
+To install this package, run:
+```bash
+pip install 'qkit[jupyter,analysis] @ git+https://github.com/qkitgroup/qkit.git@master'
+```
+If you are simply using qkit, changes to files in `src/` **should not** be necessary. If you find things you need to change there, this might be a bug. Please report this.
+
+If you intend to use the option to click on .h5 files in order to open them, don't install qkit in a virtual environment.
+
 ## Requirements
 This project uses python. An up to date installation of python is expected to be present.
 | Library | Usage |

data/README.md

@@ -1,3 +1,5 @@
 # Measurement File Location
 This folder is reserved for measurement files.
-These files shall not be shared and are ignored.
+These files shall not be shared and are ignored.
+
+It is recommended to back up these files to a long term storage solution.

notebooks/README.md

@@ -1,3 +1,5 @@
 # User Notebook Location
 This directory is reserved for user notebooks.
-These shall not be shared and are thus ignored.
+These shall not be shared and are thus ignored.
+
+The user may opt to create a separate git repository here to version control their notebooks.

pyproject.toml

@@ -33,6 +33,7 @@ dependencies = [
 
 [project.scripts]
 qviewkit = "qkit.gui.qviewkit.main:main"
+qkit-manual-breadcrumbs = "qkit.core.lib.file_service.breadcrumbs:manual_index"
 
 [project.optional-dependencies] # Here goes stuff like qiclib
 jupyter = [

src/qkit/core/lib/file_service/breadcrumbs.py

@@ -0,0 +1,84 @@
+"""
+This python file attempts to solve the problem of multiple machine beeing backed up to a different path, and the user still
+having to find .h5-Files using a UUID.
+
+While a human user can use intuition to find the file, even guided search fails to efficiently find UUID.h5 files. Here, a
+file based protocol is used to create hints for the search algorithm:
+
+When a machine has created its file info database, it is written to disk. It contains a mapping from the UUID to a path relative
+to the breadcrumb file. As a machine only sees its own files, and not the ones created by other machines, each machine must
+create a list of files it knows, at a location not colliding with the files of others.
+
+Here, we implement a file at `.{Node-UUID}.breadcrumb` derived fromt the MAC-Address.
+
+The file contents are untrusted. (Who knows where they come from.) They thus may not be pickled. The following format is used:
+UUID=rel_path\n
+
+The UUID is 6 symbols long. It is followed by an `=` symbol. The rest of the line is the relative path.
+"""
+
+from pathlib import Path
+import qkit
+import os
+import itertools
+
+FILE_END = ".breadcrumb"
+
+def derive_breadcrumb_filename() -> Path:
+    """
+    Derive a machine-unique breadcrumb file name in the data-directory.
+    """
+    import uuid
+    node = uuid.getnode()
+    filename = f".{node:x}{FILE_END}"
+    return Path(qkit.cfg['datadir']) / filename
+
+class BreadCrumbCreator():
+    """
+    Manages creating the initial bread crumb file and updating it after each measurement.
+    """
+
+    def __init__(self) -> None:
+        self._breadcrumb_path = derive_breadcrumb_filename()
+        if self._breadcrumb_path.exists():
+            os.remove(self._breadcrumb_path)
+        self._breadcrumb_file = open(self._breadcrumb_path, mode="w")
+
+    def append_entry(self, uuid: str, path: Path|str):
+        rel_path = Path(path).relative_to(self._breadcrumb_path.parent)
+        print(f"{uuid[:6]}={rel_path}", file=self._breadcrumb_file, flush=True) # Fails if uuid is not 6 digits
+
+def read_breadcrumb(path: Path) -> dict[str, Path]:
+    """
+    Read a breadcrumb file.
+    """
+    breadcrumb_parent = path.parent
+    uuid_map: dict[str, Path] = {}
+    with open(path, mode="r") as f:
+        for line in f.readlines():
+            if line[6] == '=': # Valid format
+                uuid = line[:6]
+                rel_path = line[7:].strip()
+                uuid_map[uuid] = breadcrumb_parent / rel_path
+    return uuid_map
+
+def read_breadcrumbs(dir: Path) -> dict[str, Path]:
+    assert dir.is_dir(), "Directory must be a directory!"
+    breadcrumbs = [f for f in dir.iterdir() if f.is_file and f.name.endswith(FILE_END)]
+    return dict(itertools.chain.from_iterable(map(dict.items, map(read_breadcrumb, breadcrumbs))))
+
+def manual_index():
+    import os
+    import qkit
+
+    qkit.cfg['fid_scan_datadir'] = True
+    current_dir = os.getcwd()
+    print("Current Directory: ", current_dir)
+    if input("Index? (y/N)") == "y":
+        qkit.cfg['datadir'] = current_dir
+        import qkit.core.s_init.S16_available_modules
+        from qkit.core.lib.file_service.file_info_database import fid
+        fid = fid()
+
+if __name__ == "__main__":
+    manual_index()

src/qkit/core/lib/file_service/file_info_database_lib.py

@@ -7,6 +7,7 @@
 import json
 import numpy as np
 import qkit.storage.hdf_DateTimeGenerator as dtg
+from qkit.core.lib.file_service.breadcrumbs import BreadCrumbCreator
 import h5py
 
 try:
@@ -72,6 +73,8 @@ class file_system_service(UUID_base):
     _h5_mtime_db_path   = os.path.join(qkit.cfg['logdir'],"h5_mtime.db")
     _h5_info_cache_path = os.path.join(qkit.cfg['logdir'],"h5_info_cache.db")
 
+    _breadcrumb_creator = BreadCrumbCreator()
+
     lock = threading.Lock()
 
     def _remove_cache_files(self):
@@ -153,6 +156,7 @@ def _inspect_and_add_Leaf(self,fname,root):
             # Note: All path entries with the same uuid are 
             # overwritten with the last found uuid indexed file
             self.h5_db[uuid] = fqpath
+            self._breadcrumb_creator.append_entry(uuid, fqpath)
 
             # we only care about the mtime of .h5 files 
             mtime = os.stat(fqpath).st_mtime
@@ -255,6 +259,7 @@ def _add(self, h5_filename):
         basename = os.path.basename(h5_filename)[:-2]
         dirname = os.path.dirname(h5_filename)
         uuid = basename[:6]
+        self._breadcrumb_creator.append_entry(uuid, h5_filename)
         if h5_filename[-3:] != '.h5':
             logging.error("Tried to add '{:s}' to the qkit.fid database: Not a .h5 filename.".format(h5_filename))
         with self.lock: