Catching exception when decoding velocity values in MIDILike (#210)

* catching exception when decoding velocity values in MIDILike

* switching to `src` directory structure, dropping support Python 3.8

* using python 3.12 for tests

* adding codecov token to tests action

.github/codecov.yml

@@ -10,7 +10,7 @@ coverage:
         target: 70%
       source:
         paths:
-          - "miditok/"
+          - "src/miditok/"
         target: 75%
         threshold: 0.5%
     patch:

.github/workflows/pytest.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.12"]
         os: [ ubuntu-latest, macos-latest, windows-latest ]
 
     steps:
@@ -37,12 +37,14 @@ jobs:
       # the `--dist worksteal` distribution mode to dynamically reassign queued tests to
       # free workers.
       - name: Test with pytest
-        run: pytest --cov=./ --cov-report=xml -n logical --dist worksteal --durations=0 -v tests
+        run: python -m pytest --cov=./ --cov-report=xml -n logical --dist worksteal --durations=0 -v tests
         env:
           HF_TOKEN_HUB_TESTS: ${{ secrets.HF_TOKEN_HUB_TESTS }}
 
       - name: Codecov
         uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   build:
     runs-on: ubuntu-latest

README.md

@@ -5,7 +5,7 @@ Python package to tokenize music files, introduced at the ISMIR 2021 LBDs.
 ![MidiTok Logo](docs/assets/miditok_logo_stroke.png?raw=true "")
 
 [![PyPI version fury.io](https://badge.fury.io/py/miditok.svg)](https://pypi.python.org/pypi/miditok/)
-[![Python 3.8](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/release/)
+[![Python 3.9](https://img.shields.io/badge/python-≥3.9-blue.svg)](https://www.python.org/downloads/release/)
 [![Documentation Status](https://readthedocs.org/projects/miditok/badge/?version=latest)](https://miditok.readthedocs.io/en/latest/?badge=latest)
 [![GitHub CI](https://github.com/Natooz/MidiTok/actions/workflows/pytest.yml/badge.svg)](https://github.com/Natooz/MidiTok/actions/workflows/pytest.yml)
 [![Codecov](https://img.shields.io/codecov/c/github/Natooz/MidiTok)](https://codecov.io/gh/Natooz/MidiTok)

benchmarks/miditok_preprocess_file/benchmark_preprocess.py

@@ -8,14 +8,14 @@
 from pathlib import Path
 from time import time
 
+import miditok
 import numpy as np
+from miditok.constants import SCORE_LOADING_EXCEPTION
 from pandas import DataFrame, read_csv
 from symusic import Score
 from tqdm import tqdm
 
-import miditok
 from benchmarks.utils import mean_std_str
-from miditok.constants import SCORE_LOADING_EXCEPTION
 
 TOKENIZER_CONFIG_KWARGS = {
     "use_tempos": True,

benchmarks/miditok_tokenize/benchmark_tokenize.py

@@ -7,14 +7,14 @@
 from pathlib import Path
 from time import time
 
+import miditok
 import numpy as np
+from miditok.constants import SCORE_LOADING_EXCEPTION
 from pandas import DataFrame, read_csv
 from symusic import Score
 from tqdm import tqdm
 
-import miditok
 from benchmarks import mean_std_str
-from miditok.constants import SCORE_LOADING_EXCEPTION
 
 TOKENIZER_CONFIG_KWARGS = {
     "use_tempos": True,

benchmarks/tokenizer_training/benchmark_training.py

@@ -9,14 +9,14 @@
 from time import time
 from typing import Literal
 
+import miditok
 import numpy as np
+from miditok.constants import SCORE_LOADING_EXCEPTION
 from pandas import DataFrame, read_csv  # requires tabulate package
 from symusic import Score
 from tqdm import tqdm
 
-import miditok
 from benchmarks.utils import mean_std_str
-from miditok.constants import SCORE_LOADING_EXCEPTION
 
 # Tokenizer
 TOKENIZER_PARAMS = {

docs/conf.py

@@ -9,7 +9,7 @@
 import tomllib
 from pathlib import Path
 
-sys.path.insert(0, str(Path("..").resolve()))
+sys.path.insert(0, str(Path("..").resolve() / "src"))
 
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information

pyproject.toml

@@ -4,11 +4,11 @@ build-backend = "hatchling.build"
 
 [project]
 name = "miditok"
-version = "3.0.4"
+version = "3.0.5"
 description = "MIDI / symbolic music tokenizers for Deep Learning models."
 readme = {file = "README.md", content-type = "text/markdown"}
 license = {file = "LICENSE"}
-requires-python = ">=3.8.0"
+requires-python = ">=3.9"
 authors = [
     { name = "Nathan Fradet" },
 ]
@@ -29,11 +29,11 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Operating System :: OS Independent",
 ]
 dependencies = [
@@ -65,13 +65,8 @@ Repository = "https://github.com/Natooz/MidiTok.git"
 Documentation = "https://miditok.readthedocs.io"
 Issues = "https://github.com/Natooz/MidiTok/issues"
 
-[tool.hatch.version]
-path = "miditok/__init__.py"
-
-[tool.hatch.build.targets.sdist]
-include = [
-    "/miditok",
-]
+[tool.hatch.build.targets.wheel]
+packages = ["src/treeval"]
 
 [mypy]
 warn_return_any = "True"
@@ -83,8 +78,9 @@ exclude = [
 ]
 
 [tool.pytest.ini_options]
-testpaths = [
-    "tests",
+pythonpath = "src"
+addopts = [
+    "--import-mode=importlib",
 ]
 
 [tool.coverage.report]
@@ -97,7 +93,7 @@ omit = [
 ]
 
 [tool.ruff]
-target-version = "py312"
+target-version = "py313"
 
 [tool.ruff.lint]
 extend-select = [
@@ -216,7 +212,7 @@ ignore = [
 # we don't use passwords in MidiTok, only HF token for the interactions with the hub.
 # However we have a lot of variables with "token"(s) in their name, which would yield a
 # lot of lint errors or require a lot of noqa exceptions.
-"miditok/**" = [
+"src/miditok/**" = [
     "S105",
 ]
 "tests/**" = [

miditok/classes.py → src/miditok/classes.py

@@ -832,6 +832,19 @@ def __init__(
         # Additional params
         self.additional_params = kwargs
 
+    # Using dataclass overly complicates all the checks performed after init and reduces
+    # the types flexibility (sequence etc...).
+    # Freezing the class could be done, but special cases (MMM) should be handled.
+    """def __setattr__(self, name, value):
+        if getattr(self, "_is_frozen", False) and name != "_is_frozen":
+            raise AttributeError(
+                f"Cannot modify frozen instance of {self.__class__.__name__}"
+            )
+        super().__setattr__(name, value)
+
+    def freeze(self):
+        object.__setattr__(self, "_is_frozen", True)"""
+
     @property
     def max_num_pos_per_beat(self) -> int:
         """

miditok/midi_tokenizer.py → src/miditok/midi_tokenizer.py

@@ -155,7 +155,7 @@ def __init__(
 
         # Tweak the tokenizer's configuration and / or attributes before creating the
         # vocabulary. This method is intended to be overridden by inheriting tokenizer
-        # classes
+        # classes.
         self._tweak_config_before_creating_voc()
 
         # Determines whether the tokenizer will produce a single sequence of tokens for

miditok/tokenizations/midi_like.py → src/miditok/tokenizations/midi_like.py

@@ -327,7 +327,10 @@ def is_track_empty(track: Track) -> bool:
                         if not self.config.use_velocities:
                             vel = DEFAULT_VELOCITY
                         elif ti + 1 < len(seq):
-                            vel = int(seq[ti + 1].split("_")[1])
+                            try:
+                                vel = int(seq[ti + 1].split("_")[1])
+                            except ValueError:  # invalid token succession
+                                continue
                         else:
                             break  # last token
 

miditok/utils/split.py → src/miditok/utils/split.py

@@ -145,12 +145,11 @@ def split_files_for_training(
                     TextMeta(0, f"miditok: chunk {_i}/{len(score_chunks) - 1}")
                 )
                 if tracks_separated:
-                    file_name = f"{file_path.stem}_t{ti}_{_i}{file_path.suffix}"
+                    file_name = f"{file_path.stem}_t{ti}_{_i}"
                 else:
-                    file_name = f"{file_path.stem}_{_i}{file_path.suffix}"
-                # use with_stem when dropping support for python 3.8
-                saving_path = (
-                    save_dir / file_path.relative_to(root_dir).parent / file_name
+                    file_name = f"{file_path.stem}_{_i}"
+                saving_path = save_dir / file_path.relative_to(root_dir).with_stem(
+                    file_name
                 )
                 saving_path.parent.mkdir(parents=True, exist_ok=True)
                 if file_path.suffix in MIDI_FILES_EXTENSIONS:

tests/test_attribute_controls.py

@@ -5,11 +5,10 @@
 from random import seed
 from typing import TYPE_CHECKING, Any, Literal
 
-import pytest
-from symusic import Score
-
 import miditok
+import pytest
 from miditok.attribute_controls import create_random_ac_indexes
+from symusic import Score
 
 from .utils_tests import (
     BARS_RANDOM_RATIO_RANGE,
@@ -63,7 +62,7 @@
 @pytest.mark.parametrize(
     "random_bars_idx", [False, True], ids=lambda r: "rand_bars" if r else "all_bars"
 )
-def test_controller_controls_computation(
+def test_attribute_controls_computation(
     file_path: Path,
     tokenization: str,
     random_tracks_idx: bool,

tests/test_data_augmentation.py

@@ -4,12 +4,11 @@
 
 from typing import TYPE_CHECKING
 
-from symusic import Score
-from tqdm import tqdm
-
 from miditok.data_augmentation import (
     augment_dataset,
 )
+from symusic import Score
+from tqdm import tqdm
 
 from .utils_tests import HERE
 

tests/test_hf_hub.py

@@ -6,11 +6,10 @@
 from time import sleep
 from typing import TYPE_CHECKING
 
+import miditok
 import pytest
 from huggingface_hub.utils import HfHubHTTPError
 
-import miditok
-
 if TYPE_CHECKING:
     from pathlib import Path
 

tests/test_io_formats.py

@@ -5,11 +5,10 @@
 from copy import deepcopy
 from typing import TYPE_CHECKING, Any
 
+import miditok
 import pytest
 from symusic import Score
 
-import miditok
-
 from .utils_tests import (
     ALL_TOKENIZATIONS,
     HERE,

tests/test_methods.py

@@ -4,6 +4,8 @@
 
 from typing import TYPE_CHECKING
 
+import miditok
+from miditok.utils.utils import miditoolkit_to_symusic
 from miditoolkit import Instrument, MidiFile, Pedal
 from tensorflow import Tensor as tfTensor
 from tensorflow import convert_to_tensor
@@ -17,9 +19,6 @@
     Tensor as ptTensor,
 )
 
-import miditok
-from miditok.utils.utils import miditoolkit_to_symusic
-
 from .utils_tests import HERE, MIDI_PATHS_ALL
 
 if TYPE_CHECKING:

tests/test_preprocess.py

@@ -2,11 +2,10 @@
 
 from pathlib import Path
 
+import miditok
 import pytest
 from symusic import Score
 
-import miditok
-
 from .utils_tests import MIDI_PATHS_ALL
 
 CONFIG_KWARGS = {

tests/test_pytorch_data_loading.py

@@ -5,12 +5,11 @@
 from time import time
 from typing import TYPE_CHECKING
 
+import miditok
 import pytest
 from torch import randint
 from torch.utils.data import DataLoader
 
-import miditok
-
 from .utils_tests import (
     ABC_PATHS,
     MAX_BAR_EMBEDDING,

tests/test_saving_loading_config.py

@@ -5,9 +5,8 @@
 from copy import deepcopy
 from typing import TYPE_CHECKING, Any
 
-import pytest
-
 import miditok
+import pytest
 
 from .utils_tests import ALL_TOKENIZATIONS, MAX_BAR_EMBEDDING, MIDI_PATHS_MULTITRACK
 

tests/test_tokenize.py

@@ -6,11 +6,10 @@
 from pathlib import Path
 from typing import Any
 
-import pytest
-from symusic import Score
-
 import miditok
+import pytest
 from miditok.constants import SCORE_LOADING_EXCEPTION, USE_NOTE_DURATION_PROGRAMS
+from symusic import Score
 
 from .utils_tests import (
     ABC_PATHS,

tests/test_toksequence.py

@@ -4,7 +4,6 @@
 from pathlib import Path
 
 import pytest
-
 from miditok import TSD, TokenizerConfig, TokSequence
 
 from .utils_tests import MIDI_PATHS_MULTITRACK

tests/test_train.py

@@ -7,11 +7,10 @@
 from time import time
 from typing import TYPE_CHECKING
 
-import pytest
-from tqdm import tqdm
-
 import miditok
+import pytest
 from miditok.constants import DEFAULT_TOKENIZER_FILE_NAME
+from tqdm import tqdm
 
 from .utils_tests import (
     MIDI_PATHS_ONE_TRACK,