write.py

# Authors: The MNE-Python contributors.
# License: BSD-3-Clause
# Copyright the MNE-Python contributors.

import datetime
import os.path as op
import re
import time
import uuid
from contextlib import contextmanager
from gzip import GzipFile

import numpy as np
from scipy.sparse import csc_array, csr_array

from ..utils import _file_like, _validate_type, logger
from ..utils.numerics import _date_to_julian
from .constants import FIFF

# We choose a "magic" date to store (because meas_date is obligatory)
# to treat as meas_date=None. This one should be impossible for systems
# to write -- the second field is microseconds, so anything >= 1e6
# should be moved into the first field (seconds).
DATE_NONE = (0, 2**31 - 1)


def _write(fid, data, kind, data_size, FIFFT_TYPE, dtype):
    """Write data."""
    if isinstance(data, np.ndarray):
        data_size *= data.size

    # XXX for string types the data size is used as
    # computed in ``write_string``.

    fid.write(np.array(kind, dtype=">i4").tobytes())
    fid.write(np.array(FIFFT_TYPE, dtype=">i4").tobytes())
    fid.write(np.array(data_size, dtype=">i4").tobytes())
    fid.write(np.array(FIFF.FIFFV_NEXT_SEQ, dtype=">i4").tobytes())
    fid.write(np.array(data, dtype=dtype).tobytes())


def _get_split_size(split_size):
    """Convert human-readable bytes to machine-readable bytes."""
    if isinstance(split_size, str):
        exp = dict(MB=20, GB=30).get(split_size[-2:], None)
        if exp is None:
            raise ValueError('split_size has to end with either "MB" or "GB"')
        split_size = int(float(split_size[:-2]) * 2**exp)

    if split_size > 2147483648:
        raise ValueError("split_size cannot be larger than 2GB")
    return split_size


_NEXT_FILE_BUFFER = 1048576  # 2 ** 20 extra cushion for last post-data tags


def write_nop(fid, last=False):
    """Write a FIFF_NOP."""
    fid.write(np.array(FIFF.FIFF_NOP, dtype=">i4").tobytes())
    fid.write(np.array(FIFF.FIFFT_VOID, dtype=">i4").tobytes())
    fid.write(np.array(0, dtype=">i4").tobytes())
    next_ = FIFF.FIFFV_NEXT_NONE if last else FIFF.FIFFV_NEXT_SEQ
    fid.write(np.array(next_, dtype=">i4").tobytes())


INT32_MAX = 2147483647


def write_int(fid, kind, data):
    """Write a 32-bit integer tag to a fif file."""
    data_size = 4
    data = np.asarray(data)
    if data.dtype.kind not in "uib" and data.size > 0:
        raise TypeError(f"Cannot safely write data with dtype {data.dtype} as int")
    max_val = data.max() if data.size > 0 else 0
    if max_val > INT32_MAX:
        raise TypeError(
            f"Value {max_val} exceeds maximum allowed ({INT32_MAX}) for tag {kind}"
        )
    data = data.astype(">i4").T
    _write(fid, data, kind, data_size, FIFF.FIFFT_INT, ">i4")


def write_double(fid, kind, data):
    """Write a double-precision floating point tag to a fif file."""
    data_size = 8
    data = np.array(data, dtype=">f8").T
    _write(fid, data, kind, data_size, FIFF.FIFFT_DOUBLE, ">f8")


def write_float(fid, kind, data):
    """Write a single-precision floating point tag to a fif file."""
    data_size = 4
    data = np.array(data, dtype=">f4").T
    _write(fid, data, kind, data_size, FIFF.FIFFT_FLOAT, ">f4")


def write_dau_pack16(fid, kind, data):
    """Write a dau_pack16 tag to a fif file."""
    data_size = 2
    data = np.array(data, dtype=">i2").T
    _write(fid, data, kind, data_size, FIFF.FIFFT_DAU_PACK16, ">i2")


def write_complex64(fid, kind, data):
    """Write a 64 bit complex floating point tag to a fif file."""
    data_size = 8
    data = np.array(data, dtype=">c8").T
    _write(fid, data, kind, data_size, FIFF.FIFFT_COMPLEX_FLOAT, ">c8")


def write_complex128(fid, kind, data):
    """Write a 128 bit complex floating point tag to a fif file."""
    data_size = 16
    data = np.array(data, dtype=">c16").T
    _write(fid, data, kind, data_size, FIFF.FIFFT_COMPLEX_FLOAT, ">c16")


def write_julian(fid, kind, data):
    """Write a Julian-formatted date to a FIF file."""
    assert isinstance(data, datetime.date), type(data)
    data_size = 4
    jd = _date_to_julian(data)
    data = np.array(jd, dtype=">i4")
    _write(fid, data, kind, data_size, FIFF.FIFFT_JULIAN, ">i4")


def write_string(fid, kind, data):
    """Write a string tag."""
    try:
        str_data = str(data).encode("latin1")
    except UnicodeEncodeError:
        str_data = str(data).encode("latin1", errors="xmlcharrefreplace")
    data_size = len(str_data)  # therefore compute size here
    if data_size > 0:
        _write(fid, str_data, kind, data_size, FIFF.FIFFT_STRING, ">S")


def write_name_list(fid, kind, data):
    """Write a colon-separated list of names.

    Parameters
    ----------
    data : list of strings
    """
    write_string(fid, kind, ":".join(data))


def write_name_list_sanitized(fid, kind, lst, name):
    """Write a sanitized, colon-separated list of names."""
    write_string(fid, kind, _safe_name_list(lst, "write", name))


def _safe_name_list(lst, operation, name):
    if operation == "write":
        assert isinstance(lst, list | tuple | np.ndarray), type(lst)
        if any("{COLON}" in val for val in lst):
            raise ValueError(f'The substring "{{COLON}}" in {name} not supported.')
        return ":".join(val.replace(":", "{COLON}") for val in lst)
    else:
        # take a sanitized string and return a list of strings
        assert operation == "read"
        assert lst is None or isinstance(lst, str)
        if not lst:  # None or empty string
            return []
        return [val.replace("{COLON}", ":") for val in lst.split(":")]


def write_float_matrix(fid, kind, mat):
    """Write a single-precision floating-point matrix tag."""
    _write_matrix_data(fid, kind, mat, FIFF.FIFFT_FLOAT)


def write_double_matrix(fid, kind, mat):
    """Write a double-precision floating-point matrix tag."""
    _write_matrix_data(fid, kind, mat, FIFF.FIFFT_DOUBLE)


def write_int_matrix(fid, kind, mat):
    """Write integer 32 matrix tag."""
    _write_matrix_data(fid, kind, mat, FIFF.FIFFT_INT)


def write_complex_float_matrix(fid, kind, mat):
    """Write complex 64 matrix tag."""
    _write_matrix_data(fid, kind, mat, FIFF.FIFFT_COMPLEX_FLOAT)


def write_complex_double_matrix(fid, kind, mat):
    """Write complex 128 matrix tag."""
    _write_matrix_data(fid, kind, mat, FIFF.FIFFT_COMPLEX_DOUBLE)


def _write_matrix_data(fid, kind, mat, data_type):
    dtype = {
        FIFF.FIFFT_FLOAT: ">f4",
        FIFF.FIFFT_DOUBLE: ">f8",
        FIFF.FIFFT_COMPLEX_FLOAT: ">c8",
        FIFF.FIFFT_COMPLEX_DOUBLE: ">c16",
        FIFF.FIFFT_INT: ">i4",
    }[data_type]
    dtype = np.dtype(dtype)
    data_size = dtype.itemsize * mat.size + 4 * (mat.ndim + 1)
    matrix_type = data_type | FIFF.FIFFT_MATRIX
    fid.write(np.array(kind, dtype=">i4").tobytes())
    fid.write(np.array(matrix_type, dtype=">i4").tobytes())
    fid.write(np.array(data_size, dtype=">i4").tobytes())
    fid.write(np.array(FIFF.FIFFV_NEXT_SEQ, dtype=">i4").tobytes())
    fid.write(np.array(mat, dtype=dtype).tobytes())
    dims = np.empty(mat.ndim + 1, dtype=np.int32)
    dims[: mat.ndim] = mat.shape[::-1]
    dims[-1] = mat.ndim
    fid.write(np.array(dims, dtype=">i4").tobytes())
    check_fiff_length(fid)


def get_machid():
    """Get (mostly) unique machine ID.

    Returns
    -------
    ids : array (length 2, int32)
        The machine identifier used in MNE.
    """
    mac = f"{uuid.getnode():012x}".encode()  # byte conversion for Py3
    mac = re.findall(b"..", mac)  # split string
    mac += [b"00", b"00"]  # add two more fields

    # Convert to integer in reverse-order (for some reason)
    from codecs import encode

    mac = b"".join([encode(h, "hex_codec") for h in mac[::-1]])
    ids = np.flipud(np.frombuffer(mac, np.int32, count=2))
    return ids


def get_new_file_id():
    """Create a new file ID tag."""
    secs, usecs = divmod(time.time(), 1.0)
    secs, usecs = int(secs), int(usecs * 1e6)
    return {
        "machid": get_machid(),
        "version": FIFF.FIFFC_VERSION,
        "secs": secs,
        "usecs": usecs,
    }


def write_id(fid, kind, id_=None):
    """Write fiff id."""
    id_ = _generate_meas_id() if id_ is None else id_

    data_size = 5 * 4  # The id comprises five integers
    fid.write(np.array(kind, dtype=">i4").tobytes())
    fid.write(np.array(FIFF.FIFFT_ID_STRUCT, dtype=">i4").tobytes())
    fid.write(np.array(data_size, dtype=">i4").tobytes())
    fid.write(np.array(FIFF.FIFFV_NEXT_SEQ, dtype=">i4").tobytes())

    # Collect the bits together for one write
    arr = np.array(
        [id_["version"], id_["machid"][0], id_["machid"][1], id_["secs"], id_["usecs"]],
        dtype=">i4",
    )
    fid.write(arr.tobytes())


def start_block(fid, kind):
    """Write a FIFF_BLOCK_START tag."""
    write_int(fid, FIFF.FIFF_BLOCK_START, kind)


def end_block(fid, kind):
    """Write a FIFF_BLOCK_END tag."""
    write_int(fid, FIFF.FIFF_BLOCK_END, kind)


def start_file(fname, id_=None):
    """Open a fif file for writing and writes the compulsory header tags.

    Parameters
    ----------
    fname : path-like | fid
        The name of the file to open. It is recommended
        that the name ends with .fif or .fif.gz. Can also be an
        already opened file.
    id_ : dict | None
        ID to use for the FIFF_FILE_ID.
    """
    if _file_like(fname):
        logger.debug(f"Writing using {type(fname)} I/O")
        fid = fname
        fid.seek(0)
    else:
        fname = str(fname)
        if op.splitext(fname)[1].lower() == ".gz":
            logger.debug("Writing using gzip")
            # defaults to compression level 9, which is barely smaller but much
            # slower. 2 offers a good compromise.
            fid = GzipFile(fname, "wb", compresslevel=2)
        else:
            logger.debug("Writing using normal I/O")
            fid = open(fname, "wb")
    #   Write the compulsory items
    write_id(fid, FIFF.FIFF_FILE_ID, id_)
    write_int(fid, FIFF.FIFF_DIR_POINTER, -1)
    write_int(fid, FIFF.FIFF_FREE_LIST, -1)
    return fid


@contextmanager
def start_and_end_file(fname, id_=None):
    """Start and (if successfully written) close the file."""
    with start_file(fname, id_=id_) as fid:
        yield fid
        end_file(fid)  # we only hit this line if the yield does not err


def check_fiff_length(fid, close=True):
    """Ensure our file hasn't grown too large to work properly."""
    if fid.tell() > 2147483648:  # 2 ** 31, FIFF uses signed 32-bit locations
        if close:
            fid.close()
        raise OSError(
            "FIFF file exceeded 2GB limit, please split file, reduce"
            " split_size (if possible), or save to a different "
            "format"
        )


def end_file(fid):
    """Write the closing tags to a fif file and closes the file."""
    write_nop(fid, last=True)
    check_fiff_length(fid)
    fid.close()


def write_coord_trans(fid, trans):
    """Write a coordinate transformation structure."""
    data_size = 4 * 2 * 12 + 4 * 2
    fid.write(np.array(FIFF.FIFF_COORD_TRANS, dtype=">i4").tobytes())
    fid.write(np.array(FIFF.FIFFT_COORD_TRANS_STRUCT, dtype=">i4").tobytes())
    fid.write(np.array(data_size, dtype=">i4").tobytes())
    fid.write(np.array(FIFF.FIFFV_NEXT_SEQ, dtype=">i4").tobytes())
    fid.write(np.array(trans["from"], dtype=">i4").tobytes())
    fid.write(np.array(trans["to"], dtype=">i4").tobytes())

    #   The transform...
    rot = trans["trans"][:3, :3]
    move = trans["trans"][:3, 3]
    fid.write(np.array(rot, dtype=">f4").tobytes())
    fid.write(np.array(move, dtype=">f4").tobytes())

    #   ...and its inverse
    trans_inv = np.linalg.inv(trans["trans"])
    rot = trans_inv[:3, :3]
    move = trans_inv[:3, 3]
    fid.write(np.array(rot, dtype=">f4").tobytes())
    fid.write(np.array(move, dtype=">f4").tobytes())


def write_ch_info(fid, ch):
    """Write a channel information record to a fif file."""
    data_size = 4 * 13 + 4 * 7 + 16

    fid.write(np.array(FIFF.FIFF_CH_INFO, dtype=">i4").tobytes())
    fid.write(np.array(FIFF.FIFFT_CH_INFO_STRUCT, dtype=">i4").tobytes())
    fid.write(np.array(data_size, dtype=">i4").tobytes())
    fid.write(np.array(FIFF.FIFFV_NEXT_SEQ, dtype=">i4").tobytes())

    #   Start writing fiffChInfoRec
    fid.write(np.array(ch["scanno"], dtype=">i4").tobytes())
    fid.write(np.array(ch["logno"], dtype=">i4").tobytes())
    fid.write(np.array(ch["kind"], dtype=">i4").tobytes())
    fid.write(np.array(ch["range"], dtype=">f4").tobytes())
    fid.write(np.array(ch["cal"], dtype=">f4").tobytes())
    fid.write(np.array(ch["coil_type"], dtype=">i4").tobytes())
    fid.write(np.array(ch["loc"], dtype=">f4").tobytes())  # writing 12 values

    #   unit and unit multiplier
    fid.write(np.array(ch["unit"], dtype=">i4").tobytes())
    fid.write(np.array(ch["unit_mul"], dtype=">i4").tobytes())

    #   Finally channel name
    ch_name = ch["ch_name"][:15]
    fid.write(np.array(ch_name, dtype=">c").tobytes())
    fid.write(b"\0" * (16 - len(ch_name)))


def write_dig_points(fid, dig, block=False, coord_frame=None):
    """Write a set of digitizer data points into a fif file."""
    if dig is not None:
        data_size = 5 * 4
        if block:
            start_block(fid, FIFF.FIFFB_ISOTRAK)
        if coord_frame is not None:
            write_int(fid, FIFF.FIFF_MNE_COORD_FRAME, coord_frame)
        for d in dig:
            fid.write(np.array(FIFF.FIFF_DIG_POINT, ">i4").tobytes())
            fid.write(np.array(FIFF.FIFFT_DIG_POINT_STRUCT, ">i4").tobytes())
            fid.write(np.array(data_size, dtype=">i4").tobytes())
            fid.write(np.array(FIFF.FIFFV_NEXT_SEQ, ">i4").tobytes())
            #   Start writing fiffDigPointRec
            fid.write(np.array(d["kind"], ">i4").tobytes())
            fid.write(np.array(d["ident"], ">i4").tobytes())
            fid.write(np.array(d["r"][:3], ">f4").tobytes())
        if block:
            end_block(fid, FIFF.FIFFB_ISOTRAK)


def write_float_sparse_rcs(fid, kind, mat):
    """Write a single-precision sparse compressed row matrix tag."""
    return write_float_sparse(fid, kind, mat, fmt="csr")


def write_float_sparse(fid, kind, mat, fmt="auto"):
    """Write a single-precision floating-point sparse matrix tag."""
    if fmt == "auto":
        fmt = "csr" if isinstance(mat, csr_array) else "csc"
    need = csr_array if fmt == "csr" else csc_array
    matrix_type = getattr(FIFF, f"FIFFT_SPARSE_{fmt[-1].upper()}CS_MATRIX")
    _validate_type(mat, need, "sparse")
    matrix_type = matrix_type | FIFF.FIFFT_MATRIX | FIFF.FIFFT_FLOAT
    nnzm = mat.nnz
    nrow = mat.shape[0]
    data_size = 4 * nnzm + 4 * nnzm + 4 * (nrow + 1) + 4 * 4

    fid.write(np.array(kind, dtype=">i4").tobytes())
    fid.write(np.array(matrix_type, dtype=">i4").tobytes())
    fid.write(np.array(data_size, dtype=">i4").tobytes())
    fid.write(np.array(FIFF.FIFFV_NEXT_SEQ, dtype=">i4").tobytes())

    fid.write(np.array(mat.data, dtype=">f4").tobytes())
    fid.write(np.array(mat.indices, dtype=">i4").tobytes())
    fid.write(np.array(mat.indptr, dtype=">i4").tobytes())

    dims = [nnzm, mat.shape[0], mat.shape[1], 2]
    fid.write(np.array(dims, dtype=">i4").tobytes())
    check_fiff_length(fid)


def _generate_meas_id():
    """Generate a new meas_id dict."""
    id_ = dict()
    id_["version"] = FIFF.FIFFC_VERSION
    id_["machid"] = get_machid()
    id_["secs"], id_["usecs"] = DATE_NONE
    return id_