Source code for datalab.gui.h5io

# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file.

"""
HDF5 I/O
========

The :mod:`datalab.gui.h5io` module provides the HDF5 file open/save into/from
DataLab data model/main window.

.. autoclass:: H5InputOutput
"""

from __future__ import annotations

import os.path as osp
from typing import TYPE_CHECKING, Any

import h5py
from datalab_capsule.hdf5 import ProvenanceFormatError, has_block
from datalab_capsule.recorder import Recorder
from guidata.qthelpers import exec_dialog
from qtpy import QtWidgets as QW
from sigima.objects import SignalObj
from sigimax.widgets.h5browser import H5BrowserDialog

from datalab.config import _
from datalab.env import execenv
from datalab.gui.panel.base import H5ImportBatch
from datalab.h5 import H5Importer
from datalab.h5.native import NativeH5Reader, atomic_native_writer
from datalab.utils.qthelpers import (
    create_progress_bar,
    must_propagate_errors,
    qt_try_loadsave_file,
)

if TYPE_CHECKING:
    from datalab_capsule.ledger import Ledger
    from sigima.objects import ImageObj
    from sigimax.h5.common import BaseNode

    from datalab.gui.main import DLMainWindow


def workspace_has_provenance(filename: str) -> bool:
    """Return True if a HDF5 file holds a provenance block (nothing is loaded).

    Args:
        filename: HDF5 filename (unreadable files give False).
    """
    try:
        with h5py.File(filename, "r") as h5file:
            return has_block(h5file)
    except (OSError, ValueError):
        return False


def append_provenance_notice() -> str:
    """Return the notice shown before appending a workspace holding provenance."""
    return _(
        "The data will be added, but their earlier provenance will not be "
        "imported. Open the file replacing the current workspace to keep it."
    )


[docs] class H5InputOutput: """Object handling HDF5 file open/save into/from DataLab data model/main window Args: mainwindow: Main window """ def __init__(self, mainwindow: DLMainWindow) -> None: self.mainwindow = mainwindow #: True when the user was already told, for the current opening, that #: the provenance of appended workspaces is not imported (no repeat). self.append_informed = False #: Files whose appending the user cancelled (see :meth:`__confirm_append`) self.cancelled_files: list[str] = [] @staticmethod def __progbartitle(fname: str) -> str: """Return progress bar title""" return _("Loading data from %s...") % osp.basename(fname)
[docs] def save_file(self, filename: str) -> None: """Save all signals and images from DataLab model into a HDF5 file The file is replaced only once completely written: if saving fails, the previous file (if any) is left untouched. """ with atomic_native_writer(filename) as writer: for panel in self.mainwindow.panels: panel.serialize_to_hdf5(writer) self.mainwindow.provenance.save(writer.h5)
@staticmethod def __read_provenance( reader: NativeH5Reader, ) -> tuple[Ledger, dict[str, str], dict[str, Any] | None] | None: """Read and validate the provenance block before anything is replaced. Returns: ``(ledger, state_status, capsule_origin)``, or None without block. Raises: ProvenanceFormatError: If the block is invalid (never a KeyError, which would turn the file into a generic HDF5 import). """ try: return Recorder.read(reader.h5) except ProvenanceFormatError as exc: reader.close() raise ProvenanceFormatError( _("The provenance data of this workspace are invalid: %s") % exc ) from exc def __data_objects(self) -> list[SignalObj | ImageObj]: """Return the objects of the signal and image panels.""" return [ obj for panel in (self.mainwindow.signalpanel, self.mainwindow.imagepanel) for obj in panel.objmodel.get_all_objects() ] def __confirm_append(self) -> bool: """Warn, before anything changes, that appended provenance is ignored. Shown once per opening, unless the question about clearing the workspace already said so. Returns False if the user cancels. """ if self.append_informed or execenv.unattended: return True box = QW.QMessageBox( QW.QMessageBox.Warning, _("Warning"), append_provenance_notice(), QW.QMessageBox.NoButton, self.mainwindow, ) add = box.addButton(_("Add without provenance"), QW.QMessageBox.AcceptRole) box.addButton(QW.QMessageBox.Cancel) box.setDefaultButton(add) exec_dialog(box) if box.clickedButton() is not add: return False self.append_informed = True return True
[docs] def open_file_headless(self, filename: str, reset_all: bool) -> bool: """Open native DataLab HDF5 file without any GUI elements. This method can be safely called from any thread (e.g., the console thread) as it does not create any Qt widgets or dialogs. When a file holding provenance is appended (*reset_all* False), its provenance is not imported and the notice ``datalab_capsule.recorder.APPEND_NOTICE`` is added to ``mainwindow.provenance.notices`` (and logged). Args: filename: HDF5 filename reset_all: Reset all application data before importing Returns: True if file was successfully opened as a native DataLab file, False if the file format is not compatible (KeyError was raised) Raises: ProvenanceFormatError: If the provenance block is invalid (raised before the workspace is replaced). """ reader = None try: reader = NativeH5Reader(filename) block = self.__read_provenance(reader) saved_version = reader.saved_version if reset_all: self.mainwindow.reset_all() before = {id(obj) for obj in self.__data_objects()} batches: list[H5ImportBatch] = [] for panel in self.mainwindow.panels: batch = panel.deserialize_from_hdf5(reader, reset_all) if isinstance(batch, H5ImportBatch): batches.append(batch) H5ImportBatch.finalize_imports(batches, refresh_panels=True) appended = [o for o in self.__data_objects() if id(o) not in before] self.mainwindow.provenance.load( block, reset_all, saved_version, appended=appended ) return True except KeyError: return False finally: if reader is not None: reader.close()
[docs] def open_file(self, filename: str, import_all: bool, reset_all: bool) -> None: """Open HDF5 file When the file holds provenance and is appended to the current workspace, the user is warned before anything changes (see :meth:`__confirm_append`) and may cancel. A file that is not a DataLab workspace is imported as a generic HDF5 file. """ if not self.__open_native(filename, reset_all): self.import_files([filename], import_all, reset_all)
[docs] def open_workspace_file(self, filename: str) -> None: """Open a DataLab workspace file, replacing the workspace. Unlike :meth:`open_file`, a file that is not a DataLab workspace is never imported as a generic HDF5 file. Raises: ValueError: If the file is not a DataLab workspace. ProvenanceFormatError: If the provenance block is invalid. """ if not self.__open_native(filename, True): raise ValueError(_("This file is not a DataLab workspace"))
def __open_native(self, filename: str, reset_all: bool) -> bool: """Open a native DataLab HDF5 file (see :meth:`open_file`). Returns: False if the file is not a DataLab workspace (KeyError), else True (also when the user cancels appending it). """ progress = None reader = None try: reader = NativeH5Reader(filename) block = self.__read_provenance(reader) saved_version = reader.saved_version if block is not None and not reset_all and not self.__confirm_append(): self.cancelled_files.append(filename) return True if reset_all: self.mainwindow.reset_all() before = {id(obj) for obj in self.__data_objects()} batches: list[H5ImportBatch] = [] with create_progress_bar( self.mainwindow, self.__progbartitle(filename), 2 ) as progress: for idx, panel in enumerate(self.mainwindow.panels): progress.setValue(idx + 1) QW.QApplication.processEvents() batch = panel.deserialize_from_hdf5(reader, reset_all) if isinstance(batch, H5ImportBatch): batches.append(batch) if progress.wasCanceled(): break H5ImportBatch.finalize_imports(batches, refresh_panels=True) appended = [o for o in self.__data_objects() if id(o) not in before] self.mainwindow.provenance.load( block, reset_all, saved_version, appended=appended ) return True except KeyError: if progress is not None: # KeyError was encoutered when deserializing datasets (DataLab data # model is not compatible with this version) progress.close() return False finally: if reader is not None: reader.close() def __add_object_from_node(self, node: BaseNode) -> None: """Add DataLab object from h5 node""" obj = node.get_native_object() if obj is None: return if isinstance(obj, SignalObj): self.mainwindow.signalpanel.add_object(obj) else: self.mainwindow.imagepanel.add_object(obj)
[docs] def import_files( self, filenames: list[str], import_all: bool, reset_all: bool ) -> None: """Import HDF5 files""" h5browser = H5BrowserDialog(self.mainwindow) try: self.__import_files_from_browser( h5browser, filenames, import_all, reset_all ) finally: # Close the HDF5 files, even if an import error is raised h5browser.cleanup()
def __import_files_from_browser( self, h5browser: H5BrowserDialog, filenames: list[str], import_all: bool, reset_all: bool, ) -> None: """Open HDF5 files in browser, then import the selected datasets""" for filename in filenames: with qt_try_loadsave_file(self.mainwindow, filename, "load"): h5browser.open_file(filename) if h5browser.is_empty(): if not execenv.unattended: QW.QMessageBox.warning( self.mainwindow, _("Warning"), _("No supported data available in HDF5 file(s)."), ) return if execenv.unattended: # Unattended mode: import all datasets (for testing) import_all = True if import_all or exec_dialog(h5browser): if import_all: nodes = h5browser.get_all_nodes() else: nodes = h5browser.get_nodes() if nodes is not None: if reset_all: self.mainwindow.reset_all() errors: list[str] = [] with qt_try_loadsave_file(self.mainwindow, "*.h5", "load"): with create_progress_bar(self.mainwindow, "", len(nodes)) as prog: for idx, node in enumerate(nodes): prog.setLabelText(self.__progbartitle(node.h5file.filename)) prog.setValue(idx + 1) QW.QApplication.processEvents() if prog.wasCanceled(): break try: self.__add_object_from_node(node) except Exception as exc: # pylint: disable=broad-except # Data of third-party files: import the other # datasets, then report all errors at once errors.append(f"{node.id}: {exc}") self.__report_import_errors(errors) def __report_import_errors(self, errors: list[str]) -> None: """Report the datasets that could not be imported Args: errors: description of each error ("dataset: reason") Raises: ValueError: in unattended mode, or when the errors are reported by the caller (e.g. remote request), if any dataset could not be imported """ if not errors: return if execenv.unattended or must_propagate_errors(): raise ValueError("\n".join(errors)) shown = errors[:10] if len(errors) > len(shown): shown.append(_("... and %d more") % (len(errors) - len(shown))) QW.QMessageBox.warning( self.mainwindow, _("Warning"), _("Some datasets could not be imported:") + "\n\n" + "\n".join(shown), )
[docs] def import_dataset_from_file(self, filename: str, dsetname: str) -> None: """Import dataset from HDF5 file""" h5importer = H5Importer(filename) try: node = h5importer.get(dsetname) self.__add_object_from_node(node) except KeyError as exc: raise KeyError(f"Dataset not found: {dsetname}") from exc finally: h5importer.close()