From 70bee9048beaeb26e62f6aeb0cdcbdb099c39ebe Mon Sep 17 00:00:00 2001 From: Henri Payno <payno@linazimov.esrf.fr> Date: Mon, 8 Nov 2021 14:27:29 +0100 Subject: [PATCH 1/9] [persitence] add '.nexus' amd '.nx' to the valid extensions for the NexusProxy --- ewokscore/persistence/nexus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ewokscore/persistence/nexus.py b/ewokscore/persistence/nexus.py index 26e9e62..e6fa54a 100644 --- a/ewokscore/persistence/nexus.py +++ b/ewokscore/persistence/nexus.py @@ -19,7 +19,7 @@ def h5_item_exists(path, item): class NexusProxy(FileProxy): SCHEME = "nexus" - EXTENSIONS = [".nx", ".h5", ".hdf5"] + EXTENSIONS = [".nx", "nxs", ".h5", ".hdf5", ".nexus"] ALLOW_PATH_IN_FILE = True def exists(self) -> bool: -- GitLab From c15609b89f9eff03778d412e9527deed1d4c060d Mon Sep 17 00:00:00 2001 From: Henri Payno <payno@linazimov.esrf.fr> Date: Mon, 8 Nov 2021 14:28:11 +0100 Subject: [PATCH 2/9] [persistence] add some simple documentation to the class DataProxy --- ewokscore/persistence/proxy.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ewokscore/persistence/proxy.py b/ewokscore/persistence/proxy.py index 229239f..bc24f4c 100644 --- a/ewokscore/persistence/proxy.py +++ b/ewokscore/persistence/proxy.py @@ -168,6 +168,12 @@ class DataProxy(Registered, HasUhash, register=False): @property def uri(self) -> Optional[DataUri]: + """ + Return a Unified Resource Identifier. Defined as: + URI = scheme ":" "//" authority path ["?" query] ["#" fragment] + + https://en.wikipedia.org/wiki/Uniform_Resource_Identifier + """ if self.fixed_uri: return self.__fixed_uri return self._generate_uri() @@ -177,10 +183,13 @@ class DataProxy(Registered, HasUhash, register=False): raise NotImplementedError def exists(self) -> bool: + """return True if the data exists""" raise NotImplementedError def load(self, raise_error: bool = True) -> Any: + """Load data from the uri""" raise NotImplementedError def dump(self, data: Any) -> bool: + """Dump data to the uri""" raise NotImplementedError -- GitLab From b183a1d5f4a808795f2f82a33da059c929d1c238 Mon Sep 17 00:00:00 2001 From: Henri Payno <payno@linazimov.esrf.fr> Date: Mon, 8 Nov 2021 14:28:35 +0100 Subject: [PATCH 3/9] [persistence] try to improve documentation --- ewokscore/persistence/file.py | 35 ++++++++++++++++++++++++++++++++-- ewokscore/persistence/nexus.py | 2 +- ewokscore/persistence/proxy.py | 2 +- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/ewokscore/persistence/file.py b/ewokscore/persistence/file.py index 1b71223..35923ad 100644 --- a/ewokscore/persistence/file.py +++ b/ewokscore/persistence/file.py @@ -4,6 +4,7 @@ from urllib.parse import ParseResult from .uri import path_from_uri from . import proxy from ..hashing import UniversalHashable +from silx.utils.proxy import docstring class FileProxy(proxy.DataProxy, register=False): @@ -14,11 +15,17 @@ class FileProxy(proxy.DataProxy, register=False): """ EXTENSIONS = NotImplemented + """Valid file extensions. Expected to be an iterable""" ALLOW_PATH_IN_FILE = NotImplemented + """Does format allow path inside files. Expected to be a boolean""" SEP_IN_FILE = "/" + """Separator used inside files. Expected to be a string""" @property def path(self) -> Optional[Path]: + """return 'authority path' from the foloowing URI representation: + URI = scheme ":" "//" authority path ["?" query] ["#" fragment] + """ if self.fixed_uri: return path_from_uri(self.uri.parse()) parsed_root_uri = self.parsed_root_uri @@ -65,23 +72,40 @@ class FileProxy(proxy.DataProxy, register=False): @property def root_uri_path_in_file(self) -> str: + """ + return '**data root** path query' result + for data at "file://path/to/name.ext?path=/path/in/file" return "/path/in" + """ return self.root_uri_query.get("path", "") def path_in_file_parts(self) -> Optional[List[str]]: + """Return list of 'path' query parts as a LIST WITHOUT the root""" if self.ALLOW_PATH_IN_FILE: return self._path_in_file_parts() else: return None @property - def path_in_file(self) -> Optional[str]: + def path_in_file( + self, + ) -> Optional[str]: + """ + return '**data** path query' result + for data at "file://path/to/name.ext?path=/path/in/file" return "path/in/file" + """ if self.ALLOW_PATH_IN_FILE: return self.SEP_IN_FILE.join(self._path_in_file_parts()) else: return None @property - def path_in_file_parent(self) -> Optional[str]: + def path_in_file_parent( + self, + ) -> Optional[str]: + """ + return '**data** path query' result + for data at "file://path/to/name.ext?path=/path/in/file" return "path/in" + """ if self.ALLOW_PATH_IN_FILE: parts = self._path_in_file_parts()[:-1] return self.SEP_IN_FILE.join(parts) @@ -90,6 +114,10 @@ class FileProxy(proxy.DataProxy, register=False): @property def path_in_file_name(self) -> Optional[str]: + """ + Return '**data** path query' last part + for data at "file://path/to/name.ext?path=/path/in/file" return "file" + """ if self.ALLOW_PATH_IN_FILE: return self._path_in_file_parts()[-1] else: @@ -111,12 +139,14 @@ class FileProxy(proxy.DataProxy, register=False): uri = ParseResult(self.SCHEME, str(path), "", "", query, "") return proxy.DataUri(uri, self.uhash) + @docstring(proxy.DataProxy) def exists(self) -> bool: path = self.path if path is None: return False return path.exists() + @docstring(proxy.DataProxy) def dump(self, data, **kw): path = self.path if path is None: @@ -124,6 +154,7 @@ class FileProxy(proxy.DataProxy, register=False): self._dump(path, data, **kw) return True + @docstring(proxy.DataProxy) def load(self, raise_error=True, **kw): path = self.path if path is None: diff --git a/ewokscore/persistence/nexus.py b/ewokscore/persistence/nexus.py index e6fa54a..f0ef92a 100644 --- a/ewokscore/persistence/nexus.py +++ b/ewokscore/persistence/nexus.py @@ -13,7 +13,7 @@ def h5_item_exists(path, item): try: with h5py_utils.File(path) as f: return item in f - except Exception: + except Exception: # is this robust enought ? Maybe ony some OS error should be filtered return False diff --git a/ewokscore/persistence/proxy.py b/ewokscore/persistence/proxy.py index bc24f4c..89e9612 100644 --- a/ewokscore/persistence/proxy.py +++ b/ewokscore/persistence/proxy.py @@ -163,7 +163,7 @@ class DataProxy(Registered, HasUhash, register=False): return dict() @property - def fixed_uri(self) -> bool: + def fixed_uri(self) -> bool: # shuold be rename is_uri_fixed return self.__fixed_uri is not None @property -- GitLab From da48e6646899010bc0e9b36659a1174b7397e30d Mon Sep 17 00:00:00 2001 From: Henri Payno <payno@linazimov.esrf.fr> Date: Mon, 8 Nov 2021 17:04:51 +0100 Subject: [PATCH 4/9] [persistence] improve doc of NexusProxy --- ewokscore/persistence/nexus.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ewokscore/persistence/nexus.py b/ewokscore/persistence/nexus.py index f0ef92a..cee3433 100644 --- a/ewokscore/persistence/nexus.py +++ b/ewokscore/persistence/nexus.py @@ -6,6 +6,7 @@ from .file import FileProxy from . import atomic from silx.io.dictdump import dicttonx, nxtodict from silx.io import h5py_utils +from silx.utils.proxy import docstring # @h5py_utils.retry(retry_period=1) @@ -22,6 +23,7 @@ class NexusProxy(FileProxy): EXTENSIONS = [".nx", "nxs", ".h5", ".hdf5", ".nexus"] ALLOW_PATH_IN_FILE = True + @docstring(FileProxy) def exists(self) -> bool: if not super().exists(): return False -- GitLab From f1be4f6a73e98c5b4482a17fdc67762211e6bb6e Mon Sep 17 00:00:00 2001 From: Henri Payno <payno@linazimov.esrf.fr> Date: Mon, 8 Nov 2021 17:08:21 +0100 Subject: [PATCH 5/9] [persistence] improve documentation of DatAProxy --- ewokscore/persistence/proxy.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ewokscore/persistence/proxy.py b/ewokscore/persistence/proxy.py index 89e9612..fb2e251 100644 --- a/ewokscore/persistence/proxy.py +++ b/ewokscore/persistence/proxy.py @@ -68,6 +68,7 @@ class DataUri(HasUhash): class DataProxy(Registered, HasUhash, register=False): SCHEME = NotImplemented + """name of the DataProxy scheme like json or nexus""" def __init__( self, @@ -146,6 +147,7 @@ class DataProxy(Registered, HasUhash, register=False): @property def identifier(self) -> Optional[str]: + """Return identifier DataProxy to be used as a string""" uhash = self.uhash if uhash is None: return None -- GitLab From c0bd9d2b897963864329dc50d0a46bf9b90acf00 Mon Sep 17 00:00:00 2001 From: Henri Payno <payno@linazimov.esrf.fr> Date: Mon, 8 Nov 2021 17:38:19 +0100 Subject: [PATCH 6/9] [DataProxy] rename fixed_uri -> is_fixed_uri --- ewokscore/persistence/proxy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ewokscore/persistence/proxy.py b/ewokscore/persistence/proxy.py index fb2e251..8d916eb 100644 --- a/ewokscore/persistence/proxy.py +++ b/ewokscore/persistence/proxy.py @@ -136,7 +136,7 @@ class DataProxy(Registered, HasUhash, register=False): @property def uhash(self) -> Optional[UniversalHash]: - if self.fixed_uri: + if self.is_fixed_uri: return self.__fixed_uri.uhash elif isinstance(self.__uhash_source, HasUhash): return self.__uhash_source.uhash @@ -165,7 +165,7 @@ class DataProxy(Registered, HasUhash, register=False): return dict() @property - def fixed_uri(self) -> bool: # shuold be rename is_uri_fixed + def is_fixed_uri(self) -> bool: # shuold be rename is_uri_fixed return self.__fixed_uri is not None @property @@ -176,7 +176,7 @@ class DataProxy(Registered, HasUhash, register=False): https://en.wikipedia.org/wiki/Uniform_Resource_Identifier """ - if self.fixed_uri: + if self.is_fixed_uri: return self.__fixed_uri return self._generate_uri() -- GitLab From 73690bdb527bea0d42c7cedb4f0764bdf5995c65 Mon Sep 17 00:00:00 2001 From: Henri Payno <payno@linazimov.esrf.fr> Date: Tue, 9 Nov 2021 08:14:58 +0100 Subject: [PATCH 7/9] [DataProxy] rename fixed_uri -> is_fixed_uri --- ewokscore/persistence/file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ewokscore/persistence/file.py b/ewokscore/persistence/file.py index 35923ad..01e2af2 100644 --- a/ewokscore/persistence/file.py +++ b/ewokscore/persistence/file.py @@ -26,7 +26,7 @@ class FileProxy(proxy.DataProxy, register=False): """return 'authority path' from the foloowing URI representation: URI = scheme ":" "//" authority path ["?" query] ["#" fragment] """ - if self.fixed_uri: + if self.is_fixed_uri: return path_from_uri(self.uri.parse()) parsed_root_uri = self.parsed_root_uri if parsed_root_uri is None: @@ -63,7 +63,7 @@ class FileProxy(proxy.DataProxy, register=False): def _path_in_file_parts(self) -> List[str]: parts = [s for s in self.root_uri_path_in_file.split(self.SEP_IN_FILE) if s] - if self.fixed_uri: + if self.is_fixed_uri: return parts identifier = self.identifier if identifier is not None: -- GitLab From a01b356c6abf2a7ed86ff374047d8c5139ffb9d7 Mon Sep 17 00:00:00 2001 From: Henri Payno <payno@linazimov.esrf.fr> Date: Tue, 9 Nov 2021 08:15:33 +0100 Subject: [PATCH 8/9] [persistence] rename root_uri_path_in_file -> _root_uri_path_in_file --- ewokscore/persistence/file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ewokscore/persistence/file.py b/ewokscore/persistence/file.py index 01e2af2..ecbcf99 100644 --- a/ewokscore/persistence/file.py +++ b/ewokscore/persistence/file.py @@ -62,7 +62,7 @@ class FileProxy(proxy.DataProxy, register=False): return path.with_suffix(extension) def _path_in_file_parts(self) -> List[str]: - parts = [s for s in self.root_uri_path_in_file.split(self.SEP_IN_FILE) if s] + parts = [s for s in self._root_uri_path_in_file.split(self.SEP_IN_FILE) if s] if self.is_fixed_uri: return parts identifier = self.identifier @@ -71,7 +71,7 @@ class FileProxy(proxy.DataProxy, register=False): return parts @property - def root_uri_path_in_file(self) -> str: + def _root_uri_path_in_file(self) -> str: """ return '**data root** path query' result for data at "file://path/to/name.ext?path=/path/in/file" return "/path/in" -- GitLab From adc91d832a9075865c1b5cd062a31ae78b5f1e49 Mon Sep 17 00:00:00 2001 From: Henri Payno <payno@linazimov.esrf.fr> Date: Mon, 8 Nov 2021 17:39:20 +0100 Subject: [PATCH 9/9] [persistence] fix typo in doc --- ewokscore/persistence/file.py | 4 ++-- ewokscore/persistence/nexus.py | 2 +- ewokscore/persistence/proxy.py | 10 ++++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ewokscore/persistence/file.py b/ewokscore/persistence/file.py index ecbcf99..3991d57 100644 --- a/ewokscore/persistence/file.py +++ b/ewokscore/persistence/file.py @@ -23,8 +23,8 @@ class FileProxy(proxy.DataProxy, register=False): @property def path(self) -> Optional[Path]: - """return 'authority path' from the foloowing URI representation: - URI = scheme ":" "//" authority path ["?" query] ["#" fragment] + """return 'path' from the following URI representation: + URI = scheme ":" "//" path ["?" query] ["#" fragment] """ if self.is_fixed_uri: return path_from_uri(self.uri.parse()) diff --git a/ewokscore/persistence/nexus.py b/ewokscore/persistence/nexus.py index cee3433..ba482df 100644 --- a/ewokscore/persistence/nexus.py +++ b/ewokscore/persistence/nexus.py @@ -14,7 +14,7 @@ def h5_item_exists(path, item): try: with h5py_utils.File(path) as f: return item in f - except Exception: # is this robust enought ? Maybe ony some OS error should be filtered + except Exception: return False diff --git a/ewokscore/persistence/proxy.py b/ewokscore/persistence/proxy.py index 8d916eb..b7116d4 100644 --- a/ewokscore/persistence/proxy.py +++ b/ewokscore/persistence/proxy.py @@ -165,16 +165,18 @@ class DataProxy(Registered, HasUhash, register=False): return dict() @property - def is_fixed_uri(self) -> bool: # shuold be rename is_uri_fixed + def is_fixed_uri(self) -> bool: return self.__fixed_uri is not None @property def uri(self) -> Optional[DataUri]: """ - Return a Unified Resource Identifier. Defined as: - URI = scheme ":" "//" authority path ["?" query] ["#" fragment] + Return an Unified Resource Identifier. Defined as: + URI = scheme ":" "//" path ["?" query] ["#" fragment] - https://en.wikipedia.org/wiki/Uniform_Resource_Identifier + see https://en.wikipedia.org/wiki/Uniform_Resource_Identifier + + .. warning:: query can be ?path= which is different from path """ if self.is_fixed_uri: return self.__fixed_uri -- GitLab