scan_saving.py 55.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# -*- coding: utf-8 -*-
#
# This file is part of the bliss project
#
# Copyright (c) 2015-2019 Beamline Control Unit, ESRF
# Distributed under the GNU LGPLv3. See LICENSE for more info.

import getpass
import gevent
import os
import string
import time
import tabulate
import uuid
import importlib
16
17
18
19
20
import re
import itertools
import traceback
from functools import wraps
import logging
21
import datetime
22
import enum
23
24
25

from bliss import current_session
from bliss.config.settings import ParametersWardrobe
26
from bliss.config.conductor.client import get_redis_proxy
27
from bliss.data.node import datanode_factory
28
29
30
from bliss.scanning.writer.null import Writer as NullWriter
from bliss.scanning import writer as writer_module
from bliss.common.proxy import Proxy
31
from bliss.common import logtools
32
from bliss.icat.ingester import IcatIngesterProxy
33
34
from bliss.config.static import get_config
from bliss.config.settings import scan as scan_redis
Linus Pithan's avatar
Linus Pithan committed
35
from bliss.common.utils import autocomplete_property
36
from bliss.icat.proposal import Proposal
37
from bliss.icat.dataset_collection import DatasetCollection
38
39
from bliss.icat.dataset import Dataset

40
41
42
43
44
45
46
47
48
49
50

_SCAN_SAVING_CLASS = None

ScanSaving = Proxy(lambda: _SCAN_SAVING_CLASS or BasicScanSaving)


def set_scan_saving_class(klass):
    global _SCAN_SAVING_CLASS
    _SCAN_SAVING_CLASS = klass


51
52
53
54
55
56
class ESRFDataPolicyEvent(enum.Enum):
    Enable = "enabled"
    Disable = "disabled"
    Change = "changed"


57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
logger = logging.getLogger(__name__)


class MissingParameter(ValueError):
    pass


class CircularReference(ValueError):
    pass


def with_eval_dict(method):
    """This passes a dictionary as named argument `eval_dict` to the method
    when it is not passed by the caller. This dictionary is used for caching
    parameter evaluations (user attributes and properties) in `EvalParametersWardrobe`.

    :param callable method: unbound method of `EvalParametersWardrobe`
    :returns callable:
    """

77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
    @wraps(method)
    def eval_func(self, *args, **kwargs):
        # Create a cache dictionary if not provided by caller
        if "eval_dict" in kwargs:
            eval_dict = kwargs.get("eval_dict")
        else:
            eval_dict = None
        if eval_dict is None:
            logger.debug("create eval_dict (method {})".format(repr(method.__name__)))
            # Survives only for the duration of the call
            eval_dict = kwargs["eval_dict"] = {}
        if not eval_dict:
            self._update_eval_dict(eval_dict)
            logger.debug("filled eval_dict (method {})".format(repr(method.__name__)))
        # Evaluate method (passes eval_dict)
        return method(self, *args, **kwargs)
93

94
    return eval_func
95
96


97
class property_with_eval_dict(autocomplete_property):
98
99
100
    """Combine the `with_eval_dict` and `property` decorators
    """

101
102
103
104
105
106
107
108
109
110
    def __init__(self, fget=None, fset=None, fdel=None, doc=None):
        if fget is not None:
            name = "_eval_getter_" + fget.__name__
            fget = with_eval_dict(fget)
            fget.__name__ = name
        if fset is not None:
            name = "_eval_setter_" + fset.__name__
            fset = with_eval_dict(fset)
            fset.__name__ = name
        super().__init__(fget=fget, fset=fset, fdel=fdel, doc=doc)
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149


def is_circular_call(funcname):
    """Check whether a function is called recursively

    :param str funcname:
    :returns bool:
    """
    # This is good enough for our purpose
    return any(f.name == funcname for f in traceback.extract_stack())


class EvalParametersWardrobe(ParametersWardrobe):
    """A parameter value in the Wardrobe can be:

        - literal string: do nothing
        - template string: fill with other parameters (recursive)
        - callable: unbound method of this class with signature
                    `method(self)` or `method(self, eval_dict=...)`
        - other: converted to string

    Methods with the `with_eval_dict` decorator will cache the evaluation
    of these parameter values (user attributes and properties).

    Properties with the `with_eval_dict` decorator need to be called with
    `get_cached_property` or `set_cached_property` to pass the cache dictionary.
    When used as a normal property, a temporary cache dictionary is created.

    The evaluation cache is shared by recursive calls (passed as an argument).
    It is not persistant unless you pass it explicitely as an argument on the
    first call to a `with_eval_dict` decorated method.

    Parameter evaluation is done with the method `eval_template`, which can
    also be used externally to evaluate any string template that contains
    wardrobe parameter fields.
    """

    FORMATTER = string.Formatter()

150
    NO_EVAL_PROPERTIES = set()
Linus Pithan's avatar
Linus Pithan committed
151

152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
    def _template_named_fields(self, template):
        """Get all the named fields in a template.
        For example "a{}bc{d}efg{h}ij{:04d}k" has two named fields.

        :pram str template:
        :returns set(str):
        """
        return {
            fieldname
            for _, fieldname, _, _ in self.FORMATTER.parse(template)
            if fieldname is not None
        }

    @with_eval_dict
    def eval_template(self, template, eval_dict=None):
        """Equivalent to `template.format(**eval_dict)` with additional properties:
            - The values in `eval_dict` can be callable or template strings themselves.
            - They will be evaluated recursively and replaced in `eval_dict`.

        :param str or callable template:
        :param dict eval_dict:
        """
        eval_dict.setdefault("__evaluated__", set())

        # Evaluate callable and throw exception on empty value
        if callable(template):
            try:
                template = template(self, eval_dict=eval_dict)
            except TypeError:
                template = template(self)
            if template is None:
                raise MissingParameter("Parameters value generator returned `None`")
            if not isinstance(template, str):
                template = str(template)
        else:
            if template is None:
                raise MissingParameter
            if not isinstance(template, str):
                template = str(template)

        # Evaluate fields that have not been evaluated yet
        fields = self._template_named_fields(template)
        already_evaluated = eval_dict["__evaluated__"].copy()
        eval_dict["__evaluated__"] |= fields
        for field in fields - already_evaluated:
            value = eval_dict.get(field)
            try:
                eval_dict[field] = self.eval_template(value, eval_dict=eval_dict)
            except MissingParameter:
201
202
203
204
205
206
                if hasattr(self, field):
                    self.get_cached_property(field, eval_dict)
                if field not in eval_dict:
                    raise MissingParameter(
                        f"Parameter {repr(field)} is missing in {repr(template)}"
                    ) from None
207
208
209
210
211
212
213
214
215
216
217
218
219
220

        # Evaluate string template while avoiding circular references
        fill_dict = {}
        for field in fields:
            value = eval_dict[field]
            ffield = "{{{}}}".format(field)
            if ffield in value:
                # Stop evaluating circular reference
                # raise CircularReference("Parameter {} contains a circular reference".format(repr(field)))
                fill_dict[field] = ffield
            else:
                fill_dict[field] = value
        return template.format(**fill_dict)

221
    def _update_eval_dict(self, eval_dict):
222
223
224
225
226
227
228
229
230
231
        """Update the evaluation dictionary with user attributes (from Redis)
        and properties when missing.

        :param dict eval_dict:
        :returns dict:
        """
        fromredis = self.to_dict(export_properties=False)
        for k, v in fromredis.items():
            if k not in eval_dict:
                eval_dict[k] = v
Linus Pithan's avatar
Linus Pithan committed
232
        for prop in self._iter_eval_properties():
233
234
            if prop in eval_dict:
                continue
235
            self.get_cached_property(prop, eval_dict)
236

Linus Pithan's avatar
Linus Pithan committed
237
    def _iter_eval_properties(self):
238
239
        """Yield all properties that will be cached when updating the
        evaluation dictionary
Linus Pithan's avatar
Linus Pithan committed
240
241
        """
        for prop in self._property_attributes:
242
            if prop not in self.NO_EVAL_PROPERTIES:
Linus Pithan's avatar
Linus Pithan committed
243
244
                yield prop

245
246
247
248
249
250
251
252
253
254
255
256
    def get_cached_property(self, name, eval_dict):
        """Pass `eval_dict` to a property getter. If the property has
        already been evaluated before (meaning it is in `eval_dict`)
        then that value will be used without calling the property getter.

        :param str name: property name
        :param dict eval_dict:
        :returns any:
        """
        if name in eval_dict:
            return eval_dict[name]
        _prop = getattr(self.__class__, name)
257
258
259
260
261
262
263
264
265
266
267
        if isinstance(_prop, property_with_eval_dict):
            logger.debug("fget eval property " + repr(name))
            if is_circular_call(_prop.fget.__name__):
                raise CircularReference(
                    "Property {} contains a circular reference".format(repr(name))
                )
            r = _prop.fget(self, eval_dict=eval_dict)
        elif isinstance(_prop, property):
            logger.debug("fget normal property " + repr(name))
            r = _prop.fget(self)
        else:
268
269
270
            # Not a property
            r = getattr(self, name)
        eval_dict[name] = r
271
        logger.debug(f"     eval_dict[{repr(name)}] = {repr(r)}")
272
273
274
275
276
277
278
279
280
281
        return r

    def set_cached_property(self, name, value, eval_dict):
        """Pass `eval_dict` to a property setter.

        :param str name: property name
        :param any value:
        :param dict eval_dict:
        """
        _prop = getattr(self.__class__, name)
282
283
284
285
286
287
288
289
290
291
292
        if isinstance(_prop, property_with_eval_dict):
            logger.debug("fset eval property " + repr(name))
            if is_circular_call(_prop.fset.__name__):
                raise CircularReference(
                    "Property {} contains a circular reference".format(repr(name))
                )
            _prop.fset(self, value, eval_dict=eval_dict)
        elif isinstance(_prop, property):
            logger.debug("fset normal property " + repr(name))
            _prop.fset(self, value)
        else:
293
294
295
296
297
298
299
300
301
302
303
304
305
306
            # Not a property
            setattr(self, name, value)
        eval_dict[name] = value


class BasicScanSaving(EvalParametersWardrobe):
    """Parameterized representation of the scan data file path

        base_path/template/data_filename+file_extension

    where each part (except for the file extension) is generated
    from user attributes and properties.
    """

307
308
309
310
311
312
313
314
315
316
    DEFAULT_VALUES = {
        # default and not removable values
        "base_path": "/tmp/scans",
        "data_filename": "data",
        "template": "{session}/",
        "images_path_relative": True,
        "images_path_template": "scan{scan_number}",
        "images_prefix": "{img_acq_device}_",
        "date_format": "%Y%m%d",
        "scan_number_format": "%04d",
317
        # saved properties in Redis:
318
319
320
321
322
323
        "_writer_module": "hdf5",
    }
    # read only attributes implemented with python properties
    PROPERTY_ATTRIBUTES = [
        "session",
        "date",
324
        "user_name",
325
326
327
328
        "scan_name",
        "scan_number",
        "img_acq_device",
        "writer",
329
        "data_policy",
330
331
    ]
    REDIS_SETTING_PREFIX = "scan_saving"
332
    SLOTS = []
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358

    def __init__(self, name=None):
        """
        This class hold the saving structure for a session.

        This class generate the *root path* of scans and the *parent* node use
        to publish data.

        The *root path* is generate using *base path* argument as the first part
        and use the *template* argument as the final part.

        The *template* argument is basically a (python) string format use to
        generate the final part of the root_path.

        i.e: a template like "{session}/{date}" will use the session and the date attribute
        of this class.

        Attribute used in this template can also be a function with one argument
        (scan_data) which return a string.

        i.e: date argument can point to this method
             def get_date(scan_data): datetime.datetime.now().strftime("%Y/%m/%d")
             scan_data.add('date',get_date)

        The *parent* node should be use as parameters for the Scan.
        """
359
360
        if not name:
            name = str(uuid.uuid4().hex)
361
        super().__init__(
362
            f"{self.REDIS_SETTING_PREFIX}:{name}",
363
364
365
            default_values=self.DEFAULT_VALUES,
            property_attributes=self.PROPERTY_ATTRIBUTES,
            not_removable=self.DEFAULT_VALUES.keys(),
366
            connection=get_redis_proxy(caching=True),
367
368
369
        )

    def __dir__(self):
370
371
372
373
374
        keys = list(self.PROPERTY_ATTRIBUTES)
        keys.extend([p for p in self.DEFAULT_VALUES if not p.startswith("_")])
        keys.extend(
            [
                "clone",
375
                "get",
376
                "get_data_info",
377
378
379
380
                "get_path",
                "get_parent_node",
                "filename",
                "root_path",
381
382
                "data_path",
                "data_fullpath",
383
                "images_path",
384
385
386
387
                "writer_object",
                "file_extension",
                "scan_parent_db_name",
                "newproposal",
388
                "newcollection",
389
                "newsample",
390
                "newdataset",
Linus Pithan's avatar
Linus Pithan committed
391
                "on_scan_run",
392
393
            ]
        )
394
        return keys
395
396

    def __info__(self):
397
398
        d = {}
        self._update_eval_dict(d)
399
400
        d["img_acq_device"] = "<images_* only> acquisition device name"
        info_str = super()._repr(d)
401
402
        extra = self.get_data_info(eval_dict=d)
        info_str += tabulate.tabulate(tuple(extra))
403
404
        return info_str

405
406
407
408
409
410
    @with_eval_dict
    def get_data_info(self, eval_dict=None):
        """
        :returns list:
        """
        writer = self.get_cached_property("writer_object", eval_dict)
411
412
413
414
415
416
417
418
        info_table = list()
        if isinstance(writer, NullWriter):
            info_table.append(("NO SAVING",))
        else:
            data_file = writer.filename
            data_dir = os.path.dirname(data_file)

            if os.path.exists(data_file):
419
                label = "exists"
420
            else:
421
422
                label = "does not exist"
            info_table.append((label, "filename", data_file))
423
424

            if os.path.exists(data_dir):
425
                label = "exists"
426
            else:
427
428
                label = "does not exist"
            info_table.append((label, "directory", data_dir))
429

430
        return info_table
431
432
433
434
435
436
437
438
439

    @property
    def scan_name(self):
        return "{scan_name}"

    @property
    def scan_number(self):
        return "{scan_number}"

440
441
442
443
    @property
    def data_policy(self):
        return "None"

444
445
446
447
    @property
    def img_acq_device(self):
        return "{img_acq_device}"

448
449
450
451
452
    @property
    def name(self):
        """This is the init name or a uuid"""
        return self._wardr_name.split(self.REDIS_SETTING_PREFIX + ":")[-1]

453
454
    @property
    def session(self):
455
456
457
458
459
        """This give the name of the current session or 'default' if no current session is defined """
        try:
            return current_session.name
        except AttributeError:
            return "default"
460
461
462
463
464

    @property
    def date(self):
        return time.strftime(self.date_format)

465
466
467
468
    @property
    def user_name(self):
        return getpass.getuser()

469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
    @property
    def writer(self):
        """
        Scan writer object.
        """
        return self._writer_module

    @writer.setter
    def writer(self, value):
        try:
            if value is not None:
                self._get_writer_class(value)
        except ImportError as exc:
            raise ImportError(
                "Writer module **%s** does not"
                " exist or cannot be loaded (%s)"
                " possible module are %s" % (value, exc, writer_module.__all__)
            )
        except AttributeError as exc:
            raise AttributeError(
                "Writer module **%s** does have"
                " class named Writer (%s)" % (value, exc)
            )
        else:
            self._writer_module = value

495
496
    def get_path(self):
        return self.root_path
497

498
499
500
    @property_with_eval_dict
    def root_path(self, eval_dict=None):
        """Directory of the scan *data file*
501
        """
502
        base_path = self.get_cached_property("base_path", eval_dict)
503
        return self._get_root_path(base_path, eval_dict=eval_dict)
504
505
506
507
508
509
510

    @property_with_eval_dict
    def data_path(self, eval_dict=None):
        """Full path for the scan *data file* without the extension
        This is before the writer modifies the name (given by `self.filename`)
        """
        root_path = self.get_cached_property("root_path", eval_dict)
511
        return self._get_data_path(root_path, eval_dict=eval_dict)
512
513
514
515
516

    @property_with_eval_dict
    def data_fullpath(self, eval_dict=None):
        """Full path for the scan *data file* with the extension.
        This is before the writer modifies the name (given by `self.filename`)
517
        """
518
        data_path = self.get_cached_property("data_path", eval_dict)
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
        return self._get_data_fullpath(data_path, eval_dict=eval_dict)

    @with_eval_dict
    def _get_root_path(self, base_path, eval_dict=None):
        """Directory of the scan *data file*
        """
        template = os.path.join(base_path, self.template)
        return os.path.abspath(self.eval_template(template, eval_dict=eval_dict))

    @with_eval_dict
    def _get_data_path(self, root_path, eval_dict=None):
        """Full path for the scan *data file* without the extension
        This is before the writer modifies the name (given by `self.filename`)
        """
        data_filename = self.get_cached_property("eval_data_filename", eval_dict)
        return os.path.join(root_path, data_filename)

    @with_eval_dict
    def _get_data_fullpath(self, data_path, eval_dict=None):
        """Full path for the scan *data file* with the extension.
        This is before the writer modifies the name (given by `self.filename`)
        """
541
542
543
544
545
546
547
        unknowns = self._template_named_fields(data_path)
        data_path = data_path.format(**{f: "{" + f + "}" for f in unknowns})
        return os.path.extsep.join((data_path, self.file_extension))

    @property_with_eval_dict
    def eval_data_filename(self, eval_dict=None):
        """The evaluated version of data_filename
548
        """
549
        return self.eval_template(self.data_filename, eval_dict=eval_dict)
550

551
552
553
554
555
556
557
558
559
560
561
    @property_with_eval_dict
    def filename(self, eval_dict=None):
        """Full path for the scan *data file* with the extension.
        Could be modified by the writer instance.
        """
        return self.get_cached_property("writer_object", eval_dict).filename

    @property_with_eval_dict
    def images_path(self, eval_dict=None):
        """Path to be used by external devices (normally a string template)
        """
562
563
        images_template = self.images_path_template
        images_prefix = self.images_prefix
564
565
        images_sub_path = self.eval_template(images_template, eval_dict=eval_dict)
        images_prefix = self.eval_template(images_prefix, eval_dict=eval_dict)
566
        if self.images_path_relative:
567
568
            root_path = self.get_cached_property("root_path", eval_dict)
            return os.path.join(root_path, images_sub_path, images_prefix)
569
        else:
570
            return os.path.join(images_sub_path, images_prefix)
571

572
573
    @with_eval_dict
    def get(self, eval_dict=None):
574
        """
575
        This method will compute all configurations needed for a new scan.
576
577
578
579
580
        It will return a dictionary with:
            root_path -- compute root path with *base_path* and *template* attribute
            images_path -- compute images path with *base_path* and *images_path_template* attribute
                If images_path_relative is set to True (default), the path
                template is relative to the scan path, otherwise the
581
582
583
                images_path_template has to be an absolute path.
            db_path_items -- information needed to create the parent node in Redis for the new scan
            writer -- a writer instance
584
585
        """
        return {
586
587
588
589
590
            "root_path": self.get_cached_property("root_path", eval_dict),
            "data_path": self.get_cached_property("data_path", eval_dict),
            "images_path": self.get_cached_property("images_path", eval_dict),
            "db_path_items": self.get_cached_property("_db_path_items", eval_dict),
            "writer": self.get_cached_property("writer_object", eval_dict),
591
592
        }

593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
    @property_with_eval_dict
    def scan_parent_db_name(self, eval_dict=None):
        """The Redis name of a scan's parent node is a concatenation of session
        name and data directory (e.g. "session_name:tmp:scans")
        """
        return ":".join(self.get_cached_property("_db_path_keys", eval_dict))

    @property_with_eval_dict
    def _db_path_keys(self, eval_dict=None):
        """The Redis name of a scan's parent node is a concatenation of session
        name and data directory (e.g. ["session_name", "tmp", "scans"])

        Duplicate occurences of "session_name" are removed.

        :returns list(str):
        """
        session = self.session
        parts = self.get_cached_property("root_path", eval_dict).split(os.path.sep)
        return [session] + [p for p in parts if p and p != session]

    @property_with_eval_dict
    def _db_path_items(self, eval_dict=None):
        """For scan's parent node creation (see `get_parent_node`)

        :returns list(tuple):
        """
        parts = self.get_cached_property("_db_path_keys", eval_dict)
620
621
        types = ["container"] * len(parts)
        return list(zip(parts, types))
622
623
624
625
626
627
628
629
630
631

    @property_with_eval_dict
    def writer_object(self, eval_dict=None):
        """This instantiates the writer class

        :returns bliss.scanning.writer.File:
        """
        root_path = self.get_cached_property("root_path", eval_dict)
        images_path = self.get_cached_property("images_path", eval_dict)
        data_filename = self.get_cached_property("eval_data_filename", eval_dict)
632
        klass = self._get_writer_class(self.writer)
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
        writer = klass(root_path, images_path, data_filename)
        s = root_path + images_path + data_filename
        writer.template.update(
            {f: "{" + f + "}" for f in self._template_named_fields(s)}
        )
        return writer

    @property
    def file_extension(self):
        """As determined by the writer
        """
        return self._get_writer_class(self.writer).FILE_EXTENSION

    def get_writer_object(self):
        """This instantiates the writer class
        :returns bliss.scanning.writer.File:
        """
        return self.writer_object

    def create_path(self, path):
        """The path is created by the writer if the path if part
        of the data root, else by Bliss (subdir or outside data root).

        :param str path:
        """
        self.writer_object.create_path(os.path.abspath(path))

    def create_root_path(self):
        """Create the scan data directory
        """
        self.create_path(self.root_path)
664

Linus Pithan's avatar
Linus Pithan committed
665
    def get_parent_node(self, create=True):
666
        """This method returns the parent node which should be used to publish new data
667

Linus Pithan's avatar
Linus Pithan committed
668
669
        :param bool create:
        :returns DatasetNode or None: can only return `None` when `create=False`
670
        """
671
672
673
674
675
676
677
678
679
680
        return self._get_node(self._db_path_items, create=create)

    def _get_node(self, db_path_items, create=True):
        """This method returns the parent node which should be used to publish new data

        :param list((str,str)) db_path_items:
        :param bool create:
        :returns DatasetNode or None: can only return `None` when `create=False`
        """
        node = None
Linus Pithan's avatar
Linus Pithan committed
681
682
        if create:
            for item_name, node_type in db_path_items:
683
                node = datanode_factory(
684
                    item_name, node_type=node_type, parent=node, create_not_state=True
685
                )
686
                self._fill_node_info(node, node_type)
Linus Pithan's avatar
Linus Pithan committed
687
688
        else:
            for item_name, node_type in db_path_items:
689
690
691
                node = datanode_factory(
                    item_name, node_type=node_type, parent=node, create_not_state=False
                )
692
                if node is None:
Linus Pithan's avatar
Linus Pithan committed
693
                    return None
694
695
696
697
698
699
        return node

    def _fill_node_info(self, node, node_type):
        """Add missing keys to node info
        """
        pass
700
701
702
703
704
705

    def _get_writer_class(self, writer_module_name):
        module_name = f"{writer_module.__name__}.{writer_module_name}"
        module = importlib.import_module(module_name)
        return getattr(module, "Writer")

706
707
708
    def newproposal(self, proposal_name):
        raise NotImplementedError("No data policy enabled")

709
    def newcollection(self, collection_name, **kw):
710
711
        raise NotImplementedError("No data policy enabled")

712
713
714
715
    def newsample(self, collection_name, **kw):
        raise NotImplementedError("No data policy enabled")

    def newdataset(self, dataset_name, **kw):
716
717
        raise NotImplementedError("No data policy enabled")

718
    def clone(self):
719
        new_scan_saving = self.__class__(self.name)
720
721
722
723
        for s in self.SLOTS:
            setattr(new_scan_saving, s, getattr(self, s))
        return new_scan_saving

724
725
726
727
    @property
    def elogbook(self):
        return None

Linus Pithan's avatar
Linus Pithan committed
728
729
730
731
732
    def on_scan_run(self, save):
        """Called at the start of a scan (in Scan.run)
        """
        pass

733
734

class ESRFScanSaving(BasicScanSaving):
735
736
737
738
739
740
    """Parameterized representation of the scan data file path
    according to the ESRF data policy

        base_path/template/data_filename+file_extension

    where the base_path is determined by the proposal name,
741
742
    the template is fixed to "{proposal_name}/{beamline}/{collection_name}/{collection_name}_{dataset_name}"
    and the data_filename is fixed to "{collection_name}_{dataset_name}".
743
744
    """

745
746
747
748
749
750
    DEFAULT_VALUES = {
        # default and not removable values
        "images_path_template": "scan{scan_number}",
        "images_prefix": "{img_acq_device}_",
        "date_format": "%Y%m%d",
        "scan_number_format": "%04d",
751
752
        "dataset_number_format": "%04d",
        # saved properties in Redis:
753
754
        "_writer_module": "nexus",
        "_proposal": "",
755
        "_ESRFScanSaving__proposal_timestamp": 0,
756
        "_collection": "",
757
        "_dataset": "",
758
        "_mount": "",
759
        "_reserved_dataset": "",
760
    }
761
    # Order important for resolving dependencies
762
    PROPERTY_ATTRIBUTES = BasicScanSaving.PROPERTY_ATTRIBUTES + [
763
        "template",
764
        "beamline",
765
        "proposal_name",
766
        "base_path",
767
        "collection_name",
768
        "dataset_name",
769
        "data_filename",
770
        "images_path_relative",
771
        "mount_point",
772
        "proposal",
773
        "collection",
774
775
776
777
778
        "dataset",
    ]
    SLOTS = BasicScanSaving.SLOTS + [
        "_icat_proxy",
        "_proposal_object",
779
        "_collection_object",
780
        "_dataset_object",
781
782
    ]
    REDIS_SETTING_PREFIX = "esrf_scan_saving"
783
    NO_EVAL_PROPERTIES = BasicScanSaving.NO_EVAL_PROPERTIES | {
784
        "proposal",
785
        "collection",
786
787
        "dataset",
    }
788

789
790
    def __init__(self, name):
        super().__init__(name)
791
        self._icat_proxy = None
792
        self._proposal_object = None
793
        self._collection_object = None
Linus Pithan's avatar
Linus Pithan committed
794
        self._dataset_object = None
Wout De Nolf's avatar
Wout De Nolf committed
795
796
797
        self._remove_deprecated()

    def _remove_deprecated(self):
798
        """Remove deprecated items from existing Redis databases"""
Wout De Nolf's avatar
Wout De Nolf committed
799
800
        stored = self.to_dict()
        if "_sample" in stored:
801
            # Deprecated in Bliss > 1.7.0
Wout De Nolf's avatar
Wout De Nolf committed
802
803
804
            value = stored["_sample"]
            self.remove("._sample")
            self._collection = value
805
806
807
        if "technique" in stored:
            # Deprecated in Bliss > 1.8.0
            self.remove("technique")
808

809
810
811
    def __dir__(self):
        keys = super().__dir__()
        keys.extend(
812
            ["proposal_type", "icat_root_path", "icat_data_path", "icat_data_fullpath"]
813
814
815
        )
        return keys

816
    @property
817
818
819
820
821
822
823
824
825
826
    def _session_config(self):
        """Current session config or static session config if no current session"""
        try:
            session_name = current_session.name
            config = current_session.config
        except AttributeError:
            # This may not be a session (and that's ok)
            session_name = self.name
            config = get_config()
        session_config = config.get_config(session_name)
827
        if session_config is None:
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
            return {}
        else:
            return session_config

    @property
    def _config_root(self):
        """Static config root"""
        try:
            return current_session.config.root
        except AttributeError:
            return get_config().root

    @property
    def scan_saving_config(self):
        return self._session_config.get(
            "scan_saving", self._config_root.get("scan_saving", {})
844
845
        )

846
847
848
849
    @property
    def data_policy(self):
        return "ESRF"

850
851
852
853
854
855
    @property
    def icat_proxy(self):
        if self._icat_proxy is None:
            self._icat_proxy = IcatIngesterProxy(self.beamline, self.session)
        return self._icat_proxy

856
857
    @property
    def images_path_relative(self):
858
        # Always relative due to the data policy
859
860
        return True

Linus Pithan's avatar
Linus Pithan committed
861
862
        # todo remove images_path_relative completely from here!

863
864
    @property
    def beamline(self):
865
        bl = self.scan_saving_config.get("beamline")
866
867
868
869
870
871
        if not bl:
            return "{beamline}"
        # Alphanumeric, space, dash and underscore
        if not re.match(r"^[0-9a-zA-Z_\s\-]+$", bl):
            raise ValueError("Beamline name is invalid")
        return re.sub(r"[^0-9a-z]", "", bl.lower())
872

Linus Pithan's avatar
Linus Pithan committed
873
    @autocomplete_property
874
875
876
877
878
879
880
881
882
883
    def proposal(self):
        """Nothing is created in Redis for the moment.
        """
        if self._proposal_object is None:
            # This is just for caching purposes
            self._ensure_proposal()
            self._proposal_object = self._get_proposal_object(create=True)
        return self._proposal_object

    @autocomplete_property
884
    def collection(self):
885
886
        """Nothing is created in Redis for the moment.
        """
887
        if self._collection_object is None:
888
            # This is just for caching purposes
889
890
891
892
            self._ensure_collection()
            self._collection_object = self._get_collection_object(create=True)
        return self._collection_object

893
    @autocomplete_property
894
895
896
    def sample(self):
        return self.collection

897
898
899
900
    @property_with_eval_dict
    def sample_name(self, eval_dict=None):
        # Property of ESRFScanSaving so that it can be used in a template
        return self.get_cached_property("dataset", eval_dict).sample_name
901

902
903
    @property_with_eval_dict
    def dataset(self, eval_dict=None):
Linus Pithan's avatar
Linus Pithan committed
904
905
906
        """The dataset will be created in Redis when it does not exist yet.
        """
        if self._dataset_object is None:
907
908
            # This is just for caching purposes
            self._ensure_dataset()
909
910
911
            self._dataset_object = self._get_dataset_object(
                create=True, eval_dict=eval_dict
            )
Linus Pithan's avatar
Linus Pithan committed
912
913
        return self._dataset_object

914
    @property
915
    def template(self):
916
        return "{proposal_name}/{beamline}/{collection_name}/{collection_name}_{dataset_name}"
917
918
919
920
921
922
923

    @property
    def _icat_proposal_path(self):
        # See template
        return os.sep.join(self.icat_root_path.split(os.sep)[:-3])

    @property
924
    def _icat_collection_path(self):
925
926
927
928
929
930
931
932
933
934
935
936
        # See template
        return os.sep.join(self.icat_root_path.split(os.sep)[:-1])

    @property
    def _icat_dataset_path(self):
        # See template
        return self.icat_root_path

    @property_with_eval_dict
    def _db_path_keys(self, eval_dict=None):
        session = self.session
        base_path = self.get_cached_property("base_path", eval_dict).split(os.sep)
937
        base_path = [p for p in base_path if p]
938
        proposal = self.get_cached_property("proposal_name", eval_dict)
939
        collection = self.get_cached_property("collection_name", eval_dict)
940
941
942
943
        # When dataset="0001" the DataNode.name will be the integer 1
        # so use the file name instead.
        # dataset = self.get_cached_property("dataset", eval_dict)
        data_filename = self.get_cached_property("eval_data_filename", eval_dict)
944
        return [session] + base_path + [proposal, collection, data_filename]
945
946
947
948
949
950
951
952
953
954
955

    @property_with_eval_dict
    def _db_path_items(self, eval_dict=None):
        """For scan's parent node creation (see `get_parent_node`)

        :returns list(tuple):
        """
        parts = self.get_cached_property("_db_path_keys", eval_dict)
        types = ["container"] * len(parts)
        # See template:
        types[-3] = "proposal"
956
        types[-2] = "dataset_collection"
957
958
959
        types[-1] = "dataset"
        return list(zip(parts, types))

960
961
962
    @property_with_eval_dict
    def _db_proposal_items(self, eval_dict=None):
        return self.get_cached_property("_db_path_items", eval_dict)[:-2]
963

964
965
966
    @property_with_eval_dict
    def _db_collection_items(self, eval_dict=None):
        return self.get_cached_property("_db_path_items", eval_dict)[:-1]
967

968
969
970
    @property_with_eval_dict
    def _db_dataset_items(self, eval_dict=None):
        return self.get_cached_property("_db_path_items", eval_dict)
971
972
973
974
975
976
977
978
979

    def _fill_node_info(self, node, node_type):
        """Add missing keys to node info
        """
        if node_type == "proposal":
            info = {
                "__name__": self.proposal_name,
                "__path__": self._icat_proposal_path,
            }
980
981
982
983
984
985
        elif node_type == "dataset_collection":
            info = {
                "__name__": self.collection_name,
                "__path__": self._icat_collection_path,
                "Sample_name": self.collection_name,
            }
986
987
988
989
990
991
992
993
994
995
996
997
998
        elif node_type == "dataset":
            info = {
                "__name__": self.dataset_name,
                "__path__": self._icat_dataset_path,
                "__closed__": False,
            }
        else:
            return
        existing = list(node.info.keys())
        info = {k: v for k, v in info.items() if k not in existing}
        if info:
            node.info.update(info)

999
1000
    @with_eval_dict
    def _get_proposal_node(self, create=True, eval_dict=None):