Commit c9a73209 authored by Matias Guijarro's avatar Matias Guijarro
Browse files

config: Node object refactoring to allow on-demand references evaluation, and...

config: Node object refactoring to allow on-demand references evaluation, and improvements on cloning and conversion to dictionary

* introduction of the Reference object
    - reference object has an 'eval' function to evaluate a reference
    - evaluation returns an object from config, or the value of an object attribute
* '$' prefix in YAML values now creates a Reference
    - no reference evaluation at YAML parsing time
* backward compatibility
    - getting a key with a reference returns the value, not the Reference
        * => introduction of ConfigList object to represent lists
    - '.raw_*' methods returns the raw contents, ie. with Reference objects instead of reference evaluated values
* node filenames are kept in Node objects
    - removed 'node2file' dictionary
* nodes are indexed automatically when inserting a 'name' key
    - no need for 'create_index'
    - same for tags
* '.to_dict()' method relies on the Python JSON decoder
    - simplification of the code
* '.deep_copy()' is renamed to '.clone()'
* '.deep_copy()' now relies on pickle
    - simplification of the code
    - nodes need a reference to the Config object => a trick is used to allow pickling/unpickling
        * the config object is saved to a tmp dict on pickling, and popped out on unpickling
* removed call to 'gc.collect()'
    - unclear why it was there at the first place
* removed weakref to Config object in nodes
    - nodes need the config, the config has to stay alive as long as nodes are there
parent 153046a1
......@@ -25,15 +25,15 @@ def main(host=None, user=None, passwd=None):
node = server2nodes.get(server)
if node is None:
node = static.Node(
config, filename="tango/%s.yml" % server.replace("/", "_")
node = static.ConfigNode(
config.root, filename="tango/%s.yml" % server.replace("/", "_")
)
exe_name, personal = server.split("/")
node["server"] = exe_name
node["personal_name"] = personal
server2nodes[server] = node
device_node = static.Node(config, parent=node)
device_node = static.ConfigNode(node)
device_node["tango_name"] = name
device_node["class"] = klass
device2nodes[name] = device_node
......@@ -53,7 +53,7 @@ def main(host=None, user=None, passwd=None):
continue
properties = device_node.get("properties")
if properties is None:
properties = static.Node(config, parent=device_node)
properties = static.ConfigNode(device_node)
device_node["properties"] = properties
values = properties.get(name)
......
......@@ -31,8 +31,8 @@ class Proxy(object):
def __init__(self, config):
if "tcp" in config:
tcp_config = config.get("tcp")
if hasattr(config, "deep_copy"):
self._config = tcp_config.deep_copy()
if hasattr(config, "clone"):
self._config = tcp_config.clone()
else:
self._config = tcp_config.copy()
self._mode = self.TCP
......@@ -80,8 +80,8 @@ class Proxy(object):
def _check_connection(self):
if self._mode == self.TCP:
if self._cnx is None or not self._cnx._connected:
if hasattr(self._config, "deep_copy"):
local_cfg = self._config.deep_copy()
if hasattr(self._config, "clone"):
local_cfg = self._config.clone()
else:
local_cfg = self._config.copy()
url = local_cfg.pop("url")
......
......@@ -10,8 +10,10 @@ import functools
import fnmatch
import typeguard
from sortedcontainers import SortedKeyList
from collections.abc import MutableSequence
from bliss.config import settings
from bliss.config.static import ConfigList
from bliss import current_session
from bliss import global_map
from bliss.common.proxy import Proxy
......@@ -257,7 +259,7 @@ class MeasurementGroup:
self.__name = name
self.__config = config_tree
if not isinstance(config_tree.get("counters"), list):
if not isinstance(config_tree.get("counters"), MutableSequence):
raise ValueError("MeasurementGroup: should have a counters list")
self._config_counters = config_tree.get("counters")
self._extra_counters = []
......
......@@ -301,7 +301,7 @@ class Session:
aliases.update(child_session._aliases_info())
for alias_cfg in self.__config_aliases:
cfg = alias_cfg.deep_copy()
cfg = alias_cfg.clone()
aliases[cfg.pop("original_name")] = cfg
return aliases
......
......@@ -12,7 +12,7 @@ from bliss.config.channels import Cache, EventChannel
from bliss.common import event
from bliss.common.utils import Null, autocomplete_property
from bliss.config.conductor.client import remote_open
from bliss.config.static import Node
from bliss.config.static import ConfigNode
from bliss.config.static import get_config_dict, _find_list, _find_dict, _find_subconfig
......@@ -443,9 +443,9 @@ class BeaconObject:
assert hasattr(value, "name")
obj_name = value.name
assert (
obj_name in self.config._config.names_list
obj_name in self.config.config.names_list
), f"{obj_name} does not exist in beacon config!"
return self.config._config.get(obj_name)
return self.config.config.get(obj_name)
def set_marshalling(self, value):
if value is None:
......
......@@ -340,7 +340,7 @@ def __get_plugins():
def _get_config_user_tags(config_item):
user_tag = config_item.get(static.Config.USER_TAG_KEY, [])
user_tag = config_item.get(static.ConfigNode.USER_TAG_KEY, [])
if not isinstance(user_tag, (tuple, list)):
user_tag = [user_tag]
return user_tag
......@@ -573,7 +573,7 @@ def add_folder():
folder = flask.request.form["folder"]
filename = os.path.join(folder, "__init__.yml")
node = static.Node(cfg, filename=filename)
node = static.ConfigNode(cfg.root, filename=filename)
node.save()
return flask.json.dumps(dict(message="Folder created!", type="success"))
......@@ -582,7 +582,7 @@ def add_folder():
def add_file():
cfg = __config.get_config()
filename = flask.request.form["file"]
node = static.Node(cfg, filename=filename)
node = static.ConfigNode(cfg.root, filename=filename)
node.save()
return flask.json.dumps(dict(message="File created!", type="success"))
......@@ -605,7 +605,7 @@ def copy_file():
if dst_path.endswith(os.path.pathsep):
dst_path = os.path.join(dst_path, os.path.split(src_path)[1])
node = static.Node(cfg, filename=dst_path)
node = static.ConfigNode(cfg.root, filename=dst_path)
node.save()
db_files = dict(client.get_config_db_files())
......
......@@ -6,57 +6,26 @@
# Distributed under the GNU LGPLv3. See LICENSE for more info.
from .utils import find_class_and_node, replace_reference_by_object
def _find_name_in_list(l, name):
for item in l:
if isinstance(item, dict):
find_node = _find_name_in_node(item, name)
if find_node is not None:
return find_node
elif isinstance(item, list):
find_node = _find_name_in_list(item, name)
if find_node is not None:
return find_node
def _find_name_in_node(node, name):
for key, value in node.items():
if key == "name" and value == name:
return node
if isinstance(value, dict):
find_node = _find_name_in_node(value, name)
if find_node is not None:
return find_node
elif isinstance(value, list):
find_node = _find_name_in_list(value, name)
if find_node is not None:
return find_node
from bliss.config.plugins.utils import find_class_and_node
from bliss.config.static import ConfigNode, ConfigReference
def create_objects_from_config_node(config, cfg_node):
klass, node = find_class_and_node(cfg_node)
node = node.deep_copy()
item_name = cfg_node["name"]
referenced_objects = dict()
if node.get("name") != item_name:
cfg_node = _find_name_in_node(node, item_name)
assert cfg_node is not None
cfg_node = ConfigNode.indexed_nodes[item_name]
else:
cfg_node = node
replace_reference_by_object(config, node, referenced_objects)
o = klass(item_name, cfg_node)
o = klass(item_name, cfg_node.clone())
for name, object in referenced_objects.items():
if hasattr(o, name):
continue
# raise RuntimeError("'%s` member would be shadowed by reference in yml config file." % name)
else:
setattr(o, name, object) # add_property(o, name, object)
for key, value in cfg_node.items():
if isinstance(value, ConfigReference):
if hasattr(o, key):
continue
else:
setattr(o, key, value.dereference())
return {item_name: o}
......@@ -5,12 +5,8 @@
# Copyright (c) 2015-2020 Beamline Control Unit, ESRF
# Distributed under the GNU LGPLv3. See LICENSE for more info.
from .utils import replace_reference_by_object
def create_objects_from_config_node(config, item_cfg_node):
item_name = item_cfg_node["name"]
replace_reference_by_object(config, item_cfg_node)
return {item_name: item_cfg_node}
......@@ -16,9 +16,9 @@ from bliss.common import encoder as encoder_module
from bliss.common.axis import Axis
from bliss.common.utils import auto_coerce
from bliss.common.encoder import Encoder
from bliss.config.static import Config
from bliss.config.static import ConfigNode, ConfigReference
from bliss.common.tango import DeviceProxy, _DeviceProxy
from bliss.config.plugins.utils import find_class, replace_reference_by_object
from bliss.config.plugins.utils import find_class
import bliss.controllers.motors
from bliss.controllers.motor import CalcController
......@@ -105,7 +105,7 @@ def get_axis_html(cfg):
if key not in __KNOWN_AXIS_PARAMS:
extra_params[key] = dict(name=key, label=key.capitalize(), value=value)
tags = cfg.get(Config.USER_TAG_KEY, [])
tags = cfg.get(ConfigNode.USER_TAG_KEY, [])
if not isinstance(tags, (tuple, list)):
tags = [tags]
vars["tags"] = tags
......@@ -335,7 +335,6 @@ def create_objects_from_config_node(config, node):
encoders = dict()
switches = dict()
shutters = dict()
node = node.to_dict()
cache_dict = dict()
for (
......@@ -351,11 +350,10 @@ def create_objects_from_config_node(config, node):
(switches, OBJECT_TYPE.SWITCH, None, "Switch", node.get("switches", [])),
):
for config_dict in config_nodes_list:
config_dict = config_dict.copy()
object_name = config_dict.get("name")
if object_name.startswith("$"):
object_name = config_dict.raw_get("name")
if isinstance(object_name, ConfigReference):
object_class = None
object_name = object_name.strip("$")
object_name = object_name.object_name
else:
cache_dict[object_name] = object_type, config_dict
object_class_name = config_dict.get("class")
......@@ -380,8 +378,6 @@ def create_objects_from_config_node(config, node):
raise
objects[object_name] = object_class, config_dict
referenced_objects = dict()
replace_reference_by_object(config, node, referenced_objects, greedy=True)
controller = controller_class(
controller_name, node, axes, encoders, shutters, switches
)
......@@ -415,7 +411,6 @@ def create_objects_from_config_node(config, node):
def create_object_from_cache(config, name, cache_objects):
controller, object_type, config_dict = cache_objects
replace_reference_by_object(config, config_dict)
if object_type == OBJECT_TYPE.AXIS:
return controller.get_axis(name)
elif object_type == OBJECT_TYPE.ENCODER:
......
......@@ -15,8 +15,7 @@ from bliss.common.regulation import (
ExternalInput,
ExternalOutput,
)
from bliss.config.plugins.utils import find_class, replace_reference_by_object
from bliss.config.plugins.utils import find_class
TYPE = enum.Enum("TYPE", "INPUT OUTPUT LOOP")
......@@ -51,8 +50,6 @@ def create_objects_from_config_node(config, node):
else: # <= else it is an object without a controller (like ExternalIn/out or SoftLoop)
replace_reference_by_object(config, node)
if node.get("class") in ["SoftLoop", "Loop"]:
new_obj = SoftLoop(node)
......@@ -83,13 +80,13 @@ def create_objects_from_config_node(config, node):
# --- for each subnode of a given type, store info in cache
for nd in child_nodes:
name2cacheditems[nd["name"]] = (node_type, nd.deep_copy(), controller)
name2cacheditems[nd["name"]] = (node_type, nd, controller)
# --- add the controller to stored items if it has a name
if controller_name:
name2items[controller_name] = controller
# update the config cache dict NOW to avoid cyclic instanciation (i.e. config.get => create_object_from_... => replace_reference_by_object => config.get )
# update the config cache dict NOW to avoid cyclic instanciation (i.e. config.get => create_object_from_... => config.get )
yield name2items, name2cacheditems
# --- don't forget to instanciate the object for which this function has been called (if not a controller)
......@@ -106,7 +103,6 @@ def create_object_from_cache(config, name, object_info):
# for a better understanding of this function, see bliss.config.static => config.get( obj_name )
node_type, node, controller = object_info
replace_reference_by_object(config, node)
controller_module = sys.modules[controller.__module__]
object_class_name = node.get("class", DEFAULT_CLASS_NAME[node_type])
......
......@@ -8,8 +8,8 @@
import os
import collections
from .utils import find_class
from ...common.measurementgroup import MeasurementGroup
from bliss.config.plugins.utils import find_class
from bliss.common.measurementgroup import MeasurementGroup
def create_objects_from_config_node(config, item_cfg_node):
......@@ -19,7 +19,7 @@ def create_objects_from_config_node(config, item_cfg_node):
if issubclass(klass, MeasurementGroup):
available_counters = _get_available_counters(config, item_cfg_node)
if available_counters != item_cfg_node.get("counters", list()):
item_cfg_node = item_cfg_node.deep_copy()
item_cfg_node = item_cfg_node.clone()
item_cfg_node["counters"] = available_counters
return {item_name: klass(item_name, item_cfg_node)}
......
......@@ -7,8 +7,9 @@
import sys
import itertools
from bliss.config.static import ConfigReference
from bliss.common.temperature import Input, Output, Loop
from bliss.config.plugins.utils import find_class, replace_reference_by_object
from bliss.config.plugins.utils import find_class
def create_objects_from_config_node(config, node):
......@@ -27,7 +28,6 @@ def create_objects_from_config_node(config, node):
inputs = dict()
outputs = dict()
loops = dict()
node = node.to_dict()
cache_dict = dict()
for (objects, default_class, config_nodes_list) in (
......@@ -36,11 +36,10 @@ def create_objects_from_config_node(config, node):
(loops, Loop, node.get("ctrl_loops", [])),
):
for config_dict in config_nodes_list:
config_dict = config_dict.copy()
object_name = config_dict.get("name")
if object_name.startswith("$"):
object_name = config_dict.raw_get("name")
if isinstance(object_name, ConfigReference):
object_class = None
object_name = object_name.strip("$")
object_name = object_name.ref
else:
cache_dict[object_name] = config_dict
object_class_name = config_dict.get("class")
......@@ -72,5 +71,4 @@ def create_objects_from_config_node(config, node):
def create_object_from_cache(config, name, cache_objects):
controller, config_dict = cache_objects
replace_reference_by_object(config, config_dict)
return controller.get_object(name)
......@@ -56,93 +56,3 @@ def find_class_and_node(cfg_node, base_path="bliss.controllers"):
klass = getattr(module, klass_name.title())
return klass, node
def _checkref(config, item_cfg_node, referenced_objects, name, value, placeholder):
if isinstance(value, str) and value.startswith("$"):
# convert reference to item from config
value = value.lstrip("$")
if placeholder:
obj = placeholder(value)
else:
obj = config.get(value)
item_cfg_node[name] = obj
referenced_objects[name] = obj
return True
else:
return False
def _parse_dict(config, item_cfg_node, referenced_objects, subdict, placeholder):
for name, value in tuple(subdict.items()):
if _checkref(config, subdict, referenced_objects, name, value, placeholder):
continue
elif isinstance(value, dict):
childdict = dict()
childref = dict()
_parse_dict(config, childdict, childref, value, placeholder)
if childref:
value.update(childref)
referenced_objects[name] = value
subdict[name].update(childdict)
elif isinstance(value, list):
return_list = _parse_list(config, value, placeholder)
if return_list:
referenced_objects[name] = return_list
item_cfg_node[name] = return_list
def _parse_list(config, value, placeholder):
object_list = list()
for node in value:
if isinstance(node, str) and node.startswith("$"):
node = node.lstrip("$")
if placeholder:
object_list.append(placeholder(node))
else:
object_list.append(config.get(node))
elif isinstance(node, dict):
subdict = dict()
subref = dict()
_parse_dict(config, subdict, subref, node, placeholder)
if subdict:
node.update(subdict)
object_list.append(node)
elif isinstance(node, list):
return_list = _parse_list(config, node, placeholder)
if return_list:
object_list.append(return_list)
else:
object_list.append(node)
return object_list
def replace_reference_by_object(
config, item_cfg_node, ref_objects=None, placeholder=None, greedy=False
):
"""
Args:
greedy (false): if True replaces only objects references at first level
not going deeper
"""
referenced_objects = ref_objects if ref_objects is not None else dict()
for name, value in tuple(item_cfg_node.items()):
if _checkref(
config, item_cfg_node, referenced_objects, name, value, placeholder
):
continue
if not greedy:
# enter subnodes
if isinstance(value, list):
return_list = _parse_list(config, value, placeholder)
if return_list:
referenced_objects[name] = return_list
item_cfg_node[name] = return_list
elif isinstance(value, dict):
subdict = dict()
subref = dict()
_parse_dict(config, subdict, subref, value, placeholder)
if subref:
referenced_objects[name] = subref
item_cfg_node[name].update(subdict)
This diff is collapsed.
......@@ -402,7 +402,7 @@ class Lima(CounterController):
self.__prepare_timeout = config_tree.get("prepare_timeout", None)
self.__bpm = None
self.__roi_counters = None
self._instrument_name = config_tree._config.root.get("instrument", "")
self._instrument_name = config_tree.root.get("instrument", "")
self.__bg_sub = None
self.__last = None
self._camera = None
......
......@@ -51,7 +51,7 @@ class MachInfo(BeaconObject):
self.tango_uri = config["uri"]
self.__counters = []
for cnt_name, attr_name in self.COUNTERS:
counter_config = config.deep_copy()
counter_config = config.clone()
counter_config["attr_name"] = attr_name
controller = TangoCounterController(
name, self.tango_uri, global_map_register=False
......
......@@ -44,7 +44,6 @@ from bliss.common.counter import SamplingCounter, SamplingMode
from bliss.common import tango
from bliss import global_map
from bliss.common.logtools import log_debug, log_warning
from bliss.config.static import Node
from bliss.controllers.counter import SamplingCounterController
......
......@@ -41,7 +41,7 @@ import logging
from bliss import release
from bliss.config import get_sessions_list
from bliss.config import static
from bliss.config.static import Node
from bliss.config.static import ConfigNode
from bliss.config.conductor import client
from bliss import logging_startup
from bliss import current_session, global_map
......@@ -128,10 +128,13 @@ def create_session(session_name):
"""
print(("Creating '%s' BLISS session" % session_name))
config = static.get_config()
config.set_config_db_file("sessions/__init__.yml", "plugin: session\n")
# <session_name>.yml: config file created as a config Node.
file_name = "sessions/%s.yml" % session_name
new_session_node = Node(filename=file_name)
print(("Creating %s" % file_name))
filename = "sessions/%s.yml" % session_name
new_session_node = ConfigNode(config.root, filename=filename)
print(("Creating %s" % filename))
new_session_node.update(
{
"class": "Session",
......
......@@ -12,6 +12,7 @@ import logging
import re
import weakref
import datetime
from collections.abc import MutableSequence, MutableMapping
import tango
from tango.databaseds.db_errors import *
......@@ -107,16 +108,16 @@ class beacon:
self._index()
# Trick to start
self._beacon_dserver_node = static.Node(self._config)
self._beacon_dserver_node = static.ConfigNode(self._config.root)
self._beacon_dserver_node["server"] = "DataBaseds"
self._beacon_dserver_node["personal_name"] = personal_name
tango_name = "sys/database/%s" % personal_name
databse_device_node = static.Node(self._config, self._beacon_dserver_node)
databse_device_node["class"] = "DataBase"
databse_device_node["tango_name"] = tango_name
self._beacon_dserver_node["device"] = [databse_device_node]
database_device_node = static.ConfigNode(self._beacon_dserver_node)
database_device_node["class"] = "DataBase"
database_device_node["tango_name"] = tango_name
self._beacon_dserver_node["device"] = [database_device_node]
self._beacon_dserver_node["tango_name"] = tango_name
self._tango_name_2_node[tango_name] = databse_device_node
self._tango_name_2_node[tango_name] = database_device_node
tango_name = "dserver/databaseds/%s" % personal_name
self._tango_name_2_node[tango_name] = self._beacon_dserver_node
server_name = "DataBaseds/%s" % personal_name
......@@ -129,32 +130,36 @@ class beacon:
self._tango_name_2_node = CaseInsensitiveDict()
self._class_name_2_node = CaseInsensitiveDict()
for key, values in self._config.root.items():
for key, values in self._config.root.raw_items():
indexing_flag = key == "tango"
if isinstance(values, list):
if isinstance(values, MutableSequence):
self._parse_list(values, indexing_flag)
elif isinstance(values, dict):
elif isinstance(values, MutableMapping):
self._parse_dict(values, indexing_flag)
self._index_tango(values)
def _parse_list(self, l, indexing_flag):
try:
l = l.raw_list
except AttributeError:
pass
for v in l:
if isinstance(v, list):
if isinstance(v, MutableSequence):
self._parse_list(v, indexing_flag)
elif isinstance(v, dict):
elif isinstance(v, MutableMapping):
if indexing_flag:
self._index_tango(v)
self._parse_dict(v, indexing_flag)
def _parse_dict(self, d, indexing_flag):
for k, v in d.items():
if isinstance(v, dict):
for k, v in d.raw_items():