Commit 34a3b843 authored by Piergiorgio Pancino's avatar Piergiorgio Pancino
Browse files

changed yaml parser dependency:

 - comment saving is now possible
 - updated all bliss modules using pyyaml
parent e1550de9
......@@ -18,7 +18,8 @@ import hashlib
import numpy
from tabulate import tabulate
import yaml
from ruamel.yaml import YAML
from ruamel.yaml.compat import StringIO
from .conductor import client
from bliss.config.conductor.client import set_config_db_file, remote_open
......@@ -1473,7 +1474,11 @@ class ParametersWardrobe(metaclass=ParametersType):
)
data_to_dump = {"WardrobeName": self._wardr_name, "instances": _instances}
return yaml.dump(data_to_dump, default_flow_style=False, sort_keys=False)
stream = StringIO()
yaml = YAML(pure=True)
yaml.default_flow_style = False
yaml.dump(data_to_dump, stream=stream)
return stream.getvalue()
def to_file(self, fullpath: str, *instances) -> None:
"""
......@@ -1501,7 +1506,8 @@ class ParametersWardrobe(metaclass=ParametersType):
yml: string containing yml data
instance_name: the name of the instance that you want to import
"""
dict_in = yaml.load(yml, Loader=yaml.FullLoader)
yaml = YAML(pure=True)
dict_in = yaml.load(yml)
if dict_in.get("WardrobeName") != self._wardr_name:
logger.warning("Wardrobe Names are different")
try:
......
......@@ -54,11 +54,13 @@ import collections
from collections.abc import MutableMapping, MutableSequence
import types
import yaml
from yaml.loader import Reader, Scanner, Parser, Composer, SafeConstructor, Resolver
import ruamel
from ruamel.yaml import YAML
from ruamel.yaml.compat import StringIO
from bliss.config.conductor import client
from bliss.config import channels
from bliss.common.utils import prudent_update
CONFIG = None
......@@ -102,36 +104,6 @@ def _find_subconfig(d, path):
return sub
class BlissYamlResolver(Resolver):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
new_resolvers = collections.defaultdict(list)
for k, resolver in self.__class__.yaml_implicit_resolvers.items():
for item in resolver:
tag, regexp = item
if tag.endswith("2002:bool"):
regexp = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$", re.X)
new_resolvers[k].append((tag, regexp))
self.__class__.yaml_implicit_resolvers = new_resolvers
class BlissSafeConstructor(SafeConstructor):
bool_values = {"true": True, "false": False}
class BlissSafeYamlLoader(
Reader, Scanner, Parser, Composer, BlissSafeConstructor, BlissYamlResolver
):
def __init__(self, stream):
Reader.__init__(self, stream)
Scanner.__init__(self)
Parser.__init__(self)
Composer.__init__(self)
BlissSafeConstructor.__init__(self)
BlissYamlResolver.__init__(self)
def _replace_object_with_ref(obj):
try:
obj_name = obj.name
......@@ -219,8 +191,9 @@ def get_config_dict(fullname, node_name):
"""
with client.remote_open(fullname) as f:
d = yaml.safe_load(f.read())
if isinstance(d, dict):
yaml = YAML(pure=True)
yaml.allow_duplicate_keys = True
d = yaml.load(f.read())
if isinstance(d, MutableMapping):
d = _find_dict(node_name, d)
elif isinstance(d, MutableSequence):
......@@ -363,12 +336,29 @@ class Node(dict):
if filename is None:
return # Memory
nodes_2_save = self._config._file2node[filename]
# for save_nodes in self._get_save_list(nodes_2_save, filename):
# yaml_contents can be a CommentedMap (hashmap)
if len(nodes_2_save) == 1:
node = tuple(nodes_2_save)[0]
save_nodes = self._get_save_dict(node, filename)
else:
save_nodes = self._get_save_list(nodes_2_save, filename)
file_content = yaml.dump(save_nodes, default_flow_style=False, sort_keys=False)
yaml = YAML(pure=True)
yaml.allow_duplicate_keys = True
yaml.default_flow_style = False
try:
yaml_contents = yaml.load(
client.get_text_file(filename, self._config._connection)
)
except RuntimeError:
# file does not exist
yaml_contents = save_nodes
else:
yaml_contents = prudent_update(yaml_contents, save_nodes)
string_stream = StringIO()
yaml.dump(yaml_contents, stream=string_stream)
file_content = string_stream.getvalue()
self._config.set_config_db_file(filename, file_content)
def deep_copy(self):
......@@ -565,8 +555,14 @@ class Config:
continue
try:
d = yaml.load(file_content, BlissSafeYamlLoader)
except yaml.scanner.ScannerError as exp:
# typ='safe' -> Gives dict instead of OrderedDict subclass
# (removing comments)
# pure=True -> if False 052 is interpreted as octal (using C engine)
yaml = YAML(pure=True)
yaml.allow_duplicate_keys = True
d = yaml.load(file_content)
except ruamel.yaml.scanner.ScannerError as exp:
exp.note = "Error in YAML parsing:\n"
exp.note += "----------------\n"
exp.note += f"{file_content}\n"
......@@ -574,7 +570,8 @@ class Config:
exp.note += "Hint: You can check your configuration with an on-line YAML validator like http://www.yamllint.com/ \n\n"
exp.problem_mark.name = path
raise exp
except yaml.error.MarkedYAMLError as exp:
# from ruamel.yaml.parser import ParserError
except ruamel.yaml.error.MarkedYAMLError as exp:
if exp.problem_mark is not None:
exp.problem_mark.name = path
raise
......
......@@ -6,7 +6,8 @@
# Distributed under the GNU LGPLv3. See LICENSE for more info.
import re
import yaml
from ruamel.yaml import YAML
from ruamel.yaml.compat import StringIO
from collections import namedtuple
from itertools import zip_longest
......@@ -489,7 +490,11 @@ def specfile_to_yml(iterable):
c_y["dac_offset"] = c.dac_offset
yml["interlocks"][-1]["channels"].append(c_y)
return yaml.dump(yml, default_flow_style=False, sort_keys=False)
yaml = YAML()
yaml.default_flow_style = False
string_stream = StringIO()
yaml.dump(yml, stream=string_stream)
return string_stream.getvalue()
def interlock_to_yml(interlock_list):
......@@ -527,7 +532,11 @@ def interlock_to_yml(interlock_list):
r_d["channels"].append(c_d)
d_i["interlocks"].append(r_d)
return yaml.dump(d_i, default_flow_style=False, sort_keys=False)
yaml = YAML()
yaml.default_flow_style = False
string_stream = StringIO()
yaml.dump(d_i, string_stream)
return string_stream.getvalue()
def _interlock_relay_info(
......
......@@ -28,7 +28,7 @@ pygments
pygraphviz >= 1.5
pyserial > 2
python >= 3.7
pyyaml >= 5.1
ruamel.yaml
redis >= 5.0
redis-py >= 3.3.7
hiredis
......
import yaml
import subprocess
import os
from pprint import pprint
import re
from ruamel.yaml import YAML
from ruamel.yaml.compat import StringIO
try:
import conda.cli.python_api as conda
from conda.cli import main_info
......@@ -178,6 +180,8 @@ def main():
license_family: GPL
"""
yaml = YAML()
yaml.default_flow_style = False
head = yaml.load(template_head)
body = yaml.load(template_body)
......@@ -215,8 +219,10 @@ def main():
# writing meta.yaml
with open(META, "w") as f:
f.write(yaml.dump(head, default_flow_style=False))
f.write(yaml.dump(body, default_flow_style=False))
stream = StringIO()
yaml.dump(head, stream=stream)
yaml.dump(body, stream=stream)
f.write(stream.getvalue())
if __name__ == "__main__":
......
......@@ -135,6 +135,7 @@ def main():
"tabulate",
"pyserial > 2",
"pyyaml >= 5.1",
"ruamel.yaml",
"msgpack >= 0.6.1",
"msgpack_numpy >= 0.4.4.2",
"blessings",
......
......@@ -17,7 +17,7 @@ import serial as pyserial
# from bliss.common.logtools import debugon
import socket
from contextlib import contextmanager
import yaml
from ruamel.yaml import YAML
from bliss.common.utils import deep_update
SOCAT = os.path.join(os.environ.get("CONDA_PREFIX", "/"), "bin", "socat")
......@@ -82,9 +82,10 @@ def _inspect_serial_port_settings(fake_port):
.replace(" ", "':'")
+ "'}"
)
deep_update(options, yaml.load(tmp, Loader=yaml.Loader))
yaml = YAML()
deep_update(options, yaml.load(tmp))
else:
params += yaml.load("['" + s.replace(" ", "','") + "']", Loader=yaml.Loader)
params += yaml.load("['" + s.replace(" ", "','") + "']")
return (options, params)
......
......@@ -9,6 +9,7 @@ from bliss.config.conductor import client
from bliss.config.plugins.utils import replace_reference_by_object
import pytest
import sys, os
import ruamel
@pytest.mark.parametrize(
......@@ -33,13 +34,19 @@ def test_config_save(beacon, beacon_directory, file_name, node_name, copy):
assert rw_cfg["one"][0]["pink"] == "martini"
assert rw_cfg["one"][0]["red"] == "apples"
for comment in ("comment{}".format(i) for i in range(1, 6)):
assert comment in test_file_contents
rw_cfg["one"][0]["red"] = "strawberry"
rw_cfg["one"][0]["pink"] = "raspberry"
try:
rw_cfg.save()
beacon.reload()
with open(test_file_path) as f:
content = f.read()
for comment in ("comment{}".format(i) for i in range(1, 6)):
assert comment in content
rw_cfg2 = beacon.get_config(node_name)
......@@ -51,6 +58,24 @@ def test_config_save(beacon, beacon_directory, file_name, node_name, copy):
f.write(test_file_contents)
def test_yml_load_exception(beacon, beacon_directory):
new_file = "%s/bad.yml" % beacon_directory
try:
with open(new_file, "w") as f:
f.write(
"""- name: bad_yml
let:
- 1
- 2
"""
)
assert pytest.raises(ruamel.yaml.scanner.ScannerError, beacon.reload)
finally:
os.unlink(new_file)
def test_empty_yml(beacon, beacon_directory):
new_file = "%s/toto.yml" % beacon_directory
......
......@@ -707,7 +707,7 @@ def test_wardrobe_from_yml_file(session):
copper_reload.from_file("/tmp/materials_copper.yml", instance_name="copper")
assert copper_reload.color == "yellow-brown"
assert copper_reload.specific_weight == 8.96
assert copper_reload.dimensions == (5, 10, 15)
assert copper_reload.dimensions == [5, 10, 15]
assert copper_reload.pieces == {"first": 40.3, "second": 27.2, "count": [1, 2, 3]}
assert copper_reload.precious == False
assert copper_reload.motor == session.config.get("robz")
......@@ -730,7 +730,7 @@ def test_wardrobe_from_yml_file(session):
materials_reload.switch("gold")
assert materials_reload.color == "gold"
assert materials_reload.specific_weight == 19.32
assert materials_reload.dimensions == (1, 2, 3)
assert materials_reload.dimensions == [1, 2, 3]
assert materials_reload.pieces == {
"first": 10.3,
"second": 20.2,
......@@ -742,7 +742,7 @@ def test_wardrobe_from_yml_file(session):
materials_reload.switch("copper")
assert materials_reload.color == "yellow-brown"
assert materials_reload.specific_weight == 8.96
assert materials_reload.dimensions == (5, 10, 15)
assert materials_reload.dimensions == [5, 10, 15]
assert materials_reload.pieces == {
"first": 40.3,
"second": 27.2,
......@@ -846,7 +846,7 @@ def test_from_beacon(session):
assert beacon_material.color == "gold"
assert beacon_material.specific_weight == 19.32
assert beacon_material.dimensions == (1, 2, 3)
assert beacon_material.dimensions == [1, 2, 3]
assert beacon_material.pieces == {"first": 10.3, "second": 20.2, "count": [5, 2, 5]}
assert beacon_material.precious == True
assert beacon_material.motor.name == session.config.get("roby").name
......@@ -854,7 +854,7 @@ def test_from_beacon(session):
beacon_material.switch("copper")
assert beacon_material.color == "yellow-brown"
assert beacon_material.specific_weight == 8.96
assert beacon_material.dimensions == (5, 10, 15)
assert beacon_material.dimensions == [5, 10, 15]
assert beacon_material.pieces == {"first": 40.3, "second": 27.2, "count": [1, 2, 3]}
assert beacon_material.precious == False
assert beacon_material.motor.name == session.config.get("robz").name
......
......@@ -4,7 +4,7 @@ import pytest
import random
import numpy
import yaml
from ruamel.yaml import YAML
from bliss.controllers.wago.helpers import (
bytestring_to_wordarray,
......@@ -514,7 +514,8 @@ relay intlckf2 STICKY name Interlock 2
modules_config = wago.controller.modules_config
yml = specfile_to_yml(specfile)
dict_ = yaml.safe_load(yml)
yaml = YAML()
dict_ = yaml.load(yml)
beacon_interlock_parsing(dict_["interlocks"], modules_config)
......@@ -538,7 +539,8 @@ def test_export_to_yml(default_session):
assert word in yml
for word in "esTf1 esTf2 -10 10 50 50.5 TC".split():
assert word in yml
interlock_dict_from_yml = yaml.safe_load(yml)["interlocks"]
yaml = YAML()
interlock_dict_from_yml = yaml.load(yml)["interlocks"]
# creating again a list of interlocks from yml
interlock_list_from_yml = beacon_interlock_parsing(
......
name: rw_test
test: this is just a test.
# this is a comment!
# comment1
one:
- pink: martini
red: apples
blue: berry
- pink: martini # comment2
red: apples # comment3
blue: berry # comment4
two:
- a: true
# comment5
b: false
c: null
name: rw_test_2
# comment1
one:
- pink: martini
red: apples
red: apples # comment2
blue: berry
two:
- a: true
# comment3
b: false
c: null
test_list:
- $diode
- $diode # comment4
dict_list:
# comment5
- cnt_channel: a
instance: $diode2
- cnt_channel: b
instance: $diode3
\ No newline at end of file
instance: $diode3
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment