settings.py 53.5 KB
Newer Older
1 2 3 4
# -*- coding: utf-8 -*-
#
# This file is part of the bliss project
#
5
# Copyright (c) 2015-2019 Beamline Control Unit, ESRF
6 7
# Distributed under the GNU LGPLv3. See LICENSE for more info.

8
from contextlib import contextmanager
Matias Guijarro's avatar
Matias Guijarro committed
9 10
import weakref
import pickle
11 12
import keyword
import re
13
import reprlib
14 15
import datetime
import logging
16

17
import numpy
18
from tabulate import tabulate
19
import yaml
Matias Guijarro's avatar
Matias Guijarro committed
20

21
from .conductor import client
22
from bliss.config.conductor.client import set_config_db_file, remote_open
23 24
from bliss.common.utils import Null
from bliss import setup_globals
25

26 27
logger = logging.getLogger(__name__)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
28

Matias Guijarro's avatar
Matias Guijarro committed
29 30 31
class InvalidValue(Null):
    def __str__(self):
        raise ValueError
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
32

Matias Guijarro's avatar
Matias Guijarro committed
33
    def __repr__(self):
34
        return "#ERR"
35

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
36

37
class DefaultValue:
38 39 40 41 42 43
    def __init__(self, wrapped_value):
        self.__value = wrapped_value

    @property
    def value(self):
        return self.__value
Matias Guijarro's avatar
Matias Guijarro committed
44

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
45 46

def boolify(s, **keys):
47
    if s in ("True", "true"):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
48
        return True
49
    if s in ("False", "false"):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
50
        return False
51
    raise ValueError("Not Boolean Value!")
Matias Guijarro's avatar
Matias Guijarro committed
52

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
53

54 55 56
def auto_coerce_from_redis(s):
    """Convert variable to a new type from the str representation"""
    if s is None:
Matias Guijarro's avatar
Matias Guijarro committed
57
        return None
58 59 60 61 62 63 64 65
    # Default is unicode string
    try:
        if isinstance(s, bytes):
            s = s.decode()
    # Pickled data fails at first byte
    except UnicodeDecodeError:
        pass
    # Cast to standard types
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
66
    for caster in (boolify, int, float):
Matias Guijarro's avatar
Matias Guijarro committed
67
        try:
68
            return caster(s)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
69
        except (ValueError, TypeError):
Matias Guijarro's avatar
Matias Guijarro committed
70
            pass
71
    return s
Matias Guijarro's avatar
Matias Guijarro committed
72

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
73

74
def pickle_loads(var):
Matias Guijarro's avatar
Matias Guijarro committed
75 76
    if var is None:
        return None
Matias Guijarro's avatar
Matias Guijarro committed
77 78 79 80
    try:
        return pickle.loads(var)
    except Exception:
        return InvalidValue()
Matias Guijarro's avatar
Matias Guijarro committed
81

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
82

83 84
def get_redis_connection():
    return client.get_redis_connection(db=0)
85 86


Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
87
def ttl_func(cnx, name, value=-1):
Matias Guijarro's avatar
Matias Guijarro committed
88 89 90 91 92
    if value is None:
        return cnx.persist(name)
    elif value is -1:
        return cnx.ttl(name)
    else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
93 94
        return cnx.expire(name, value)

Matias Guijarro's avatar
Matias Guijarro committed
95

96
def read_decorator(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
97 98
    def _read(self, *args, **keys):
        value = func(self, *args, **keys)
Matias Guijarro's avatar
Matias Guijarro committed
99
        if self._read_type_conversion:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
100
            if isinstance(value, list):
Matias Guijarro's avatar
Matias Guijarro committed
101
                value = [self._read_type_conversion(x) for x in value]
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
102
            elif isinstance(value, dict):
Vincent Michel's avatar
Vincent Michel committed
103
                for k, v in value.items():
Matias Guijarro's avatar
Matias Guijarro committed
104
                    value[k] = self._read_type_conversion(v)
105 106 107
                if hasattr(self, "default_values") and isinstance(
                    self.default_values, dict
                ):
108 109 110
                    tmp = dict(self._default_values)
                    tmp.update(value)
                    value = tmp
Matias Guijarro's avatar
Matias Guijarro committed
111
            else:
112 113 114
                if isinstance(value, DefaultValue):
                    value = value.value
                elif value is not None:
115
                    value = self._read_type_conversion(value)
116
        if value is None:
117
            if hasattr(self, "_default_value"):
118
                value = self._default_value
119 120 121
            elif hasattr(self, "_default_values") and hasattr(
                self._default_values, "get"
            ):
122
                value = self._default_values.get(args[0])
Matias Guijarro's avatar
Matias Guijarro committed
123
        return value
124

Matias Guijarro's avatar
Matias Guijarro committed
125 126
    return _read

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
127

128
def write_decorator_dict(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
129
    def _write(self, values, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
130
        if self._write_type_conversion:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
131
            if not isinstance(values, dict) and values is not None:
132
                raise TypeError("can only be dict")
Matias Guijarro's avatar
Matias Guijarro committed
133 134

            if values is not None:
135
                new_dict = dict()
Vincent Michel's avatar
Vincent Michel committed
136
                for k, v in values.items():
137 138
                    new_dict[k] = self._write_type_conversion(v)
                values = new_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
139
        return func(self, values, **keys)
140

Matias Guijarro's avatar
Matias Guijarro committed
141 142
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
143

144
def write_decorator_multiple(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
145
    def _write(self, values, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
146
        if self._write_type_conversion:
147 148 149 150 151
            if (
                not isinstance(values, (list, tuple, numpy.ndarray))
                and values is not None
            ):
                raise TypeError("Can only be tuple, list or numpy array")
Matias Guijarro's avatar
Matias Guijarro committed
152 153
            if values is not None:
                values = [self._write_type_conversion(x) for x in values]
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
154
        return func(self, values, **keys)
155

Matias Guijarro's avatar
Matias Guijarro committed
156 157
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
158

159
def write_decorator(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
160
    def _write(self, value, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
161 162
        if self._write_type_conversion and value is not None:
            value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
163
        return func(self, value, **keys)
164

Matias Guijarro's avatar
Matias Guijarro committed
165 166
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
167

168
def scan(match="*", count=1000, connection=None):
169
    if connection is None:
170
        connection = get_redis_connection()
171 172
    cursor = 0
    while 1:
173
        cursor, values = connection.scan(cursor=cursor, match=match, count=count)
174
        for val in values:
175
            yield val.decode()
176 177
        if int(cursor) == 0:
            break
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
178 179


180 181 182 183 184 185
def _get_connection(setting_object):
    """
    Return the connection of a setting_object
    """
    if isinstance(setting_object, Struct):
        return setting_object._proxy._cnx
186
    elif isinstance(setting_object, (SimpleSetting, QueueSetting, BaseHashSetting)):
187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
        return setting_object._cnx
    else:
        raise TypeError(
            f"Setting object should be one of: Struct, SimpleSetting, QueueSetting or HashSetting instead of {setting_object!r}"
        )


def _set_connection(setting_object, new_cnx):
    """
    change the connection of a setting_object
    and return the previous connection
    """
    if isinstance(setting_object, Struct):
        cnx = setting_object._proxy._cnx
        setting_object._proxy._cnx = new_cnx
202
    elif isinstance(setting_object, (SimpleSetting, QueueSetting, BaseHashSetting)):
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237
        cnx = setting_object._cnx
        setting_object._cnx = new_cnx
    else:
        raise TypeError(
            f"Setting object should be one of: Struct, SimpleSetting, QueueSetting or HashSetting instead of {setting_object!r}"
        )
    return cnx


@contextmanager
def pipeline(*settings):
    """
    Contextmanager which create a redis pipeline to group redis commands
    on settings.

    IN CASE OF you execute the pipeline, it will return raw database values
    (byte strings).
    """
    first_settings = settings[0]
    cnx = _get_connection(first_settings)()
    # check they have the same connection
    for s in settings[1:]:
        if _get_connection(s)() != cnx:
            raise RuntimeError("Cannot groupe redis commands in a pipeline")

    pipeline = cnx.pipeline()
    try:
        # replace settings connection with the pipeline
        previous_cnx = [_set_connection(s, weakref.ref(pipeline)) for s in settings]
        yield pipeline
    finally:
        [_set_connection(s, c) for c, s in zip(previous_cnx, settings)]
        pipeline.execute()


238 239
class SimpleSetting:
    """
240
    Class to manage a setting that is stored as a string on Redis
241
    """
242

243 244 245 246
    def __init__(
        self,
        name,
        connection=None,
247 248
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
249 250
        default_value=None,
    ):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
251
        if connection is None:
252
            connection = get_redis_connection()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
253 254 255 256
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
257
        self._default_value = default_value
Matias Guijarro's avatar
Matias Guijarro committed
258

259 260 261
    @property
    def name(self):
        return self._name
262

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
263
    @read_decorator
264
    def get(self):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
265 266 267 268 269
        cnx = self._cnx()
        value = cnx.get(self._name)
        return value

    @write_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
270
    def set(self, value):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
271
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
272 273 274 275
        cnx.set(self._name, value)

    def ttl(self, value=-1):
        return ttl_func(self._cnx(), self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
276

277 278 279
    def clear(self):
        cnx = self._cnx()
        cnx.delete(self._name)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
280 281

    def __add__(self, other):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
282
        value = self.get()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
283
        if isinstance(other, SimpleSetting):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
284 285 286
            other = other.get()
        return value + other

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
287
    def __iadd__(self, other):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
288 289
        cnx = self._cnx()
        if cnx is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
290
            if isinstance(other, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
291 292
                if other == 1:
                    cnx.incr(self._name)
Matias Guijarro's avatar
Matias Guijarro committed
293
                else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
294 295 296
                    cnx.incrby(self._name, other)
            elif isinstance(other, float):
                cnx.incrbyfloat(self._name, other)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
297
            else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
298
                cnx.append(self._name, other)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
299
            return self
Matias Guijarro's avatar
Matias Guijarro committed
300

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
301
    def __isub__(self, other):
Vincent Michel's avatar
Vincent Michel committed
302
        if isinstance(other, str):
303 304 305
            raise TypeError(
                "unsupported operand type(s) for -=: %s" % type(other).__name__
            )
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
306
        return self.__iadd__(-other)
Matias Guijarro's avatar
Matias Guijarro committed
307

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
308
    def __getitem__(self, ran):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
309 310 311
        cnx = self._cnx()
        if cnx is not None:
            step = None
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
312 313
            if isinstance(ran, slice):
                i, j = ran.start, ran.stop
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
314
                step = ran.step
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
315
            elif isinstance(ran, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
316 317
                i = j = ran
            else:
318
                raise TypeError("indices must be integers")
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
319

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
320
            value = cnx.getrange(self._name, i, j)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
321 322 323 324 325 326 327
            if step is not None:
                value = value[0:-1:step]
            return value

    def __repr__(self):
        cnx = self._cnx()
        value = cnx.get(self._name)
328
        return "<SimpleSetting name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
329

Matias Guijarro's avatar
Matias Guijarro committed
330

331 332 333 334 335 336
class SimpleSettingProp:
    """
    A python's property implementation for SimpleSetting
    To be used inside user defined classes
    """

337 338 339 340
    def __init__(
        self,
        name,
        connection=None,
341 342
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
343 344 345
        default_value=None,
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
346
        self._name = name
347
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
348 349
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
350
        self._default_value = default_value
Matias Guijarro's avatar
Matias Guijarro committed
351 352
        self._use_object_name = use_object_name

353 354 355 356
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
357
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
358
        if self._use_object_name:
359
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
360 361
        else:
            name = self._name
362 363 364 365 366 367 368
        return SimpleSetting(
            name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_value,
        )
Matias Guijarro's avatar
Matias Guijarro committed
369

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
370 371 372
    def __set__(self, obj, value):
        if isinstance(value, SimpleSetting):
            return
Matias Guijarro's avatar
Matias Guijarro committed
373 374

        if self._use_object_name:
375
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
376 377 378 379 380 381 382 383
        else:
            name = self._name

        if value is None:
            self._cnx.delete(name)
        else:
            if self._write_type_conversion:
                value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
384 385
            self._cnx.set(name, value)

Matias Guijarro's avatar
Matias Guijarro committed
386

387 388
class QueueSetting:
    """
389
    Class to manage a setting that is stored as a list on Redis
390 391
    """

392 393 394 395
    def __init__(
        self,
        name,
        connection=None,
396 397
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
398
    ):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
399
        if connection is None:
400
            connection = get_redis_connection()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
401 402 403 404
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
Matias Guijarro's avatar
Matias Guijarro committed
405

406 407 408 409
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
410
    @read_decorator
411 412 413
    def get(self, first=0, last=-1, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
414
        if first == last:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
415
            l = cnx.lindex(self._name, first)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
416
        else:
417 418
            if last != -1:
                last -= 1
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
419
            l = cnx.lrange(self._name, first, last)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
420
        return l
Matias Guijarro's avatar
Matias Guijarro committed
421

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
422
    @write_decorator
423 424 425
    def append(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
426
        return cnx.rpush(self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
427

428 429 430
    def clear(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
431 432
        cnx.delete(self._name)

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
433
    @write_decorator
434 435 436
    def prepend(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
437
        return cnx.lpush(self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
438

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
439
    @write_decorator_multiple
440
    def extend(self, values, cnx=None):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
441
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
442
        return cnx.rpush(self._name, *values)
Matias Guijarro's avatar
Matias Guijarro committed
443

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
444
    @write_decorator
445 446 447
    def remove(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
448
        cnx.lrem(self._name, 0, value)
Matias Guijarro's avatar
Matias Guijarro committed
449

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
450
    @write_decorator_multiple
451 452 453
    def set(self, values, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
454 455
        cnx.delete(self._name)
        if values is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
456
            cnx.rpush(self._name, *values)
Matias Guijarro's avatar
Matias Guijarro committed
457

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
458
    @write_decorator
459 460 461
    def set_item(self, value, pos=0, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
462
        cnx.lset(self._name, pos, value)
Matias Guijarro's avatar
Matias Guijarro committed
463

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
464
    @read_decorator
465 466 467
    def pop_front(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
468 469 470 471 472 473
        value = cnx.lpop(self._name)
        if self._read_type_conversion:
            value = self._read_type_conversion(value)
        return value

    @read_decorator
474 475 476
    def pop_back(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
477 478 479 480 481
        value = cnx.rpop(self._name)
        if self._read_type_conversion:
            value = self._read_type_conversion(value)
        return value

482 483 484 485
    def ttl(self, value=-1, cnx=None):
        if cnx is None:
            cnx = self._cnx()
        return ttl_func(cnx, self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
486

487 488 489
    def __len__(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
490 491
        return cnx.llen(self._name)

492 493 494
    def __repr__(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
495
        value = cnx.lrange(self._name, 0, -1)
496
        return "<QueueSetting name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
497

498 499
    def __iadd__(self, other, cnx=None):
        self.extend(other, cnx)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
500 501
        return self

502
    def __getitem__(self, ran, cnx=None):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
503
        if isinstance(ran, slice):
504 505
            i = ran.start is not None and ran.start or 0
            j = ran.stop is not None and ran.stop or -1
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
506
        elif isinstance(ran, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
507 508
            i = j = ran
        else:
509
            raise TypeError("indices must be integers")
510
        value = self.get(first=i, last=j, cnx=cnx)
511
        if value is None:
512
            raise IndexError
513 514
        else:
            return value
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
515

516
    def __iter__(self, cnx=None):
517 518
        if cnx is None:
            cnx = self._cnx()
519
        lsize = cnx.llen(self._name)
Vincent Michel's avatar
Vincent Michel committed
520
        for first in range(0, lsize, 1024):
521
            last = first + 1024
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
522 523 524
            if last >= lsize:
                last = -1
            for value in self.get(first, last):
525
                yield value
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
526

527
    def __setitem__(self, ran, value, cnx=None):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
528
        if isinstance(ran, slice):
529
            for i, v in zip(range(ran.start, ran.stop), value):
530
                self.set_item(v, pos=i, cnx=cnx)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
531
        elif isinstance(ran, int):
532
            self.set_item(value, pos=ran, cnx=cnx)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
533
        else:
534
            raise TypeError("indices must be integers")
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
535 536
        return self

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
537

538 539 540 541 542 543
class QueueSettingProp:
    """
    A python's property implementation for QueueSetting
    To be used inside user defined classes
    """

544 545 546 547
    def __init__(
        self,
        name,
        connection=None,
548 549
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
550 551
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
552
        self._name = name
553
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
554 555 556 557
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
        self._use_object_name = use_object_name

558 559 560 561
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
562
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
563
        if self._use_object_name:
564
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
565 566 567
        else:
            name = self._name

568 569 570
        return QueueSetting(
            name, self._cnx, self._read_type_conversion, self._write_type_conversion
        )
Matias Guijarro's avatar
Matias Guijarro committed
571

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
572 573 574
    def __set__(self, obj, values):
        if isinstance(values, QueueSetting):
            return
Matias Guijarro's avatar
Matias Guijarro committed
575 576

        if self._use_object_name:
577
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
578 579 580
        else:
            name = self._name

581 582 583
        proxy = QueueSetting(
            name, self._cnx, self._read_type_conversion, self._write_type_conversion
        )
Matias Guijarro's avatar
Matias Guijarro committed
584 585
        proxy.set(values)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
586

587 588 589 590 591 592 593 594 595 596 597 598 599
class BaseHashSetting:
    """
    A Setting stored as a key,value pair in Redis

    Args:
        name: name of the BaseHashSetting (used on Redis)
        connection: Redis connection object
            read_type_conversion: conversion of data applyed
                after reading
            write_type_conversion: conversion of data applyed
                before writing
    """

600 601 602 603
    def __init__(
        self,
        name,
        connection=None,
604 605
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
606
    ):
Matias Guijarro's avatar
Matias Guijarro committed
607
        if connection is None:
608
            connection = get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
609 610 611 612 613
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion

614 615 616 617
    @property
    def name(self):
        return self._name

618 619
    def __repr__(self):
        value = self.get_all()
620
        return f"<{type(self).__name__} name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
621 622

    def __delitem__(self, key):
623
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
624
        cnx.hdel(self._name, key)
Matias Guijarro's avatar
Matias Guijarro committed
625

626
    def __len__(self):
Matias Guijarro's avatar
Matias Guijarro committed
627 628 629
        cnx = self._cnx()
        return cnx.hlen(self._name)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
630 631
    def ttl(self, value=-1):
        return ttl_func(self._cnx(), self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
632

633
    def raw_get(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
634
        cnx = self._cnx()
635
        return cnx.hget(self._name, *keys)
Matias Guijarro's avatar
Matias Guijarro committed
636 637

    @read_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
638
    def get(self, key, default=None):
Matias Guijarro's avatar
Matias Guijarro committed
639 640
        v = self.raw_get(key)
        if v is None:
641
            v = DefaultValue(default)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
642
        return v
Matias Guijarro's avatar
Matias Guijarro committed
643

Matias Guijarro's avatar
Matias Guijarro committed
644 645 646
    def _raw_get_all(self):
        cnx = self._cnx()
        return cnx.hgetall(self._name)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
647

Matias Guijarro's avatar
Matias Guijarro committed
648
    def get_all(self):
649
        all_dict = dict()
Vincent Michel's avatar
Vincent Michel committed
650
        for k, raw_v in self._raw_get_all().items():
651
            k = k.decode()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
652 653 654
            v = self._read_type_conversion(raw_v)
            if isinstance(v, InvalidValue):
                raise ValueError(
655 656 657
                    "%s: Invalid value '%s` (cannot deserialize %r)"
                    % (self._name, k, raw_v)
                )
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
658
            all_dict[k] = v
Matias Guijarro's avatar
Matias Guijarro committed
659
        return all_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
660

Matias Guijarro's avatar
Matias Guijarro committed
661
    @read_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
662
    def pop(self, key, default=Null()):
663
        cnx = self._cnx().pipeline()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
664 665 666
        cnx.hget(self._name, key)
        cnx.hdel(self._name, key)
        (value, worked) = cnx.execute()
667
        if not worked:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
668
            if isinstance(default, Null):
669 670 671
                raise KeyError(key)
            else:
                value = default
Matias Guijarro's avatar
Matias Guijarro committed
672
        return value
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
673

Matias Guijarro's avatar
Matias Guijarro committed
674
    def remove(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
675
        cnx = self._cnx()
Matias Guijarro's avatar
Matias Guijarro committed
676
        cnx.hdel(self._name, *keys)
Matias Guijarro's avatar
Matias Guijarro committed
677

678
    def clear(self):
Matias Guijarro's avatar
Matias Guijarro committed
679 680 681
        cnx = self._cnx()
        cnx.delete(self._name)

682 683
    # def copy(self):
    #    return self.get()
Matias Guijarro's avatar
Matias Guijarro committed
684 685

    @write_decorator_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
686
    def set(self, values):
Matias Guijarro's avatar
Matias Guijarro committed
687 688 689
        cnx = self._cnx()
        cnx.delete(self._name)
        if values is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
690
            cnx.hmset(self._name, values)
Matias Guijarro's avatar
Matias Guijarro committed
691 692

    @write_decorator_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
693
    def update(self, values):
Matias Guijarro's avatar
Matias Guijarro committed
694
        cnx = self._cnx()
695
        if values:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
696
            cnx.hmset(self._name, values)
Matias Guijarro's avatar
Matias Guijarro committed
697 698

    @read_decorator
699
    def fromkeys(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
700
        cnx = self._cnx()
701
        return cnx.hmget(self._name, *keys)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
702 703

    def has_key(self, key):
Matias Guijarro's avatar
Matias Guijarro committed
704
        cnx = self._cnx()
705
        return cnx.hexists(self._name, key)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
706

707
    def keys(self):
Vincent Michel's avatar
Vincent Michel committed
708
        for k, v in self.items():
709 710
            yield k

711
    def values(self):
Vincent Michel's avatar
Vincent Michel committed
712
        for k, v in self.items():
713
            yield v
Matias Guijarro's avatar
Matias Guijarro committed
714

715
    def items(self):
Matias Guijarro's avatar
Matias Guijarro committed
716 717
        cnx = self._cnx()
        next_id = 0
718
        seen_keys = set()
Matias Guijarro's avatar
Matias Guijarro committed
719
        while True:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
720
            next_id, pd = cnx.hscan(self._name, next_id)
Vincent Michel's avatar
Vincent Michel committed
721
            for k, v in pd.items():
722 723
                # Add key conversion
                k = k.decode()
Matias Guijarro's avatar
Matias Guijarro committed
724 725
                if self._read_type_conversion:
                    v = self._read_type_conversion(v)
726
                seen_keys.add(k)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
727
                yield k, v
728
            if not next_id or next_id is "0":
Matias Guijarro's avatar
Matias Guijarro committed
729 730
                break

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
731
    def __getitem__(self, key):
Matias Guijarro's avatar
Matias Guijarro committed
732 733
        value = self.get(key)
        if value is None:
734
            raise KeyError(key)
Matias Guijarro's avatar
Matias Guijarro committed
735 736
        return value

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
737
    def __setitem__(self, key, value):
Matias Guijarro's avatar
Matias Guijarro committed
738
        cnx = self._cnx()
739
        if value is None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
740
            cnx.hdel(self._name, key)
741
            return
Matias Guijarro's avatar
Matias Guijarro committed
742 743
        if self._write_type_conversion:
            value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
744
        cnx.hset(self._name, key, value)
Matias Guijarro's avatar
Matias Guijarro committed
745

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
746
    def __contains__(self, key):
747 748 749 750 751 752 753
        try:
            self[key]
            return True
        except KeyError:
            return False


754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840
class HashSetting(BaseHashSetting):
    """
    A Setting stored as a key,value pair in Redis
    with a default_value dictionary to serve as a callback
    when elements lookup fails

    Args:
        name: name of the HashSetting (used on Redis)
        connection: Redis connection object
            read_type_conversion: conversion of data applyed
                after reading
            write_type_conversion: conversion of data applyed
                before writing
    kwargs:
        default_values: dictionary of default values retrieved
            on fallback
    """

    def __init__(
        self,
        name,
        connection=None,
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
        default_values={},
    ):
        super().__init__(
            name,
            connection=connection,
            read_type_conversion=read_type_conversion,
            write_type_conversion=write_type_conversion,
        )
        self._default_values = default_values

    @read_decorator
    def get(self, key, default=None):
        v = super().raw_get(key)
        if v is None:
            v = DefaultValue(default)
        return v

    def has_key(self, key):
        return super().has_key(key) or key in self._default_values

    def __getitem__(self, key):
        value = self.get(key)
        if value is None:
            if key not in self._default_values:
                raise KeyError(key)
        return value

    def get_all(self):
        all_dict = dict(self._default_values)
        for k, raw_v in self._raw_get_all().items():
            k = k.decode()
            v = self._read_type_conversion(raw_v)
            if isinstance(v, InvalidValue):
                raise ValueError(
                    "%s: Invalid value '%s` (cannot deserialize %r)"
                    % (self._name, k, raw_v)
                )
            all_dict[k] = v
        return all_dict

    def items(self):
        cnx = self._cnx()
        next_id = 0
        seen_keys = set()
        while True:
            next_id, pd = cnx.hscan(self._name, next_id)
            for k, v in pd.items():
                # Add key conversion
                k = k.decode()
                if self._read_type_conversion:
                    v = self._read_type_conversion(v)
                seen_keys.add(k)
                yield k, v
            if not next_id or next_id is "0":
                break

        for k, v in self._default_values.items():
            if k in seen_keys:
                continue
            yield k, v


class HashSettingProp:
841 842 843 844
    def __init__(
        self,
        name,
        connection=None,
845 846
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
847 848 849
        default_values={},
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
850
        self._name = name
851
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
852 853
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
854
        self._default_values = default_values
Matias Guijarro's avatar
Matias Guijarro committed
855 856
        self._use_object_name = use_object_name

857 858 859 860
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
861
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
862
        if self._use_object_name:
863
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
864 865
        else:
            name = self._name
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
866

867 868 869 870 871 872 873
        return HashSetting(
            name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_values,
        )
Matias Guijarro's avatar
Matias Guijarro committed
874