settings.py 51.2 KB
Newer Older
1 2 3 4
# -*- coding: utf-8 -*-
#
# This file is part of the bliss project
#
5
# Copyright (c) 2015-2019 Beamline Control Unit, ESRF
6 7
# Distributed under the GNU LGPLv3. See LICENSE for more info.

8
from contextlib import contextmanager
Matias Guijarro's avatar
Matias Guijarro committed
9 10
import weakref
import pickle
11 12
import keyword
import re
13
import reprlib
14 15
import datetime
import logging
16

17
import numpy
18
from tabulate import tabulate
19
import yaml
Matias Guijarro's avatar
Matias Guijarro committed
20

21 22 23
from .conductor import client
from bliss.common.utils import Null
from bliss import setup_globals
24

25 26
logger = logging.getLogger(__name__)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
27

Matias Guijarro's avatar
Matias Guijarro committed
28 29 30
class InvalidValue(Null):
    def __str__(self):
        raise ValueError
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
31

Matias Guijarro's avatar
Matias Guijarro committed
32
    def __repr__(self):
33
        return "#ERR"
34

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
35

36
class DefaultValue:
37 38 39 40 41 42
    def __init__(self, wrapped_value):
        self.__value = wrapped_value

    @property
    def value(self):
        return self.__value
Matias Guijarro's avatar
Matias Guijarro committed
43

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
44 45

def boolify(s, **keys):
46
    if s in ("True", "true"):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
47
        return True
48
    if s in ("False", "false"):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
49
        return False
50
    raise ValueError("Not Boolean Value!")
Matias Guijarro's avatar
Matias Guijarro committed
51

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
52

53 54 55
def auto_coerce_from_redis(s):
    """Convert variable to a new type from the str representation"""
    if s is None:
Matias Guijarro's avatar
Matias Guijarro committed
56
        return None
57 58 59 60 61 62 63 64
    # Default is unicode string
    try:
        if isinstance(s, bytes):
            s = s.decode()
    # Pickled data fails at first byte
    except UnicodeDecodeError:
        pass
    # Cast to standard types
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
65
    for caster in (boolify, int, float):
Matias Guijarro's avatar
Matias Guijarro committed
66
        try:
67
            return caster(s)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
68
        except (ValueError, TypeError):
Matias Guijarro's avatar
Matias Guijarro committed
69
            pass
70
    return s
Matias Guijarro's avatar
Matias Guijarro committed
71

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
72

73
def pickle_loads(var):
Matias Guijarro's avatar
Matias Guijarro committed
74 75
    if var is None:
        return None
Matias Guijarro's avatar
Matias Guijarro committed
76 77 78 79
    try:
        return pickle.loads(var)
    except Exception:
        return InvalidValue()
Matias Guijarro's avatar
Matias Guijarro committed
80

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
81

82 83
def get_redis_connection():
    return client.get_redis_connection(db=0)
84 85


Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
86
def ttl_func(cnx, name, value=-1):
Matias Guijarro's avatar
Matias Guijarro committed
87 88 89 90 91
    if value is None:
        return cnx.persist(name)
    elif value is -1:
        return cnx.ttl(name)
    else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
92 93
        return cnx.expire(name, value)

Matias Guijarro's avatar
Matias Guijarro committed
94

95
def read_decorator(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
96 97
    def _read(self, *args, **keys):
        value = func(self, *args, **keys)
Matias Guijarro's avatar
Matias Guijarro committed
98
        if self._read_type_conversion:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
99
            if isinstance(value, list):
Matias Guijarro's avatar
Matias Guijarro committed
100
                value = [self._read_type_conversion(x) for x in value]
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
101
            elif isinstance(value, dict):
Vincent Michel's avatar
Vincent Michel committed
102
                for k, v in value.items():
Matias Guijarro's avatar
Matias Guijarro committed
103
                    value[k] = self._read_type_conversion(v)
104 105 106
                if hasattr(self, "default_values") and isinstance(
                    self.default_values, dict
                ):
107 108 109
                    tmp = dict(self._default_values)
                    tmp.update(value)
                    value = tmp
Matias Guijarro's avatar
Matias Guijarro committed
110
            else:
111 112 113
                if isinstance(value, DefaultValue):
                    value = value.value
                elif value is not None:
114
                    value = self._read_type_conversion(value)
115
        if value is None:
116
            if hasattr(self, "_default_value"):
117
                value = self._default_value
118 119 120
            elif hasattr(self, "_default_values") and hasattr(
                self._default_values, "get"
            ):
121
                value = self._default_values.get(args[0])
Matias Guijarro's avatar
Matias Guijarro committed
122
        return value
123

Matias Guijarro's avatar
Matias Guijarro committed
124 125
    return _read

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
126

127
def write_decorator_dict(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
128
    def _write(self, values, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
129
        if self._write_type_conversion:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
130
            if not isinstance(values, dict) and values is not None:
131
                raise TypeError("can only be dict")
Matias Guijarro's avatar
Matias Guijarro committed
132 133

            if values is not None:
134
                new_dict = dict()
Vincent Michel's avatar
Vincent Michel committed
135
                for k, v in values.items():
136 137
                    new_dict[k] = self._write_type_conversion(v)
                values = new_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
138
        return func(self, values, **keys)
139

Matias Guijarro's avatar
Matias Guijarro committed
140 141
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
142

143
def write_decorator_multiple(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
144
    def _write(self, values, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
145
        if self._write_type_conversion:
146 147 148 149 150
            if (
                not isinstance(values, (list, tuple, numpy.ndarray))
                and values is not None
            ):
                raise TypeError("Can only be tuple, list or numpy array")
Matias Guijarro's avatar
Matias Guijarro committed
151 152
            if values is not None:
                values = [self._write_type_conversion(x) for x in values]
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
153
        return func(self, values, **keys)
154

Matias Guijarro's avatar
Matias Guijarro committed
155 156
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
157

158
def write_decorator(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
159
    def _write(self, value, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
160 161
        if self._write_type_conversion and value is not None:
            value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
162
        return func(self, value, **keys)
163

Matias Guijarro's avatar
Matias Guijarro committed
164 165
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
166

167
def scan(match="*", count=1000, connection=None):
168
    if connection is None:
169
        connection = get_redis_connection()
170 171
    cursor = 0
    while 1:
172
        cursor, values = connection.scan(cursor=cursor, match=match, count=count)
173
        for val in values:
174
            yield val.decode()
175 176
        if int(cursor) == 0:
            break
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
177 178


179 180 181 182 183 184
def _get_connection(setting_object):
    """
    Return the connection of a setting_object
    """
    if isinstance(setting_object, Struct):
        return setting_object._proxy._cnx
185
    elif isinstance(setting_object, (SimpleSetting, QueueSetting, BaseHashSetting)):
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
        return setting_object._cnx
    else:
        raise TypeError(
            f"Setting object should be one of: Struct, SimpleSetting, QueueSetting or HashSetting instead of {setting_object!r}"
        )


def _set_connection(setting_object, new_cnx):
    """
    change the connection of a setting_object
    and return the previous connection
    """
    if isinstance(setting_object, Struct):
        cnx = setting_object._proxy._cnx
        setting_object._proxy._cnx = new_cnx
201
    elif isinstance(setting_object, (SimpleSetting, QueueSetting, BaseHashSetting)):
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236
        cnx = setting_object._cnx
        setting_object._cnx = new_cnx
    else:
        raise TypeError(
            f"Setting object should be one of: Struct, SimpleSetting, QueueSetting or HashSetting instead of {setting_object!r}"
        )
    return cnx


@contextmanager
def pipeline(*settings):
    """
    Contextmanager which create a redis pipeline to group redis commands
    on settings.

    IN CASE OF you execute the pipeline, it will return raw database values
    (byte strings).
    """
    first_settings = settings[0]
    cnx = _get_connection(first_settings)()
    # check they have the same connection
    for s in settings[1:]:
        if _get_connection(s)() != cnx:
            raise RuntimeError("Cannot groupe redis commands in a pipeline")

    pipeline = cnx.pipeline()
    try:
        # replace settings connection with the pipeline
        previous_cnx = [_set_connection(s, weakref.ref(pipeline)) for s in settings]
        yield pipeline
    finally:
        [_set_connection(s, c) for c, s in zip(previous_cnx, settings)]
        pipeline.execute()


237 238
class SimpleSetting:
    """
239
    Class to manage a setting that is stored as a string on Redis
240
    """
241

242 243 244 245
    def __init__(
        self,
        name,
        connection=None,
246 247
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
248 249
        default_value=None,
    ):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
250
        if connection is None:
251
            connection = get_redis_connection()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
252 253 254 255
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
256
        self._default_value = default_value
Matias Guijarro's avatar
Matias Guijarro committed
257

258 259 260
    @property
    def name(self):
        return self._name
261

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
262
    @read_decorator
263
    def get(self):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
264 265 266 267 268
        cnx = self._cnx()
        value = cnx.get(self._name)
        return value

    @write_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
269
    def set(self, value):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
270
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
271 272 273 274
        cnx.set(self._name, value)

    def ttl(self, value=-1):
        return ttl_func(self._cnx(), self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
275

276 277 278
    def clear(self):
        cnx = self._cnx()
        cnx.delete(self._name)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
279 280

    def __add__(self, other):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
281
        value = self.get()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
282
        if isinstance(other, SimpleSetting):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
283 284 285
            other = other.get()
        return value + other

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
286
    def __iadd__(self, other):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
287 288
        cnx = self._cnx()
        if cnx is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
289
            if isinstance(other, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
290 291
                if other == 1:
                    cnx.incr(self._name)
Matias Guijarro's avatar
Matias Guijarro committed
292
                else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
293 294 295
                    cnx.incrby(self._name, other)
            elif isinstance(other, float):
                cnx.incrbyfloat(self._name, other)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
296
            else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
297
                cnx.append(self._name, other)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
298
            return self
Matias Guijarro's avatar
Matias Guijarro committed
299

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
300
    def __isub__(self, other):
Vincent Michel's avatar
Vincent Michel committed
301
        if isinstance(other, str):
302 303 304
            raise TypeError(
                "unsupported operand type(s) for -=: %s" % type(other).__name__
            )
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
305
        return self.__iadd__(-other)
Matias Guijarro's avatar
Matias Guijarro committed
306

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
307
    def __getitem__(self, ran):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
308 309 310
        cnx = self._cnx()
        if cnx is not None:
            step = None
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
311 312
            if isinstance(ran, slice):
                i, j = ran.start, ran.stop
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
313
                step = ran.step
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
314
            elif isinstance(ran, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
315 316
                i = j = ran
            else:
317
                raise TypeError("indices must be integers")
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
318

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
319
            value = cnx.getrange(self._name, i, j)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
320 321 322 323 324 325 326
            if step is not None:
                value = value[0:-1:step]
            return value

    def __repr__(self):
        cnx = self._cnx()
        value = cnx.get(self._name)
327
        return "<SimpleSetting name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
328

Matias Guijarro's avatar
Matias Guijarro committed
329

330 331 332 333 334 335
class SimpleSettingProp:
    """
    A python's property implementation for SimpleSetting
    To be used inside user defined classes
    """

336 337 338 339
    def __init__(
        self,
        name,
        connection=None,
340 341
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
342 343 344
        default_value=None,
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
345
        self._name = name
346
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
347 348
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
349
        self._default_value = default_value
Matias Guijarro's avatar
Matias Guijarro committed
350 351
        self._use_object_name = use_object_name

352 353 354 355
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
356
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
357
        if self._use_object_name:
358
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
359 360
        else:
            name = self._name
361 362 363 364 365 366 367
        return SimpleSetting(
            name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_value,
        )
Matias Guijarro's avatar
Matias Guijarro committed
368

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
369 370 371
    def __set__(self, obj, value):
        if isinstance(value, SimpleSetting):
            return
Matias Guijarro's avatar
Matias Guijarro committed
372 373

        if self._use_object_name:
374
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
375 376 377 378 379 380 381 382
        else:
            name = self._name

        if value is None:
            self._cnx.delete(name)
        else:
            if self._write_type_conversion:
                value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
383 384
            self._cnx.set(name, value)

Matias Guijarro's avatar
Matias Guijarro committed
385

386 387
class QueueSetting:
    """
388
    Class to manage a setting that is stored as a list on Redis
389 390
    """

391 392 393 394
    def __init__(
        self,
        name,
        connection=None,
395 396
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
397
    ):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
398
        if connection is None:
399
            connection = get_redis_connection()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
400 401 402 403
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
Matias Guijarro's avatar
Matias Guijarro committed
404

405 406 407 408
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
409
    @read_decorator
410 411 412
    def get(self, first=0, last=-1, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
413
        if first == last:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
414
            l = cnx.lindex(self._name, first)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
415
        else:
416 417
            if last != -1:
                last -= 1
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
418
            l = cnx.lrange(self._name, first, last)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
419
        return l
Matias Guijarro's avatar
Matias Guijarro committed
420

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
421
    @write_decorator
422 423 424
    def append(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
425
        return cnx.rpush(self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
426

427 428 429
    def clear(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
430 431
        cnx.delete(self._name)

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
432
    @write_decorator
433 434 435
    def prepend(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
436
        return cnx.lpush(self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
437

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
438
    @write_decorator_multiple
439
    def extend(self, values, cnx=None):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
440
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
441
        return cnx.rpush(self._name, *values)
Matias Guijarro's avatar
Matias Guijarro committed
442

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
443
    @write_decorator
444 445 446
    def remove(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
447
        cnx.lrem(self._name, 0, value)
Matias Guijarro's avatar
Matias Guijarro committed
448

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
449
    @write_decorator_multiple
450 451 452
    def set(self, values, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
453 454
        cnx.delete(self._name)
        if values is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
455
            cnx.rpush(self._name, *values)
Matias Guijarro's avatar
Matias Guijarro committed
456

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
457
    @write_decorator
458 459 460
    def set_item(self, value, pos=0, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
461
        cnx.lset(self._name, pos, value)
Matias Guijarro's avatar
Matias Guijarro committed
462

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
463
    @read_decorator
464 465 466
    def pop_front(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
467 468 469 470 471 472
        value = cnx.lpop(self._name)
        if self._read_type_conversion:
            value = self._read_type_conversion(value)
        return value

    @read_decorator
473 474 475
    def pop_back(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
476 477 478 479 480
        value = cnx.rpop(self._name)
        if self._read_type_conversion:
            value = self._read_type_conversion(value)
        return value

481 482 483 484
    def ttl(self, value=-1, cnx=None):
        if cnx is None:
            cnx = self._cnx()
        return ttl_func(cnx, self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
485

486 487 488
    def __len__(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
489 490
        return cnx.llen(self._name)

491 492 493
    def __repr__(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
494
        value = cnx.lrange(self._name, 0, -1)
495
        return "<QueueSetting name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
496

497 498
    def __iadd__(self, other, cnx=None):
        self.extend(other, cnx)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
499 500
        return self

501
    def __getitem__(self, ran, cnx=None):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
502
        if isinstance(ran, slice):
503 504
            i = ran.start is not None and ran.start or 0
            j = ran.stop is not None and ran.stop or -1
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
505
        elif isinstance(ran, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
506 507
            i = j = ran
        else:
508
            raise TypeError("indices must be integers")
509
        value = self.get(first=i, last=j, cnx=cnx)
510
        if value is None:
511
            raise IndexError
512 513
        else:
            return value
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
514

515
    def __iter__(self, cnx=None):
516 517
        if cnx is None:
            cnx = self._cnx()
518
        lsize = cnx.llen(self._name)
Vincent Michel's avatar
Vincent Michel committed
519
        for first in range(0, lsize, 1024):
520
            last = first + 1024
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
521 522 523
            if last >= lsize:
                last = -1
            for value in self.get(first, last):
524
                yield value
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
525

526
    def __setitem__(self, ran, value, cnx=None):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
527
        if isinstance(ran, slice):
528
            for i, v in zip(range(ran.start, ran.stop), value):
529
                self.set_item(v, pos=i, cnx=cnx)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
530
        elif isinstance(ran, int):
531
            self.set_item(value, pos=ran, cnx=cnx)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
532
        else:
533
            raise TypeError("indices must be integers")
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
534 535
        return self

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
536

537 538 539 540 541 542
class QueueSettingProp:
    """
    A python's property implementation for QueueSetting
    To be used inside user defined classes
    """

543 544 545 546
    def __init__(
        self,
        name,
        connection=None,
547 548
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
549 550
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
551
        self._name = name
552
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
553 554 555 556
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
        self._use_object_name = use_object_name

557 558 559 560
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
561
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
562
        if self._use_object_name:
563
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
564 565 566
        else:
            name = self._name

567 568 569
        return QueueSetting(
            name, self._cnx, self._read_type_conversion, self._write_type_conversion
        )
Matias Guijarro's avatar
Matias Guijarro committed
570

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
571 572 573
    def __set__(self, obj, values):
        if isinstance(values, QueueSetting):
            return
Matias Guijarro's avatar
Matias Guijarro committed
574 575

        if self._use_object_name:
576
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
577 578 579
        else:
            name = self._name

580 581 582
        proxy = QueueSetting(
            name, self._cnx, self._read_type_conversion, self._write_type_conversion
        )
Matias Guijarro's avatar
Matias Guijarro committed
583 584
        proxy.set(values)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
585

586 587 588 589 590 591 592 593 594 595 596 597 598
class BaseHashSetting:
    """
    A Setting stored as a key,value pair in Redis

    Args:
        name: name of the BaseHashSetting (used on Redis)
        connection: Redis connection object
            read_type_conversion: conversion of data applyed
                after reading
            write_type_conversion: conversion of data applyed
                before writing
    """

599 600 601 602
    def __init__(
        self,
        name,
        connection=None,
603 604
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
605
    ):
Matias Guijarro's avatar
Matias Guijarro committed
606
        if connection is None:
607
            connection = get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
608 609 610 611 612
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion

613 614 615 616
    @property
    def name(self):
        return self._name

617 618
    def __repr__(self):
        value = self.get_all()
619
        return f"<{type(self).__name__} name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
620 621

    def __delitem__(self, key):
622
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
623
        cnx.hdel(self._name, key)
Matias Guijarro's avatar
Matias Guijarro committed
624

625
    def __len__(self):
Matias Guijarro's avatar
Matias Guijarro committed
626 627 628
        cnx = self._cnx()
        return cnx.hlen(self._name)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
629 630
    def ttl(self, value=-1):
        return ttl_func(self._cnx(), self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
631

632
    def raw_get(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
633
        cnx = self._cnx()
634
        return cnx.hget(self._name, *keys)
Matias Guijarro's avatar
Matias Guijarro committed
635 636

    @read_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
637
    def get(self, key, default=None):
Matias Guijarro's avatar
Matias Guijarro committed
638 639
        v = self.raw_get(key)
        if v is None:
640
            v = DefaultValue(default)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
641
        return v
Matias Guijarro's avatar
Matias Guijarro committed
642

Matias Guijarro's avatar
Matias Guijarro committed
643 644 645
    def _raw_get_all(self):
        cnx = self._cnx()
        return cnx.hgetall(self._name)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
646

Matias Guijarro's avatar
Matias Guijarro committed
647
    def get_all(self):
648
        all_dict = dict()
Vincent Michel's avatar
Vincent Michel committed
649
        for k, raw_v in self._raw_get_all().items():
650
            k = k.decode()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
651 652 653
            v = self._read_type_conversion(raw_v)
            if isinstance(v, InvalidValue):
                raise ValueError(
654 655 656
                    "%s: Invalid value '%s` (cannot deserialize %r)"
                    % (self._name, k, raw_v)
                )
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
657
            all_dict[k] = v
Matias Guijarro's avatar
Matias Guijarro committed
658
        return all_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
659

Matias Guijarro's avatar
Matias Guijarro committed
660
    @read_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
661
    def pop(self, key, default=Null()):
662
        cnx = self._cnx().pipeline()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
663 664 665
        cnx.hget(self._name, key)
        cnx.hdel(self._name, key)
        (value, worked) = cnx.execute()
666
        if not worked:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
667
            if isinstance(default, Null):
668 669 670
                raise KeyError(key)
            else:
                value = default
Matias Guijarro's avatar
Matias Guijarro committed
671
        return value
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
672

Matias Guijarro's avatar
Matias Guijarro committed
673
    def remove(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
674
        cnx = self._cnx()
Matias Guijarro's avatar
Matias Guijarro committed
675
        cnx.hdel(self._name, *keys)
Matias Guijarro's avatar
Matias Guijarro committed
676

677
    def clear(self):
Matias Guijarro's avatar
Matias Guijarro committed
678 679 680
        cnx = self._cnx()
        cnx.delete(self._name)

681 682
    # def copy(self):
    #    return self.get()
Matias Guijarro's avatar
Matias Guijarro committed
683 684

    @write_decorator_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
685
    def set(self, values):
Matias Guijarro's avatar
Matias Guijarro committed
686 687 688
        cnx = self._cnx()
        cnx.delete(self._name)
        if values is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
689
            cnx.hmset(self._name, values)
Matias Guijarro's avatar
Matias Guijarro committed
690 691

    @write_decorator_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
692
    def update(self, values):
Matias Guijarro's avatar
Matias Guijarro committed
693
        cnx = self._cnx()
694
        if values:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
695
            cnx.hmset(self._name, values)
Matias Guijarro's avatar
Matias Guijarro committed
696 697

    @read_decorator
698
    def fromkeys(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
699
        cnx = self._cnx()
700
        return cnx.hmget(self._name, *keys)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
701 702

    def has_key(self, key):
Matias Guijarro's avatar
Matias Guijarro committed
703
        cnx = self._cnx()
704
        return cnx.hexists(self._name, key)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
705

706
    def keys(self):
Vincent Michel's avatar
Vincent Michel committed
707
        for k, v in self.items():
708 709
            yield k

710
    def values(self):
Vincent Michel's avatar
Vincent Michel committed
711
        for k, v in self.items():
712
            yield v
Matias Guijarro's avatar
Matias Guijarro committed
713

714
    def items(self):
Matias Guijarro's avatar
Matias Guijarro committed
715 716
        cnx = self._cnx()
        next_id = 0
717
        seen_keys = set()
Matias Guijarro's avatar
Matias Guijarro committed
718
        while True:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
719
            next_id, pd = cnx.hscan(self._name, next_id)
Vincent Michel's avatar
Vincent Michel committed
720
            for k, v in pd.items():
721 722
                # Add key conversion
                k = k.decode()
Matias Guijarro's avatar
Matias Guijarro committed
723 724
                if self._read_type_conversion:
                    v = self._read_type_conversion(v)
725
                seen_keys.add(k)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
726
                yield k, v
727
            if not next_id or next_id is "0":
Matias Guijarro's avatar
Matias Guijarro committed
728 729
                break

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
730
    def __getitem__(self, key):
Matias Guijarro's avatar
Matias Guijarro committed
731 732
        value = self.get(key)
        if value is None:
733
            raise KeyError(key)
Matias Guijarro's avatar
Matias Guijarro committed
734 735
        return value

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
736
    def __setitem__(self, key, value):
Matias Guijarro's avatar
Matias Guijarro committed
737
        cnx = self._cnx()
738
        if value is None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
739
            cnx.hdel(self._name, key)
740
            return
Matias Guijarro's avatar
Matias Guijarro committed
741 742
        if self._write_type_conversion:
            value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
743
        cnx.hset(self._name, key, value)
Matias Guijarro's avatar
Matias Guijarro committed
744

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
745
    def __contains__(self, key):
746 747 748 749 750 751 752
        try:
            self[key]
            return True
        except KeyError:
            return False


753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839
class HashSetting(BaseHashSetting):
    """
    A Setting stored as a key,value pair in Redis
    with a default_value dictionary to serve as a callback
    when elements lookup fails

    Args:
        name: name of the HashSetting (used on Redis)
        connection: Redis connection object
            read_type_conversion: conversion of data applyed
                after reading
            write_type_conversion: conversion of data applyed
                before writing
    kwargs:
        default_values: dictionary of default values retrieved
            on fallback
    """

    def __init__(
        self,
        name,
        connection=None,
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
        default_values={},
    ):
        super().__init__(
            name,
            connection=connection,
            read_type_conversion=read_type_conversion,
            write_type_conversion=write_type_conversion,
        )
        self._default_values = default_values

    @read_decorator
    def get(self, key, default=None):
        v = super().raw_get(key)
        if v is None:
            v = DefaultValue(default)
        return v

    def has_key(self, key):
        return super().has_key(key) or key in self._default_values

    def __getitem__(self, key):
        value = self.get(key)
        if value is None:
            if key not in self._default_values:
                raise KeyError(key)
        return value

    def get_all(self):
        all_dict = dict(self._default_values)
        for k, raw_v in self._raw_get_all().items():
            k = k.decode()
            v = self._read_type_conversion(raw_v)
            if isinstance(v, InvalidValue):
                raise ValueError(
                    "%s: Invalid value '%s` (cannot deserialize %r)"
                    % (self._name, k, raw_v)
                )
            all_dict[k] = v
        return all_dict

    def items(self):
        cnx = self._cnx()
        next_id = 0
        seen_keys = set()
        while True:
            next_id, pd = cnx.hscan(self._name, next_id)
            for k, v in pd.items():
                # Add key conversion
                k = k.decode()
                if self._read_type_conversion:
                    v = self._read_type_conversion(v)
                seen_keys.add(k)
                yield k, v
            if not next_id or next_id is "0":
                break

        for k, v in self._default_values.items():
            if k in seen_keys:
                continue
            yield k, v


class HashSettingProp:
840 841 842 843
    def __init__(
        self,
        name,
        connection=None,
844 845
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
846 847 848
        default_values={},
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
849
        self._name = name
850
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
851 852
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
853
        self._default_values = default_values
Matias Guijarro's avatar
Matias Guijarro committed
854 855
        self._use_object_name = use_object_name

856 857 858 859
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
860
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
861
        if self._use_object_name:
862
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
863 864
        else:
            name = self._name
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
865

866 867 868 869 870 871 872
        return HashSetting(
            name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_values,
        )
Matias Guijarro's avatar
Matias Guijarro committed
873

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
874
    def __set__(self, obj, values):
Matias Guijarro's avatar
Matias Guijarro committed
875
        if self._use_object_name: