settings.py 47.8 KB
Newer Older
1 2 3 4
# -*- coding: utf-8 -*-
#
# This file is part of the bliss project
#
5
# Copyright (c) 2015-2019 Beamline Control Unit, ESRF
6 7
# Distributed under the GNU LGPLv3. See LICENSE for more info.

8
from contextlib import contextmanager
Matias Guijarro's avatar
Matias Guijarro committed
9 10
import weakref
import pickle
11 12
import keyword
import re
13
import reprlib
14 15
import datetime
import logging
16

17
import numpy
Matias Guijarro's avatar
Matias Guijarro committed
18

19 20 21
from .conductor import client
from bliss.common.utils import Null
from bliss import setup_globals
22 23
from tabulate import tabulate

24 25
logger = logging.getLogger(__name__)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
26

Matias Guijarro's avatar
Matias Guijarro committed
27 28 29
class InvalidValue(Null):
    def __str__(self):
        raise ValueError
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
30

Matias Guijarro's avatar
Matias Guijarro committed
31
    def __repr__(self):
32
        return "#ERR"
33

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
34

35
class DefaultValue:
36 37 38 39 40 41
    def __init__(self, wrapped_value):
        self.__value = wrapped_value

    @property
    def value(self):
        return self.__value
Matias Guijarro's avatar
Matias Guijarro committed
42

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
43 44

def boolify(s, **keys):
45
    if s in ("True", "true"):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
46
        return True
47
    if s in ("False", "false"):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
48
        return False
49
    raise ValueError("Not Boolean Value!")
Matias Guijarro's avatar
Matias Guijarro committed
50

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
51

52 53 54
def auto_coerce_from_redis(s):
    """Convert variable to a new type from the str representation"""
    if s is None:
Matias Guijarro's avatar
Matias Guijarro committed
55
        return None
56 57 58 59 60 61 62 63
    # Default is unicode string
    try:
        if isinstance(s, bytes):
            s = s.decode()
    # Pickled data fails at first byte
    except UnicodeDecodeError:
        pass
    # Cast to standard types
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
64
    for caster in (boolify, int, float):
Matias Guijarro's avatar
Matias Guijarro committed
65
        try:
66
            return caster(s)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
67
        except (ValueError, TypeError):
Matias Guijarro's avatar
Matias Guijarro committed
68
            pass
69
    return s
Matias Guijarro's avatar
Matias Guijarro committed
70

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
71

72
def pickle_loads(var):
Matias Guijarro's avatar
Matias Guijarro committed
73 74
    if var is None:
        return None
Matias Guijarro's avatar
Matias Guijarro committed
75 76 77 78
    try:
        return pickle.loads(var)
    except Exception:
        return InvalidValue()
Matias Guijarro's avatar
Matias Guijarro committed
79

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
80

81 82
def get_redis_connection():
    return client.get_redis_connection(db=0)
83 84


Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
85
def ttl_func(cnx, name, value=-1):
Matias Guijarro's avatar
Matias Guijarro committed
86 87 88 89 90
    if value is None:
        return cnx.persist(name)
    elif value is -1:
        return cnx.ttl(name)
    else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
91 92
        return cnx.expire(name, value)

Matias Guijarro's avatar
Matias Guijarro committed
93

94
def read_decorator(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
95 96
    def _read(self, *args, **keys):
        value = func(self, *args, **keys)
Matias Guijarro's avatar
Matias Guijarro committed
97
        if self._read_type_conversion:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
98
            if isinstance(value, list):
Matias Guijarro's avatar
Matias Guijarro committed
99
                value = [self._read_type_conversion(x) for x in value]
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
100
            elif isinstance(value, dict):
Vincent Michel's avatar
Vincent Michel committed
101
                for k, v in value.items():
Matias Guijarro's avatar
Matias Guijarro committed
102
                    value[k] = self._read_type_conversion(v)
103 104 105
                if hasattr(self, "default_values") and isinstance(
                    self.default_values, dict
                ):
106 107 108
                    tmp = dict(self._default_values)
                    tmp.update(value)
                    value = tmp
Matias Guijarro's avatar
Matias Guijarro committed
109
            else:
110 111 112
                if isinstance(value, DefaultValue):
                    value = value.value
                elif value is not None:
113
                    value = self._read_type_conversion(value)
114
        if value is None:
115
            if hasattr(self, "_default_value"):
116
                value = self._default_value
117 118 119
            elif hasattr(self, "_default_values") and hasattr(
                self._default_values, "get"
            ):
120
                value = self._default_values.get(args[0])
Matias Guijarro's avatar
Matias Guijarro committed
121
        return value
122

Matias Guijarro's avatar
Matias Guijarro committed
123 124
    return _read

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
125

126
def write_decorator_dict(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
127
    def _write(self, values, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
128
        if self._write_type_conversion:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
129
            if not isinstance(values, dict) and values is not None:
130
                raise TypeError("can only be dict")
Matias Guijarro's avatar
Matias Guijarro committed
131 132

            if values is not None:
133
                new_dict = dict()
Vincent Michel's avatar
Vincent Michel committed
134
                for k, v in values.items():
135 136
                    new_dict[k] = self._write_type_conversion(v)
                values = new_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
137
        return func(self, values, **keys)
138

Matias Guijarro's avatar
Matias Guijarro committed
139 140
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
141

142
def write_decorator_multiple(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
143
    def _write(self, values, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
144
        if self._write_type_conversion:
145 146 147 148 149
            if (
                not isinstance(values, (list, tuple, numpy.ndarray))
                and values is not None
            ):
                raise TypeError("Can only be tuple, list or numpy array")
Matias Guijarro's avatar
Matias Guijarro committed
150 151
            if values is not None:
                values = [self._write_type_conversion(x) for x in values]
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
152
        return func(self, values, **keys)
153

Matias Guijarro's avatar
Matias Guijarro committed
154 155
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
156

157
def write_decorator(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
158
    def _write(self, value, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
159 160
        if self._write_type_conversion and value is not None:
            value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
161
        return func(self, value, **keys)
162

Matias Guijarro's avatar
Matias Guijarro committed
163 164
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
165

166
def scan(match="*", count=1000, connection=None):
167
    if connection is None:
168
        connection = get_redis_connection()
169 170
    cursor = 0
    while 1:
171
        cursor, values = connection.scan(cursor=cursor, match=match, count=count)
172
        for val in values:
173
            yield val.decode()
174 175
        if int(cursor) == 0:
            break
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
176 177


178 179 180 181 182 183
def _get_connection(setting_object):
    """
    Return the connection of a setting_object
    """
    if isinstance(setting_object, Struct):
        return setting_object._proxy._cnx
184
    elif isinstance(setting_object, (SimpleSetting, QueueSetting, BaseHashSetting)):
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
        return setting_object._cnx
    else:
        raise TypeError(
            f"Setting object should be one of: Struct, SimpleSetting, QueueSetting or HashSetting instead of {setting_object!r}"
        )


def _set_connection(setting_object, new_cnx):
    """
    change the connection of a setting_object
    and return the previous connection
    """
    if isinstance(setting_object, Struct):
        cnx = setting_object._proxy._cnx
        setting_object._proxy._cnx = new_cnx
200
    elif isinstance(setting_object, (SimpleSetting, QueueSetting, BaseHashSetting)):
201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
        cnx = setting_object._cnx
        setting_object._cnx = new_cnx
    else:
        raise TypeError(
            f"Setting object should be one of: Struct, SimpleSetting, QueueSetting or HashSetting instead of {setting_object!r}"
        )
    return cnx


@contextmanager
def pipeline(*settings):
    """
    Contextmanager which create a redis pipeline to group redis commands
    on settings.

    IN CASE OF you execute the pipeline, it will return raw database values
    (byte strings).
    """
    first_settings = settings[0]
    cnx = _get_connection(first_settings)()
    # check they have the same connection
    for s in settings[1:]:
        if _get_connection(s)() != cnx:
            raise RuntimeError("Cannot groupe redis commands in a pipeline")

    pipeline = cnx.pipeline()
    try:
        # replace settings connection with the pipeline
        previous_cnx = [_set_connection(s, weakref.ref(pipeline)) for s in settings]
        yield pipeline
    finally:
        [_set_connection(s, c) for c, s in zip(previous_cnx, settings)]
        pipeline.execute()


236 237
class SimpleSetting:
    """
238
    Class to manage a setting that is stored as a string on Redis
239
    """
240

241 242 243 244
    def __init__(
        self,
        name,
        connection=None,
245 246
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
247 248
        default_value=None,
    ):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
249
        if connection is None:
250
            connection = get_redis_connection()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
251 252 253 254
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
255
        self._default_value = default_value
Matias Guijarro's avatar
Matias Guijarro committed
256

257 258 259
    @property
    def name(self):
        return self._name
260

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
261
    @read_decorator
262
    def get(self):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
263 264 265 266 267
        cnx = self._cnx()
        value = cnx.get(self._name)
        return value

    @write_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
268
    def set(self, value):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
269
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
270 271 272 273
        cnx.set(self._name, value)

    def ttl(self, value=-1):
        return ttl_func(self._cnx(), self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
274

275 276 277
    def clear(self):
        cnx = self._cnx()
        cnx.delete(self._name)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
278 279

    def __add__(self, other):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
280
        value = self.get()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
281
        if isinstance(other, SimpleSetting):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
282 283 284
            other = other.get()
        return value + other

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
285
    def __iadd__(self, other):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
286 287
        cnx = self._cnx()
        if cnx is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
288
            if isinstance(other, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
289 290
                if other == 1:
                    cnx.incr(self._name)
Matias Guijarro's avatar
Matias Guijarro committed
291
                else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
292 293 294
                    cnx.incrby(self._name, other)
            elif isinstance(other, float):
                cnx.incrbyfloat(self._name, other)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
295
            else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
296
                cnx.append(self._name, other)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
297
            return self
Matias Guijarro's avatar
Matias Guijarro committed
298

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
299
    def __isub__(self, other):
Vincent Michel's avatar
Vincent Michel committed
300
        if isinstance(other, str):
301 302 303
            raise TypeError(
                "unsupported operand type(s) for -=: %s" % type(other).__name__
            )
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
304
        return self.__iadd__(-other)
Matias Guijarro's avatar
Matias Guijarro committed
305

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
306
    def __getitem__(self, ran):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
307 308 309
        cnx = self._cnx()
        if cnx is not None:
            step = None
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
310 311
            if isinstance(ran, slice):
                i, j = ran.start, ran.stop
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
312
                step = ran.step
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
313
            elif isinstance(ran, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
314 315
                i = j = ran
            else:
316
                raise TypeError("indices must be integers")
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
317

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
318
            value = cnx.getrange(self._name, i, j)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
319 320 321 322 323 324 325
            if step is not None:
                value = value[0:-1:step]
            return value

    def __repr__(self):
        cnx = self._cnx()
        value = cnx.get(self._name)
326
        return "<SimpleSetting name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
327

Matias Guijarro's avatar
Matias Guijarro committed
328

329 330 331 332 333 334
class SimpleSettingProp:
    """
    A python's property implementation for SimpleSetting
    To be used inside user defined classes
    """

335 336 337 338
    def __init__(
        self,
        name,
        connection=None,
339 340
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
341 342 343
        default_value=None,
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
344
        self._name = name
345
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
346 347
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
348
        self._default_value = default_value
Matias Guijarro's avatar
Matias Guijarro committed
349 350
        self._use_object_name = use_object_name

351 352 353 354
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
355
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
356
        if self._use_object_name:
357
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
358 359
        else:
            name = self._name
360 361 362 363 364 365 366
        return SimpleSetting(
            name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_value,
        )
Matias Guijarro's avatar
Matias Guijarro committed
367

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
368 369 370
    def __set__(self, obj, value):
        if isinstance(value, SimpleSetting):
            return
Matias Guijarro's avatar
Matias Guijarro committed
371 372

        if self._use_object_name:
373
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
374 375 376 377 378 379 380 381
        else:
            name = self._name

        if value is None:
            self._cnx.delete(name)
        else:
            if self._write_type_conversion:
                value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
382 383
            self._cnx.set(name, value)

Matias Guijarro's avatar
Matias Guijarro committed
384

385 386
class QueueSetting:
    """
387
    Class to manage a setting that is stored as a list on Redis
388 389
    """

390 391 392 393
    def __init__(
        self,
        name,
        connection=None,
394 395
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
396
    ):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
397
        if connection is None:
398
            connection = get_redis_connection()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
399 400 401 402
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
Matias Guijarro's avatar
Matias Guijarro committed
403

404 405 406 407
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
408
    @read_decorator
409 410 411
    def get(self, first=0, last=-1, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
412
        if first == last:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
413
            l = cnx.lindex(self._name, first)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
414
        else:
415 416
            if last != -1:
                last -= 1
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
417
            l = cnx.lrange(self._name, first, last)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
418
        return l
Matias Guijarro's avatar
Matias Guijarro committed
419

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
420
    @write_decorator
421 422 423
    def append(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
424
        return cnx.rpush(self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
425

426 427 428
    def clear(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
429 430
        cnx.delete(self._name)

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
431
    @write_decorator
432 433 434
    def prepend(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
435
        return cnx.lpush(self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
436

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
437
    @write_decorator_multiple
438
    def extend(self, values, cnx=None):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
439
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
440
        return cnx.rpush(self._name, *values)
Matias Guijarro's avatar
Matias Guijarro committed
441

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
442
    @write_decorator
443 444 445
    def remove(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
446
        cnx.lrem(self._name, 0, value)
Matias Guijarro's avatar
Matias Guijarro committed
447

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
448
    @write_decorator_multiple
449 450 451
    def set(self, values, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
452 453
        cnx.delete(self._name)
        if values is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
454
            cnx.rpush(self._name, *values)
Matias Guijarro's avatar
Matias Guijarro committed
455

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
456
    @write_decorator
457 458 459
    def set_item(self, value, pos=0, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
460
        cnx.lset(self._name, pos, value)
Matias Guijarro's avatar
Matias Guijarro committed
461

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
462
    @read_decorator
463 464 465
    def pop_front(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
466 467 468 469 470 471
        value = cnx.lpop(self._name)
        if self._read_type_conversion:
            value = self._read_type_conversion(value)
        return value

    @read_decorator
472 473 474
    def pop_back(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
475 476 477 478 479
        value = cnx.rpop(self._name)
        if self._read_type_conversion:
            value = self._read_type_conversion(value)
        return value

480 481 482 483
    def ttl(self, value=-1, cnx=None):
        if cnx is None:
            cnx = self._cnx()
        return ttl_func(cnx, self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
484

485 486 487
    def __len__(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
488 489
        return cnx.llen(self._name)

490 491 492
    def __repr__(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
493
        value = cnx.lrange(self._name, 0, -1)
494
        return "<QueueSetting name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
495

496 497
    def __iadd__(self, other, cnx=None):
        self.extend(other, cnx)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
498 499
        return self

500
    def __getitem__(self, ran, cnx=None):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
501
        if isinstance(ran, slice):
502 503
            i = ran.start is not None and ran.start or 0
            j = ran.stop is not None and ran.stop or -1
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
504
        elif isinstance(ran, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
505 506
            i = j = ran
        else:
507
            raise TypeError("indices must be integers")
508
        value = self.get(first=i, last=j, cnx=cnx)
509
        if value is None:
510
            raise IndexError
511 512
        else:
            return value
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
513

514
    def __iter__(self, cnx=None):
515 516
        if cnx is None:
            cnx = self._cnx()
517
        lsize = cnx.llen(self._name)
Vincent Michel's avatar
Vincent Michel committed
518
        for first in range(0, lsize, 1024):
519
            last = first + 1024
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
520 521 522
            if last >= lsize:
                last = -1
            for value in self.get(first, last):
523
                yield value
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
524

525
    def __setitem__(self, ran, value, cnx=None):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
526
        if isinstance(ran, slice):
527
            for i, v in zip(range(ran.start, ran.stop), value):
528
                self.set_item(v, pos=i, cnx=cnx)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
529
        elif isinstance(ran, int):
530
            self.set_item(value, pos=ran, cnx=cnx)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
531
        else:
532
            raise TypeError("indices must be integers")
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
533 534
        return self

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
535

536 537 538 539 540 541
class QueueSettingProp:
    """
    A python's property implementation for QueueSetting
    To be used inside user defined classes
    """

542 543 544 545
    def __init__(
        self,
        name,
        connection=None,
546 547
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
548 549
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
550
        self._name = name
551
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
552 553 554 555
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
        self._use_object_name = use_object_name

556 557 558 559
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
560
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
561
        if self._use_object_name:
562
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
563 564 565
        else:
            name = self._name

566 567 568
        return QueueSetting(
            name, self._cnx, self._read_type_conversion, self._write_type_conversion
        )
Matias Guijarro's avatar
Matias Guijarro committed
569

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
570 571 572
    def __set__(self, obj, values):
        if isinstance(values, QueueSetting):
            return
Matias Guijarro's avatar
Matias Guijarro committed
573 574

        if self._use_object_name:
575
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
576 577 578
        else:
            name = self._name

579 580 581
        proxy = QueueSetting(
            name, self._cnx, self._read_type_conversion, self._write_type_conversion
        )
Matias Guijarro's avatar
Matias Guijarro committed
582 583
        proxy.set(values)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
584

585 586 587 588 589 590 591 592 593 594 595 596 597
class BaseHashSetting:
    """
    A Setting stored as a key,value pair in Redis

    Args:
        name: name of the BaseHashSetting (used on Redis)
        connection: Redis connection object
            read_type_conversion: conversion of data applyed
                after reading
            write_type_conversion: conversion of data applyed
                before writing
    """

598 599 600 601
    def __init__(
        self,
        name,
        connection=None,
602 603
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
604
    ):
Matias Guijarro's avatar
Matias Guijarro committed
605
        if connection is None:
606
            connection = get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
607 608 609 610 611
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion

612 613 614 615
    @property
    def name(self):
        return self._name

616 617
    def __repr__(self):
        value = self.get_all()
618
        return f"<{type(self).__name__} name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
619 620

    def __delitem__(self, key):
621
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
622
        cnx.hdel(self._name, key)
Matias Guijarro's avatar
Matias Guijarro committed
623

624
    def __len__(self):
Matias Guijarro's avatar
Matias Guijarro committed
625 626 627
        cnx = self._cnx()
        return cnx.hlen(self._name)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
628 629
    def ttl(self, value=-1):
        return ttl_func(self._cnx(), self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
630

631
    def raw_get(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
632
        cnx = self._cnx()
633
        return cnx.hget(self._name, *keys)
Matias Guijarro's avatar
Matias Guijarro committed
634 635

    @read_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
636
    def get(self, key, default=None):
Matias Guijarro's avatar
Matias Guijarro committed
637 638
        v = self.raw_get(key)
        if v is None:
639
            v = DefaultValue(default)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
640
        return v
Matias Guijarro's avatar
Matias Guijarro committed
641

Matias Guijarro's avatar
Matias Guijarro committed
642 643 644
    def _raw_get_all(self):
        cnx = self._cnx()
        return cnx.hgetall(self._name)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
645

Matias Guijarro's avatar
Matias Guijarro committed
646
    def get_all(self):
647
        all_dict = dict()
Vincent Michel's avatar
Vincent Michel committed
648
        for k, raw_v in self._raw_get_all().items():
649
            k = k.decode()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
650 651 652
            v = self._read_type_conversion(raw_v)
            if isinstance(v, InvalidValue):
                raise ValueError(
653 654 655
                    "%s: Invalid value '%s` (cannot deserialize %r)"
                    % (self._name, k, raw_v)
                )
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
656
            all_dict[k] = v
Matias Guijarro's avatar
Matias Guijarro committed
657
        return all_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
658

Matias Guijarro's avatar
Matias Guijarro committed
659
    @read_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
660
    def pop(self, key, default=Null()):
661
        cnx = self._cnx().pipeline()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
662 663 664
        cnx.hget(self._name, key)
        cnx.hdel(self._name, key)
        (value, worked) = cnx.execute()
665
        if not worked:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
666
            if isinstance(default, Null):
667 668 669
                raise KeyError(key)
            else:
                value = default
Matias Guijarro's avatar
Matias Guijarro committed
670
        return value
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
671

Matias Guijarro's avatar
Matias Guijarro committed
672
    def remove(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
673
        cnx = self._cnx()
Matias Guijarro's avatar
Matias Guijarro committed
674
        cnx.hdel(self._name, *keys)
Matias Guijarro's avatar
Matias Guijarro committed
675

676
    def clear(self):
Matias Guijarro's avatar
Matias Guijarro committed
677 678 679
        cnx = self._cnx()
        cnx.delete(self._name)

680 681
    # def copy(self):
    #    return self.get()
Matias Guijarro's avatar
Matias Guijarro committed
682 683

    @write_decorator_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
684
    def set(self, values):
Matias Guijarro's avatar
Matias Guijarro committed
685 686 687
        cnx = self._cnx()
        cnx.delete(self._name)
        if values is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
688
            cnx.hmset(self._name, values)
Matias Guijarro's avatar
Matias Guijarro committed
689 690

    @write_decorator_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
691
    def update(self, values):
Matias Guijarro's avatar
Matias Guijarro committed
692
        cnx = self._cnx()
693
        if values:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
694
            cnx.hmset(self._name, values)
Matias Guijarro's avatar
Matias Guijarro committed
695 696

    @read_decorator
697
    def fromkeys(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
698
        cnx = self._cnx()
699
        return cnx.hmget(self._name, *keys)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
700 701

    def has_key(self, key):
Matias Guijarro's avatar
Matias Guijarro committed
702
        cnx = self._cnx()
703
        return cnx.hexists(self._name, key)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
704

705
    def keys(self):
Vincent Michel's avatar
Vincent Michel committed
706
        for k, v in self.items():
707 708
            yield k

709
    def values(self):
Vincent Michel's avatar
Vincent Michel committed
710
        for k, v in self.items():
711
            yield v
Matias Guijarro's avatar
Matias Guijarro committed
712

713
    def items(self):
Matias Guijarro's avatar
Matias Guijarro committed
714 715
        cnx = self._cnx()
        next_id = 0
716
        seen_keys = set()
Matias Guijarro's avatar
Matias Guijarro committed
717
        while True:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
718
            next_id, pd = cnx.hscan(self._name, next_id)
Vincent Michel's avatar
Vincent Michel committed
719
            for k, v in pd.items():
720 721
                # Add key conversion
                k = k.decode()
Matias Guijarro's avatar
Matias Guijarro committed
722 723
                if self._read_type_conversion:
                    v = self._read_type_conversion(v)
724
                seen_keys.add(k)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
725
                yield k, v
726
            if not next_id or next_id is "0":
Matias Guijarro's avatar
Matias Guijarro committed
727 728
                break

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
729
    def __getitem__(self, key):
Matias Guijarro's avatar
Matias Guijarro committed
730 731
        value = self.get(key)
        if value is None:
732
            raise KeyError(key)
Matias Guijarro's avatar
Matias Guijarro committed
733 734
        return value

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
735
    def __setitem__(self, key, value):
Matias Guijarro's avatar
Matias Guijarro committed
736
        cnx = self._cnx()
737
        if value is None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
738
            cnx.hdel(self._name, key)
739
            return
Matias Guijarro's avatar
Matias Guijarro committed
740 741
        if self._write_type_conversion:
            value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
742
        cnx.hset(self._name, key, value)
Matias Guijarro's avatar
Matias Guijarro committed
743

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
744
    def __contains__(self, key):
745 746 747 748 749 750 751
        try:
            self[key]
            return True
        except KeyError:
            return False


752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838
class HashSetting(BaseHashSetting):
    """
    A Setting stored as a key,value pair in Redis
    with a default_value dictionary to serve as a callback
    when elements lookup fails

    Args:
        name: name of the HashSetting (used on Redis)
        connection: Redis connection object
            read_type_conversion: conversion of data applyed
                after reading
            write_type_conversion: conversion of data applyed
                before writing
    kwargs:
        default_values: dictionary of default values retrieved
            on fallback
    """

    def __init__(
        self,
        name,
        connection=None,
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
        default_values={},
    ):
        super().__init__(
            name,
            connection=connection,
            read_type_conversion=read_type_conversion,
            write_type_conversion=write_type_conversion,
        )
        self._default_values = default_values

    @read_decorator
    def get(self, key, default=None):
        v = super().raw_get(key)
        if v is None:
            v = DefaultValue(default)
        return v

    def has_key(self, key):
        return super().has_key(key) or key in self._default_values

    def __getitem__(self, key):
        value = self.get(key)
        if value is None:
            if key not in self._default_values:
                raise KeyError(key)
        return value

    def get_all(self):
        all_dict = dict(self._default_values)
        for k, raw_v in self._raw_get_all().items():
            k = k.decode()
            v = self._read_type_conversion(raw_v)
            if isinstance(v, InvalidValue):
                raise ValueError(
                    "%s: Invalid value '%s` (cannot deserialize %r)"
                    % (self._name, k, raw_v)
                )
            all_dict[k] = v
        return all_dict

    def items(self):
        cnx = self._cnx()
        next_id = 0
        seen_keys = set()
        while True:
            next_id, pd = cnx.hscan(self._name, next_id)
            for k, v in pd.items():
                # Add key conversion
                k = k.decode()
                if self._read_type_conversion:
                    v = self._read_type_conversion(v)
                seen_keys.add(k)
                yield k, v
            if not next_id or next_id is "0":
                break

        for k, v in self._default_values.items():
            if k in seen_keys:
                continue
            yield k, v


class HashSettingProp:
839 840 841 842
    def __init__(
        self,
        name,
        connection=None,
843 844
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
845 846 847
        default_values={},
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
848
        self._name = name
849
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
850 851
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
852
        self._default_values = default_values
Matias Guijarro's avatar
Matias Guijarro committed
853 854
        self._use_object_name = use_object_name

855 856 857 858
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
859
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
860
        if self._use_object_name:
861
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
862 863
        else:
            name = self._name
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
864

865 866 867 868 869 870 871
        return HashSetting(
            name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_values,
        )
Matias Guijarro's avatar
Matias Guijarro committed
872

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
873
    def __set__(self, obj, values):
Matias Guijarro's avatar
Matias Guijarro committed
874
        if self._use_object_name:
875
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
876 877