settings.py 28.9 KB
Newer Older
1 2 3 4 5 6 7
# -*- coding: utf-8 -*-
#
# This file is part of the bliss project
#
# Copyright (c) 2016 Beamline Control Unit, ESRF
# Distributed under the GNU LGPLv3. See LICENSE for more info.

8
from contextlib import contextmanager
Matias Guijarro's avatar
Matias Guijarro committed
9 10
import weakref
import pickle
11 12 13
import keyword
import re

14
import numpy
Matias Guijarro's avatar
Matias Guijarro committed
15

16 17 18
from .conductor import client
from bliss.common.utils import Null
from bliss import setup_globals
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
19

Matias Guijarro's avatar
Matias Guijarro committed
20 21 22
class InvalidValue(Null):
    def __str__(self):
        raise ValueError
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
23

Matias Guijarro's avatar
Matias Guijarro committed
24
    def __repr__(self):
25
        return "#ERR"
26

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
27

28
class DefaultValue:
29 30 31 32 33 34
    def __init__(self, wrapped_value):
        self.__value = wrapped_value

    @property
    def value(self):
        return self.__value
Matias Guijarro's avatar
Matias Guijarro committed
35

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
36 37

def boolify(s, **keys):
38
    if s in ("True", "true"):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
39
        return True
40
    if s in ("False", "false"):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
41
        return False
42
    raise ValueError("Not Boolean Value!")
Matias Guijarro's avatar
Matias Guijarro committed
43

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
44

45 46 47
def auto_coerce_from_redis(s):
    """Convert variable to a new type from the str representation"""
    if s is None:
Matias Guijarro's avatar
Matias Guijarro committed
48
        return None
49 50 51 52 53 54 55 56
    # Default is unicode string
    try:
        if isinstance(s, bytes):
            s = s.decode()
    # Pickled data fails at first byte
    except UnicodeDecodeError:
        pass
    # Cast to standard types
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
57
    for caster in (boolify, int, float):
Matias Guijarro's avatar
Matias Guijarro committed
58
        try:
59
            return caster(s)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
60
        except (ValueError, TypeError):
Matias Guijarro's avatar
Matias Guijarro committed
61
            pass
62
    return s
Matias Guijarro's avatar
Matias Guijarro committed
63

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
64

65
def pickle_loads(var):
Matias Guijarro's avatar
Matias Guijarro committed
66 67
    if var is None:
        return None
Matias Guijarro's avatar
Matias Guijarro committed
68 69 70 71
    try:
        return pickle.loads(var)
    except Exception:
        return InvalidValue()
Matias Guijarro's avatar
Matias Guijarro committed
72

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
73

74 75
def get_redis_connection():
    return client.get_redis_connection(db=0)
76 77


Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
78
def ttl_func(cnx, name, value=-1):
Matias Guijarro's avatar
Matias Guijarro committed
79 80 81 82 83
    if value is None:
        return cnx.persist(name)
    elif value is -1:
        return cnx.ttl(name)
    else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
84 85
        return cnx.expire(name, value)

Matias Guijarro's avatar
Matias Guijarro committed
86

87
def read_decorator(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
88 89
    def _read(self, *args, **keys):
        value = func(self, *args, **keys)
Matias Guijarro's avatar
Matias Guijarro committed
90
        if self._read_type_conversion:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
91
            if isinstance(value, list):
Matias Guijarro's avatar
Matias Guijarro committed
92
                value = [self._read_type_conversion(x) for x in value]
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
93
            elif isinstance(value, dict):
Vincent Michel's avatar
Vincent Michel committed
94
                for k, v in value.items():
Matias Guijarro's avatar
Matias Guijarro committed
95
                    value[k] = self._read_type_conversion(v)
96 97 98
                if hasattr(self, "default_values") and isinstance(
                    self.default_values, dict
                ):
99 100 101
                    tmp = dict(self._default_values)
                    tmp.update(value)
                    value = tmp
Matias Guijarro's avatar
Matias Guijarro committed
102
            else:
103 104 105
                if isinstance(value, DefaultValue):
                    value = value.value
                elif value is not None:
106
                    value = self._read_type_conversion(value)
107
        if value is None:
108
            if hasattr(self, "_default_value"):
109
                value = self._default_value
110 111 112
            elif hasattr(self, "_default_values") and hasattr(
                self._default_values, "get"
            ):
113
                value = self._default_values.get(args[0])
Matias Guijarro's avatar
Matias Guijarro committed
114
        return value
115

Matias Guijarro's avatar
Matias Guijarro committed
116 117
    return _read

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
118

119
def write_decorator_dict(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
120
    def _write(self, values, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
121
        if self._write_type_conversion:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
122
            if not isinstance(values, dict) and values is not None:
123
                raise TypeError("can only be dict")
Matias Guijarro's avatar
Matias Guijarro committed
124 125

            if values is not None:
126
                new_dict = dict()
Vincent Michel's avatar
Vincent Michel committed
127
                for k, v in values.items():
128 129
                    new_dict[k] = self._write_type_conversion(v)
                values = new_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
130
        return func(self, values, **keys)
131

Matias Guijarro's avatar
Matias Guijarro committed
132 133
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
134

135
def write_decorator_multiple(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
136
    def _write(self, values, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
137
        if self._write_type_conversion:
138 139 140 141 142
            if (
                not isinstance(values, (list, tuple, numpy.ndarray))
                and values is not None
            ):
                raise TypeError("Can only be tuple, list or numpy array")
Matias Guijarro's avatar
Matias Guijarro committed
143 144
            if values is not None:
                values = [self._write_type_conversion(x) for x in values]
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
145
        return func(self, values, **keys)
146

Matias Guijarro's avatar
Matias Guijarro committed
147 148
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
149

150
def write_decorator(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
151
    def _write(self, value, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
152 153
        if self._write_type_conversion and value is not None:
            value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
154
        return func(self, value, **keys)
155

Matias Guijarro's avatar
Matias Guijarro committed
156 157
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
158

159
def scan(match="*", count=1000, connection=None):
160
    if connection is None:
161
        connection = get_redis_connection()
162 163
    cursor = 0
    while 1:
164
        cursor, values = connection.scan(cursor=cursor, match=match, count=count)
165
        for val in values:
166
            yield val.decode()
167 168
        if int(cursor) == 0:
            break
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
169 170


171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228
def _get_connection(setting_object):
    """
    Return the connection of a setting_object
    """
    if isinstance(setting_object, Struct):
        return setting_object._proxy._cnx
    elif isinstance(setting_object, (SimpleSetting, QueueSetting, HashSetting)):
        return setting_object._cnx
    else:
        raise TypeError(
            f"Setting object should be one of: Struct, SimpleSetting, QueueSetting or HashSetting instead of {setting_object!r}"
        )


def _set_connection(setting_object, new_cnx):
    """
    change the connection of a setting_object
    and return the previous connection
    """
    if isinstance(setting_object, Struct):
        cnx = setting_object._proxy._cnx
        setting_object._proxy._cnx = new_cnx
    elif isinstance(setting_object, (SimpleSetting, QueueSetting, HashSetting)):
        cnx = setting_object._cnx
        setting_object._cnx = new_cnx
    else:
        raise TypeError(
            f"Setting object should be one of: Struct, SimpleSetting, QueueSetting or HashSetting instead of {setting_object!r}"
        )
    return cnx


@contextmanager
def pipeline(*settings):
    """
    Contextmanager which create a redis pipeline to group redis commands
    on settings.

    IN CASE OF you execute the pipeline, it will return raw database values
    (byte strings).
    """
    first_settings = settings[0]
    cnx = _get_connection(first_settings)()
    # check they have the same connection
    for s in settings[1:]:
        if _get_connection(s)() != cnx:
            raise RuntimeError("Cannot groupe redis commands in a pipeline")

    pipeline = cnx.pipeline()
    try:
        # replace settings connection with the pipeline
        previous_cnx = [_set_connection(s, weakref.ref(pipeline)) for s in settings]
        yield pipeline
    finally:
        [_set_connection(s, c) for c, s in zip(previous_cnx, settings)]
        pipeline.execute()


229 230 231 232
class SimpleSetting:
    """
    Class to manage a setting that is stored as string on redis
    """
233 234 235 236
    def __init__(
        self,
        name,
        connection=None,
237 238
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
239 240
        default_value=None,
    ):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
241
        if connection is None:
242
            connection = get_redis_connection()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
243 244 245 246
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
247
        self._default_value = default_value
Matias Guijarro's avatar
Matias Guijarro committed
248

249 250 251
    @property
    def name(self):
        return self._name
252

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
253
    @read_decorator
254
    def get(self):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
255 256 257 258 259
        cnx = self._cnx()
        value = cnx.get(self._name)
        return value

    @write_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
260
    def set(self, value):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
261
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
262 263 264 265
        cnx.set(self._name, value)

    def ttl(self, value=-1):
        return ttl_func(self._cnx(), self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
266

267 268 269
    def clear(self):
        cnx = self._cnx()
        cnx.delete(self._name)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
270 271

    def __add__(self, other):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
272
        value = self.get()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
273
        if isinstance(other, SimpleSetting):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
274 275 276
            other = other.get()
        return value + other

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
277
    def __iadd__(self, other):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
278 279
        cnx = self._cnx()
        if cnx is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
280
            if isinstance(other, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
281 282
                if other == 1:
                    cnx.incr(self._name)
Matias Guijarro's avatar
Matias Guijarro committed
283
                else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
284 285 286
                    cnx.incrby(self._name, other)
            elif isinstance(other, float):
                cnx.incrbyfloat(self._name, other)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
287
            else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
288
                cnx.append(self._name, other)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
289
            return self
Matias Guijarro's avatar
Matias Guijarro committed
290

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
291
    def __isub__(self, other):
Vincent Michel's avatar
Vincent Michel committed
292
        if isinstance(other, str):
293 294 295
            raise TypeError(
                "unsupported operand type(s) for -=: %s" % type(other).__name__
            )
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
296
        return self.__iadd__(-other)
Matias Guijarro's avatar
Matias Guijarro committed
297

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
298
    def __getitem__(self, ran):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
299 300 301
        cnx = self._cnx()
        if cnx is not None:
            step = None
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
302 303
            if isinstance(ran, slice):
                i, j = ran.start, ran.stop
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
304
                step = ran.step
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
305
            elif isinstance(ran, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
306 307
                i = j = ran
            else:
308
                raise TypeError("indices must be integers")
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
309

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
310
            value = cnx.getrange(self._name, i, j)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
311 312 313 314 315 316 317
            if step is not None:
                value = value[0:-1:step]
            return value

    def __repr__(self):
        cnx = self._cnx()
        value = cnx.get(self._name)
318
        return "<SimpleSetting name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
319

Matias Guijarro's avatar
Matias Guijarro committed
320

321 322 323 324 325 326
class SimpleSettingProp:
    """
    A python's property implementation for SimpleSetting
    To be used inside user defined classes
    """

327 328 329 330
    def __init__(
        self,
        name,
        connection=None,
331 332
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
333 334 335
        default_value=None,
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
336
        self._name = name
337
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
338 339
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
340
        self._default_value = default_value
Matias Guijarro's avatar
Matias Guijarro committed
341 342
        self._use_object_name = use_object_name

343 344 345 346
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
347
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
348
        if self._use_object_name:
349
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
350 351
        else:
            name = self._name
352 353 354 355 356 357 358
        return SimpleSetting(
            name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_value,
        )
Matias Guijarro's avatar
Matias Guijarro committed
359

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
360 361 362
    def __set__(self, obj, value):
        if isinstance(value, SimpleSetting):
            return
Matias Guijarro's avatar
Matias Guijarro committed
363 364

        if self._use_object_name:
365
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
366 367 368 369 370 371 372 373
        else:
            name = self._name

        if value is None:
            self._cnx.delete(name)
        else:
            if self._write_type_conversion:
                value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
374 375
            self._cnx.set(name, value)

Matias Guijarro's avatar
Matias Guijarro committed
376

377 378 379 380 381
class QueueSetting:
    """
    Class to manage a setting that is stored as list on redis
    """

382 383 384 385
    def __init__(
        self,
        name,
        connection=None,
386 387
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
388
    ):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
389
        if connection is None:
390
            connection = get_redis_connection()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
391 392 393 394
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
Matias Guijarro's avatar
Matias Guijarro committed
395

396 397 398 399
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
400
    @read_decorator
401 402 403
    def get(self, first=0, last=-1, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
404
        if first == last:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
405
            l = cnx.lindex(self._name, first)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
406
        else:
407 408
            if last != -1:
                last -= 1
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
409
            l = cnx.lrange(self._name, first, last)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
410
        return l
Matias Guijarro's avatar
Matias Guijarro committed
411

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
412
    @write_decorator
413 414 415
    def append(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
416
        return cnx.rpush(self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
417

418 419 420
    def clear(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
421 422
        cnx.delete(self._name)

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
423
    @write_decorator
424 425 426
    def prepend(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
427
        return cnx.lpush(self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
428

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
429
    @write_decorator_multiple
430
    def extend(self, values, cnx=None):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
431
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
432
        return cnx.rpush(self._name, *values)
Matias Guijarro's avatar
Matias Guijarro committed
433

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
434
    @write_decorator
435 436 437
    def remove(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
438
        cnx.lrem(self._name, 0, value)
Matias Guijarro's avatar
Matias Guijarro committed
439

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
440
    @write_decorator_multiple
441 442 443
    def set(self, values, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
444 445
        cnx.delete(self._name)
        if values is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
446
            cnx.rpush(self._name, *values)
Matias Guijarro's avatar
Matias Guijarro committed
447

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
448
    @write_decorator
449 450 451
    def set_item(self, value, pos=0, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
452
        cnx.lset(self._name, pos, value)
Matias Guijarro's avatar
Matias Guijarro committed
453

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
454
    @read_decorator
455 456 457
    def pop_front(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
458 459 460 461 462 463
        value = cnx.lpop(self._name)
        if self._read_type_conversion:
            value = self._read_type_conversion(value)
        return value

    @read_decorator
464 465 466
    def pop_back(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
467 468 469 470 471
        value = cnx.rpop(self._name)
        if self._read_type_conversion:
            value = self._read_type_conversion(value)
        return value

472 473 474 475
    def ttl(self, value=-1, cnx=None):
        if cnx is None:
            cnx = self._cnx()
        return ttl_func(cnx, self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
476

477 478 479
    def __len__(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
480 481
        return cnx.llen(self._name)

482 483 484
    def __repr__(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
485
        value = cnx.lrange(self._name, 0, -1)
486
        return "<QueueSetting name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
487

488 489
    def __iadd__(self, other, cnx=None):
        self.extend(other, cnx)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
490 491
        return self

492
    def __getitem__(self, ran, cnx=None):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
493
        if isinstance(ran, slice):
494 495
            i = ran.start is not None and ran.start or 0
            j = ran.stop is not None and ran.stop or -1
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
496
        elif isinstance(ran, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
497 498
            i = j = ran
        else:
499
            raise TypeError("indices must be integers")
500
        value = self.get(first=i, last=j, cnx=cnx)
501
        if value is None:
502
            raise IndexError
503 504
        else:
            return value
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
505

506
    def __iter__(self, cnx=None):
507 508
        if cnx is None:
            cnx = self._cnx()
509
        lsize = cnx.llen(self._name)
Vincent Michel's avatar
Vincent Michel committed
510
        for first in range(0, lsize, 1024):
511
            last = first + 1024
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
512 513 514
            if last >= lsize:
                last = -1
            for value in self.get(first, last):
515
                yield value
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
516

517
    def __setitem__(self, ran, value, cnx=None):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
518
        if isinstance(ran, slice):
519
            for i, v in zip(range(ran.start, ran.stop), value):
520
                self.set_item(v, pos=i, cnx=cnx)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
521
        elif isinstance(ran, int):
522
            self.set_item(value, pos=ran, cnx=cnx)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
523
        else:
524
            raise TypeError("indices must be integers")
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
525 526
        return self

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
527

528 529 530 531 532 533
class QueueSettingProp:
    """
    A python's property implementation for QueueSetting
    To be used inside user defined classes
    """

534 535 536 537
    def __init__(
        self,
        name,
        connection=None,
538 539
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
540 541
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
542
        self._name = name
543
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
544 545 546 547
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
        self._use_object_name = use_object_name

548 549 550 551
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
552
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
553
        if self._use_object_name:
554
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
555 556 557
        else:
            name = self._name

558 559 560
        return QueueSetting(
            name, self._cnx, self._read_type_conversion, self._write_type_conversion
        )
Matias Guijarro's avatar
Matias Guijarro committed
561

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
562 563 564
    def __set__(self, obj, values):
        if isinstance(values, QueueSetting):
            return
Matias Guijarro's avatar
Matias Guijarro committed
565 566

        if self._use_object_name:
567
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
568 569 570
        else:
            name = self._name

571 572 573
        proxy = QueueSetting(
            name, self._cnx, self._read_type_conversion, self._write_type_conversion
        )
Matias Guijarro's avatar
Matias Guijarro committed
574 575
        proxy.set(values)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
576

Matias Guijarro's avatar
Matias Guijarro committed
577
class HashSetting(object):
578 579 580 581
    def __init__(
        self,
        name,
        connection=None,
582 583
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
584 585
        default_values={},
    ):
Matias Guijarro's avatar
Matias Guijarro committed
586
        if connection is None:
587
            connection = get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
588 589 590 591
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
592
        self._default_values = default_values
Matias Guijarro's avatar
Matias Guijarro committed
593

594 595 596 597
    @property
    def name(self):
        return self._name

598 599
    def __repr__(self):
        value = self.get_all()
600
        return "<HashSetting name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
601 602

    def __delitem__(self, key):
603
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
604
        cnx.hdel(self._name, key)
Matias Guijarro's avatar
Matias Guijarro committed
605

606
    def __len__(self):
Matias Guijarro's avatar
Matias Guijarro committed
607 608 609
        cnx = self._cnx()
        return cnx.hlen(self._name)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
610 611
    def ttl(self, value=-1):
        return ttl_func(self._cnx(), self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
612

613
    def raw_get(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
614
        cnx = self._cnx()
615
        return cnx.hget(self._name, *keys)
Matias Guijarro's avatar
Matias Guijarro committed
616 617

    @read_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
618
    def get(self, key, default=None):
Matias Guijarro's avatar
Matias Guijarro committed
619 620
        v = self.raw_get(key)
        if v is None:
621
            v = DefaultValue(default)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
622
        return v
Matias Guijarro's avatar
Matias Guijarro committed
623

Matias Guijarro's avatar
Matias Guijarro committed
624 625 626
    def _raw_get_all(self):
        cnx = self._cnx()
        return cnx.hgetall(self._name)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
627

Matias Guijarro's avatar
Matias Guijarro committed
628 629
    def get_all(self):
        all_dict = dict(self._default_values)
Vincent Michel's avatar
Vincent Michel committed
630
        for k, raw_v in self._raw_get_all().items():
631
            k = k.decode()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
632 633 634
            v = self._read_type_conversion(raw_v)
            if isinstance(v, InvalidValue):
                raise ValueError(
635 636 637
                    "%s: Invalid value '%s` (cannot deserialize %r)"
                    % (self._name, k, raw_v)
                )
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
638
            all_dict[k] = v
Matias Guijarro's avatar
Matias Guijarro committed
639
        return all_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
640

Matias Guijarro's avatar
Matias Guijarro committed
641
    @read_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
642
    def pop(self, key, default=Null()):
643
        cnx = self._cnx().pipeline()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
644 645 646
        cnx.hget(self._name, key)
        cnx.hdel(self._name, key)
        (value, worked) = cnx.execute()
647
        if not worked:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
648
            if isinstance(default, Null):
649 650 651
                raise KeyError(key)
            else:
                value = default
Matias Guijarro's avatar
Matias Guijarro committed
652
        return value
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
653

Matias Guijarro's avatar
Matias Guijarro committed
654
    def remove(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
655
        cnx = self._cnx()
Matias Guijarro's avatar
Matias Guijarro committed
656
        cnx.hdel(self._name, *keys)
Matias Guijarro's avatar
Matias Guijarro committed
657

658
    def clear(self):
Matias Guijarro's avatar
Matias Guijarro committed
659 660 661
        cnx = self._cnx()
        cnx.delete(self._name)

662
    def copy(self):
Matias Guijarro's avatar
Matias Guijarro committed
663 664 665
        return self.get()

    @write_decorator_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
666
    def set(self, values):
Matias Guijarro's avatar
Matias Guijarro committed
667 668 669
        cnx = self._cnx()
        cnx.delete(self._name)
        if values is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
670
            cnx.hmset(self._name, values)
Matias Guijarro's avatar
Matias Guijarro committed
671 672

    @write_decorator_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
673
    def update(self, values):
Matias Guijarro's avatar
Matias Guijarro committed
674
        cnx = self._cnx()
675
        if values:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
676
            cnx.hmset(self._name, values)
Matias Guijarro's avatar
Matias Guijarro committed
677 678

    @read_decorator
679
    def fromkeys(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
680
        cnx = self._cnx()
681
        return cnx.hmget(self._name, *keys)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
682 683

    def has_key(self, key):
Matias Guijarro's avatar
Matias Guijarro committed
684
        cnx = self._cnx()
Vincent Michel's avatar
Vincent Michel committed
685
        return cnx.hexists(self._name, key) or key in self._default_values
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
686

687
    def keys(self):
Vincent Michel's avatar
Vincent Michel committed
688
        for k, v in self.items():
689 690
            yield k

691
    def values(self):
Vincent Michel's avatar
Vincent Michel committed
692
        for k, v in self.items():
693
            yield v
Matias Guijarro's avatar
Matias Guijarro committed
694

695
    def items(self):
Matias Guijarro's avatar
Matias Guijarro committed
696 697
        cnx = self._cnx()
        next_id = 0
698
        seen_keys = set()
Matias Guijarro's avatar
Matias Guijarro committed
699
        while True:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
700
            next_id, pd = cnx.hscan(self._name, next_id)
Vincent Michel's avatar
Vincent Michel committed
701
            for k, v in pd.items():
702 703
                # Add key conversion
                k = k.decode()
Matias Guijarro's avatar
Matias Guijarro committed
704 705
                if self._read_type_conversion:
                    v = self._read_type_conversion(v)
706
                seen_keys.add(k)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
707
                yield k, v
708
            if not next_id or next_id is "0":
Matias Guijarro's avatar
Matias Guijarro committed
709 710
                break

Vincent Michel's avatar
Vincent Michel committed
711
        for k, v in self._default_values.items():
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
712 713 714
            if k in seen_keys:
                continue
            yield k, v
715

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
716
    def __getitem__(self, key):
Matias Guijarro's avatar
Matias Guijarro committed
717 718
        value = self.get(key)
        if value is None:
Vincent Michel's avatar
Vincent Michel committed
719
            if key not in self._default_values:
720
                raise KeyError(key)
Matias Guijarro's avatar
Matias Guijarro committed
721 722
        return value

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
723
    def __setitem__(self, key, value):
Matias Guijarro's avatar
Matias Guijarro committed
724
        cnx = self._cnx()
725
        if value is None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
726
            cnx.hdel(self._name, key)
727
            return
Matias Guijarro's avatar
Matias Guijarro committed
728 729
        if self._write_type_conversion:
            value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
730
        cnx.hset(self._name, key, value)
Matias Guijarro's avatar
Matias Guijarro committed
731

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
732
    def __contains__(self, key):
733 734 735 736 737 738 739
        try:
            self[key]
            return True
        except KeyError:
            return False


Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
740
class HashSettingProp(object):
741 742 743 744
    def __init__(
        self,
        name,
        connection=None,
745 746
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
747 748 749
        default_values={},
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
750
        self._name = name
751
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
752 753
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
754
        self._default_values = default_values
Matias Guijarro's avatar
Matias Guijarro committed
755 756
        self._use_object_name = use_object_name

757 758 759 760
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
761
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
762
        if self._use_object_name:
763
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
764 765
        else:
            name = self._name
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
766

767 768 769 770 771 772 773
        return HashSetting(
            name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_values,
        )
Matias Guijarro's avatar
Matias Guijarro committed
774

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
775
    def __set__(self, obj, values):
Matias Guijarro's avatar
Matias Guijarro committed
776
        if self._use_object_name:
777
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
778 779 780
        else:
            name = self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
781 782
        if isinstance(values, HashSetting):
            return
Matias Guijarro's avatar
Matias Guijarro committed
783

784 785 786 787 788 789 790
        proxy = HashSetting(
            name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_values,
        )
Matias Guijarro's avatar
Matias Guijarro committed
791
        proxy.set(values)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
792

793
    def get_proxy(self):
794 795 796 797 798 799 800 801 802
        return HashSetting(
            self._name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_values,
        )


Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
803 804
# helper

Matias Guijarro's avatar
Matias Guijarro committed
805

806
def _change_to_obj_marshalling(keys):
807 808 809 810 811 812 813 814
    read_type_conversion = keys.pop("read_type_conversion", pickle_loads)
    write_type_conversion = keys.pop("write_type_conversion", pickle.dumps)
    keys.update(
        {
            "read_type_conversion": read_type_conversion,
            "write_type_conversion": write_type_conversion,
        }
    )
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
815

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
816

817
class HashObjSetting(HashSetting):
818 819 820 821 822
    """
    Class to manage a setting that is stored as a dictionary on redis
    where values of the dictionary are pickled
    """

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
823
    def __init__(self, name, **keys):
824
        _change_to_obj_marshalling(keys)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
825 826
        HashSetting.__init__(self, name, **keys)

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
827

828
class HashObjSettingProp(HashSettingProp):
829 830 831 832 833
    """
    A python's property implementation for HashObjSetting
    To be used inside user defined classes
    """

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
834
    def __init__(self, name, **keys):
Matias Guijarro's avatar