settings.py 27.5 KB
Newer Older
1 2 3 4 5 6 7
# -*- coding: utf-8 -*-
#
# This file is part of the bliss project
#
# Copyright (c) 2016 Beamline Control Unit, ESRF
# Distributed under the GNU LGPLv3. See LICENSE for more info.

8
from contextlib import contextmanager
9
from .conductor import client
10
from bliss.common.utils import Null
11
from bliss import setup_globals
Matias Guijarro's avatar
Matias Guijarro committed
12 13
import weakref
import pickle
14
import numpy
Matias Guijarro's avatar
Matias Guijarro committed
15

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
16

Matias Guijarro's avatar
Matias Guijarro committed
17 18 19
class InvalidValue(Null):
    def __str__(self):
        raise ValueError
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
20

Matias Guijarro's avatar
Matias Guijarro committed
21
    def __repr__(self):
22
        return "#ERR"
23

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
24

25 26 27 28 29 30 31
class DefaultValue(object):
    def __init__(self, wrapped_value):
        self.__value = wrapped_value

    @property
    def value(self):
        return self.__value
Matias Guijarro's avatar
Matias Guijarro committed
32

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
33 34

def boolify(s, **keys):
35
    if s == "True" or s == "true":
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
36
        return True
37
    if s == "False" or s == "false":
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
38
        return False
39
    raise ValueError("Not Boolean Value!")
Matias Guijarro's avatar
Matias Guijarro committed
40

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
41

42 43 44
def auto_coerce_from_redis(s):
    """Convert variable to a new type from the str representation"""
    if s is None:
Matias Guijarro's avatar
Matias Guijarro committed
45
        return None
46 47 48 49 50 51 52 53
    # Default is unicode string
    try:
        if isinstance(s, bytes):
            s = s.decode()
    # Pickled data fails at first byte
    except UnicodeDecodeError:
        pass
    # Cast to standard types
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
54
    for caster in (boolify, int, float):
Matias Guijarro's avatar
Matias Guijarro committed
55
        try:
56
            return caster(s)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
57
        except (ValueError, TypeError):
Matias Guijarro's avatar
Matias Guijarro committed
58
            pass
59
    return s
Matias Guijarro's avatar
Matias Guijarro committed
60

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
61

62
def pickle_loads(var):
Matias Guijarro's avatar
Matias Guijarro committed
63 64
    if var is None:
        return None
Matias Guijarro's avatar
Matias Guijarro committed
65 66 67 68
    try:
        return pickle.loads(var)
    except Exception:
        return InvalidValue()
Matias Guijarro's avatar
Matias Guijarro committed
69

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
70

71 72
def get_redis_connection():
    return client.get_redis_connection(db=0)
73 74


Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
75
def ttl_func(cnx, name, value=-1):
Matias Guijarro's avatar
Matias Guijarro committed
76 77 78 79 80
    if value is None:
        return cnx.persist(name)
    elif value is -1:
        return cnx.ttl(name)
    else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
81 82
        return cnx.expire(name, value)

Matias Guijarro's avatar
Matias Guijarro committed
83

84
def read_decorator(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
85 86
    def _read(self, *args, **keys):
        value = func(self, *args, **keys)
Matias Guijarro's avatar
Matias Guijarro committed
87
        if self._read_type_conversion:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
88
            if isinstance(value, list):
Matias Guijarro's avatar
Matias Guijarro committed
89
                value = [self._read_type_conversion(x) for x in value]
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
90
            elif isinstance(value, dict):
Vincent Michel's avatar
Vincent Michel committed
91
                for k, v in value.items():
Matias Guijarro's avatar
Matias Guijarro committed
92
                    value[k] = self._read_type_conversion(v)
93 94 95
                if hasattr(self, "default_values") and isinstance(
                    self.default_values, dict
                ):
96 97 98
                    tmp = dict(self._default_values)
                    tmp.update(value)
                    value = tmp
Matias Guijarro's avatar
Matias Guijarro committed
99
            else:
100 101 102
                if isinstance(value, DefaultValue):
                    value = value.value
                elif value is not None:
103
                    value = self._read_type_conversion(value)
104
        if value is None:
105
            if hasattr(self, "_default_value"):
106
                value = self._default_value
107 108 109
            elif hasattr(self, "_default_values") and hasattr(
                self._default_values, "get"
            ):
110
                value = self._default_values.get(args[0])
Matias Guijarro's avatar
Matias Guijarro committed
111
        return value
112

Matias Guijarro's avatar
Matias Guijarro committed
113 114
    return _read

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
115

116
def write_decorator_dict(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
117
    def _write(self, values, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
118
        if self._write_type_conversion:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
119
            if not isinstance(values, dict) and values is not None:
120
                raise TypeError("can only be dict")
Matias Guijarro's avatar
Matias Guijarro committed
121 122

            if values is not None:
123
                new_dict = dict()
Vincent Michel's avatar
Vincent Michel committed
124
                for k, v in values.items():
125 126
                    new_dict[k] = self._write_type_conversion(v)
                values = new_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
127
        return func(self, values, **keys)
128

Matias Guijarro's avatar
Matias Guijarro committed
129 130
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
131

132
def write_decorator_multiple(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
133
    def _write(self, values, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
134
        if self._write_type_conversion:
135 136 137 138 139
            if (
                not isinstance(values, (list, tuple, numpy.ndarray))
                and values is not None
            ):
                raise TypeError("Can only be tuple, list or numpy array")
Matias Guijarro's avatar
Matias Guijarro committed
140 141
            if values is not None:
                values = [self._write_type_conversion(x) for x in values]
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
142
        return func(self, values, **keys)
143

Matias Guijarro's avatar
Matias Guijarro committed
144 145
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
146

147
def write_decorator(func):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
148
    def _write(self, value, **keys):
Matias Guijarro's avatar
Matias Guijarro committed
149 150
        if self._write_type_conversion and value is not None:
            value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
151
        return func(self, value, **keys)
152

Matias Guijarro's avatar
Matias Guijarro committed
153 154
    return _write

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
155

156
def scan(match="*", count=1000, connection=None):
157
    if connection is None:
158
        connection = get_redis_connection()
159 160
    cursor = 0
    while 1:
161
        cursor, values = connection.scan(cursor=cursor, match=match, count=count)
162
        for val in values:
163
            yield val.decode()
164 165
        if int(cursor) == 0:
            break
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
166 167


168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
def _get_connection(setting_object):
    """
    Return the connection of a setting_object
    """
    if isinstance(setting_object, Struct):
        return setting_object._proxy._cnx
    elif isinstance(setting_object, (SimpleSetting, QueueSetting, HashSetting)):
        return setting_object._cnx
    else:
        raise TypeError(
            f"Setting object should be one of: Struct, SimpleSetting, QueueSetting or HashSetting instead of {setting_object!r}"
        )


def _set_connection(setting_object, new_cnx):
    """
    change the connection of a setting_object
    and return the previous connection
    """
    if isinstance(setting_object, Struct):
        cnx = setting_object._proxy._cnx
        setting_object._proxy._cnx = new_cnx
    elif isinstance(setting_object, (SimpleSetting, QueueSetting, HashSetting)):
        cnx = setting_object._cnx
        setting_object._cnx = new_cnx
    else:
        raise TypeError(
            f"Setting object should be one of: Struct, SimpleSetting, QueueSetting or HashSetting instead of {setting_object!r}"
        )
    return cnx


@contextmanager
def pipeline(*settings):
    """
    Contextmanager which create a redis pipeline to group redis commands
    on settings.

    IN CASE OF you execute the pipeline, it will return raw database values
    (byte strings).
    """
    first_settings = settings[0]
    cnx = _get_connection(first_settings)()
    # check they have the same connection
    for s in settings[1:]:
        if _get_connection(s)() != cnx:
            raise RuntimeError("Cannot groupe redis commands in a pipeline")

    pipeline = cnx.pipeline()
    try:
        # replace settings connection with the pipeline
        previous_cnx = [_set_connection(s, weakref.ref(pipeline)) for s in settings]
        yield pipeline
    finally:
        [_set_connection(s, c) for c, s in zip(previous_cnx, settings)]
        pipeline.execute()


226
class SimpleSetting(object):
227 228 229 230
    def __init__(
        self,
        name,
        connection=None,
231 232
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
233 234
        default_value=None,
    ):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
235
        if connection is None:
236
            connection = get_redis_connection()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
237 238 239 240
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
241
        self._default_value = default_value
Matias Guijarro's avatar
Matias Guijarro committed
242

243 244 245
    @property
    def name(self):
        return self._name
246

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
247
    @read_decorator
248
    def get(self):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
249 250 251 252 253
        cnx = self._cnx()
        value = cnx.get(self._name)
        return value

    @write_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
254
    def set(self, value):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
255
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
256 257 258 259
        cnx.set(self._name, value)

    def ttl(self, value=-1):
        return ttl_func(self._cnx(), self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
260

261 262 263
    def clear(self):
        cnx = self._cnx()
        cnx.delete(self._name)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
264 265

    def __add__(self, other):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
266
        value = self.get()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
267
        if isinstance(other, SimpleSetting):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
268 269 270
            other = other.get()
        return value + other

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
271
    def __iadd__(self, other):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
272 273
        cnx = self._cnx()
        if cnx is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
274
            if isinstance(other, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
275 276
                if other == 1:
                    cnx.incr(self._name)
Matias Guijarro's avatar
Matias Guijarro committed
277
                else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
278 279 280
                    cnx.incrby(self._name, other)
            elif isinstance(other, float):
                cnx.incrbyfloat(self._name, other)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
281
            else:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
282
                cnx.append(self._name, other)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
283
            return self
Matias Guijarro's avatar
Matias Guijarro committed
284

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
285
    def __isub__(self, other):
Vincent Michel's avatar
Vincent Michel committed
286
        if isinstance(other, str):
287 288 289
            raise TypeError(
                "unsupported operand type(s) for -=: %s" % type(other).__name__
            )
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
290
        return self.__iadd__(-other)
Matias Guijarro's avatar
Matias Guijarro committed
291

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
292
    def __getitem__(self, ran):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
293 294 295
        cnx = self._cnx()
        if cnx is not None:
            step = None
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
296 297
            if isinstance(ran, slice):
                i, j = ran.start, ran.stop
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
298
                step = ran.step
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
299
            elif isinstance(ran, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
300 301
                i = j = ran
            else:
302
                raise TypeError("indices must be integers")
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
303

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
304
            value = cnx.getrange(self._name, i, j)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
305 306 307 308 309 310 311
            if step is not None:
                value = value[0:-1:step]
            return value

    def __repr__(self):
        cnx = self._cnx()
        value = cnx.get(self._name)
312
        return "<SimpleSetting name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
313

Matias Guijarro's avatar
Matias Guijarro committed
314

315
class SimpleSettingProp(object):
316 317 318 319
    def __init__(
        self,
        name,
        connection=None,
320 321
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
322 323 324
        default_value=None,
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
325
        self._name = name
326
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
327 328
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
329
        self._default_value = default_value
Matias Guijarro's avatar
Matias Guijarro committed
330 331
        self._use_object_name = use_object_name

332 333 334 335
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
336
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
337
        if self._use_object_name:
338
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
339 340
        else:
            name = self._name
341 342 343 344 345 346 347
        return SimpleSetting(
            name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_value,
        )
Matias Guijarro's avatar
Matias Guijarro committed
348

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
349 350 351
    def __set__(self, obj, value):
        if isinstance(value, SimpleSetting):
            return
Matias Guijarro's avatar
Matias Guijarro committed
352 353

        if self._use_object_name:
354
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
355 356 357 358 359 360 361 362
        else:
            name = self._name

        if value is None:
            self._cnx.delete(name)
        else:
            if self._write_type_conversion:
                value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
363 364
            self._cnx.set(name, value)

Matias Guijarro's avatar
Matias Guijarro committed
365 366

class QueueSetting(object):
367 368 369 370
    def __init__(
        self,
        name,
        connection=None,
371 372
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
373
    ):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
374
        if connection is None:
375
            connection = get_redis_connection()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
376 377 378 379
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
Matias Guijarro's avatar
Matias Guijarro committed
380

381 382 383 384
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
385
    @read_decorator
386 387 388
    def get(self, first=0, last=-1, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
389
        if first == last:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
390
            l = cnx.lindex(self._name, first)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
391
        else:
392 393
            if last != -1:
                last -= 1
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
394
            l = cnx.lrange(self._name, first, last)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
395
        return l
Matias Guijarro's avatar
Matias Guijarro committed
396

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
397
    @write_decorator
398 399 400
    def append(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
401
        return cnx.rpush(self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
402

403 404 405
    def clear(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
406 407
        cnx.delete(self._name)

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
408
    @write_decorator
409 410 411
    def prepend(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
412
        return cnx.lpush(self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
413

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
414
    @write_decorator_multiple
415
    def extend(self, values, cnx=None):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
416
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
417
        return cnx.rpush(self._name, *values)
Matias Guijarro's avatar
Matias Guijarro committed
418

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
419
    @write_decorator
420 421 422
    def remove(self, value, cnx=None):
        if cnx is None:
            cnx = self._cnx()
423
        cnx.lrem(self._name, 0, value)
Matias Guijarro's avatar
Matias Guijarro committed
424

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
425
    @write_decorator_multiple
426 427 428
    def set(self, values, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
429 430
        cnx.delete(self._name)
        if values is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
431
            cnx.rpush(self._name, *values)
Matias Guijarro's avatar
Matias Guijarro committed
432

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
433
    @write_decorator
434 435 436
    def set_item(self, value, pos=0, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
437
        cnx.lset(self._name, pos, value)
Matias Guijarro's avatar
Matias Guijarro committed
438

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
439
    @read_decorator
440 441 442
    def pop_front(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
443 444 445 446 447 448
        value = cnx.lpop(self._name)
        if self._read_type_conversion:
            value = self._read_type_conversion(value)
        return value

    @read_decorator
449 450 451
    def pop_back(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
452 453 454 455 456
        value = cnx.rpop(self._name)
        if self._read_type_conversion:
            value = self._read_type_conversion(value)
        return value

457 458 459 460
    def ttl(self, value=-1, cnx=None):
        if cnx is None:
            cnx = self._cnx()
        return ttl_func(cnx, self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
461

462 463 464
    def __len__(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
465 466
        return cnx.llen(self._name)

467 468 469
    def __repr__(self, cnx=None):
        if cnx is None:
            cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
470
        value = cnx.lrange(self._name, 0, -1)
471
        return "<QueueSetting name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
472

473 474
    def __iadd__(self, other, cnx=None):
        self.extend(other, cnx)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
475 476
        return self

477
    def __getitem__(self, ran, cnx=None):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
478
        if isinstance(ran, slice):
479 480
            i = ran.start is not None and ran.start or 0
            j = ran.stop is not None and ran.stop or -1
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
481
        elif isinstance(ran, int):
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
482 483
            i = j = ran
        else:
484
            raise TypeError("indices must be integers")
485
        value = self.get(first=i, last=j, cnx=cnx)
486
        if value is None:
487
            raise IndexError
488 489
        else:
            return value
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
490

491
    def __iter__(self, cnx=None):
492 493
        if cnx is None:
            cnx = self._cnx()
494
        lsize = cnx.llen(self._name)
Vincent Michel's avatar
Vincent Michel committed
495
        for first in range(0, lsize, 1024):
496
            last = first + 1024
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
497 498 499
            if last >= lsize:
                last = -1
            for value in self.get(first, last):
500
                yield value
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
501

502
    def __setitem__(self, ran, value, cnx=None):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
503
        if isinstance(ran, slice):
504
            for i, v in zip(range(ran.start, ran.stop), value):
505
                self.set_item(v, pos=i, cnx=cnx)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
506
        elif isinstance(ran, int):
507
            self.set_item(value, pos=ran, cnx=cnx)
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
508
        else:
509
            raise TypeError("indices must be integers")
Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
510 511
        return self

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
512

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
513
class QueueSettingProp(object):
514 515 516 517
    def __init__(
        self,
        name,
        connection=None,
518 519
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
520 521
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
522
        self._name = name
523
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
524 525 526 527
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
        self._use_object_name = use_object_name

528 529 530 531
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
532
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
533
        if self._use_object_name:
534
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
535 536 537
        else:
            name = self._name

538 539 540
        return QueueSetting(
            name, self._cnx, self._read_type_conversion, self._write_type_conversion
        )
Matias Guijarro's avatar
Matias Guijarro committed
541

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
542 543 544
    def __set__(self, obj, values):
        if isinstance(values, QueueSetting):
            return
Matias Guijarro's avatar
Matias Guijarro committed
545 546

        if self._use_object_name:
547
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
548 549 550
        else:
            name = self._name

551 552 553
        proxy = QueueSetting(
            name, self._cnx, self._read_type_conversion, self._write_type_conversion
        )
Matias Guijarro's avatar
Matias Guijarro committed
554 555
        proxy.set(values)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
556

Matias Guijarro's avatar
Matias Guijarro committed
557
class HashSetting(object):
558 559 560 561
    def __init__(
        self,
        name,
        connection=None,
562 563
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
564 565
        default_values={},
    ):
Matias Guijarro's avatar
Matias Guijarro committed
566
        if connection is None:
567
            connection = get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
568 569 570 571
        self._cnx = weakref.ref(connection)
        self._name = name
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
572
        self._default_values = default_values
Matias Guijarro's avatar
Matias Guijarro committed
573

574 575 576 577
    @property
    def name(self):
        return self._name

578 579
    def __repr__(self):
        value = self.get_all()
580
        return "<HashSetting name=%s value=%s>" % (self._name, value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
581 582

    def __delitem__(self, key):
583
        cnx = self._cnx()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
584
        cnx.hdel(self._name, key)
Matias Guijarro's avatar
Matias Guijarro committed
585

586
    def __len__(self):
Matias Guijarro's avatar
Matias Guijarro committed
587 588 589
        cnx = self._cnx()
        return cnx.hlen(self._name)

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
590 591
    def ttl(self, value=-1):
        return ttl_func(self._cnx(), self._name, value)
Matias Guijarro's avatar
Matias Guijarro committed
592

593
    def raw_get(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
594
        cnx = self._cnx()
595
        return cnx.hget(self._name, *keys)
Matias Guijarro's avatar
Matias Guijarro committed
596 597

    @read_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
598
    def get(self, key, default=None):
Matias Guijarro's avatar
Matias Guijarro committed
599 600
        v = self.raw_get(key)
        if v is None:
601
            v = DefaultValue(default)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
602
        return v
Matias Guijarro's avatar
Matias Guijarro committed
603

Matias Guijarro's avatar
Matias Guijarro committed
604 605 606
    def _raw_get_all(self):
        cnx = self._cnx()
        return cnx.hgetall(self._name)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
607

Matias Guijarro's avatar
Matias Guijarro committed
608 609
    def get_all(self):
        all_dict = dict(self._default_values)
Vincent Michel's avatar
Vincent Michel committed
610
        for k, raw_v in self._raw_get_all().items():
611
            k = k.decode()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
612 613 614
            v = self._read_type_conversion(raw_v)
            if isinstance(v, InvalidValue):
                raise ValueError(
615 616 617
                    "%s: Invalid value '%s` (cannot deserialize %r)"
                    % (self._name, k, raw_v)
                )
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
618
            all_dict[k] = v
Matias Guijarro's avatar
Matias Guijarro committed
619
        return all_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
620

Matias Guijarro's avatar
Matias Guijarro committed
621
    @read_decorator
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
622
    def pop(self, key, default=Null()):
623
        cnx = self._cnx().pipeline()
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
624 625 626
        cnx.hget(self._name, key)
        cnx.hdel(self._name, key)
        (value, worked) = cnx.execute()
627
        if not worked:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
628
            if isinstance(default, Null):
629 630 631
                raise KeyError(key)
            else:
                value = default
Matias Guijarro's avatar
Matias Guijarro committed
632
        return value
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
633

Matias Guijarro's avatar
Matias Guijarro committed
634
    def remove(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
635
        cnx = self._cnx()
Matias Guijarro's avatar
Matias Guijarro committed
636
        cnx.hdel(self._name, *keys)
Matias Guijarro's avatar
Matias Guijarro committed
637

638
    def clear(self):
Matias Guijarro's avatar
Matias Guijarro committed
639 640 641
        cnx = self._cnx()
        cnx.delete(self._name)

642
    def copy(self):
Matias Guijarro's avatar
Matias Guijarro committed
643 644 645
        return self.get()

    @write_decorator_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
646
    def set(self, values):
Matias Guijarro's avatar
Matias Guijarro committed
647 648 649
        cnx = self._cnx()
        cnx.delete(self._name)
        if values is not None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
650
            cnx.hmset(self._name, values)
Matias Guijarro's avatar
Matias Guijarro committed
651 652

    @write_decorator_dict
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
653
    def update(self, values):
Matias Guijarro's avatar
Matias Guijarro committed
654
        cnx = self._cnx()
655
        if values:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
656
            cnx.hmset(self._name, values)
Matias Guijarro's avatar
Matias Guijarro committed
657 658

    @read_decorator
659
    def fromkeys(self, *keys):
Matias Guijarro's avatar
Matias Guijarro committed
660
        cnx = self._cnx()
661
        return cnx.hmget(self._name, *keys)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
662 663

    def has_key(self, key):
Matias Guijarro's avatar
Matias Guijarro committed
664
        cnx = self._cnx()
Vincent Michel's avatar
Vincent Michel committed
665
        return cnx.hexists(self._name, key) or key in self._default_values
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
666

667
    def keys(self):
Vincent Michel's avatar
Vincent Michel committed
668
        for k, v in self.items():
669 670
            yield k

671
    def values(self):
Vincent Michel's avatar
Vincent Michel committed
672
        for k, v in self.items():
673
            yield v
Matias Guijarro's avatar
Matias Guijarro committed
674

675
    def items(self):
Matias Guijarro's avatar
Matias Guijarro committed
676 677
        cnx = self._cnx()
        next_id = 0
678
        seen_keys = set()
Matias Guijarro's avatar
Matias Guijarro committed
679
        while True:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
680
            next_id, pd = cnx.hscan(self._name, next_id)
Vincent Michel's avatar
Vincent Michel committed
681
            for k, v in pd.items():
682 683
                # Add key conversion
                k = k.decode()
Matias Guijarro's avatar
Matias Guijarro committed
684 685
                if self._read_type_conversion:
                    v = self._read_type_conversion(v)
686
                seen_keys.add(k)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
687
                yield k, v
688
            if not next_id or next_id is "0":
Matias Guijarro's avatar
Matias Guijarro committed
689 690
                break

Vincent Michel's avatar
Vincent Michel committed
691
        for k, v in self._default_values.items():
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
692 693 694
            if k in seen_keys:
                continue
            yield k, v
695

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
696
    def __getitem__(self, key):
Matias Guijarro's avatar
Matias Guijarro committed
697 698
        value = self.get(key)
        if value is None:
Vincent Michel's avatar
Vincent Michel committed
699
            if key not in self._default_values:
700
                raise KeyError(key)
Matias Guijarro's avatar
Matias Guijarro committed
701 702
        return value

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
703
    def __setitem__(self, key, value):
Matias Guijarro's avatar
Matias Guijarro committed
704
        cnx = self._cnx()
705
        if value is None:
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
706
            cnx.hdel(self._name, key)
707
            return
Matias Guijarro's avatar
Matias Guijarro committed
708 709
        if self._write_type_conversion:
            value = self._write_type_conversion(value)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
710
        cnx.hset(self._name, key, value)
Matias Guijarro's avatar
Matias Guijarro committed
711

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
712
    def __contains__(self, key):
713 714 715 716 717 718 719
        try:
            self[key]
            return True
        except KeyError:
            return False


Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
720
class HashSettingProp(object):
721 722 723 724
    def __init__(
        self,
        name,
        connection=None,
725 726
        read_type_conversion=auto_coerce_from_redis,
        write_type_conversion=str,
727 728 729
        default_values={},
        use_object_name=True,
    ):
Matias Guijarro's avatar
Matias Guijarro committed
730
        self._name = name
731
        self._cnx = connection or get_redis_connection()
Matias Guijarro's avatar
Matias Guijarro committed
732 733
        self._read_type_conversion = read_type_conversion
        self._write_type_conversion = write_type_conversion
734
        self._default_values = default_values
Matias Guijarro's avatar
Matias Guijarro committed
735 736
        self._use_object_name = use_object_name

737 738 739 740
    @property
    def name(self):
        return self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
741
    def __get__(self, obj, type=None):
Matias Guijarro's avatar
Matias Guijarro committed
742
        if self._use_object_name:
743
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
744 745
        else:
            name = self._name
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
746

747 748 749 750 751 752 753
        return HashSetting(
            name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_values,
        )
Matias Guijarro's avatar
Matias Guijarro committed
754

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
755
    def __set__(self, obj, values):
Matias Guijarro's avatar
Matias Guijarro committed
756
        if self._use_object_name:
757
            name = obj.name + ":" + self._name
Matias Guijarro's avatar
Matias Guijarro committed
758 759 760
        else:
            name = self._name

Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
761 762
        if isinstance(values, HashSetting):
            return
Matias Guijarro's avatar
Matias Guijarro committed
763

764 765 766 767 768 769 770
        proxy = HashSetting(
            name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_values,
        )
Matias Guijarro's avatar
Matias Guijarro committed
771
        proxy.set(values)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
772

773
    def get_proxy(self):
774 775 776 777 778 779 780 781 782
        return HashSetting(
            self._name,
            self._cnx,
            self._read_type_conversion,
            self._write_type_conversion,
            self._default_values,
        )


Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
783 784
# helper

Matias Guijarro's avatar
Matias Guijarro committed
785

786
def _change_to_obj_marshalling(keys):
787 788 789 790 791 792 793 794
    read_type_conversion = keys.pop("read_type_conversion", pickle_loads)
    write_type_conversion = keys.pop("write_type_conversion", pickle.dumps)
    keys.update(
        {
            "read_type_conversion": read_type_conversion,
            "write_type_conversion": write_type_conversion,
        }
    )
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
795

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
796

797
class HashObjSetting(HashSetting):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
798
    def __init__(self, name, **keys):
799
        _change_to_obj_marshalling(keys)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
800 801
        HashSetting.__init__(self, name, **keys)

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
802

803
class HashObjSettingProp(HashSettingProp):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
804
    def __init__(self, name, **keys):
805
        _change_to_obj_marshalling(keys)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
806 807
        HashSettingProp.__init__(self, name, **keys)

Sebastien Petitdemange's avatar
Sebastien Petitdemange committed
808

809
class QueueObjSetting(QueueSetting):
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
810
    def __init__(self, name, **keys):
811
        _change_to_obj_marshalling(keys)
Sebastien Petitdemange's avatar
pep8  
Sebastien Petitdemange committed
812 813