/usr/lib/python3/dist-packages/partd/buffer.py is in python3-partd 0.3.7-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | from .core import Interface
from threading import Lock
from toolz import merge_with, topk, accumulate, pluck
from operator import add
from bisect import bisect
from collections import defaultdict
from .compatibility import Queue, Empty
def zero():
return 0
class Buffer(Interface):
def __init__(self, fast, slow, available_memory=1e9):
self.lock = Lock()
self.fast = fast
self.slow = slow
self.available_memory = available_memory
self.lengths = defaultdict(zero)
self.memory_usage = 0
Interface.__init__(self)
def __getstate__(self):
return {'fast': self.fast,
'slow': self.slow,
'memory_usage': self.memory_usage,
'lengths': self.lengths,
'available_memory': self.available_memory}
def __setstate__(self, state):
Interface.__setstate__(self, state)
self.lock = Lock()
self.__dict__.update(state)
def append(self, data, lock=True, **kwargs):
if lock: self.lock.acquire()
try:
for k, v in data.items():
self.lengths[k] += len(v)
self.memory_usage += len(v)
self.fast.append(data, lock=False, **kwargs)
while self.memory_usage > self.available_memory:
keys = keys_to_flush(self.lengths, 0.1, maxcount=20)
self.flush(keys)
finally:
if lock: self.lock.release()
def _get(self, keys, lock=True, **kwargs):
if lock: self.lock.acquire()
try:
result = list(map(add, self.fast.get(keys, lock=False),
self.slow.get(keys, lock=False)))
finally:
if lock: self.lock.release()
return result
def _iset(self, key, value, lock=True):
""" Idempotent set """
if lock: self.lock.acquire()
try:
self.fast.iset(key, value, lock=False)
finally:
if lock: self.lock.release()
def _delete(self, keys, lock=True):
if lock: self.lock.acquire()
try:
self.fast.delete(keys, lock=False)
self.slow.delete(keys, lock=False)
finally:
if lock: self.lock.release()
def drop(self):
self._iset_seen.clear()
self.fast.drop()
self.slow.drop()
def __exit__(self, *args):
self.drop()
def flush(self, keys=None, block=None):
""" Flush keys to disk
Parameters
----------
keys: list or None
list of keys to flush
block: bool (defaults to None)
Whether or not to block until all writing is complete
If no keys are given then flush all keys
"""
if keys is None:
keys = list(self.lengths)
self.slow.append(dict(zip(keys, self.fast.get(keys))))
self.fast.delete(keys)
for key in keys:
self.memory_usage -= self.lengths[key]
del self.lengths[key]
def keys_to_flush(lengths, fraction=0.1, maxcount=100000):
""" Which keys to remove
>>> lengths = {'a': 20, 'b': 10, 'c': 15, 'd': 15,
... 'e': 10, 'f': 25, 'g': 5}
>>> keys_to_flush(lengths, 0.5)
['f', 'a']
"""
top = topk(max(len(lengths) // 2, 1),
lengths.items(),
key=1)
total = sum(lengths.values())
cutoff = min(maxcount, max(1,
bisect(list(accumulate(add, pluck(1, top))),
total * fraction)))
result = [k for k, v in top[:cutoff]]
assert result
return result
|