/usr/lib/python3/dist-packages/requests_cache/backends/storage/dbdict.py is in python3-requests-cache 0.4.13-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 | #!/usr/bin/python
# -*- coding: utf-8 -*-
"""
requests_cache.backends.dbdict
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Dictionary-like objects for saving large data sets to `sqlite` database
"""
from collections import MutableMapping
import sqlite3 as sqlite
from contextlib import contextmanager
try:
import threading
except ImportError:
import dummy_threading as threading
try:
import cPickle as pickle
except ImportError:
import pickle
from requests_cache.compat import bytes
class DbDict(MutableMapping):
""" DbDict - a dictionary-like object for saving large datasets to `sqlite` database
It's possible to create multiply DbDict instances, which will be stored as separate
tables in one database::
d1 = DbDict('test', 'table1')
d2 = DbDict('test', 'table2')
d3 = DbDict('test', 'table3')
all data will be stored in ``test.sqlite`` database into
correspondent tables: ``table1``, ``table2`` and ``table3``
"""
def __init__(self, filename, table_name='data', fast_save=False, **options):
"""
:param filename: filename for database (without extension)
:param table_name: table name
:param fast_save: If it's True, then sqlite will be configured with
`"PRAGMA synchronous = 0;" <http://www.sqlite.org/pragma.html#pragma_synchronous>`_
to speedup cache saving, but be careful, it's dangerous.
Tests showed that insertion order of records can be wrong with this option.
"""
self.filename = filename
self.table_name = table_name
self.fast_save = fast_save
#: Transactions can be commited if this property is set to `True`
self.can_commit = True
self._bulk_commit = False
self._pending_connection = None
self._lock = threading.RLock()
with self.connection() as con:
con.execute("create table if not exists `%s` (key PRIMARY KEY, value)" % self.table_name)
@contextmanager
def connection(self, commit_on_success=False):
with self._lock:
if self._bulk_commit:
if self._pending_connection is None:
self._pending_connection = sqlite.connect(self.filename)
con = self._pending_connection
else:
con = sqlite.connect(self.filename)
try:
if self.fast_save:
con.execute("PRAGMA synchronous = 0;")
yield con
if commit_on_success and self.can_commit:
con.commit()
finally:
if not self._bulk_commit:
con.close()
def commit(self, force=False):
"""
Commits pending transaction if :attr:`can_commit` or `force` is `True`
:param force: force commit, ignore :attr:`can_commit`
"""
if force or self.can_commit:
if self._pending_connection is not None:
self._pending_connection.commit()
@contextmanager
def bulk_commit(self):
"""
Context manager used to speedup insertion of big number of records
::
>>> d1 = DbDict('test')
>>> with d1.bulk_commit():
... for i in range(1000):
... d1[i] = i * 2
"""
self._bulk_commit = True
self.can_commit = False
try:
yield
self.commit(True)
finally:
self._bulk_commit = False
self.can_commit = True
self._pending_connection.close()
self._pending_connection = None
def __getitem__(self, key):
with self.connection() as con:
row = con.execute("select value from `%s` where key=?" %
self.table_name, (key,)).fetchone()
if not row:
raise KeyError
return row[0]
def __setitem__(self, key, item):
with self.connection(True) as con:
con.execute("insert or replace into `%s` (key,value) values (?,?)" %
self.table_name, (key, item))
def __delitem__(self, key):
with self.connection(True) as con:
cur = con.execute("delete from `%s` where key=?" %
self.table_name, (key,))
if not cur.rowcount:
raise KeyError
def __iter__(self):
with self.connection() as con:
for row in con.execute("select key from `%s`" %
self.table_name):
yield row[0]
def __len__(self):
with self.connection() as con:
return con.execute("select count(key) from `%s`" %
self.table_name).fetchone()[0]
def clear(self):
with self.connection(True) as con:
con.execute("drop table `%s`" % self.table_name)
con.execute("create table `%s` (key PRIMARY KEY, value)" %
self.table_name)
def __str__(self):
return str(dict(self.items()))
class DbPickleDict(DbDict):
""" Same as :class:`DbDict`, but pickles values before saving
"""
def __setitem__(self, key, item):
super(DbPickleDict, self).__setitem__(key,
sqlite.Binary(pickle.dumps(item)))
def __getitem__(self, key):
return pickle.loads(bytes(super(DbPickleDict, self).__getitem__(key)))
|