/usr/share/pyshared/TileStache/Caches.py is in tilestache 1.31.0-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 | """ The cache bits of TileStache.
A Cache is the part of TileStache that stores static files to speed up future
requests. A few default caches are found here, but it's possible to define your
own and pull them into TileStache dynamically by class name.
Built-in providers:
- test
- disk
- multi
- memcache
- s3
Example built-in cache, for JSON configuration file:
"cache": {
"name": "Disk",
"path": "/tmp/stache",
"umask": "0000"
}
Example external cache, for JSON configuration file:
"cache": {
"class": "Module:Classname",
"kwargs": {"frob": "yes"}
}
- The "class" value is split up into module and classname, and dynamically
included. If this doesn't work for some reason, TileStache will fail loudly
to let you know.
- The "kwargs" value is fed to the class constructor as a dictionary of keyword
args. If your defined class doesn't accept any of these keyword arguments,
TileStache will throw an exception.
A cache must provide these methods: lock(), unlock(), read(), and save().
Each method accepts three arguments:
- layer: instance of a Layer.
- coord: single Coordinate that represents a tile.
- format: string like "png" or "jpg" that is used as a filename extension.
The save() method accepts an additional argument before the others:
- body: raw content to save to the cache.
TODO: add stale_lock_timeout and cache_lifespan to cache API in v2.
"""
import os
import sys
import time
import gzip
from tempfile import mkstemp
from os.path import isdir, exists, dirname, basename, join as pathjoin
from .Core import KnownUnknown
from . import Memcache
from . import S3
def getCacheByName(name):
""" Retrieve a cache object by name.
Raise an exception if the name doesn't work out.
"""
if name.lower() == 'test':
return Test
elif name.lower() == 'disk':
return Disk
elif name.lower() == 'multi':
return Multi
elif name.lower() == 'memcache':
return Memcache.Cache
elif name.lower() == 's3':
return S3.Cache
raise Exception('Unknown cache name: "%s"' % name)
class Test:
""" Simple cache that doesn't actually cache anything.
Activity is optionally logged, though.
Example configuration:
"cache": {
"name": "Test",
"verbose": True
}
Extra configuration parameters:
- verbose: optional boolean flag to write cache activities to a logging
function, defaults to False if omitted.
"""
def __init__(self, logfunc=None):
self.logfunc = logfunc
def _description(self, layer, coord, format):
"""
"""
name = layer.name()
tile = '%(zoom)d/%(column)d/%(row)d' % coord.__dict__
return ' '.join( (name, tile, format) )
def lock(self, layer, coord, format):
""" Pretend to acquire a cache lock for this tile.
"""
name = self._description(layer, coord, format)
if self.logfunc:
self.logfunc('Test cache lock: ' + name)
def unlock(self, layer, coord, format):
""" Pretend to release a cache lock for this tile.
"""
name = self._description(layer, coord, format)
if self.logfunc:
self.logfunc('Test cache unlock: ' + name)
def remove(self, layer, coord, format):
""" Pretend to remove a cached tile.
"""
name = self._description(layer, coord, format)
if self.logfunc:
self.logfunc('Test cache remove: ' + name)
def read(self, layer, coord, format):
""" Pretend to read a cached tile.
"""
name = self._description(layer, coord, format)
if self.logfunc:
self.logfunc('Test cache read: ' + name)
return None
def save(self, body, layer, coord, format):
""" Pretend to save a cached tile.
"""
name = self._description(layer, coord, format)
if self.logfunc:
self.logfunc('Test cache save: %d bytes to %s' % (len(body), name))
class Disk:
""" Caches files to disk.
Example configuration:
"cache": {
"name": "Disk",
"path": "/tmp/stache",
"umask": "0000",
"dirs": "portable"
}
Extra parameters:
- path: required local directory path where files should be stored.
- umask: optional string representation of octal permission mask
for stored files. Defaults to 0022.
- dirs: optional string saying whether to create cache directories that
are safe or portable. For an example tile 12/656/1582.png, "portable"
creates matching directory trees while "portable" guarantees directories
with fewer files, e.g. 12/000/656/001/582.png. Defaults to safe.
- gzip: optional list of file formats that should be stored in a
compressed form. Defaults to "txt", "text", "json", and "xml".
Provide an empty list in the configuration for no compression.
If your configuration file is loaded from a remote location, e.g.
"http://example.com/tilestache.cfg", the path *must* be an unambiguous
filesystem path, e.g. "file:///tmp/cache"
"""
def __init__(self, path, umask=0022, dirs='safe', gzip='txt text json xml'.split()):
self.cachepath = path
self.umask = umask
self.dirs = dirs
self.gzip = [format.lower() for format in gzip]
def _is_compressed(self, format):
return format.lower() in self.gzip
def _filepath(self, layer, coord, format):
"""
"""
l = layer.name()
z = '%d' % coord.zoom
e = format.lower()
e += self._is_compressed(format) and '.gz' or ''
if self.dirs == 'safe':
x = '%06d' % coord.column
y = '%06d' % coord.row
x1, x2 = x[:3], x[3:]
y1, y2 = y[:3], y[3:]
filepath = os.sep.join( (l, z, x1, x2, y1, y2 + '.' + e) )
elif self.dirs == 'portable':
x = '%d' % coord.column
y = '%d' % coord.row
filepath = os.sep.join( (l, z, x, y + '.' + e) )
else:
raise KnownUnknown('Please provide a valid "dirs" parameter to the Disk cache, either "safe" or "portable" but not "%s"' % self.dirs)
return filepath
def _fullpath(self, layer, coord, format):
"""
"""
filepath = self._filepath(layer, coord, format)
fullpath = pathjoin(self.cachepath, filepath)
return fullpath
def _lockpath(self, layer, coord, format):
"""
"""
return self._fullpath(layer, coord, format) + '.lock'
def lock(self, layer, coord, format):
""" Acquire a cache lock for this tile.
Returns nothing, but blocks until the lock has been acquired.
Lock is implemented as an empty directory next to the tile file.
"""
lockpath = self._lockpath(layer, coord, format)
due = time.time() + layer.stale_lock_timeout
while True:
# try to acquire a directory lock, repeating if necessary.
try:
umask_old = os.umask(self.umask)
if time.time() > due:
# someone left the door locked.
try:
os.rmdir(lockpath)
except OSError:
# Oh - no they didn't.
pass
os.makedirs(lockpath, 0777&~self.umask)
break
except OSError, e:
if e.errno != 17:
raise
time.sleep(.2)
finally:
os.umask(umask_old)
def unlock(self, layer, coord, format):
""" Release a cache lock for this tile.
Lock is implemented as an empty directory next to the tile file.
"""
lockpath = self._lockpath(layer, coord, format)
try:
os.rmdir(lockpath)
except OSError:
# Ok, someone else deleted it already
pass
def remove(self, layer, coord, format):
""" Remove a cached tile.
"""
fullpath = self._fullpath(layer, coord, format)
try:
os.remove(fullpath)
except OSError, e:
# errno=2 means that the file does not exist, which is fine
if e.errno != 2:
raise
def read(self, layer, coord, format):
""" Read a cached tile.
"""
fullpath = self._fullpath(layer, coord, format)
if not exists(fullpath):
return None
age = time.time() - os.stat(fullpath).st_mtime
if layer.cache_lifespan and age > layer.cache_lifespan:
return None
elif self._is_compressed(format):
return gzip.open(fullpath, 'r').read()
else:
body = open(fullpath, 'rb').read()
return body
def save(self, body, layer, coord, format):
""" Save a cached tile.
"""
fullpath = self._fullpath(layer, coord, format)
try:
umask_old = os.umask(self.umask)
os.makedirs(dirname(fullpath), 0777&~self.umask)
except OSError, e:
if e.errno != 17:
raise
finally:
os.umask(umask_old)
suffix = '.' + format.lower()
suffix += self._is_compressed(format) and '.gz' or ''
fh, tmp_path = mkstemp(dir=self.cachepath, suffix=suffix)
if self._is_compressed(format):
os.close(fh)
tmp_file = gzip.open(tmp_path, 'w')
tmp_file.write(body)
tmp_file.close()
else:
os.write(fh, body)
os.close(fh)
try:
os.rename(tmp_path, fullpath)
except OSError:
os.unlink(fullpath)
os.rename(tmp_path, fullpath)
os.chmod(fullpath, 0666&~self.umask)
class Multi:
""" Caches tiles to multiple, ordered caches.
Multi cache is well-suited for a speed-to-capacity gradient, for
example a combination of Memcache and S3 to take advantage of the high
speed of memcache and the high capacity of S3. Each tier of caching is
checked sequentially when reading from the cache, while all tiers are
used together for writing. Locks are only used with the first cache.
Example configuration:
"cache": {
"name": "Multi",
"tiers": [
{
"name": "Memcache",
"servers": ["127.0.0.1:11211"]
},
{
"name": "Disk",
"path": "/tmp/stache"
}
]
}
Multi cache parameters:
tiers
Required list of cache configurations. The fastest, most local
cache should be at the beginning of the list while the slowest or
most remote cache should be at the end. Memcache and S3 together
make a great pair.
"""
def __init__(self, tiers):
self.tiers = tiers
def lock(self, layer, coord, format):
""" Acquire a cache lock for this tile in the first tier.
Returns nothing, but blocks until the lock has been acquired.
"""
return self.tiers[0].lock(layer, coord, format)
def unlock(self, layer, coord, format):
""" Release a cache lock for this tile in the first tier.
"""
return self.tiers[0].unlock(layer, coord, format)
def remove(self, layer, coord, format):
""" Remove a cached tile from every tier.
"""
for (index, cache) in enumerate(self.tiers):
cache.remove(layer, coord, format)
def read(self, layer, coord, format):
""" Read a cached tile.
Start at the first tier and work forwards until a cached tile
is found. When found, save it back to the earlier tiers for faster
access on future requests.
"""
for (index, cache) in enumerate(self.tiers):
body = cache.read(layer, coord, format)
if body:
# save the body in earlier tiers for speedier access
for cache in self.tiers[:index]:
cache.save(body, layer, coord, format)
return body
return None
def save(self, body, layer, coord, format):
""" Save a cached tile.
Every tier gets a saved copy.
"""
for (index, cache) in enumerate(self.tiers):
cache.save(body, layer, coord, format)
|