/usr/share/pyshared/scrapy/contrib/memdebug.py is in python-scrapy 0.14.4-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | """
MemoryDebugger extension
See documentation in docs/topics/extensions.rst
"""
import gc
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.conf import settings
from scrapy.stats import stats
from scrapy.utils.trackref import live_refs
class MemoryDebugger(object):
def __init__(self):
try:
import libxml2
self.libxml2 = libxml2
except ImportError:
self.libxml2 = None
if not settings.getbool('MEMDEBUG_ENABLED'):
raise NotConfigured
dispatcher.connect(self.engine_started, signals.engine_started)
dispatcher.connect(self.engine_stopped, signals.engine_stopped)
def engine_started(self):
if self.libxml2:
self.libxml2.debugMemory(1)
def engine_stopped(self):
if self.libxml2:
self.libxml2.cleanupParser()
stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1))
gc.collect()
stats.set_value('memdebug/gc_garbage_count', len(gc.garbage))
if settings.getbool('TRACK_REFS'):
for cls, wdict in live_refs.iteritems():
if not wdict:
continue
stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict))
|