/usr/share/pyshared/xappy/fieldactions.py is in python-xappy 0.5-5.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 | #!/usr/bin/env python
#
# Copyright (C) 2007 Lemur Consulting Ltd
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
r"""fieldactions.py: Definitions and implementations of field actions.
"""
__docformat__ = "restructuredtext en"
import _checkxapian
import errors
import marshall
from replaylog import log
import xapian
import parsedate
def _act_store_content(fieldname, doc, value, context):
"""Perform the STORE_CONTENT action.
"""
try:
fielddata = doc.data[fieldname]
except KeyError:
fielddata = []
doc.data[fieldname] = fielddata
fielddata.append(value)
def _act_index_exact(fieldname, doc, value, context):
"""Perform the INDEX_EXACT action.
"""
doc.add_term(fieldname, value, 0)
def _act_tag(fieldname, doc, value, context):
"""Perform the TAG action.
"""
doc.add_term(fieldname, value.lower(), 0)
def _act_facet(fieldname, doc, value, context, type=None):
"""Perform the FACET action.
"""
if type is None or type == 'string':
value = value.lower()
doc.add_term(fieldname, value, 0)
serialiser = log(xapian.StringListSerialiser,
doc.get_value(fieldname, 'facet'))
serialiser.append(value)
doc.add_value(fieldname, serialiser.get(), 'facet')
else:
marshaller = SortableMarshaller()
fn = marshaller.get_marshall_function(fieldname, type)
doc.add_value(fieldname, fn(fieldname, value), 'facet')
def _act_index_freetext(fieldname, doc, value, context, weight=1,
language=None, stop=None, spell=False,
nopos=False,
allow_field_specific=True,
search_by_default=True):
"""Perform the INDEX_FREETEXT action.
"""
termgen = log(xapian.TermGenerator)
if language is not None:
termgen.set_stemmer(log(xapian.Stem, language))
if stop is not None:
stopper = log(xapian.SimpleStopper)
for term in stop:
stopper.add (term)
termgen.set_stopper (stopper)
if spell:
termgen.set_database(context.index)
termgen.set_flags(termgen.FLAG_SPELLING)
termgen.set_document(doc._doc)
if search_by_default:
termgen.set_termpos(context.current_position)
# Store a copy of the field without a prefix, for non-field-specific
# searches.
if nopos:
termgen.index_text_without_positions(value, weight, '')
else:
termgen.index_text(value, weight, '')
if allow_field_specific:
# Store a second copy of the term with a prefix, for field-specific
# searches.
prefix = doc._fieldmappings.get_prefix(fieldname)
if len(prefix) != 0:
termgen.set_termpos(context.current_position)
if nopos:
termgen.index_text_without_positions(value, weight, prefix)
else:
termgen.index_text(value, weight, prefix)
# Add a gap between each field instance, so that phrase searches don't
# match across instances.
termgen.increase_termpos(10)
context.current_position = termgen.get_termpos()
class SortableMarshaller(object):
"""Implementation of marshalling for sortable values.
"""
def __init__(self, indexing=True):
if indexing:
self._err = errors.IndexerError
else:
self._err = errors.SearchError
def marshall_string(self, fieldname, value):
"""Marshall a value for sorting in lexicograpical order.
This returns the input as the output, since strings already sort in
lexicographical order.
"""
return value
def marshall_float(self, fieldname, value):
"""Marshall a value for sorting as a floating point value.
"""
# convert the value to a float
try:
value = float(value)
except ValueError:
raise self._err("Value supplied to field %r must be a "
"valid floating point number: was %r" %
(fieldname, value))
return marshall.float_to_string(value)
def marshall_date(self, fieldname, value):
"""Marshall a value for sorting as a date.
"""
try:
value = parsedate.date_from_string(value)
except ValueError, e:
raise self._err("Value supplied to field %r must be a "
"valid date: was %r: error is '%s'" %
(fieldname, value, str(e)))
return marshall.date_to_string(value)
def get_marshall_function(self, fieldname, sorttype):
"""Get a function used to marshall values of a given sorttype.
"""
try:
return {
None: self.marshall_string,
'string': self.marshall_string,
'float': self.marshall_float,
'date': self.marshall_date,
}[sorttype]
except KeyError:
raise self._err("Unknown sort type %r for field %r" %
(sorttype, fieldname))
def _act_sort_and_collapse(fieldname, doc, value, context, type=None):
"""Perform the SORTABLE action.
"""
marshaller = SortableMarshaller()
fn = marshaller.get_marshall_function(fieldname, type)
value = fn(fieldname, value)
doc.add_value(fieldname, value, 'collsort')
class ActionContext(object):
"""The context in which an action is performed.
This is just used to pass term generators, word positions, and the like
around.
"""
def __init__(self, index):
self.current_language = None
self.current_position = 0
self.index = index
class FieldActions(object):
"""An object describing the actions to be performed on a field.
The supported actions are:
- `STORE_CONTENT`: store the unprocessed content of the field in the search
engine database. All fields which need to be displayed or used when
displaying the search results need to be given this action.
- `INDEX_EXACT`: index the exact content of the field as a single search
term. Fields whose contents need to be searchable as an "exact match"
need to be given this action.
- `INDEX_FREETEXT`: index the content of this field as text. The content
will be split into terms, allowing free text searching of the field. Four
optional parameters may be supplied:
- 'weight' is a multiplier to apply to the importance of the field. This
must be an integer, and the default value is 1.
- 'language' is the language to use when processing the field. This can
be expressed as an ISO 2-letter language code. The supported languages
are those supported by the xapian core in use.
- 'stop' is an iterable of stopwords to filter out of the generated
terms. Note that due to Xapian design, only non-positional terms are
affected, so this is of limited use.
- 'spell' is a boolean flag - if true, the contents of the field will be
used for spelling correction.
- 'nopos' is a boolean flag - if true, positional information is not
stored.
- 'allow_field_specific' is a boolean flag - if False, prevents terms with the field
prefix being generated. This means that searches specific to this
field will not work, and thus should only be used when only non-field
specific searches are desired. Defaults to True.
- 'search_by_default' is a boolean flag - if False, the field will not be
searched by non-field specific searches. If True, or omitted, the
field will be included in searches for non field-specific searches.
- `SORTABLE`: index the content of the field such that it can be used to
sort result sets. It also allows result sets to be restricted to those
documents with a field values in a given range. One optional parameter
may be supplied:
- 'type' is a value indicating how to sort the field. It has several
possible values:
- 'string' - sort in lexicographic (ie, alphabetical) order.
This is the default, used if no type is set.
- 'float' - treat the values as (decimal representations of) floating
point numbers, and sort in numerical order. The values in the field
must be valid floating point numbers (according to Python's float()
function).
- 'date' - sort in date order. The values must be valid dates (either
Python datetime.date objects, or ISO 8601 format (ie, YYYYMMDD or
YYYY-MM-DD).
- `COLLAPSE`: index the content of the field such that it can be used to
"collapse" result sets, such that only the highest result with each value
of the field will be returned.
- `TAG`: the field contains tags; these are strings, which will be matched
in a case insensitive way, but otherwise must be exact matches. Tag
fields can be searched for by making an explict query (ie, using
query_field(), but not with query_parse()). A list of the most frequent
tags in a result set can also be accessed easily.
- `FACET`: the field represents a classification facet; these are strings
which will be matched exactly, but a list of all the facets present in
the result set can also be accessed easily - in addition, a suitable
subset of the facets, and a selection of the facet values, present in the
result set can be calculated. One optional parameter may be supplied:
- 'type' is a value indicating the type of facet contained in the field:
- 'string' - the facet values are exact binary strings.
- 'float' - the facet values are floating point numbers.
"""
# See the class docstring for the meanings of the following constants.
STORE_CONTENT = 1
INDEX_EXACT = 2
INDEX_FREETEXT = 3
SORTABLE = 4
COLLAPSE = 5
TAG = 6
FACET = 7
# Sorting and collapsing store the data in a value, but the format depends
# on the sort type. Easiest way to implement is to treat them as the same
# action.
SORT_AND_COLLAPSE = -1
_unsupported_actions = []
if 'tags' in _checkxapian.missing_features:
_unsupported_actions.append(TAG)
if 'facets' in _checkxapian.missing_features:
_unsupported_actions.append(FACET)
def __init__(self, fieldname):
# Dictionary of actions, keyed by type.
self._actions = {}
self._fieldname = fieldname
def add(self, field_mappings, action, **kwargs):
"""Add an action to perform on a field.
"""
if action in self._unsupported_actions:
raise errors.IndexerError("Action unsupported with this release of xapian")
if action not in (FieldActions.STORE_CONTENT,
FieldActions.INDEX_EXACT,
FieldActions.INDEX_FREETEXT,
FieldActions.SORTABLE,
FieldActions.COLLAPSE,
FieldActions.TAG,
FieldActions.FACET,
):
raise errors.IndexerError("Unknown field action: %r" % action)
info = self._action_info[action]
# Check parameter names
for key in kwargs.keys():
if key not in info[1]:
raise errors.IndexerError("Unknown parameter name for action %r: %r" % (info[0], key))
# Fields cannot be indexed both with "EXACT" and "FREETEXT": whilst we
# could implement this, the query parser wouldn't know what to do with
# searches.
if action == FieldActions.INDEX_EXACT:
if FieldActions.INDEX_FREETEXT in self._actions:
raise errors.IndexerError("Field %r is already marked for indexing "
"as free text: cannot mark for indexing "
"as exact text as well" % self._fieldname)
if action == FieldActions.INDEX_FREETEXT:
if FieldActions.INDEX_EXACT in self._actions:
raise errors.IndexerError("Field %r is already marked for indexing "
"as exact text: cannot mark for indexing "
"as free text as well" % self._fieldname)
# Fields cannot be indexed as more than one type for "SORTABLE": to
# implement this, we'd need to use a different prefix for each sortable
# type, but even then the search end wouldn't know what to sort on when
# searching. Also, if they're indexed as "COLLAPSE", the value must be
# stored in the right format for the type "SORTABLE".
if action == FieldActions.SORTABLE or action == FieldActions.COLLAPSE:
if action == FieldActions.COLLAPSE:
sorttype = None
else:
try:
sorttype = kwargs['type']
except KeyError:
sorttype = 'string'
kwargs['type'] = sorttype
action = FieldActions.SORT_AND_COLLAPSE
try:
oldsortactions = self._actions[FieldActions.SORT_AND_COLLAPSE]
except KeyError:
oldsortactions = ()
if len(oldsortactions) > 0:
for oldsortaction in oldsortactions:
oldsorttype = oldsortaction['type']
if sorttype == oldsorttype or oldsorttype is None:
# Use new type
self._actions[action] = []
elif sorttype is None:
# Use old type
return
else:
raise errors.IndexerError("Field %r is already marked for "
"sorting, with a different "
"sort type" % self._fieldname)
if 'prefix' in info[3]:
field_mappings.add_prefix(self._fieldname)
if 'slot' in info[3]:
purposes = info[3]['slot']
if isinstance(purposes, basestring):
field_mappings.add_slot(self._fieldname, purposes)
else:
slotnum = None
for purpose in purposes:
slotnum = field_mappings.get_slot(self._fieldname, purpose)
if slotnum is not None:
break
for purpose in purposes:
field_mappings.add_slot(self._fieldname, purpose, slotnum=slotnum)
# Make an entry for the action
if action not in self._actions:
self._actions[action] = []
# Check for repetitions of actions
for old_action in self._actions[action]:
if old_action == kwargs:
return
# Append the action to the list of actions
self._actions[action].append(kwargs)
def perform(self, doc, value, context):
"""Perform the actions on the field.
- `doc` is a ProcessedDocument to store the result of the actions in.
- `value` is a string holding the value of the field.
- `context` is an ActionContext object used to keep state in.
"""
for type, actionlist in self._actions.iteritems():
info = self._action_info[type]
for kwargs in actionlist:
info[2](self._fieldname, doc, value, context, **kwargs)
_action_info = {
STORE_CONTENT: ('STORE_CONTENT', (), _act_store_content, {}, ),
INDEX_EXACT: ('INDEX_EXACT', (), _act_index_exact, {'prefix': True}, ),
INDEX_FREETEXT: ('INDEX_FREETEXT', ('weight', 'language', 'stop', 'spell', 'nopos', 'allow_field_specific', 'search_by_default', ),
_act_index_freetext, {'prefix': True, }, ),
SORTABLE: ('SORTABLE', ('type', ), None, {'slot': 'collsort',}, ),
COLLAPSE: ('COLLAPSE', (), None, {'slot': 'collsort',}, ),
TAG: ('TAG', (), _act_tag, {'prefix': True,}, ),
FACET: ('FACET', ('type', ), _act_facet, {'prefix': True, 'slot': 'facet',}, ),
SORT_AND_COLLAPSE: ('SORT_AND_COLLAPSE', ('type', ), _act_sort_and_collapse, {'slot': 'collsort',}, ),
}
if __name__ == '__main__':
import doctest, sys
doctest.testmod (sys.modules[__name__])
|