#!/usr/bin/env python # # Copyright (C) 2007 Lemur Consulting Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. r"""fieldactions.py: Definitions and implementations of field actions. """ __docformat__ = "restructuredtext en" import errors as _errors import marshall as _marshall import xapian as _xapian import parsedate as _parsedate def _act_store_content(fieldname, doc, value, context): """Perform the STORE_CONTENT action. """ try: fielddata = doc.data[fieldname] except KeyError: fielddata = [] doc.data[fieldname] = fielddata fielddata.append(value) def _act_index_exact(fieldname, doc, value, context): """Perform the INDEX_EXACT action. """ doc.add_term(fieldname, value, 0) def _act_tag(fieldname, doc, value, context): """Perform the TAG action. """ doc.add_term(fieldname, value.lower(), 0) def _act_facet(fieldname, doc, value, context, type=None): """Perform the FACET action. """ marshaller = SortableMarshaller() fn = marshaller.get_marshall_function(fieldname, type) doc.add_term(fieldname, value.lower(), 0) value = fn(fieldname, value) doc.add_value(fieldname, value) def _act_index_freetext(fieldname, doc, value, context, weight=1, language=None, stop=None, spell=False, nopos=False, noprefix=False): """Perform the INDEX_FREETEXT action. """ termgen = _xapian.TermGenerator() if language is not None: termgen.set_stemmer(_xapian.Stem(language)) if stop is not None: stopper = _xapian.SimpleStopper() for term in stop: stopper.add (term) termgen.set_stopper (stopper) if spell: termgen.set_database(context.index) termgen.set_flags(termgen.FLAG_SPELLING) termgen.set_document(doc._doc) termgen.set_termpos(context.current_position) if nopos: termgen.index_text_without_positions(value, weight, '') else: termgen.index_text(value, weight, '') if not noprefix: # Store a second copy of the term with a prefix, for field-specific # searches. prefix = doc._fieldmappings.get_prefix(fieldname) if len(prefix) != 0: termgen.set_termpos(context.current_position) if nopos: termgen.index_text_without_positions(value, weight, prefix) else: termgen.index_text(value, weight, prefix) # Add a gap between each field instance, so that phrase searches don't # match across instances. termgen.increase_termpos(10) context.current_position = termgen.get_termpos() class SortableMarshaller(object): """Implementation of marshalling for sortable values. """ def __init__(self, indexing=True): if indexing: self._err = _errors.IndexerError else: self._err = _errors.SearchError def marshall_string(self, fieldname, value): """Marshall a value for sorting in lexicograpical order. This returns the input as the output, since strings already sort in lexicographical order. """ return value def marshall_float(self, fieldname, value): """Marshall a value for sorting as a floating point value. """ # convert the value to a float try: value = float(value) except ValueError: raise self._err("Value supplied to field %r must be a " "valid floating point number: was %r" % (fieldname, value)) return _marshall.float_to_string(value) def marshall_date(self, fieldname, value): """Marshall a value for sorting as a date. """ try: value = _parsedate.date_from_string(value) except ValueError, e: raise self._err("Value supplied to field %r must be a " "valid date: was %r: error is '%s'" % (fieldname, value, str(e))) return _marshall.date_to_string(value) def get_marshall_function(self, fieldname, sorttype): """Get a function used to marshall values of a given sorttype. """ try: return { None: self.marshall_string, 'string': self.marshall_string, 'float': self.marshall_float, 'date': self.marshall_date, }[sorttype] except KeyError: raise self._err("Unknown sort type %r for field %r" % (sorttype, fieldname)) def _act_sort_and_collapse(fieldname, doc, value, context, type=None): """Perform the SORTABLE action. """ marshaller = SortableMarshaller() fn = marshaller.get_marshall_function(fieldname, type) value = fn(fieldname, value) doc.add_value(fieldname, value) class ActionContext(object): """The context in which an action is performed. This is just used to pass term generators, word positions, and the like around. """ def __init__(self, index): self.current_language = None self.current_position = 0 self.index = index class FieldActions(object): """An object describing the actions to be performed on a field. The supported actions are: - `STORE_CONTENT`: store the unprocessed content of the field in the search engine database. All fields which need to be displayed or used when displaying the search results need to be given this action. - `INDEX_EXACT`: index the exact content of the field as a single search term. Fields whose contents need to be searchable as an "exact match" need to be given this action. - `INDEX_FREETEXT`: index the content of this field as text. The content will be split into terms, allowing free text searching of the field. Four optional parameters may be supplied: - 'weight' is a multiplier to apply to the importance of the field. This must be an integer, and the default value is 1. - 'language' is the language to use when processing the field. This can be expressed as an ISO 2-letter language code. The supported languages are those supported by the xapian core in use. - 'stop' is an iterable of stopwords to filter out of the generated terms. Note that due to Xapian design, only non-positional terms are affected, so this is of limited use. - 'spell' is a boolean flag - if true, the contents of the field will be used for spelling correction. - 'nopos' is a boolean flag - if true, positional information is not stored. - 'noprefix' is a boolean flag - if true, prevents terms with the field prefix being generated. This means that searches specific to this field will not work, and thus should only be used for special cases. - `SORTABLE`: index the content of the field such that it can be used to sort result sets. It also allows result sets to be restricted to those documents with a field values in a given range. One optional parameter may be supplied: - 'type' is a value indicating how to sort the field. It has several possible values: - 'string' - sort in lexicographic (ie, alphabetical) order. This is the default, used if no type is set. - 'float' - treat the values as (decimal representations of) floating point numbers, and sort in numerical order. The values in the field must be valid floating point numbers (according to Python's float() function). - 'date' - sort in date order. The values must be valid dates (either Python datetime.date objects, or ISO 8601 format (ie, YYYYMMDD or YYYY-MM-DD). - `COLLAPSE`: index the content of the field such that it can be used to "collapse" result sets, such that only the highest result with each value of the field will be returned. - `TAG`: the field contains tags; these are strings, which will be matched in a case insensitive way, but otherwise must be exact matches. Tag fields can be searched for by making an explict query (ie, using query_field(), but not with query_parse()). A list of the most frequent tags in a result set can also be accessed easily. - `FACET`: the field represents a classification facet; these are strings which will be matched exactly, but a list of all the facets present in the result set can also be accessed easily - in addition, a suitable subset of the facets, and a selection of the facet values, present in the result set can be calculated. One optional parameter may be supplied: - 'type' is a value indicating the type of facet contained in the field: - 'string' - the facet values are exact binary strings. - 'float' - the facet values are floating point numbers. """ # See the class docstring for the meanings of the following constants. STORE_CONTENT = 1 INDEX_EXACT = 2 INDEX_FREETEXT = 3 SORTABLE = 4 COLLAPSE = 5 TAG = 6 FACET = 7 # Sorting and collapsing store the data in a value, but the format depends # on the sort type. Easiest way to implement is to treat them as the same # action. SORT_AND_COLLAPSE = -1 # NEED_SLOT is a flag used to indicate that an action needs a slot number NEED_SLOT = 1 # NEED_PREFIX is a flag used to indicate that an action needs a prefix NEED_PREFIX = 2 def __init__(self, fieldname): # Dictionary of actions, keyed by type. self._actions = {} self._fieldname = fieldname def add(self, field_mappings, action, **kwargs): """Add an action to perform on a field. """ if action not in (FieldActions.STORE_CONTENT, FieldActions.INDEX_EXACT, FieldActions.INDEX_FREETEXT, FieldActions.SORTABLE, FieldActions.COLLAPSE, FieldActions.TAG, FieldActions.FACET, ): raise _errors.IndexerError("Unknown field action: %r" % action) info = self._action_info[action] # Check parameter names for key in kwargs.keys(): if key not in info[1]: raise _errors.IndexerError("Unknown parameter name for action %r: %r" % (info[0], key)) # Fields cannot be indexed both with "EXACT" and "FREETEXT": whilst we # could implement this, the query parser wouldn't know what to do with # searches. if action == FieldActions.INDEX_EXACT: if FieldActions.INDEX_FREETEXT in self._actions: raise _errors.IndexerError("Field %r is already marked for indexing " "as free text: cannot mark for indexing " "as exact text as well" % self._fieldname) if action == FieldActions.INDEX_FREETEXT: if FieldActions.INDEX_EXACT in self._actions: raise _errors.IndexerError("Field %r is already marked for indexing " "as exact text: cannot mark for indexing " "as free text as well" % self._fieldname) # Fields cannot be indexed as more than one type for "SORTABLE": to # implement this, we'd need to use a different prefix for each sortable # type, but even then the search end wouldn't know what to sort on when # searching. Also, if they're indexed as "COLLAPSE", the value must be # stored in the right format for the type "SORTABLE". if action == FieldActions.SORTABLE or action == FieldActions.COLLAPSE: if action == FieldActions.COLLAPSE: sorttype = None else: try: sorttype = kwargs['type'] except KeyError: sorttype = 'string' kwargs['type'] = sorttype action = FieldActions.SORT_AND_COLLAPSE try: oldsortactions = self._actions[FieldActions.SORT_AND_COLLAPSE] except KeyError: oldsortactions = () if len(oldsortactions) > 0: for oldsortaction in oldsortactions: oldsorttype = oldsortaction['type'] if sorttype == oldsorttype or oldsorttype is None: # Use new type self._actions[action] = [] elif sorttype is None: # Use old type return else: raise _errors.IndexerError("Field %r is already marked for " "sorting, with a different " "sort type" % self._fieldname) if self.NEED_PREFIX in info[3]: field_mappings.add_prefix(self._fieldname) if self.NEED_SLOT in info[3]: field_mappings.add_slot(self._fieldname) # Make an entry for the action if action not in self._actions: self._actions[action] = [] # Check for repetitions of actions for old_action in self._actions[action]: if old_action == kwargs: return # Append the action to the list of actions self._actions[action].append(kwargs) def perform(self, doc, value, context): """Perform the actions on the field. - `doc` is a ProcessedDocument to store the result of the actions in. - `value` is a string holding the value of the field. - `context` is an ActionContext object used to keep state in. """ for type, actionlist in self._actions.iteritems(): info = self._action_info[type] for kwargs in actionlist: info[2](self._fieldname, doc, value, context, **kwargs) _action_info = { STORE_CONTENT: ('STORE_CONTENT', (), _act_store_content, (), ), INDEX_EXACT: ('INDEX_EXACT', (), _act_index_exact, (NEED_PREFIX,), ), INDEX_FREETEXT: ('INDEX_FREETEXT', ('weight', 'language', 'stop', 'spell', 'nopos', 'noprefix', ), _act_index_freetext, (NEED_PREFIX, ), ), SORTABLE: ('SORTABLE', ('type', ), None, (NEED_SLOT,), ), COLLAPSE: ('COLLAPSE', (), None, (NEED_SLOT,), ), SORT_AND_COLLAPSE: ('SORT_AND_COLLAPSE', ('type', ), _act_sort_and_collapse, (NEED_SLOT,), ), TAG: ('TAG', (), _act_tag, (NEED_PREFIX,), ), FACET: ('FACET', ('type', ), _act_facet, (NEED_PREFIX, NEED_SLOT,), ), } if __name__ == '__main__': import doctest, sys doctest.testmod (sys.modules[__name__])