diff options
author | Benjamin Saller <bcsaller@objectrealms.net> | 2007-10-08 20:09:28 (GMT) |
---|---|---|
committer | Benjamin Saller <bcsaller@objectrealms.net> | 2007-10-08 20:09:28 (GMT) |
commit | 1838f586c6dfa0541dfac2a2d7c0033e59e089bd (patch) | |
tree | c6f84d4a9f0fed441acf2bf806fa48945e84aa06 | |
parent | b603f340b26c57bb93f35188036e474cc87a5a3b (diff) |
external props
-rw-r--r-- | secore/datastructures.py | 4 | ||||
-rw-r--r-- | src/olpc/datastore/backingstore.py | 21 | ||||
-rw-r--r-- | src/olpc/datastore/model.py | 35 | ||||
-rw-r--r-- | src/olpc/datastore/xapianindex.py | 15 |
4 files changed, 60 insertions, 15 deletions
diff --git a/secore/datastructures.py b/secore/datastructures.py index 6006739..588ac06 100644 --- a/secore/datastructures.py +++ b/secore/datastructures.py @@ -192,9 +192,11 @@ class ProcessedDocument(object): term = tl.skip_to('Q').term if len(term) == 0 or term[0] != 'Q': return None + self._id = term[1:] except StopIteration: + self._id = None return None - self._id = term[1:] + return self._id def _set_id(self, id): tl = self._doc.termlist() diff --git a/src/olpc/datastore/backingstore.py b/src/olpc/datastore/backingstore.py index 04ef20e..5b35b65 100644 --- a/src/olpc/datastore/backingstore.py +++ b/src/olpc/datastore/backingstore.py @@ -20,6 +20,7 @@ import subprocess import time import threading +import dbus import xapian from olpc.datastore.xapianindex import IndexManager @@ -556,8 +557,28 @@ class FileBackingStore(BackingStore): def get_uniquevaluesfor(self, propertyname): return self.indexmanager.get_uniquevaluesfor(propertyname) + + + def get_external_property(self, doc_id, key): + # external properties default to the following storage + # <repo>/key/uid which is the file containing the external + # data. its contents is returned by this call + # when missing or absent '' is returned + pfile = os.path.join(self.base, key, str(doc_id)) + if os.path.exists(pfile): v = open(pfile, 'r').read() + else: v = '' + return dbus.ByteArray(v) + def set_external_property(self, doc_id, key, value): + pdir = os.path.join(self.base, key) + if not os.path.exists(pdir): os.mkdir(pdir) + pfile = os.path.join(pdir, doc_id) + fp = open(pfile, 'w') + fp.write(value) + fp.close() + + def find(self, query, order_by=None, limit=None, offset=0): if not limit: limit = 4069 return self.indexmanager.search(query, start_index=offset, end_index=limit, order_by=order_by) diff --git a/src/olpc/datastore/model.py b/src/olpc/datastore/model.py index b141269..6eed2a5 100644 --- a/src/olpc/datastore/model.py +++ b/src/olpc/datastore/model.py @@ -194,23 +194,33 @@ class Content(object): if isinstance(result, list) and len(result) == 1: result = result[0] field = self._model.fields.get(key) - kind = propertyByKind(field[1]) - # Errors here usually property request for a missing field - return kind.from_xapian(result) - + if field[1] == "external": + return self.get_external_property(key) + else: + kind = propertyByKind(field[1]) + # Errors here usually property request for a missing field + return kind.from_xapian(result) + + def get_external_property(self, key): + return self._backingstore.get_external_property(self.id, key) @property def properties(self): d = {} - for k, v in self.data.iteritems(): + for k in self._model.fields: if k in EXCLUDED_PROPERTIES: continue - if isinstance(v, list) and len(v) == 1: - v = v[0] field = self._model.fields.get(k) if field: - kind = propertyByKind(field[1]) - v = kind.from_xapian(v) + if field[1] == "external": + v = self.get_external_property(k) + else: + v = self.data.get(k, _marker) + if v is _marker: continue + if isinstance(v, list) and len(v) == 1: + v = v[0] + kind = propertyByKind(field[1]) + v = kind.from_xapian(v) else: # do some generic property handling if v: v = str(v) @@ -344,6 +354,11 @@ registerPropertyType('date', dateenc, datedec, 'float', {'store' : True, from_xapain=decode_datetime) +registerPropertyType('external', noop, noop, 'string', {'external' : True, + 'store' : False, + 'exact' : False, + 'fulltext' : False, + }) defaultModel = Model().addFields( ('fulltext', 'text'), @@ -374,7 +389,7 @@ defaultModel = Model().addFields( ('title_set_by_user', 'text'), ('keep', 'int'), ('icon-color', 'string'), - ('preview', 'binary'), + ('preview', 'external'), ('buddies', 'text'), ) diff --git a/src/olpc/datastore/xapianindex.py b/src/olpc/datastore/xapianindex.py index 8a5970f..c1217d5 100644 --- a/src/olpc/datastore/xapianindex.py +++ b/src/olpc/datastore/xapianindex.py @@ -258,7 +258,8 @@ class IndexManager(object): type='string', collapse=False, **kwargs): language = kwargs.pop('language', self.language) - + external = kwargs.pop('external', False) + xi = self.write_index.add_field_action if store: xi(key, secore.FieldActions.STORE_CONTENT) @@ -348,9 +349,15 @@ class IndexManager(object): warnings.warn("""Missing field configuration for %s""" % k, RuntimeWarning) continue - - add(secore.Field(k, value)) - + + # XXX: wrap this in a proper API + if self.datamodel.fields[k][1] == "external": + # XXX: this is done directly inline and could block, + # but the expected datasize is still small + self.backingstore.set_external_property(uid, k, value) + else: + add(secore.Field(k, value)) + # queue the document for processing self.enque(uid, vid, doc, operation, filestuff) |