external props

4 files changed, 60 insertions, 15 deletions

diff --git a/secore/datastructures.py b/secore/datastructures.py
index 6006739..588ac06 100644
--- a/secore/datastructures.py
+++ b/secore/datastructures.py
@@ -192,9 +192,11 @@ class ProcessedDocument(object):
                 term = tl.skip_to('Q').term
                 if len(term) == 0 or term[0] != 'Q':
                     return None
+                self._id = term[1:]
             except StopIteration:
+                self._id = None
                 return None
-            self._id = term[1:]
+
         return self._id
     def _set_id(self, id):
         tl = self._doc.termlist()
diff --git a/src/olpc/datastore/backingstore.py b/src/olpc/datastore/backingstore.py
index 04ef20e..5b35b65 100644
--- a/src/olpc/datastore/backingstore.py
+++ b/src/olpc/datastore/backingstore.py
@@ -20,6 +20,7 @@ import subprocess
 import time
 import threading
 
+import dbus
 import xapian
 
 from olpc.datastore.xapianindex import IndexManager
@@ -556,8 +557,28 @@ class FileBackingStore(BackingStore):
         
     def get_uniquevaluesfor(self, propertyname):
         return self.indexmanager.get_uniquevaluesfor(propertyname)
+
+
+    def get_external_property(self, doc_id, key):
+        # external properties default to the following storage
+        # <repo>/key/uid which is the file containing the external
+        # data. its contents is returned by this call
+        # when missing or absent '' is returned
+        pfile = os.path.join(self.base, key, str(doc_id))
+        if os.path.exists(pfile): v = open(pfile, 'r').read()
+        else: v = ''
+        return dbus.ByteArray(v)
     
 
+    def set_external_property(self, doc_id, key, value):
+        pdir = os.path.join(self.base, key)
+        if not os.path.exists(pdir): os.mkdir(pdir)
+        pfile = os.path.join(pdir, doc_id)
+        fp = open(pfile, 'w')
+        fp.write(value)
+        fp.close()
+        
+        
     def find(self, query, order_by=None, limit=None, offset=0):
         if not limit: limit = 4069
         return self.indexmanager.search(query, start_index=offset, end_index=limit, order_by=order_by)
diff --git a/src/olpc/datastore/model.py b/src/olpc/datastore/model.py
index b141269..6eed2a5 100644
--- a/src/olpc/datastore/model.py
+++ b/src/olpc/datastore/model.py
@@ -194,23 +194,33 @@ class Content(object):
         if isinstance(result, list) and len(result) == 1:
             result = result[0]
         field = self._model.fields.get(key)
-        kind = propertyByKind(field[1])
-        # Errors here usually property request for a missing field
-        return kind.from_xapian(result)
-    
+        if field[1] == "external":
+            return self.get_external_property(key)
+        else:
+            kind = propertyByKind(field[1])
+            # Errors here usually property request for a missing field
+            return kind.from_xapian(result)
+
+    def get_external_property(self, key):
+        return self._backingstore.get_external_property(self.id, key)
         
     @property
     def properties(self):
         d = {}
-        for k, v in self.data.iteritems():
+        for k in self._model.fields:
             if k in EXCLUDED_PROPERTIES: continue
             
-            if isinstance(v, list) and len(v) == 1:
-                v = v[0]
             field = self._model.fields.get(k)
             if field:
-                kind = propertyByKind(field[1])
-                v = kind.from_xapian(v)
+                if field[1] == "external":
+                    v = self.get_external_property(k)
+                else:
+                    v = self.data.get(k, _marker)
+                    if v is _marker: continue
+                    if isinstance(v, list) and len(v) == 1:
+                        v = v[0]
+                    kind = propertyByKind(field[1])
+                    v = kind.from_xapian(v)
             else:
                 # do some generic property handling
                 if v: v = str(v)
@@ -344,6 +354,11 @@ registerPropertyType('date', dateenc, datedec, 'float', {'store' : True,
                      from_xapain=decode_datetime)
 
 
+registerPropertyType('external', noop, noop, 'string', {'external' : True,
+                                                        'store' : False,
+                                                        'exact' : False,
+                                                        'fulltext' : False,
+                                                        })
 
 defaultModel = Model().addFields(    
     ('fulltext', 'text'),
@@ -374,7 +389,7 @@ defaultModel = Model().addFields(
     ('title_set_by_user', 'text'),
     ('keep', 'int'),
     ('icon-color', 'string'),
-    ('preview', 'binary'),
+    ('preview', 'external'),
     ('buddies', 'text'),
     )
 
diff --git a/src/olpc/datastore/xapianindex.py b/src/olpc/datastore/xapianindex.py
index 8a5970f..c1217d5 100644
--- a/src/olpc/datastore/xapianindex.py
+++ b/src/olpc/datastore/xapianindex.py
@@ -258,7 +258,8 @@ class IndexManager(object):
                  type='string', collapse=False,
                  **kwargs):
         language = kwargs.pop('language', self.language)
-        
+        external = kwargs.pop('external', False)
+
         xi = self.write_index.add_field_action
         
         if store: xi(key, secore.FieldActions.STORE_CONTENT)
@@ -348,9 +349,15 @@ class IndexManager(object):
                 warnings.warn("""Missing field configuration for %s""" % k,
                               RuntimeWarning)
                 continue
-            
-            add(secore.Field(k, value))
-            
+
+            # XXX: wrap this in a proper API
+            if self.datamodel.fields[k][1] == "external":
+                # XXX: this is done directly inline and could block,
+                # but the expected datasize is still small
+                self.backingstore.set_external_property(uid, k, value)
+            else:
+                add(secore.Field(k, value))
+                
         # queue the document for processing
         self.enque(uid, vid, doc, operation, filestuff)