minor search changes pulled from command branch

3 files changed, 42 insertions, 7 deletions

diff --git a/src/olpc/datastore/converter.py b/src/olpc/datastore/converter.py
index 730613a..9f32a24 100644
--- a/src/olpc/datastore/converter.py
+++ b/src/olpc/datastore/converter.py
@@ -91,10 +91,15 @@ class subprocessconverter(object):
             # reading
             if os.path.exists(target):
                 os.unlink(target)
-    
+
 class noop(object):
     def verify(self): return True
     def __call__(self, filename):
+        return open(filename, 'r')
+
+class decoder(object):
+    def verify(self): return True
+    def __call__(self, filename):
         return codecs.open(filename, 'r', 'utf-8')
         
 class Converter(object):
diff --git a/src/olpc/datastore/hg_backingstore.py b/src/olpc/datastore/hg_backingstore.py
index 78bd175..831da6c 100644
--- a/src/olpc/datastore/hg_backingstore.py
+++ b/src/olpc/datastore/hg_backingstore.py
@@ -223,6 +223,10 @@ class HgBackingStore(FileBackingStore):
         if not self.repo:
             self.repo = FileRepo(self.base)
 
+
+    def tip(self, uid):
+        return self.repo.tip(uid)
+    
     # File Management API
     def create(self, props, filelike):
         # generate the uid ourselves. we do this so we can track the
@@ -306,7 +310,21 @@ class HgBackingStore(FileBackingStore):
     def checkin(self, props, filelike):
         """create or update the content object, creating a new
         version"""
-        uid = props.setdefault('uid', create_uid())
+        uid = props.get("uid")
+        if uid is None:
+            uid = create_uid()
+        else:
+            # is there an existing object with this uid?
+            # XXX: if there isn't it should it be an error?
+            r, count = self.indexmanager.get_by_uid_prop(uid, 'tip')
+            if count == 1:
+                c = r.next()
+                # copy the value forward
+                old_props = c.properties.copy()
+                old_props.update(props)
+                props = old_props
+                
+        props['uid'] = uid
         if filelike:
             message = props.setdefault('message', 'initial')
             parent = props.pop('parent', None)
diff --git a/src/olpc/datastore/xapianindex.py b/src/olpc/datastore/xapianindex.py
index f8a26b4..84b3ebd 100644
--- a/src/olpc/datastore/xapianindex.py
+++ b/src/olpc/datastore/xapianindex.py
@@ -156,7 +156,14 @@ class IndexManager(object):
             if not filestuff:
                 # In this case we are done
                 return
-
+        elif operation is DELETE:
+            # sync deletes
+            with self._write_lock:
+                self.write_index.delete(uid)
+                logger.info("deleted content %s" % (uid,))
+            self.flush()
+            return
+        
         self.queue.put((uid, vid, doc, operation, filestuff))
 
     def indexThread(self):
@@ -216,9 +223,11 @@ class IndexManager(object):
                                 
                             fp = converter(filename, mimetype=mimetype)
                             if fp:
-                                # read in at a fixed block size, try to
-                                # conserve memory. If this doesn't work
-                                # we can make doc.fields a generator
+                                # fixed size doesn't make sense, we
+                                # shouldn't be adding fulltext unless
+                                # it converted down to plain text in
+                                # the first place
+                                
                                 while True:
                                     chunk = fp.read(2048)
                                     if not chunk: break
@@ -397,6 +406,9 @@ class IndexManager(object):
         ri =  self.read_index
         q = ri.query_field('uid', uid)
         if rev:
+            if rev == "tip":
+                rev = self.backingstore.tip(uid)
+                
             q = ri.query_filter(q, ri.query_field('vid', str(rev)))
         results, count = self._search(q, 0, 1000)
         
@@ -465,7 +477,7 @@ class IndexManager(object):
 
         # map the result set to model.Content items
         return ContentMappingIter(results, self.backingstore, self.datamodel), count
-    
+
 
     def get_uniquevaluesfor(self, property):
         # XXX: this is very sketchy code