Merge branch 'master' of git+ssh://dev.laptop.org/git/projects/datastore into version_prototype

Conflicts:

	src/olpc/datastore/backingstore.py

5 files changed, 172 insertions, 7 deletions

diff --git a/NEWS b/NEWS
index dcad2ee..d7dc243 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,10 @@
+Snapshot a111996299
+
+Snapshot 356b35a278
+
+* #2559 Fix copy of files on a removable device. (bcsaller)
+* Fix abiword documents indexing. (bcsaller)
+
 Snapshot 4a4283978f
 
 * More usb devices fixes. (bcsaller)
diff --git a/bin/test-inplace.py b/bin/test-inplace.py
new file mode 100755
index 0000000..2d0b1af
--- /dev/null
+++ b/bin/test-inplace.py
@@ -0,0 +1,106 @@
+#!/usr/bin/python
+
+import dbus
+import dbus.glib
+import os
+import time
+
+import tempfile
+
+def tmpData(data):
+    """Put data into a temporary file returning the filename """
+    fd, fn = tempfile.mkstemp()
+    os.write(fd, data)
+    os.close(fd)
+    return fn
+
+
+
+DS_DBUS_SERVICE = "org.laptop.sugar.DataStore"
+DS_DBUS_INTERFACE = "org.laptop.sugar.DataStore"
+DS_DBUS_PATH = "/org/laptop/sugar/DataStore"
+
+
+# clean up any old tests
+assert os.system('rm -rf /tmp/store1') == 0
+
+_bus = dbus.SessionBus()
+_data_store = dbus.Interface(_bus.get_object(DS_DBUS_SERVICE, DS_DBUS_PATH), DS_DBUS_INTERFACE)
+
+mount_id = _data_store.mount('inplace:/tmp/store1', dict(title='pen drive'))
+
+props = {'activity_id': dbus.String(u'461c7467f9ef6478b205a687579843fc36a98e7a'),
+         'title_set_by_user': '0', 
+         'ctime': '2007-07-28T11:57:57.909689',
+         'title': 'Google News',
+         'mtime': '2007-07-28T11:58:22.460331',
+         'keep': '0',
+         'icon-color': '#00EA11,#00588C',
+         'activity': 'org.laptop.WebActivity',
+         'mime_type': 'text/plain'}
+
+jsonData = '''{"history":[{"url":"http://www.google.com/","title":"Google"},
+              {"url":"http://news.google.com/nwshp?tab=wn","title":"Google News"}]}'''
+
+uid = _data_store.create(props, '')
+file_name = os.path.join('/tmp', str(time.time()))
+fn = tmpData(jsonData)
+
+_data_store.update(uid, props, fn)
+
+
+
+
+props = _data_store.get_properties(uid)
+file_name = _data_store.get_filename(uid)
+props['mountpoint'] = mount_id
+# suggest a filename
+props['filename'] = "history.json"
+uid2 = _data_store.create(props, file_name)
+
+# the file name related to the new copy.
+fn2 = _data_store.get_filename(uid2)
+
+assert fn2
+
+contents = open(fn2, 'r').read()
+assert contents == jsonData
+
+# Now unmount the store, remount it and read the file
+
+_data_store.unmount(mount_id)
+
+
+mount_id = _data_store.mount('inplace:/tmp/store1', dict(title='pen drive'))
+
+fn2 = _data_store.get_filename(uid2)
+
+assert fn2
+
+contents = open(fn2, 'r').read()
+assert contents == jsonData
+
+print "ALL GOOD"
+
+print "Trying with Abidoc"
+
+props = {'mountpoint' : mount_id,
+         'title' : 'Abidoc',
+
+         }
+
+uid = _data_store.create(props, '')
+# now update it with the document
+fn = os.path.abspath("tests/test.odt")
+abidoc = open(fn, 'r').read()
+
+
+props['filename'] =  'test.odt'
+_data_store.update(uid, props, fn)
+
+fn2 = _data_store.get_filename(uid)
+
+contents = open(fn2, 'r').read()
+assert contents == abidoc
+
+print "Abiword ok"
diff --git a/src/olpc/datastore/backingstore.py b/src/olpc/datastore/backingstore.py
index 87b1ef1..0d04b4a 100644
--- a/src/olpc/datastore/backingstore.py
+++ b/src/olpc/datastore/backingstore.py
@@ -353,7 +353,7 @@ class FileBackingStore(BackingStore):
             
         if replace is False and os.path.exists(path):
             raise KeyError("objects with path:%s for uid:%s exists" %(
-                            path, uid))
+                path, uid))
 
         bin_copy.bin_copy(filelike, path)
         
@@ -441,6 +441,7 @@ class InplaceFileBackingStore(FileBackingStore):
         # now map/update the existing data into the indexes
         # but do it async
         self.walker = threading.Thread(target=self._walk)
+        self._runWalker = True
         self.walker.setDaemon(True)
         self.walker.start()
 
@@ -460,6 +461,8 @@ class InplaceFileBackingStore(FileBackingStore):
                 
             
             for fn in filenames:
+                # give the thread a chance to exit
+                if not self._runWalker: break
                 # blacklist files
                 #   ignore conventionally hidden files
                 if fn.startswith("."): continue
@@ -534,7 +537,29 @@ class InplaceFileBackingStore(FileBackingStore):
         # the file would have already been changed inplace
         # don't touch it
         props['uid'] = uid
+
+        proposed_name = None
+        if filelike:
+            if isinstance(filelike, basestring):
+                # lets treat it as a filename
+                filelike = open(filelike, "r")
+            filelike.seek(0)
+            # usually with USB drives and the like the file we are
+            # indexing is already on it, however in the case of moving
+            # files to these devices we need to detect this case and
+            # place the file
+            proposed_name = props.get('filename', None)
+            if not proposed_name:
+                proposed_name = os.path.split(filelike.name)[1]
+            # record the name before qualifying it to the store
+            props['filename'] = proposed_name
+            proposed_name = os.path.join(self.uri, proposed_name)
+
         self.indexmanager.index(props, filelike)
+
+        if proposed_name:
+            self._writeContent(uid, filelike, replace=True, target=proposed_name)
+        
         
     def delete(self, uid):
         c = self.indexmanager.get(uid)
@@ -548,8 +573,9 @@ class InplaceFileBackingStore(FileBackingStore):
         
     def stop(self):
         if self.walker and self.walker.isAlive():
-            self.walker.join()
-        self.indexmanager.stop()
+            # XXX: just force the unmount, flush the index queue
+            self._runWalker = False
+        self.indexmanager.stop(force=True)
 
     def complete_indexing(self):
         if self.walker and self.walker.isAlive():
diff --git a/src/olpc/datastore/xapianindex.py b/src/olpc/datastore/xapianindex.py
index 4212ad3..ceb7d56 100644
--- a/src/olpc/datastore/xapianindex.py
+++ b/src/olpc/datastore/xapianindex.py
@@ -14,6 +14,7 @@ __license__  = 'The GNU Public License V2+'
 
 
 from Queue import Queue, Empty
+import gc
 import logging
 import os
 import re
@@ -104,11 +105,19 @@ class IndexManager(object):
         self.backingstore = backingstore
 
     
-    def stop(self):
-        self.stopIndexer()
+    def stop(self, force=False):
+        self.stopIndexer(force)
         self.write_index.close()
         self.read_index.close()
-
+        # XXX: work around for xapian not having close() this will
+        # change in the future in the meantime we delete the
+        # references to the indexers and then force the gc() to run
+        # which should inturn trigger the C++ destructor which forces
+        # the database shut.
+        self.write_index = None
+        self.read_index = None
+        gc.collect()
+        
     # Index thread management
     def startIndexer(self):
         self.indexer_running = True
@@ -120,6 +129,7 @@ class IndexManager(object):
         if not self.indexer_running: return 
         if not force: self.queue.join()
         self.indexer_running = False
+        # should terminate after the current task
         self.indexer.join()
 
     # flow control
diff --git a/tests/mountpoints.txt b/tests/mountpoints.txt
index 0f736be..9210906 100644
--- a/tests/mountpoints.txt
+++ b/tests/mountpoints.txt
@@ -173,6 +173,23 @@ The file was properly created in the expected place.
 
 >>> assert os.path.exists('/tmp/store3/one.txt')
 
+Now let's update that file.
+
+>>> fn = ds.get_filename(u1)
+>>> fp = open(fn, 'w')
+>>> print >>fp, "This is new content"
+>>> fp.close()
+
+>>> ds.update(pen_copy, props, fn)
+>>> ds.complete_indexing()
+
+and verify it worked.
+
+>>> result, count = ds.find(dict(query="new content"))
+>>> assert count == 1
+>>> assert result[0]['uid'] == pen_copy
+
+
 
 We also need to be sure that delete commands work on inplace
 mounts. We will delete the object from the datastore and then verify
@@ -198,5 +215,4 @@ verify that we can still remount the store.
 >>> mp = ds.mountpoints[mp3]
 >>> assert mp.descriptor()['title'] == 'Fake USB from broken'
 
-
 >>> ds.stop(); del ds