From 57e90f572b41d40f9601c50e2d7e63bc47a6279a Mon Sep 17 00:00:00 2001 From: Benjamin Saller Date: Sun, 26 Aug 2007 16:46:56 +0000 Subject: Merge branch 'master' of git+ssh://dev.laptop.org/git/projects/datastore into version_prototype Conflicts: src/olpc/datastore/backingstore.py --- diff --git a/NEWS b/NEWS index dcad2ee..d7dc243 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,10 @@ +Snapshot a111996299 + +Snapshot 356b35a278 + +* #2559 Fix copy of files on a removable device. (bcsaller) +* Fix abiword documents indexing. (bcsaller) + Snapshot 4a4283978f * More usb devices fixes. (bcsaller) diff --git a/bin/test-inplace.py b/bin/test-inplace.py new file mode 100755 index 0000000..2d0b1af --- /dev/null +++ b/bin/test-inplace.py @@ -0,0 +1,106 @@ +#!/usr/bin/python + +import dbus +import dbus.glib +import os +import time + +import tempfile + +def tmpData(data): + """Put data into a temporary file returning the filename """ + fd, fn = tempfile.mkstemp() + os.write(fd, data) + os.close(fd) + return fn + + + +DS_DBUS_SERVICE = "org.laptop.sugar.DataStore" +DS_DBUS_INTERFACE = "org.laptop.sugar.DataStore" +DS_DBUS_PATH = "/org/laptop/sugar/DataStore" + + +# clean up any old tests +assert os.system('rm -rf /tmp/store1') == 0 + +_bus = dbus.SessionBus() +_data_store = dbus.Interface(_bus.get_object(DS_DBUS_SERVICE, DS_DBUS_PATH), DS_DBUS_INTERFACE) + +mount_id = _data_store.mount('inplace:/tmp/store1', dict(title='pen drive')) + +props = {'activity_id': dbus.String(u'461c7467f9ef6478b205a687579843fc36a98e7a'), + 'title_set_by_user': '0', + 'ctime': '2007-07-28T11:57:57.909689', + 'title': 'Google News', + 'mtime': '2007-07-28T11:58:22.460331', + 'keep': '0', + 'icon-color': '#00EA11,#00588C', + 'activity': 'org.laptop.WebActivity', + 'mime_type': 'text/plain'} + +jsonData = '''{"history":[{"url":"http://www.google.com/","title":"Google"}, + {"url":"http://news.google.com/nwshp?tab=wn","title":"Google News"}]}''' + +uid = _data_store.create(props, '') +file_name = os.path.join('/tmp', str(time.time())) +fn = tmpData(jsonData) + +_data_store.update(uid, props, fn) + + + + +props = _data_store.get_properties(uid) +file_name = _data_store.get_filename(uid) +props['mountpoint'] = mount_id +# suggest a filename +props['filename'] = "history.json" +uid2 = _data_store.create(props, file_name) + +# the file name related to the new copy. +fn2 = _data_store.get_filename(uid2) + +assert fn2 + +contents = open(fn2, 'r').read() +assert contents == jsonData + +# Now unmount the store, remount it and read the file + +_data_store.unmount(mount_id) + + +mount_id = _data_store.mount('inplace:/tmp/store1', dict(title='pen drive')) + +fn2 = _data_store.get_filename(uid2) + +assert fn2 + +contents = open(fn2, 'r').read() +assert contents == jsonData + +print "ALL GOOD" + +print "Trying with Abidoc" + +props = {'mountpoint' : mount_id, + 'title' : 'Abidoc', + + } + +uid = _data_store.create(props, '') +# now update it with the document +fn = os.path.abspath("tests/test.odt") +abidoc = open(fn, 'r').read() + + +props['filename'] = 'test.odt' +_data_store.update(uid, props, fn) + +fn2 = _data_store.get_filename(uid) + +contents = open(fn2, 'r').read() +assert contents == abidoc + +print "Abiword ok" diff --git a/src/olpc/datastore/backingstore.py b/src/olpc/datastore/backingstore.py index 87b1ef1..0d04b4a 100644 --- a/src/olpc/datastore/backingstore.py +++ b/src/olpc/datastore/backingstore.py @@ -353,7 +353,7 @@ class FileBackingStore(BackingStore): if replace is False and os.path.exists(path): raise KeyError("objects with path:%s for uid:%s exists" %( - path, uid)) + path, uid)) bin_copy.bin_copy(filelike, path) @@ -441,6 +441,7 @@ class InplaceFileBackingStore(FileBackingStore): # now map/update the existing data into the indexes # but do it async self.walker = threading.Thread(target=self._walk) + self._runWalker = True self.walker.setDaemon(True) self.walker.start() @@ -460,6 +461,8 @@ class InplaceFileBackingStore(FileBackingStore): for fn in filenames: + # give the thread a chance to exit + if not self._runWalker: break # blacklist files # ignore conventionally hidden files if fn.startswith("."): continue @@ -534,7 +537,29 @@ class InplaceFileBackingStore(FileBackingStore): # the file would have already been changed inplace # don't touch it props['uid'] = uid + + proposed_name = None + if filelike: + if isinstance(filelike, basestring): + # lets treat it as a filename + filelike = open(filelike, "r") + filelike.seek(0) + # usually with USB drives and the like the file we are + # indexing is already on it, however in the case of moving + # files to these devices we need to detect this case and + # place the file + proposed_name = props.get('filename', None) + if not proposed_name: + proposed_name = os.path.split(filelike.name)[1] + # record the name before qualifying it to the store + props['filename'] = proposed_name + proposed_name = os.path.join(self.uri, proposed_name) + self.indexmanager.index(props, filelike) + + if proposed_name: + self._writeContent(uid, filelike, replace=True, target=proposed_name) + def delete(self, uid): c = self.indexmanager.get(uid) @@ -548,8 +573,9 @@ class InplaceFileBackingStore(FileBackingStore): def stop(self): if self.walker and self.walker.isAlive(): - self.walker.join() - self.indexmanager.stop() + # XXX: just force the unmount, flush the index queue + self._runWalker = False + self.indexmanager.stop(force=True) def complete_indexing(self): if self.walker and self.walker.isAlive(): diff --git a/src/olpc/datastore/xapianindex.py b/src/olpc/datastore/xapianindex.py index 4212ad3..ceb7d56 100644 --- a/src/olpc/datastore/xapianindex.py +++ b/src/olpc/datastore/xapianindex.py @@ -14,6 +14,7 @@ __license__ = 'The GNU Public License V2+' from Queue import Queue, Empty +import gc import logging import os import re @@ -104,11 +105,19 @@ class IndexManager(object): self.backingstore = backingstore - def stop(self): - self.stopIndexer() + def stop(self, force=False): + self.stopIndexer(force) self.write_index.close() self.read_index.close() - + # XXX: work around for xapian not having close() this will + # change in the future in the meantime we delete the + # references to the indexers and then force the gc() to run + # which should inturn trigger the C++ destructor which forces + # the database shut. + self.write_index = None + self.read_index = None + gc.collect() + # Index thread management def startIndexer(self): self.indexer_running = True @@ -120,6 +129,7 @@ class IndexManager(object): if not self.indexer_running: return if not force: self.queue.join() self.indexer_running = False + # should terminate after the current task self.indexer.join() # flow control diff --git a/tests/mountpoints.txt b/tests/mountpoints.txt index 0f736be..9210906 100644 --- a/tests/mountpoints.txt +++ b/tests/mountpoints.txt @@ -173,6 +173,23 @@ The file was properly created in the expected place. >>> assert os.path.exists('/tmp/store3/one.txt') +Now let's update that file. + +>>> fn = ds.get_filename(u1) +>>> fp = open(fn, 'w') +>>> print >>fp, "This is new content" +>>> fp.close() + +>>> ds.update(pen_copy, props, fn) +>>> ds.complete_indexing() + +and verify it worked. + +>>> result, count = ds.find(dict(query="new content")) +>>> assert count == 1 +>>> assert result[0]['uid'] == pen_copy + + We also need to be sure that delete commands work on inplace mounts. We will delete the object from the datastore and then verify @@ -198,5 +215,4 @@ verify that we can still remount the store. >>> mp = ds.mountpoints[mp3] >>> assert mp.descriptor()['title'] == 'Fake USB from broken' - >>> ds.stop(); del ds -- cgit v0.9.1