Merge remote branch 'refs/remotes/origin/t/testsuite' into HEAD

* refs/remotes/origin/t/testsuite: (43 commits)
  fix typo
  mention that secondary sort order is undefined; use to_native() for listing updated entries
  rename basic_api.txt to basic_api_v2.txt to allow for easier merging in t/versions
  fix typo in last commit
  fix typos
  fix sort order in ambiguous cases, fix obscure test suite breakage due to overlapping timestamp values
  test_massops.py:test_update_verify(): prevent file leak on failure, update description
  test_massops.py: test ordering of find() results (for all supported orders) and offset/limit (for default order)
  test_massops: verify update has changed metadata as well
  test_massops: factor out _filter_properties()
  add test for version 1 -> version 2 data store migration
  cosmetic change
  print test suite descriptions
  start a new data store instance for each group of tests
  start dbus-launch separately and use the PID of dbus-daemon to reap it and its children
  add option "-k" / "--keep" to keep temporary files for inspection
  fix uninstalled testing (dbus service file, dbus service binary)
  runalltests.py: add MALLOC_CHECK_ to environment whitelist
  runalltests.py: add MASSOPS_RUNS to environment whitelist
  ignore 'checksum' property until we're sure it's not supposed to be leaked
  ...

11 files changed, 844 insertions, 5 deletions

diff --git a/.topdeps b/.topdeps
new file mode 100644
index 0000000..9c9ac90
--- /dev/null
+++ b/.topdeps
@@ -0,0 +1 @@
+upstream/master
diff --git a/.topmsg b/.topmsg
new file mode 100644
index 0000000..76e731e
--- /dev/null
+++ b/.topmsg
@@ -0,0 +1,8 @@
+From: Sascha Silbe <sascha@silbe.org>
+Subject: [PATCH] add (minimal) test suite (#1438)
+
+Add a minimal test suite for the data store operating on public DBus API
+level. Checks all public API calls, including some simple performance
+measurements.
+
+Signed-off-by: Sascha Silbe <sascha@silbe.org>
diff --git a/Makefile.am b/Makefile.am
index bfebefe..d450f24 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,7 +1,3 @@
 ACLOCAL_AMFLAGS = -I m4
 
-SUBDIRS = bin etc src
-
-test:
-	@cd tests
-	$(MAKE) -C tests test
+SUBDIRS = bin etc src tests
diff --git a/src/carquinyol/indexstore.py b/src/carquinyol/indexstore.py
index dc721e7..80a1ade 100644
--- a/src/carquinyol/indexstore.py
+++ b/src/carquinyol/indexstore.py
@@ -301,8 +301,10 @@ class IndexStore(object):
 
         if order_by == '+timestamp':
             enquire.set_sort_by_value(_VALUE_TIMESTAMP, True)
+            enquire.set_docid_order(False)
         elif order_by == '-timestamp':
             enquire.set_sort_by_value(_VALUE_TIMESTAMP, False)
+            enquire.set_docid_order(True)
         elif order_by == '+title':
             enquire.set_sort_by_value(_VALUE_TITLE, True)
         elif order_by == '-title':
@@ -317,6 +319,7 @@ class IndexStore(object):
             enquire.set_sort_by_value(_VALUE_CREATION_TIME, False)
         else:
             logging.warning('Unsupported property for sorting: %s', order_by)
+            order_by = '+timestamp'
 
         query_result = enquire.get_mset(offset, limit, check_at_least)
         total_count = query_result.get_matches_estimated()
diff --git a/tests/.gitignore b/tests/.gitignore
new file mode 100644
index 0000000..2460008
--- /dev/null
+++ b/tests/.gitignore
@@ -0,0 +1 @@
+!Makefile
diff --git a/tests/Makefile b/tests/Makefile
new file mode 100644
index 0000000..5ef30e6
--- /dev/null
+++ b/tests/Makefile
@@ -0,0 +1,22 @@
+all:
+install:
+uninstall:
+
+check:
+	# bad hack to make Python extensions work from source directory
+	cp ../src/carquinyol/.libs/*.so ../src/carquinyol/
+	@./runalltests.py
+
+valgrind:
+	@echo "Profiling the process. Run kcachegrind on the output"
+	valgrind  --tool=callgrind --suppressions=valgrind-python.supp python runalltests.py
+
+distclean: clean
+clean:
+	@find . -name "*.pyc" -exec rm {} \;
+	@find . -name "*.pyo" -exec rm {} \;
+	@find . -name "*~" -exec rm {} \;
+	@find . -name "callgrind.out*" -exec rm {} \;
+
+tags:
+
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..5b3912c
--- /dev/null
+++ b/tests/__init__.py
@@ -0,0 +1 @@
+# testing package
diff --git a/tests/basic_api_v2.txt b/tests/basic_api_v2.txt
new file mode 100644
index 0000000..d9a38ac
--- /dev/null
+++ b/tests/basic_api_v2.txt
@@ -0,0 +1,124 @@
+>>> import os
+>>> import tempfile
+>>> import time
+
+Define some helper functions
+>>> def test_unique(items):
+...     return not [True for e in items if items.count(e) > 1]
+>>> def to_native(value):
+...     if isinstance(value, list):
+...         return [to_native(e) for e in value]
+...     elif isinstance(value, dict):
+...         return dict([(to_native(k), to_native(v)) for k, v in value.items()])
+...     elif isinstance(value, unicode):
+...         return unicode(value)
+...     elif isinstance(value, str):
+...         return str(value)
+...     return value
+
+
+Connect to datastore using DBus and wait for it to get ready:
+>>> import dbus
+>>> DS_DBUS_SERVICE = "org.laptop.sugar.DataStore"
+>>> DS_DBUS_INTERFACE = "org.laptop.sugar.DataStore"
+>>> DS_DBUS_PATH = "/org/laptop/sugar/DataStore"
+>>> bus = dbus.SessionBus()
+>>> ds = dbus.Interface(bus.get_object(DS_DBUS_SERVICE, DS_DBUS_PATH), DS_DBUS_INTERFACE)
+
+
+Make sure we're starting from an empty datastore:
+>>> assert ds.find({}, [], byte_arrays=True) == ([], 0)
+
+
+Create something to play with:
+>>> o1_uid = ds.create({'title': 'DS test object 1', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1'}, '', False)
+>>> assert isinstance(o1_uid, basestring)
+>>> o2_uid = ds.create({'title': 'DS test object 2', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest2'}, '', False)
+>>> assert isinstance(o2_uid, basestring)
+>>> o3_uid = ds.create({'title': 'DS test object 3', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest2'}, '', False)
+>>> assert isinstance(o3_uid, basestring)
+>>> assert test_unique([o1_uid, o2_uid, o3_uid])
+
+
+Check everything is there:
+>>> sorted(ds.find({}, ['title', 'activity'], byte_arrays=True)[0])
+[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 1', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest1', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest2', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 3', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest2', variant_level=1)}, signature=dbus.Signature('sv'))]
+>>> ds.get_filename(o1_uid, byte_arrays=True)
+dbus.String(u'')
+>>> ds.get_filename(o2_uid, byte_arrays=True)
+dbus.String(u'')
+>>> ds.get_filename(o3_uid, byte_arrays=True)
+dbus.String(u'')
+
+
+
+Test get_uniquevaluesfor().
+>>> sorted(ds.get_uniquevaluesfor('activity', {}))
+[dbus.String(u'org.sugarlabs.DataStoreTest1'), dbus.String(u'org.sugarlabs.DataStoreTest2')]
+
+
+Change some entries:
+>>> ds.update(o1_uid, {'title': 'DS test object 1 updated', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1', 'tags': 'foo'}, '', False)
+>>> ds.update(o2_uid, {'title': 'DS test object 2', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1', 'tags': 'bar baz'}, '', False)
+>>> ds.update(o3_uid, {'title': 'DS test object 2', 'mime_type': 'text/html', 'activity': 'org.sugarlabs.DataStoreTest3', 'timestamp': 10000}, '', False)
+>>> sorted(to_native(ds.find({}, ['title', 'activity'], byte_arrays=True)[0]))
+[{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}, {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'}, {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}]
+
+Retrieve metadata for a single entry, ignoring variable data:
+>>> d=dict(ds.get_properties(o3_uid, byte_arrays=True))
+>>> del d['uid'], d['timestamp']
+>>> d
+{dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/html', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest3', variant_level=1)}
+
+
+Find entries using "known" metadata:
+>>> sorted(ds.find({'mime_type': ['text/plain']}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0])
+[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest1', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/plain', variant_level=1), dbus.String(u'tags'): dbus.ByteArray('bar baz', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 1 updated', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest1', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/plain', variant_level=1), dbus.String(u'tags'): dbus.ByteArray('foo', variant_level=1)}, signature=dbus.Signature('sv'))]
+>>> sorted(ds.find({'mime_type': ['text/html']}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0])
+[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/html', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv'))]
+>>> sorted(ds.find({'uid': o3_uid}, ['title', 'activity', 'mime_type'], byte_arrays=True)[0])
+[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/html', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv'))]
+>>> sorted(ds.find({'timestamp': (9000, 11000)}, ['title', 'activity', 'mime_type'], byte_arrays=True)[0])
+[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/html', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv'))]
+
+Find entries using "unknown" metadata (=> returns all entries):
+>>> sorted(ds.find({'title': 'DS test object 2'}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0])
+[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/html', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest1', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/plain', variant_level=1), dbus.String(u'tags'): dbus.ByteArray('bar baz', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 1 updated', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest1', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/plain', variant_level=1), dbus.String(u'tags'): dbus.ByteArray('foo', variant_level=1)}, signature=dbus.Signature('sv'))]
+
+You can specify a (primary) sort order. Please note that the secondary sort order is undefined / implementation-dependent.
+>>> to_native(ds.find({'order_by': ['+title']}, ['title', 'activity'], byte_arrays=True)[0])
+[{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'}, {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}, {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}]
+>>> to_native(ds.find({'order_by': ['-title']}, ['title', 'activity'], byte_arrays=True)[0])
+[{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}, {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'}, {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}]
+
+Delete an entry:
+>>> ds.delete(o1_uid)
+>>> sorted(ds.find({}, ['title', 'activity'], byte_arrays=True)[0])
+[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest1', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.sugarlabs.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv'))]
+
+
+Create an entry with content:
+>>> dog_content = 'The quick brown dog jumped over the lazy fox.'
+>>> dog_props = {'title': 'dog/fox story', 'mime_type': 'text/plain'}
+>>> dog_file = tempfile.NamedTemporaryFile()
+>>> dog_file.write(dog_content)
+>>> dog_file.flush()
+>>> dog_uid = ds.create(dog_props, dog_file.name, False)
+
+Retrieve and verify the entry with content:
+>>> dog_retrieved = ds.get_filename(dog_uid)
+>>> assert(file(dog_retrieved).read() == dog_content)
+>>> os.remove(dog_retrieved)
+
+Update the entry content:
+>>> dog_content = 'The quick brown fox jumped over the lazy dog.'
+>>> dog_file.seek(0)
+>>> dog_file.write(dog_content)
+>>> dog_file.flush()
+>>> ds.update(dog_uid, dog_props, dog_file.name, False)
+
+Verify updated content:
+>>> dog_retrieved = ds.get_filename(dog_uid)
+>>> assert(file(dog_retrieved).read() == dog_content)
+>>> os.remove(dog_retrieved)
+>>> dog_file.close()
diff --git a/tests/runalltests.py b/tests/runalltests.py
new file mode 100755
index 0000000..f59200f
--- /dev/null
+++ b/tests/runalltests.py
@@ -0,0 +1,353 @@
+#!/usr/bin/env python
+"""Run all tests in the current directory.
+
+You can either call it without arguments to run all tests or name specific
+ones to run:
+
+  ./runalltests.py test_massops.py
+"""
+
+import doctest
+import errno
+import logging
+from optparse import OptionParser
+import os
+import os.path
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+
+import dbus
+import dbus.mainloop.glib
+import gobject
+
+
+logging.basicConfig(level=logging.WARN,
+    format="%(asctime)-15s %(name)s %(levelname)s: %(message)s",
+    stream=sys.stderr)
+
+
+DOCTESTS = [
+    "basic_api_v2.txt",
+]
+DOCTEST_OPTIONS = doctest.ELLIPSIS
+DOCTEST_OPTIONS |= doctest.REPORT_ONLY_FIRST_FAILURE
+
+DS_DBUS_SERVICE = "org.laptop.sugar.DataStore"
+DS_DBUS_INTERFACE = "org.laptop.sugar.DataStore"
+DS_DBUS_PATH = "/org/laptop/sugar/DataStore"
+
+ENVIRONMENT_WHITELIST = [
+    'LD_LIBRARY_PATH',
+    'MALLOC_CHECK_',
+    'MASSOPS_RUNS',
+    'SUGAR_LOGGER_LEVEL',
+]
+
+def setup():
+    """Prepare for testing and return environment.
+
+    Sets HOME and creates a new process group so we can clean up easily later.
+    Sets up environment variables and imports whitelisted ones.
+    """
+    environment = {}
+    for name in ENVIRONMENT_WHITELIST:
+        if name in os.environ:
+            environment[name] = os.environ[name]
+
+    environment['HOME'] = tempfile.mkdtemp(prefix='datastore-test')
+    if 'PYTHONPATH' in os.environ:
+        python_path = os.environ.get('PYTHONPATH').split(':')
+    else:
+        python_path = []
+
+    # Run tests on sources instead of on installed files.
+    basedir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), '..')
+    python_path = [os.path.join(basedir, 'src')] + python_path
+    environment['PYTHONPATH'] = ':'.join(python_path)
+    environment['PATH'] = os.path.join(basedir, 'bin')+':'+os.environ['PATH']
+
+    servicedir = os.path.join(environment['HOME'], 'dbus-1', 'services')
+    servicepath = os.path.join(servicedir, 'org.laptop.sugar.DataStore.service')
+    os.makedirs(servicedir)
+    servicefile = file(servicepath, 'w')
+    servicefile.write("""
+        [D-BUS Service]
+        Name = org.laptop.sugar.DataStore
+        Exec = %s/bin/datastore-service
+        """.replace('        ', '') % (basedir, ))
+    servicefile.close()
+    environment['XDG_DATA_DIRS'] = environment['HOME']
+
+    os.setpgid(0, 0)
+    # prevent suicide in cleanup()
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    return environment
+
+
+def wait_children():
+    """Wait for all children to exit."""
+    try:
+        while True:
+            os.wait()
+
+    except OSError, exception:
+        if exception.errno in (errno.ECHILD, errno.ESRCH):
+            # ECHILD is not documented in kill(2) and kill(2p), but used by
+            # Linux to indicate no child processes remaining to be waited for
+            return
+
+        raise
+
+
+def cleanup(home, keep_files, dbus_pid):
+    """Clean up test environment.
+
+    Kills all children and removes home directory.
+    """
+    if dbus_pid:
+        os.kill(-dbus_pid, signal.SIGTERM)
+
+    os.kill(0, signal.SIGTERM)
+    wait_children()
+
+    if not keep_files:
+        shutil.rmtree(home)
+
+
+class TestSuiteWrapper(unittest.TestCase):
+    """Wrap a test suite to clean up after it.
+
+    This ensures each test module gets a clean data store instance.
+    """
+
+    def __init__(self, suite):
+        self._wrapped_suite = suite
+        self._bus = dbus.SessionBus()
+        self._loop = None
+        unittest.TestCase.__init__(self)
+
+    def runTest(self, result=None):
+        self._wrapped_suite(result)
+
+    def run(self, result=None):
+        if result is None:
+            result = self.defaultTestResult()
+        result.startTest(self)
+        try:
+            try:
+                self.setUp()
+            except KeyboardInterrupt:
+                raise
+            except:
+                result.addError(self, self._exc_info())
+                return
+
+            ok = False
+            try:
+                self.runTest(result)
+                ok = True
+            except self.failureException:
+                result.addFailure(self, self._exc_info())
+            except KeyboardInterrupt:
+                raise
+            except:
+                result.addError(self, self._exc_info())
+
+            try:
+                self.tearDown()
+            except KeyboardInterrupt:
+                raise
+            except:
+                result.addError(self, self._exc_info())
+                ok = False
+            if ok:
+                result.addSuccess(self)
+        finally:
+            result.stopTest(self)
+
+    def shortDescription(self):
+        doc = self._wrapped_suite.__doc__
+        return doc and doc.split("\n")[0].strip() or None
+
+    def tearDown(self):
+        self._kill_data_store()
+        self._clean_data_store()
+
+    def _kill_data_store(self):
+        pgrep = subprocess.Popen(['pgrep', '-g', os.environ['DBUS_PID'],
+            '-f', 'datastore-service'],
+            close_fds=True, stdout=subprocess.PIPE)
+        stdout, stderr_ = pgrep.communicate()
+        pids = stdout.strip().split('\n')
+        if len(pids) != 1 or not pids[0]:
+            raise ValueError("Can't find (a single) data store process "
+                "(pgrep output %r)" % (stdout, ))
+
+        pid = int(pids[0])
+        self._loop = gobject.MainLoop()
+        self._bus.watch_name_owner(DS_DBUS_SERVICE, self._service_changed_cb)
+        os.kill(pid, signal.SIGTERM)
+        self._loop.run()
+
+    def _service_changed_cb(self, new_owner):
+        if not new_owner:
+            self._loop.quit()
+
+    def _clean_data_store(self):
+        profile = os.environ.get('SUGAR_PROFILE', 'default')
+        base_dir = os.path.join(os.path.expanduser('~'), '.sugar', profile)
+        root_path = os.path.join(base_dir, 'datastore')
+        shutil.rmtree(root_path)
+
+
+class TimedTestResult(unittest._TextTestResult):
+    """Store and display test results and test runtime.
+
+    Only displays actual tests, not test suite wrappers."""
+
+    # Depending on a private class is bad style, but the only alternative is
+    # copying it verbatim.
+    # pylint: disable-msg=W0212
+
+    def __init__(self, stream, descriptions, verbosity):
+        unittest._TextTestResult.__init__(self, stream, descriptions,
+            verbosity)
+        self.start_times = {}
+        self.run_times = {}
+
+    def startTest(self, test):
+        self.start_times[test] = time.time()
+        unittest.TestResult.startTest(self, test)
+        if not self.showAll:
+            return
+
+        description = self.getDescription(test)
+        if isinstance(test, TestSuiteWrapper):
+            self.stream.write('Test Suite: %s\n' % (description, ))
+        else:
+            self.stream.write('  %s ... ' % (description, ))
+
+    def stopTest(self, test):
+        if test in self.start_times and test not in self.run_times:
+            self.run_times[test] = time.time() - self.start_times[test]
+
+        unittest._TextTestResult.stopTest(self, test)
+
+    def addSuccess(self, test):
+        if test in self.start_times and test not in self.run_times:
+            self.run_times[test] = time.time() - self.start_times[test]
+
+        run_time = self.run_times.get(test, -1)
+
+        unittest.TestResult.addSuccess(self, test)
+
+        if isinstance(test, TestSuiteWrapper):
+            return
+
+        if self.showAll:
+            self.stream.writeln("ok (%.3fs)" % (run_time, ))
+        elif self.dots:
+            self.stream.write('.')
+
+
+class TimedTestRunner(unittest.TextTestRunner):
+    """Run tests, displaying test result and runtime in textual form."""
+
+    def _makeResult(self):
+        return TimedTestResult(self.stream, self.descriptions, self.verbosity)
+
+
+def test_suite(tests=None):
+    suite = unittest.TestSuite()
+    if not tests:
+        test_dir = os.path.dirname(__file__)
+        tests = DOCTESTS
+        tests += [name for name in os.listdir(test_dir)
+            if name.startswith('test') and name.endswith('.py')]
+
+    for test in tests:
+        if test.endswith('.txt'):
+            doc_suite = doctest.DocFileSuite(test, optionflags=DOCTEST_OPTIONS)
+            doc_suite.__doc__ = test
+            suite.addTest(TestSuiteWrapper(doc_suite))
+
+        elif test.endswith('.py'):
+            m = __import__(test[:-3])
+            if hasattr(m, 'suite'):
+                suite.addTest(TestSuiteWrapper(m.suite()))
+
+    return suite
+
+
+def run_tests(tests):
+    dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
+    runner = TimedTestRunner(verbosity=2)
+    suite = test_suite(tests)
+    result = runner.run(suite)
+    if result.wasSuccessful():
+        return 0
+    else:
+        return 10
+
+
+def _start_dbus(environment):
+    pipe = subprocess.Popen(['dbus-launch'], stdout=subprocess.PIPE,
+        close_fds=True, env=environment, cwd=environment['HOME'])
+    stdout, stderr_ = pipe.communicate()
+    pid = None
+    address = None
+    for line in stdout.strip().split('\n'):
+        key, value = line.split('=', 1)
+        if key == 'DBUS_SESSION_BUS_ADDRESS':
+            address = value
+        elif key == 'DBUS_SESSION_BUS_PID':
+            pid = int(value)
+        else:
+            raise ValueError('Cannot parse dbus-launch output: %r' % (line, ))
+
+    assert pid is not None
+    assert address is not None
+    return pid, address
+
+
+def _parse_options():
+    """Parse command line arguments."""
+    parser = OptionParser()
+    parser.add_option('-k', '--keep', dest='keep',
+        action='store_true', default=False,
+        help='Keep temporary files')
+    parser.add_option('', '--stage2', dest='stage2',
+        action='store_true', default=False,
+        help='For internal use only')
+    return parser.parse_args()
+
+
+def main(my_name, arguments):
+    options, tests = _parse_options()
+    if not options.stage2:
+        environment = setup()
+        dbus_pid = None
+        dbus_address = None
+        try:
+            dbus_pid, dbus_address = _start_dbus(environment)
+            environment['DBUS_SESSION_BUS_ADDRESS'] = dbus_address
+            environment['DBUS_PID'] = str(dbus_pid)
+
+            pipe = subprocess.Popen([os.path.abspath(my_name),
+                '--stage2']+arguments,
+                cwd=environment['HOME'], env=environment)
+            return pipe.wait()
+
+        finally:
+            cleanup(environment['HOME'], options.keep, dbus_pid)
+
+    return run_tests(tests)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[0], sys.argv[1:]))
diff --git a/tests/test_massops.py b/tests/test_massops.py
new file mode 100644
index 0000000..ddc5242
--- /dev/null
+++ b/tests/test_massops.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+"""Large number of operations intended for measuring performance."""
+
+import dbus
+import decorator
+import os
+import tempfile
+import time
+import unittest
+
+
+DS_DBUS_SERVICE = "org.laptop.sugar.DataStore"
+DS_DBUS_INTERFACE = "org.laptop.sugar.DataStore"
+DS_DBUS_PATH = "/org/laptop/sugar/DataStore"
+NUM_RUNS = int(os.environ.get('MASSOPS_RUNS', '100'))
+IGNORE_PROPERTIES = [
+    'checksum',
+    'number',
+    'timestamp',
+    'uid',
+]
+
+
+@decorator.decorator
+def repeat(func, *args, **kwargs):
+    """Run the decorated function NUM_RUNS times."""
+    for i_ in range(NUM_RUNS):
+        func(*args, **kwargs)
+
+
+class MassOpsTestCase(unittest.TestCase):
+    """Large number of operations intended for measuring performance."""
+
+    def setUp(self):
+        # pylint: disable-msg=C0103
+        self._bus = dbus.SessionBus()
+        self._datastore = dbus.Interface(self._bus.get_object(DS_DBUS_SERVICE,
+            DS_DBUS_PATH), DS_DBUS_INTERFACE)
+
+    _create_properties = {
+        'title': 'DS test object',
+        'mime_type': 'text/plain',
+        'activity': 'org.sugarlabs.DataStoreTest1',
+    }
+    _create_content = 'Foo bar\n'*1000
+
+    def test_create(self):
+        """Run create() lots of times to create new objects."""
+        for i in range(NUM_RUNS):
+            content_file = tempfile.NamedTemporaryFile()
+            content_file.write(self._create_content)
+            content_file.flush()
+            properties = self._create_properties.copy()
+            properties['number'] = str(i)
+            properties['timestamp'] = time.time()
+            self._datastore.create(properties, content_file.name, False)
+            content_file.close()
+
+    @repeat
+    def test_find_all(self):
+        """Run find() to list all entries."""
+        entries, total_count = self._datastore.find({}, ['number'],
+            byte_arrays=True)
+        self.assertEquals(total_count, NUM_RUNS)
+        self.assertEquals(total_count, len(entries))
+        for position, entry in enumerate(entries):
+            self.assertEquals(int(entry['number']), NUM_RUNS-position-1)
+
+    @repeat
+    def test_find_all_reverse_time(self):
+        """Run find() to list all entries in reverse chronological order."""
+        entries, total_count = self._datastore.find({'order_by':
+            ['-timestamp']}, ['number'], byte_arrays=True)
+        self.assertEquals(total_count, NUM_RUNS)
+        self.assertEquals(total_count, len(entries))
+        for position, entry in enumerate(entries):
+            self.assertEquals(int(entry['number']), position)
+
+    @repeat
+    def test_find_all_title(self):
+        """Run find() to list all entries ordered by title."""
+        entries, total_count = self._datastore.find({'order_by':
+            ['+title']}, ['tree_id'], byte_arrays=True)
+        self.assertEquals(total_count, NUM_RUNS)
+        self.assertEquals(total_count, len(entries))
+
+    @repeat
+    def test_find_all_reverse_title(self):
+        """Run find() to list all entries ordered by title (reversed)."""
+        entries, total_count = self._datastore.find({'order_by':
+            ['-title']}, ['tree_id'], byte_arrays=True)
+        self.assertEquals(total_count, NUM_RUNS)
+        self.assertEquals(total_count, len(entries))
+
+    @repeat
+    def test_find_all_chunked(self):
+        """Run find() to list all entries in small chunks."""
+        chunk_size = 30
+        for chunk_start in range(0, NUM_RUNS, 30):
+            entries, total_count = self._datastore.find({
+                'offset': chunk_start, 'limit': chunk_size},
+                ['number'], byte_arrays=True)
+            self.assertEquals(len(entries),
+                min(chunk_size, NUM_RUNS-chunk_start))
+            self.assertEquals(total_count, NUM_RUNS)
+            for position, entry in enumerate(entries):
+                self.assertEquals(int(entry['number']),
+                    NUM_RUNS-(chunk_start+position)-1)
+
+    def test_get_properties(self):
+        """Run get_properties() on all entries and verify result."""
+        for entry in self._datastore.find({}, ['uid'], byte_arrays=True)[0]:
+            properties = self._datastore.get_properties(entry['uid'],
+                byte_arrays=True)
+            self._filter_properties(properties)
+            self.assertEquals(properties, self._create_properties)
+
+    def test_get_filename(self):
+        """Run get_filename() on all entries and verify content."""
+        for entry in self._datastore.find({}, ['uid'], byte_arrays=True)[0]:
+            filename = self._datastore.get_filename(entry['uid'],
+                byte_arrays=True)
+            try:
+                self.assertEquals(file(filename).read(), self._create_content)
+            finally:
+                os.remove(filename)
+
+    _update_properties = {
+        'title': 'DS test object (updated)',
+        'mime_type': 'text/plain',
+        'activity': 'org.sugarlabs.DataStoreTest1',
+    }
+    _update_content = 'Foo bar baz\n'*1000
+
+    def test_update(self):
+        """Update the content of all existing entries"""
+        content_file = tempfile.NamedTemporaryFile()
+        content_file.write(self._update_content)
+        content_file.flush()
+        for entry in self._datastore.find({}, ['uid'], byte_arrays=True)[0]:
+            self._datastore.update(entry['uid'], self._update_properties,
+                content_file.name, False)
+
+    def test_update_verify(self):
+        """Verify test_update() has changed content and metadata of all entries."""
+        for entry in self._datastore.find({}, [], byte_arrays=True)[0]:
+            filename = self._datastore.get_filename(entry['uid'],
+                byte_arrays=True)
+            self._filter_properties(entry)
+            try:
+                self.assertEquals(entry, self._update_properties)
+                self.assertEquals(file(filename).read(), self._update_content)
+            finally:
+                os.remove(filename)
+
+    def _filter_properties(self, properties):
+        for key in IGNORE_PROPERTIES:
+            properties.pop(key, None)
+
+
+def suite():
+    test_suite = unittest.TestLoader().loadTestsFromTestCase(MassOpsTestCase)
+    test_suite.__doc__ = MassOpsTestCase.__doc__
+    return test_suite
diff --git a/tests/test_migration_v1_v2.py b/tests/test_migration_v1_v2.py
new file mode 100644
index 0000000..82c4cac
--- /dev/null
+++ b/tests/test_migration_v1_v2.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+"""Test datastore migration from version 1 to version 2."""
+
+import dbus
+import decorator
+import hashlib
+import os
+import tempfile
+import time
+import unittest
+import uuid
+
+
+DS_DBUS_SERVICE = "org.laptop.sugar.DataStore"
+DS_DBUS_INTERFACE = "org.laptop.sugar.DataStore"
+DS_DBUS_PATH = "/org/laptop/sugar/DataStore"
+IGNORE_PROPERTIES = [
+    'activity_id',
+    'checksum',
+    'ctime',
+    'mtime',
+    'number',
+    'timestamp',
+    'uid',
+]
+
+
+class MigrationV1V2TestCase(unittest.TestCase):
+    """Test datastore migration from version 1 to version 2."""
+
+    def __init__(self, *args, **kwargs):
+        unittest.TestCase.__init__(self, *args, **kwargs)
+        self._templates = self._v1_properties*10
+
+    def setUp(self):
+        # pylint: disable-msg=C0103
+        profile = os.environ.get('SUGAR_PROFILE', 'default')
+        base_dir = os.path.join(os.path.expanduser('~'), '.sugar', profile)
+        self._root_path = os.path.join(base_dir, 'datastore')
+        if not os.path.exists(self._root_path):
+            self._create_v1_datastore()
+
+        self._bus = dbus.SessionBus()
+        self._datastore = dbus.Interface(self._bus.get_object(DS_DBUS_SERVICE,
+            DS_DBUS_PATH), DS_DBUS_INTERFACE)
+
+    _v1_properties = [
+        {
+            'title': lambda number: 'DS test object %d' % (number, ),
+            'mime_type': 'text/plain',
+        },
+        {
+            'title': lambda number: 'DS test object %d' % (number, ),
+            'mime_type': 'text/html',
+        },
+        {
+            'title': lambda number: 'DS test object %d' % (number, ),
+            'title_set_by_user': '1',
+            'keep': '1',
+            'mime_type': 'text/html',
+            'activity': 'org.sugarlabs.DataStoreTest2',
+            'activity_id': lambda *args: str(uuid.uuid4()),
+            'timestamp': lambda *args: time.time(),
+            'icon-color': '#00ff00,#0000ff',
+            'buddies': '{}',
+            'description': 'DS migration test object',
+            'tags': lambda number: 'test tag%d' % (number, ),
+            'preview': dbus.ByteArray(''.join([chr(i) for i in range(255)])),
+        },
+        {
+            'title': lambda number: 'DS test object %d' % (number, ),
+            'activity': 'org.sugarlabs.DataStoreTest3',
+            'activity_id': lambda *args: str(uuid.uuid4()),
+            'ctime': lambda *args: time.strftime('%Y-%m-%dT%H:%M:%S'),
+        },
+        {
+            'title': lambda number: 'DS test object %d' % (number, ),
+            'activity': 'org.sugarlabs.DataStoreTest4',
+            'activity_id': lambda *args: str(uuid.uuid4()),
+            'mtime': lambda *args: time.strftime('%Y-%m-%dT%H:%M:%S'),
+        },
+        {},
+    ]
+    def _v1_content(self, num):
+        return ('Foo bar %d\n' % (num, ))*1000
+
+    def _create_v1_datastore(self):
+        """Create a version 1 datastore on disk."""
+        os.makedirs(self._root_path)
+        file(os.path.join(self._root_path, 'version'), 'w').write('1')
+        for i, template in enumerate(self._templates):
+            metadata = self._fill_template(template, i)
+            data = self._v1_content(i)
+            tree_id = str(uuid.uuid4())
+            metadata['uid'] = tree_id
+            metadata['number'] = i
+
+            self._create_v1_entry(tree_id, metadata, data)
+
+    def _fill_template(self, template, i):
+        metadata = {}
+        for (key, value) in template.items():
+            if callable(value):
+                value = value(i)
+
+            metadata[key] = value
+
+        return metadata
+
+    def _create_v1_entry(self, tree_id, metadata, data):
+        """Create a single version 1 datastore entry."""
+        checksum = hashlib.md5(data).hexdigest()
+        entry_dir = os.path.join(self._root_path, tree_id[:2], tree_id)
+        os.makedirs(entry_dir)
+        file(os.path.join(entry_dir, 'data'), 'w').write(data)
+        self._write_v1_metadata(os.path.join(entry_dir, 'metadata'), metadata)
+        checksum_dir = os.path.join(self._root_path, 'checksums', checksum)
+        os.makedirs(checksum_dir)
+        file(os.path.join(checksum_dir, tree_id), 'w').close()
+
+    def _write_v1_metadata(self, directory, metadata):
+        os.makedirs(directory)
+        for key, value in metadata.items():
+            file(os.path.join(directory, key), 'w').write(str(value))
+
+    def test_find_all(self):
+        """Run find() to list all migrated entries."""
+        entries, count = self._find({}, ['uid'])
+        self.assertEquals(count, len(self._templates))
+
+    def test_get_properties(self):
+        """Run get_properties() on all entries and verify result."""
+        for entry in self._find({}, ['uid'])[0]:
+            properties = self._datastore.get_properties(entry['uid'],
+                byte_arrays=True)
+            number = int(properties['number'])
+            expected = self._fill_template(self._templates[number],
+                number)
+            self._filter_properties(properties)
+            self._filter_properties(expected)
+            self.assertEquals(properties, expected)
+
+    def test_get_filename(self):
+        """Run get_filename() on all entries and verify content."""
+        for entry in self._find({}, ['number', 'uid'])[0]:
+            filename = self._datastore.get_filename(entry['uid'],
+                byte_arrays=True)
+            content = file(filename).read()
+            os.remove(filename)
+            number = int(entry['number'])
+            expected = self._v1_content(number)
+            self.assertEquals(content, expected)
+
+    def _find(self, query, properties):
+        return self._datastore.find(dbus.Dictionary(query, signature='sv'),
+            properties, byte_arrays=True)
+
+    def _filter_properties(self, properties):
+        for key in IGNORE_PROPERTIES:
+            properties.pop(key, None)
+
+
+def suite():
+    test_suite = unittest.TestLoader().loadTestsFromTestCase(MigrationV1V2TestCase)
+    test_suite.__doc__ = MigrationV1V2TestCase.__doc__
+    return test_suite