check point before prop kind/type merge

11 files changed, 137 insertions, 74 deletions

diff --git a/tests/Makefile b/tests/Makefile
index 7961b02..c2581cb 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -2,10 +2,9 @@
 # its not an option to configure
 PYTHON=python
 
-all: test
+all: clean test
 
 test: 
-	@rm -rf fulltext 
 	@${PYTHON} runalltests.py 
 
 valgrind:
@@ -17,6 +16,7 @@ profile:
 	@${PYTHON} ./profilealltests.py
 
 clean: 
+	@${PYTHON} ./cleaner.py
 	@find . -name "*.pyc" -exec rm {} \;
 	@find . -name "*~" -exec rm {} \;
 	@find . -name "hotspot*" -exec rm {} \;
diff --git a/tests/milestone_1.txt b/tests/milestone_1.txt
index bde3720..2472260 100644
--- a/tests/milestone_1.txt
+++ b/tests/milestone_1.txt
@@ -12,6 +12,10 @@ datastore.
 
 
 First, create and connect the store.
+>>> from testutils import waitforindex
+>>> import os
+>>> assert os.system('rm -rf /tmp/test_ds') == 0
+
 
 >>> from olpc.datastore import DataStore
 >>> from olpc.datastore import backingstore
@@ -35,11 +39,13 @@ Note that we retain no reference to the created documents.
 Now we should be able to test the first requirement.
 * Get the unique ids of all the objects in the store.
 
+>>> waitforindex(ds)
+
 >>> results, count = ds.find()
 
 A find command with out any parameters will return everything in the store.
 
-* Get an object from the store given his uid.
+* Get an object from the store given its uid.
 
 Here we manually cycle through the results looking for the title we
 want.
@@ -51,30 +57,24 @@ want.
 
 * Get the object metadata.
 >>> c1.properties
-[...]
+{...}
 
 * Get the object file.
 >>> c1.filename
 '/tmp/...'
 
->>> c1.data
+>>> c1.contents
 'this is the first document'
 >>> c1.file
 <open file ...>
 
 
-Or if you prefer access through the datastore (which is how DBus would
-use it)
-
->>> fn = ds.get_filename(first_uid)
->>> ds.get_data(first_uid)
-'this is the first document'
-
 
 Now we can modify that file and then 
 * Push the changes made to the file back to the store.
 * Update the metadata of an object.
 
+>>> fn = c1.filename
 >>> fp = open(fn, 'a')
 >>> print >>fp, "more content"
 >>> fp.close()
@@ -89,4 +89,4 @@ This is the basis of milestone 1.
 >>> ds.stop()
 >>> del ds
 
-
+>>> assert os.system('rm -rf /tmp/test_ds') == 0
diff --git a/tests/mountpoints.txt b/tests/mountpoints.txt
index 9a821b5..1066da0 100644
--- a/tests/mountpoints.txt
+++ b/tests/mountpoints.txt
@@ -12,7 +12,7 @@ mounting a backingstore on the datastore.
 
 >>> from olpc.datastore import DataStore
 >>> from olpc.datastore import backingstore
->>> from testutils import tmpData
+>>> from testutils import tmpData, waitforindex
 >>> import dbus
 
 
@@ -41,6 +41,7 @@ Now lets create some content
 We can now, if we wish verify which mount point this content came
 from.
 
+>>> waitforindex(ds)
 
 >>> c1 = ds.get(u1)
 >>> assert c1.backingstore.id == mountpoint
@@ -61,6 +62,8 @@ Now lets add another mount point.
 Now lets create a new content item.
 >>> u3 = ds.create(dict(title="Document 3", mountpoint=mp2), tmpData("""document three"""))
 
+>>> waitforindex(ds)
+
 We explictly passed a mount point here. Lets examine the properties of
 the object and verify this.
 >>> c3 = ds.find(dict(title="Document 3"))[0][0]
@@ -102,6 +105,8 @@ Register the filesystem type
 
 If that worked it should have imported content on load().
 
+>>> waitforindex(ds)
+
 >>> result, count = ds.find(dict(fulltext="four"))
 >>> assert count == 1
 >>> assert result[0]['mountpoint'] == mp3
@@ -114,6 +119,8 @@ as DBus data.
 
 >>> mp3 = ds.mount("inplace:/tmp/store3", dict(title=dbus.String("Fake USB again")))
 
+>>> waitforindex(ds)
+
 >>> result, count = ds.find(dict(fulltext="four"))
 >>> assert count == 1
 >>> assert result[0]['mountpoint'] == mp3
diff --git a/tests/properties.txt b/tests/properties.txt
index 689414f..dd93b69 100644
--- a/tests/properties.txt
+++ b/tests/properties.txt
@@ -8,16 +8,23 @@ properties to content and managing them.
 
 
 >>> from olpc.datastore import DataStore
->>> from olpc.datastore import backingstore
+>>> from olpc.datastore import backingstore, model
 >>> from testutils import tmpData
 >>> import dbus
 
 Set up two mount points.
 
->>> ds = DataStore(sync_index=True)
+>>> ds = DataStore()
 >>> ds.registerBackend(backingstore.FileBackingStore)
->>> mp1 = ds.mount("/tmp/store1", dict(title="Primary Storage"))
->>> mp2 = ds.mount("/tmp/store2", dict(title="Secondary Storage"))
+
+Extend the model to retain a 'year' property used below.
+
+>>> dm = model.defaultModel.addField('year', store=True, exact=True, sortable=True, type="float")
+
+Mount a couple of stores.
+
+>>> mp1 = ds.mount("/tmp/store1", {'title' : "Primary Storage", 'indexmanager.model' : dm})
+>>> mp2 = ds.mount("/tmp/store2", {'title' : "Secondary Storage", 'indexmanager.model' : dm})
 
 Create some content on each.
 
diff --git a/tests/query.txt b/tests/query.txt
index 2c58851..1e7624e 100644
--- a/tests/query.txt
+++ b/tests/query.txt
@@ -47,7 +47,7 @@ This returned a list of all properties on the Content object in which
 case we can find the property by enumeration. The other option is
 using the get_properties call on Content
 
->>> a.get_properties(key='title')
+>>> a.get_properties(dict(key='title'))
 [<TextProperty title:'New Content'>]
 
 Using the query manager API we are able to update the
@@ -57,11 +57,11 @@ that this works lets attach another property.
 >>> qm.update(a, dict(author='Benjamin'))
 
 A request for title still returns only the title property.
->>> a.get_properties(key='title')
+>>> a.get_properties(dict(key='title'))
 [<TextProperty title:'New Content'>]
 
 And a request for author works as expected.
->>> a.get_properties(key='author')
+>>> a.get_properties(dict(key='author'))
 [<Property author:'Benjamin'>]
 
 >>> qm.update(a, dict(foo='bar'))
@@ -91,11 +91,9 @@ Here we want to show that certain types of Properties map to
 specialized implemenations automatically based on their type. 'ctime'
 is a DateTime Property and we can verify that it is returned properly
 from the mapping layer with the following.
->>> ctimeProp = a.get_properties(key='ctime')[0]
->>> ctimeProp.type == "date"
+>>> ctimeProp = a.get_properties(dict(key='ctime'))[0]
+>>> ctimeProp.kind == "date"
 True
->>> type(ctimeProp)
-<class 'olpc.datastore.model.DateProperty'>
 
 Special support is needed to make dates easily addressable within the
 datastore. The properties 'ctime', creation time, and 'mtime',
@@ -144,7 +142,7 @@ refers. This is available through the 'content' attrbiute of
 properties. Only properties bound to content and synchronized with the
 database have this property.
 
->>> p = a.get_properties(key='author')[0]
+>>> p = a.get_properties(dict(key='author'))[0]
 >>> p.content
 <Content id:...>
 
diff --git a/tests/runalltests.py b/tests/runalltests.py
index bbf0f97..28802ec 100644
--- a/tests/runalltests.py
+++ b/tests/runalltests.py
@@ -14,10 +14,9 @@ import unittest
 import doctest
 from pkg_resources import resource_filename
 
-from sqlalchemy import clear_mappers
 
 doctests = [
-    resource_filename(__name__, "query.txt"),
+    resource_filename(__name__, "xapianindex.txt"),
     resource_filename(__name__, "milestone_1.txt"),
     resource_filename(__name__, "sugar_demo_may17.txt"),
     resource_filename(__name__, "milestone_2.txt"),
@@ -44,13 +43,14 @@ sys.path.insert(0, test_lib)
 
 
 def tearDownDS(test):
-    # reset the module global mappers used in SQLAlchemy between tests
-    clear_mappers()
     # and remove the test repository used in some tests
     os.system('rm -rf /tmp/test_ds')
     
 def test_suite():
     suite = unittest.TestSuite()
+    if len(sys.argv) > 1:
+        doctests = sys.argv[1:]
+        
     for dt in doctests:
         suite.addTest(doctest.DocFileSuite(dt,
     optionflags=doctest_options, tearDown=tearDownDS))
@@ -68,5 +68,6 @@ def test_suite():
 
 if __name__ == "__main__":
     runner = unittest.TextTestRunner(verbosity=1)
-    runner.run(test_suite())
+    suite = test_suite()
+    runner.run(suite)
                         
diff --git a/tests/sugar_demo_may17.txt b/tests/sugar_demo_may17.txt
index c899799..f242140 100644
--- a/tests/sugar_demo_may17.txt
+++ b/tests/sugar_demo_may17.txt
@@ -2,6 +2,7 @@ How Sugar will interact with the DS for the May 17th demo in Argentina:
 
 >>> from olpc.datastore import DataStore
 >>> from olpc.datastore import backingstore
+>>> from testutils import waitforindex
 >>> ds = DataStore()
 >>> ds.registerBackend(backingstore.FileBackingStore)
 >>> assert ds.mount("/tmp/test_ds")
@@ -9,11 +10,14 @@ How Sugar will interact with the DS for the May 17th demo in Argentina:
 
 Create an entry without data:
 >>> uid = ds.create(dict(title="New entry"), '')
+>>> waitforindex(ds)
+
 >>> ds.get_filename(uid)
 ''
 
 Update an entry without data:
 >>> ds.update(uid, dict(title="New entry still without content"), '')
+>>> waitforindex(ds)
  
 >>> ds.get_filename(uid)
 ''
@@ -23,6 +27,7 @@ Add some data to the same entry:
 >>> print >>fp, "some content"
 >>> fp.close()
 >>> ds.update(uid, dict(title="Same entry now with some content"), fp.name)
+>>> waitforindex(ds)
 
 Retrieve that data:
 >>> fn = ds.get_filename(uid)
@@ -36,6 +41,7 @@ Update again:
 >>> print >>fp, "some other content"
 >>> fp.close()
 >>> ds.update(uid, dict(title="Same entry with some other content"), fp.name)
+>>> waitforindex(ds)
 
 And retrieve again:
 >>> fn = ds.get_filename(uid)
@@ -60,6 +66,7 @@ Set content as pdf:
 >>> ds.update(uid, dict(title="Same entry with some content in pdf"), 'test.pdf')
 >>> ds.update(uid, dict(title="Same entry with some content in doc"), 'test.doc')
 >>> ds.update(uid, dict(title="Same entry with some content in odt"), 'test.odt')
+>>> waitforindex(ds)
 
 >>> ds.stop()
 >>> del ds
diff --git a/tests/test_backingstore.py b/tests/test_backingstore.py
index 28fdeba..a13e28c 100644
--- a/tests/test_backingstore.py
+++ b/tests/test_backingstore.py
@@ -1,21 +1,21 @@
 import unittest
-from StringIO import StringIO
+from testutils import tmpData, waitforindex
 
 from olpc.datastore import backingstore
-from sqlalchemy import clear_mappers
 import os
 
 DEFAULT_STORE = '/tmp/_bs_test'
 
 class Test(unittest.TestCase):
-    def tearDown(self):
+    def setUp(self):
         if os.path.exists(DEFAULT_STORE):
             os.system("rm -rf %s" % DEFAULT_STORE)
 
-        clear_mappers()
+    def tearDown(self):
+        if os.path.exists(DEFAULT_STORE):
+            os.system("rm -rf %s" % DEFAULT_STORE)
             
     def test_fsstore(self):
-        clear_mappers()
         bs = backingstore.FileBackingStore(DEFAULT_STORE)
         bs.initialize_and_load()
         bs.create_descriptor()
@@ -28,20 +28,27 @@ class Test(unittest.TestCase):
         d = """This is a test"""
         d2 = "Different"
         
-        c = bs.create(dict(title="A"), StringIO(d))
-        obj = bs.get(c.id)
+        uid = bs.create(dict(title="A"), tmpData(d))
+
+        waitforindex(bs)
+        
+        obj = bs.get(uid)
+
         assert obj.get_property('title') == "A"
         got = obj.file.read()
         assert got == d
 
-        bs.update(c.id, dict(title="B"), StringIO(d2))
-        obj = bs.get(c.id)
+        bs.update(uid, dict(title="B"), tmpData(d2))
+
+        waitforindex(bs)
+        
+        obj = bs.get(uid)
         assert obj.get_property('title') == "B"
         got = obj.file.read()
         assert got == d2
 
-        bs.delete(c.id)
-        self.failUnlessRaises(KeyError, bs.get, c.id)
+        bs.delete(uid)
+        self.failUnlessRaises(KeyError, bs.get, uid)
         
 def test_suite():
     suite = unittest.TestSuite()
diff --git a/tests/test_model.py b/tests/test_model.py
index 6e8c896..d7aea45 100644
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -1,35 +1,56 @@
 import unittest
-from testutils import tmpData
+from testutils import tmpData, waitforindex
 
 from olpc.datastore import DataStore
 from olpc.datastore import model, backingstore
 import datetime
 import os
 
+
+DEFAULT_STORE = '/tmp/test_ds'
+
 class Test(unittest.TestCase):
+    def setUp(self): os.system('rm -rf %s' % DEFAULT_STORE)
+    def tearDown(self): os.system('rm -rf %s' % DEFAULT_STORE)
+        
     def test_dateproperty(self):
         n = datetime.datetime.now()
         # we have to kill the microseconds as
         # time.strptime which we must use in 2.4 doesn't parse it
         n = n.replace(microsecond=0)
-        p = model.DateProperty('ctime', n)
+        p = model.Property('ctime', n, 'date')
         assert p.key == "ctime"
-        assert p.value.isoformat() == n.isoformat()
+        # XXX: the 'date()' is a work around for a missing secore
+        # feature right now
+        assert p.value == n.date().isoformat()
 
+        
     def test_binaryproperty(self):
         ds = DataStore()
         ds.registerBackend(backingstore.FileBackingStore)
-        ds.mount('/tmp/test_ds')
+
+        #add a custom field to the model 
+        dm = model.defaultModel.addField('thumbnail',
+                                         store=True,
+                                         exact=False,
+                                         sortable=False)
         
+        ds.mount(DEFAULT_STORE, {'indexmanager.model' : dm})
+
+
         data = open('test.jpg', 'r').read()
         # binary data with \0's in it can cause dbus errors here
-        uid = ds.create({'title' : "Document 1", 'thumbnail:binary' : data},
-                        tmpData("with image\0\0 prop"))
+        fn = tmpData("with image\0\0 prop")
+        uid = ds.create({'title' : "Document 1", 'thumbnail:binary' : data}, fn)
+        
+        waitforindex(ds)
+
         c = ds.get(uid)
         assert c.get_property('thumbnail') == data
+
         ds.stop()
 
-        os.system('rm -rf /tmp/test_ds')
+
         
 def test_suite():
     suite = unittest.TestSuite()
diff --git a/tests/testutils.py b/tests/testutils.py
index 243747a..48d1060 100644
--- a/tests/testutils.py
+++ b/tests/testutils.py
@@ -1,5 +1,9 @@
 import tempfile
 import os
+import time
+
+from olpc.datastore.xapianindex import IndexManager
+from olpc.datastore.datastore import DataStore
 
 def tmpData(data):
     """Put data into a temporary file returning the filename """
@@ -7,3 +11,17 @@ def tmpData(data):
     os.write(fd, data)
     os.close(fd)
     return fn
+
+def waitforindex(obj, interval=0.1):
+    # wait for any/all index managers associated with object to finish
+    # indexing so that tests can do there thing
+    if isinstance(obj, IndexManager):
+        obj.complete_indexing()
+    elif isinstance(obj, DataStore):
+        for mp in obj.mountpoints.values():
+            im = mp.indexmanager
+            im.complete_indexing()
+    else:
+        # backingstore
+        obj.indexmanager.complete_indexing()
+        
diff --git a/tests/xapianindex.txt b/tests/xapianindex.txt
index de495a6..5ef1d5c 100644
--- a/tests/xapianindex.txt
+++ b/tests/xapianindex.txt
@@ -16,20 +16,11 @@ First clean up any old test data.
 >>> im = IndexManager()
 >>> im.connect(index_home)
 
-A small utility method for wrapping a normal dict into proper property
-objects.
-
->>> def propsdict(**kwargs):
-...    d = {}
-...    for k,v in kwargs.iteritems(): 
-...        d[k] = model.Property(k, v)
-...    return d
-
 
 Now add the file to the index.
 
->>> props = propsdict(title="PDF Document",
-...                   mimetype="application/pdf")
+>>> props = dict(title="PDF Document",
+...              mimetype="application/pdf")
 
 
 >>> uid = im.index(props, "test.pdf")
@@ -41,36 +32,42 @@ left, when it has none we expect our content to be indexed and searchable.
 
 
 Searching on an property of the content works.
->>> assert im.search("PDF")[0] == uid
+>>> def expect(r, count=None):
+...    if count: assert r[1] == count
+...    return list(r[0])
+>>> def expect_single(r):
+...    assert r[1] == 1
+...    return r[0].next()
+>>> def expect_none(r):
+...    assert r[1] == 0
+...    assert list(r[0]) == []
+
+
+>>> assert expect_single(im.search("PDF")).id == uid
 
 Searching into the binary content of the object works as well.
->>> assert im.search("peek")[0] == uid
+>>> assert expect_single(im.search("peek")).id == uid
 
 Specifying a search that demands a document term be found only in the
 title works as well.
 
->>> assert im.search('title:PDF')[0] == uid
->>> im.search('title:peek')
-[]
+>>> assert expect_single(im.search('title:PDF')).id == uid
+>>> expect_none(im.search('title:peek'))
 
 Searching for documents that are PDF works as expected here. Here we
 use the dictionary form of the query where each field name is given
 and creates a search.
->>> assert im.search(dict(mimetype='application/pdf'))[0] == uid
-
-
-#Likewise excluding the match works as expected
-#>>> im.search('-title:PDF')
-#[]
-
+>>> assert expect_single(im.search(dict(mimetype='application/pdf'))).id == uid
 
 Punctuation is fine.
 
->>> assert im.search("Don't peek")[0] == uid
+>>> assert expect_single(im.search("Don't peek")).id == uid
 
 As well as quoted strings
 
->>> assert im.search(r'''"Don't peek"''')[0] == uid
+>>> assert expect_single(im.search(r'''"Don't peek"''')).id == uid
 
 Cleanly shut down.
 >>> im.stop()
+
+>>> assert os.system('rm -rf %s' % index_home) == 0