resolve keys through model

removed testutils.waitforindex

10 files changed, 61 insertions, 58 deletions

diff --git a/src/olpc/datastore/model.py b/src/olpc/datastore/model.py
index 011c3f4..e3f9668 100644
--- a/src/olpc/datastore/model.py
+++ b/src/olpc/datastore/model.py
@@ -60,14 +60,16 @@ class Property(object):
     >>> b = Property(key, value, 'binary')
     """
     def __init__(self, key, value, kind=None):
-        self.key = key
-        self._value = value
+
         self.kind = kind
         if kind not in propertyTypes:
             warnings.warn("Unknown property type: %s on key %s" % \
                           (kind, key), RuntimeWarning)
         else: self._impl = propertyTypes[kind]
 
+        self.key = key
+        self.value = value
+        
     @classmethod
     def fromstring(cls, key, value=''):
         kind = 'string'
@@ -105,6 +107,21 @@ class Model(object):
         m.fields = self.fields.copy()
         m.fieldnames = self.fieldnames[:]
         return m
+
+    def fromstring(self, key, value):
+        """create a property from the key name by looking it up in the
+        model."""
+        kind = None
+        if ':' in key: key, kind = key.split(':', 1)
+        
+        mkind = self.fields[key][1]
+        if kind and mkind:
+            if kind != mkind: raise ValueError("""Specified wire
+            encoding for property %s was %s, expected %s""" %(key, kind, mkind)) 
+        kind = mkind
+            
+        return Property(key, value, kind)
+
     
     def addField(self, key, kind, overrides=None):
         """ Add a field to the model.
@@ -256,19 +273,13 @@ def decode_datetime(value):
     return datetime.datetime.fromtimestamp(float(value)).isoformat()
 
 def datedec(value, dateformat=DATEFORMAT):
-    ti = time.strptime(value, dateformat)
-    dt = datetime.datetime(*(ti[:-2]))
-    dt = dt.replace(microsecond=0)
-    return dt
+    return timeparse(value, DATEFORMAT)
 
 def dateenc(value, dateformat=DATEFORMAT):
     if isinstance(value, basestring):
         # XXX: there  is an issue with microseconds not getting parsed
-        ti = time.strptime(value, dateformat)
-        value = datetime.datetime(*(ti[:-2]))
+        value = timeparse(value, DATEFORMAT)
     value = value.replace(microsecond=0)
-    # XXX: drop time for now, this is a xapian issue
-    value = value.date()
     return value.isoformat()
 
         
diff --git a/src/olpc/datastore/utils.py b/src/olpc/datastore/utils.py
index 5000cfb..2998298 100644
--- a/src/olpc/datastore/utils.py
+++ b/src/olpc/datastore/utils.py
@@ -122,7 +122,7 @@ def timeparse(t, format):
     microseconds in the time string.
     """
     try:
-        return datetime.datetime(*time.strptime(t, format)[0:6]).time()
+        return datetime.datetime(*time.strptime(t, format)[0:6])
     except ValueError, msg:
         if "%S" in format:
             msg = str(msg)
diff --git a/src/olpc/datastore/xapianindex.py b/src/olpc/datastore/xapianindex.py
index ac1fa82..1b27d79 100644
--- a/src/olpc/datastore/xapianindex.py
+++ b/src/olpc/datastore/xapianindex.py
@@ -235,7 +235,7 @@ class IndexManager(object):
         """
         d = {}
         for k,v in props.iteritems():
-            p = model.Property.fromstring(k, v)
+            p = self.datamodel.fromstring(k, v)
             d[p.key] = p
         return d
 
@@ -244,6 +244,22 @@ class IndexManager(object):
         Props must contain the following:
             key -> Property()
         """
+        #
+        # Version handling
+        #
+        # we implicitly create new versions of documents the version
+        # id should have been set by the higher level system
+        uid = props.pop('uid', None)
+        vid = props.pop('vid', None)
+
+        if not uid:
+            uid = create_uid()
+            operation = CREATE
+            
+        if vid: vid = str(float(vid.value) + 1.0)
+        else: vid = "1.0"
+        
+        # Property mapping via model
         props = self._mapProperties(props)
         doc = secore.UnprocessedDocument()
         add = doc.fields.append
@@ -259,22 +275,6 @@ class IndexManager(object):
             mimetype = mimetype and mimetype.value or 'text/plain'
             filestuff = (filename, mimetype)
 
-        #
-        # Version handling
-        #
-        # we implicitly create new versions of documents the version
-        # id should have been set by the higher level system
-        uid = props.pop('uid', None)
-        vid = props.pop('vid', None)
-
-        if uid: uid = uid.value
-        else:
-            uid = create_uid()
-            operation = CREATE
-            
-        if vid: vid = str(float(vid.value) + 1.0)
-        else: vid = "1.0"
-        
         doc.id = uid
         add(secore.Field('vid', vid))
         
diff --git a/tests/milestone_1.txt b/tests/milestone_1.txt
index 48d09bc..35e8fb6 100644
--- a/tests/milestone_1.txt
+++ b/tests/milestone_1.txt
@@ -12,7 +12,6 @@ datastore.
 
 
 First, create and connect the store.
->>> from testutils import waitforindex
 >>> import os
 >>> assert os.system('rm -rf /tmp/test_ds') == 0
 
@@ -39,7 +38,7 @@ Note that we retain no reference to the created documents.
 Now we should be able to test the first requirement.
 * Get the unique ids of all the objects in the store.
 
->>> waitforindex(ds)
+>>> ds.complete_indexing()
 
 >>> results, count = ds.find()
 
diff --git a/tests/properties.txt b/tests/properties.txt
index fe34782..4a95ec4 100644
--- a/tests/properties.txt
+++ b/tests/properties.txt
@@ -19,7 +19,7 @@ Set up two mount points.
 
 Extend the model to retain a 'year' property used below.
 
->>> dm = model.defaultModel.copy().addField('year', "int")
+>>> dm = model.defaultModel.copy().addField('year', "number")
 
 Mount a couple of stores.
 
diff --git a/tests/sugar_demo_may17.txt b/tests/sugar_demo_may17.txt
index f242140..64d49e5 100644
--- a/tests/sugar_demo_may17.txt
+++ b/tests/sugar_demo_may17.txt
@@ -2,7 +2,6 @@ How Sugar will interact with the DS for the May 17th demo in Argentina:
 
 >>> from olpc.datastore import DataStore
 >>> from olpc.datastore import backingstore
->>> from testutils import waitforindex
 >>> ds = DataStore()
 >>> ds.registerBackend(backingstore.FileBackingStore)
 >>> assert ds.mount("/tmp/test_ds")
@@ -10,15 +9,16 @@ How Sugar will interact with the DS for the May 17th demo in Argentina:
 
 Create an entry without data:
 >>> uid = ds.create(dict(title="New entry"), '')
->>> waitforindex(ds)
+>>> ds.complete_indexing()
 
 >>> ds.get_filename(uid)
 ''
 
 Update an entry without data:
 >>> ds.update(uid, dict(title="New entry still without content"), '')
->>> waitforindex(ds)
- 
+
+>>> ds.complete_indexing()
+
 >>> ds.get_filename(uid)
 ''
 
@@ -27,7 +27,7 @@ Add some data to the same entry:
 >>> print >>fp, "some content"
 >>> fp.close()
 >>> ds.update(uid, dict(title="Same entry now with some content"), fp.name)
->>> waitforindex(ds)
+>>> ds.complete_indexing()
 
 Retrieve that data:
 >>> fn = ds.get_filename(uid)
@@ -41,7 +41,7 @@ Update again:
 >>> print >>fp, "some other content"
 >>> fp.close()
 >>> ds.update(uid, dict(title="Same entry with some other content"), fp.name)
->>> waitforindex(ds)
+>>> ds.complete_indexing()
 
 And retrieve again:
 >>> fn = ds.get_filename(uid)
@@ -66,7 +66,7 @@ Set content as pdf:
 >>> ds.update(uid, dict(title="Same entry with some content in pdf"), 'test.pdf')
 >>> ds.update(uid, dict(title="Same entry with some content in doc"), 'test.doc')
 >>> ds.update(uid, dict(title="Same entry with some content in odt"), 'test.odt')
->>> waitforindex(ds)
+>>> ds.complete_indexing()
 
 >>> ds.stop()
 >>> del ds
diff --git a/tests/test_backingstore.py b/tests/test_backingstore.py
index 8aab9e6..4138219 100644
--- a/tests/test_backingstore.py
+++ b/tests/test_backingstore.py
@@ -1,5 +1,5 @@
 import unittest
-from testutils import tmpData, waitforindex
+from testutils import tmpData
 
 from olpc.datastore import backingstore
 import os
@@ -30,7 +30,7 @@ class Test(unittest.TestCase):
         
         uid = bs.create(dict(title="A"), tmpData(d))
 
-        waitforindex(bs)
+        bs.complete_indexing()
         
         obj = bs.get(uid)
 
@@ -40,7 +40,7 @@ class Test(unittest.TestCase):
 
         bs.update(uid, dict(title="B"), tmpData(d2))
 
-        waitforindex(bs)
+        bs.complete_indexing()
         
         obj = bs.get(uid)
         assert obj.get_property('title') == "B"
diff --git a/tests/test_model.py b/tests/test_model.py
index 87ed94f..145f5d8 100644
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -1,5 +1,5 @@
 import unittest
-from testutils import tmpData, waitforindex
+from testutils import tmpData
 
 from olpc.datastore import DataStore
 from olpc.datastore import model, backingstore
@@ -20,10 +20,9 @@ class Test(unittest.TestCase):
         n = n.replace(microsecond=0)
         p = model.Property('ctime', n.isoformat(), 'date')
         assert p.key == "ctime"
-        # XXX: the 'date()' is a work around for a missing secore
-        # feature right now
-        assert p.value == n.date().isoformat()
-
+        assert p.value == n.isoformat()
+        p.value = p.value
+        assert p.value == n.isoformat()
         
     def test_binaryproperty(self):
         ds = DataStore()
@@ -39,11 +38,10 @@ class Test(unittest.TestCase):
         data = open('test.jpg', 'r').read()
         # binary data with \0's in it can cause dbus errors here
         fn = tmpData("with image\0\0 prop")
-        # XXX: We should be able to remove:binary now
-        uid = ds.create({'title' : "Document 1", 'thumbnail:binary' :
-        data, 'ctime:date' : n.isoformat()}, fn)
+        # The key types are looked up in the model now
+        uid = ds.create({'title' : "Document 1", 'thumbnail' : data, 'ctime' : n.isoformat()}, fn)
         
-        waitforindex(ds)
+        ds.complete_indexing()
 
         c = ds.get(uid)
         assert c.get_property('thumbnail') == data
diff --git a/tests/test_xapianindex.py b/tests/test_xapianindex.py
index 3d39d3e..c455c44 100644
--- a/tests/test_xapianindex.py
+++ b/tests/test_xapianindex.py
@@ -16,16 +16,15 @@ def index_file(iconn, filepath):
     main, subtype = mimetype.split('/',1)
 
     stat = os.stat(filepath)
-    ctime = datetime.fromtimestamp(stat.st_ctime)
-    mtime = datetime.fromtimestamp(stat.st_mtime)
+    ctime = datetime.fromtimestamp(stat.st_ctime).isoformat()
+    mtime = datetime.fromtimestamp(stat.st_mtime).isoformat()
     
     if main in ['image']: filepath = None
     if subtype in ['x-trash', 'x-python-bytecode']: filepath = None
 
 
 
-    props = {'mime_type' : mimetype, 'mtime:date' : mtime,
-             'ctime:date' : ctime,}
+    props = {'mime_type' : mimetype, 'mtime' : mtime, 'ctime' : ctime,}
 
     if filepath:
         fn = os.path.split(filepath)[1]
diff --git a/tests/testutils.py b/tests/testutils.py
index a4efc0a..fc667db 100644
--- a/tests/testutils.py
+++ b/tests/testutils.py
@@ -8,7 +8,3 @@ def tmpData(data):
     os.close(fd)
     return fn
 
-def waitforindex(obj):
-    # wait for any/all index managers associated with object to finish
-    # indexing so that tests can do there thing
-    obj.complete_indexing()