From 6df3576dcc93acafbb4d8d22dcd4ab284b64e412 Mon Sep 17 00:00:00 2001 From: Benjamin Saller Date: Mon, 16 Jul 2007 14:18:10 +0000 Subject: resolve keys through model removed testutils.waitforindex --- diff --git a/src/olpc/datastore/model.py b/src/olpc/datastore/model.py index 011c3f4..e3f9668 100644 --- a/src/olpc/datastore/model.py +++ b/src/olpc/datastore/model.py @@ -60,14 +60,16 @@ class Property(object): >>> b = Property(key, value, 'binary') """ def __init__(self, key, value, kind=None): - self.key = key - self._value = value + self.kind = kind if kind not in propertyTypes: warnings.warn("Unknown property type: %s on key %s" % \ (kind, key), RuntimeWarning) else: self._impl = propertyTypes[kind] + self.key = key + self.value = value + @classmethod def fromstring(cls, key, value=''): kind = 'string' @@ -105,6 +107,21 @@ class Model(object): m.fields = self.fields.copy() m.fieldnames = self.fieldnames[:] return m + + def fromstring(self, key, value): + """create a property from the key name by looking it up in the + model.""" + kind = None + if ':' in key: key, kind = key.split(':', 1) + + mkind = self.fields[key][1] + if kind and mkind: + if kind != mkind: raise ValueError("""Specified wire + encoding for property %s was %s, expected %s""" %(key, kind, mkind)) + kind = mkind + + return Property(key, value, kind) + def addField(self, key, kind, overrides=None): """ Add a field to the model. @@ -256,19 +273,13 @@ def decode_datetime(value): return datetime.datetime.fromtimestamp(float(value)).isoformat() def datedec(value, dateformat=DATEFORMAT): - ti = time.strptime(value, dateformat) - dt = datetime.datetime(*(ti[:-2])) - dt = dt.replace(microsecond=0) - return dt + return timeparse(value, DATEFORMAT) def dateenc(value, dateformat=DATEFORMAT): if isinstance(value, basestring): # XXX: there is an issue with microseconds not getting parsed - ti = time.strptime(value, dateformat) - value = datetime.datetime(*(ti[:-2])) + value = timeparse(value, DATEFORMAT) value = value.replace(microsecond=0) - # XXX: drop time for now, this is a xapian issue - value = value.date() return value.isoformat() diff --git a/src/olpc/datastore/utils.py b/src/olpc/datastore/utils.py index 5000cfb..2998298 100644 --- a/src/olpc/datastore/utils.py +++ b/src/olpc/datastore/utils.py @@ -122,7 +122,7 @@ def timeparse(t, format): microseconds in the time string. """ try: - return datetime.datetime(*time.strptime(t, format)[0:6]).time() + return datetime.datetime(*time.strptime(t, format)[0:6]) except ValueError, msg: if "%S" in format: msg = str(msg) diff --git a/src/olpc/datastore/xapianindex.py b/src/olpc/datastore/xapianindex.py index ac1fa82..1b27d79 100644 --- a/src/olpc/datastore/xapianindex.py +++ b/src/olpc/datastore/xapianindex.py @@ -235,7 +235,7 @@ class IndexManager(object): """ d = {} for k,v in props.iteritems(): - p = model.Property.fromstring(k, v) + p = self.datamodel.fromstring(k, v) d[p.key] = p return d @@ -244,6 +244,22 @@ class IndexManager(object): Props must contain the following: key -> Property() """ + # + # Version handling + # + # we implicitly create new versions of documents the version + # id should have been set by the higher level system + uid = props.pop('uid', None) + vid = props.pop('vid', None) + + if not uid: + uid = create_uid() + operation = CREATE + + if vid: vid = str(float(vid.value) + 1.0) + else: vid = "1.0" + + # Property mapping via model props = self._mapProperties(props) doc = secore.UnprocessedDocument() add = doc.fields.append @@ -259,22 +275,6 @@ class IndexManager(object): mimetype = mimetype and mimetype.value or 'text/plain' filestuff = (filename, mimetype) - # - # Version handling - # - # we implicitly create new versions of documents the version - # id should have been set by the higher level system - uid = props.pop('uid', None) - vid = props.pop('vid', None) - - if uid: uid = uid.value - else: - uid = create_uid() - operation = CREATE - - if vid: vid = str(float(vid.value) + 1.0) - else: vid = "1.0" - doc.id = uid add(secore.Field('vid', vid)) diff --git a/tests/milestone_1.txt b/tests/milestone_1.txt index 48d09bc..35e8fb6 100644 --- a/tests/milestone_1.txt +++ b/tests/milestone_1.txt @@ -12,7 +12,6 @@ datastore. First, create and connect the store. ->>> from testutils import waitforindex >>> import os >>> assert os.system('rm -rf /tmp/test_ds') == 0 @@ -39,7 +38,7 @@ Note that we retain no reference to the created documents. Now we should be able to test the first requirement. * Get the unique ids of all the objects in the store. ->>> waitforindex(ds) +>>> ds.complete_indexing() >>> results, count = ds.find() diff --git a/tests/properties.txt b/tests/properties.txt index fe34782..4a95ec4 100644 --- a/tests/properties.txt +++ b/tests/properties.txt @@ -19,7 +19,7 @@ Set up two mount points. Extend the model to retain a 'year' property used below. ->>> dm = model.defaultModel.copy().addField('year', "int") +>>> dm = model.defaultModel.copy().addField('year', "number") Mount a couple of stores. diff --git a/tests/sugar_demo_may17.txt b/tests/sugar_demo_may17.txt index f242140..64d49e5 100644 --- a/tests/sugar_demo_may17.txt +++ b/tests/sugar_demo_may17.txt @@ -2,7 +2,6 @@ How Sugar will interact with the DS for the May 17th demo in Argentina: >>> from olpc.datastore import DataStore >>> from olpc.datastore import backingstore ->>> from testutils import waitforindex >>> ds = DataStore() >>> ds.registerBackend(backingstore.FileBackingStore) >>> assert ds.mount("/tmp/test_ds") @@ -10,15 +9,16 @@ How Sugar will interact with the DS for the May 17th demo in Argentina: Create an entry without data: >>> uid = ds.create(dict(title="New entry"), '') ->>> waitforindex(ds) +>>> ds.complete_indexing() >>> ds.get_filename(uid) '' Update an entry without data: >>> ds.update(uid, dict(title="New entry still without content"), '') ->>> waitforindex(ds) - + +>>> ds.complete_indexing() + >>> ds.get_filename(uid) '' @@ -27,7 +27,7 @@ Add some data to the same entry: >>> print >>fp, "some content" >>> fp.close() >>> ds.update(uid, dict(title="Same entry now with some content"), fp.name) ->>> waitforindex(ds) +>>> ds.complete_indexing() Retrieve that data: >>> fn = ds.get_filename(uid) @@ -41,7 +41,7 @@ Update again: >>> print >>fp, "some other content" >>> fp.close() >>> ds.update(uid, dict(title="Same entry with some other content"), fp.name) ->>> waitforindex(ds) +>>> ds.complete_indexing() And retrieve again: >>> fn = ds.get_filename(uid) @@ -66,7 +66,7 @@ Set content as pdf: >>> ds.update(uid, dict(title="Same entry with some content in pdf"), 'test.pdf') >>> ds.update(uid, dict(title="Same entry with some content in doc"), 'test.doc') >>> ds.update(uid, dict(title="Same entry with some content in odt"), 'test.odt') ->>> waitforindex(ds) +>>> ds.complete_indexing() >>> ds.stop() >>> del ds diff --git a/tests/test_backingstore.py b/tests/test_backingstore.py index 8aab9e6..4138219 100644 --- a/tests/test_backingstore.py +++ b/tests/test_backingstore.py @@ -1,5 +1,5 @@ import unittest -from testutils import tmpData, waitforindex +from testutils import tmpData from olpc.datastore import backingstore import os @@ -30,7 +30,7 @@ class Test(unittest.TestCase): uid = bs.create(dict(title="A"), tmpData(d)) - waitforindex(bs) + bs.complete_indexing() obj = bs.get(uid) @@ -40,7 +40,7 @@ class Test(unittest.TestCase): bs.update(uid, dict(title="B"), tmpData(d2)) - waitforindex(bs) + bs.complete_indexing() obj = bs.get(uid) assert obj.get_property('title') == "B" diff --git a/tests/test_model.py b/tests/test_model.py index 87ed94f..145f5d8 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,5 +1,5 @@ import unittest -from testutils import tmpData, waitforindex +from testutils import tmpData from olpc.datastore import DataStore from olpc.datastore import model, backingstore @@ -20,10 +20,9 @@ class Test(unittest.TestCase): n = n.replace(microsecond=0) p = model.Property('ctime', n.isoformat(), 'date') assert p.key == "ctime" - # XXX: the 'date()' is a work around for a missing secore - # feature right now - assert p.value == n.date().isoformat() - + assert p.value == n.isoformat() + p.value = p.value + assert p.value == n.isoformat() def test_binaryproperty(self): ds = DataStore() @@ -39,11 +38,10 @@ class Test(unittest.TestCase): data = open('test.jpg', 'r').read() # binary data with \0's in it can cause dbus errors here fn = tmpData("with image\0\0 prop") - # XXX: We should be able to remove:binary now - uid = ds.create({'title' : "Document 1", 'thumbnail:binary' : - data, 'ctime:date' : n.isoformat()}, fn) + # The key types are looked up in the model now + uid = ds.create({'title' : "Document 1", 'thumbnail' : data, 'ctime' : n.isoformat()}, fn) - waitforindex(ds) + ds.complete_indexing() c = ds.get(uid) assert c.get_property('thumbnail') == data diff --git a/tests/test_xapianindex.py b/tests/test_xapianindex.py index 3d39d3e..c455c44 100644 --- a/tests/test_xapianindex.py +++ b/tests/test_xapianindex.py @@ -16,16 +16,15 @@ def index_file(iconn, filepath): main, subtype = mimetype.split('/',1) stat = os.stat(filepath) - ctime = datetime.fromtimestamp(stat.st_ctime) - mtime = datetime.fromtimestamp(stat.st_mtime) + ctime = datetime.fromtimestamp(stat.st_ctime).isoformat() + mtime = datetime.fromtimestamp(stat.st_mtime).isoformat() if main in ['image']: filepath = None if subtype in ['x-trash', 'x-python-bytecode']: filepath = None - props = {'mime_type' : mimetype, 'mtime:date' : mtime, - 'ctime:date' : ctime,} + props = {'mime_type' : mimetype, 'mtime' : mtime, 'ctime' : ctime,} if filepath: fn = os.path.split(filepath)[1] diff --git a/tests/testutils.py b/tests/testutils.py index a4efc0a..fc667db 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -8,7 +8,3 @@ def tmpData(data): os.close(fd) return fn -def waitforindex(obj): - # wait for any/all index managers associated with object to finish - # indexing so that tests can do there thing - obj.complete_indexing() -- cgit v0.9.1