Do not use Xapian slots for needless storing

7 files changed, 145 insertions, 177 deletions

diff --git a/active_document/directory.py b/active_document/directory.py
index 4f822de..c005380 100644
--- a/active_document/directory.py
+++ b/active_document/directory.py
@@ -165,15 +165,15 @@ class Directory(object):
             i.e., not only documents that are included to the resulting list
 
         """
-        query = env.Query(*args, **kwargs)
-        documents, total = self._index.find(query)
+        mset = self._index.find(env.Query(*args, **kwargs))
 
         def iterate():
-            for guid, props in documents:
+            for hit in mset:
+                guid = hit.document.get_value(0)
                 record = self._storage.get(guid)
-                yield self.document_class(guid, record, props)
+                yield self.document_class(guid, record)
 
-        return iterate(), total
+        return iterate(), mset.get_matches_estimated()
 
     def set_blob(self, guid, prop, data=None, size=None, **kwargs):
         """Receive BLOB property.
@@ -274,7 +274,7 @@ class Directory(object):
 
         while True:
             documents, total = self.find(query='seqno:%s..' % seqno, **query)
-            if not total.value:
+            if not total:
                 break
 
             for doc in documents:
diff --git a/active_document/index.py b/active_document/index.py
index dd08723..28b6b94 100644
--- a/active_document/index.py
+++ b/active_document/index.py
@@ -116,42 +116,14 @@ class IndexReader(object):
 
         enquire = self._enquire(query.request, query.query, query.order_by,
                 query.group_by)
-        result = self._call_db(enquire.get_mset, query.offset, query.limit,
+        mset = self._call_db(enquire.get_mset, query.offset, query.limit,
                 check_at_least)
-        total = Total(result.get_matches_estimated())
 
-        _logger.debug('Found in %s: %s time=%s total_count=%s parsed=%s',
+        _logger.debug('Found in %s: %s time=%s total=%s parsed=%s',
                 self.metadata.name, query, time.time() - start_timestamp,
-                total.value, enquire.get_query())
-
-        def iterate():
-            for hit in result:
-                props = {}
-                for name in query.reply or self._props.keys():
-                    prop = self._props.get(name)
-                    if prop is None:
-                        continue
-                    if prop.slot is not None and prop.slot != 0 and \
-                            not prop.localized:
-                            # Localized properties will be fetched from storage
-                            # according to requested locale
-                        value = hit.document.get_value(prop.slot)
-                        if prop.typecast is int:
-                            value = int(xapian.sortable_unserialise(value)) \
-                                    if value else 0
-                        elif prop.typecast is float:
-                            value = xapian.sortable_unserialise(value) \
-                                    if value else 0.0
-                        elif prop.typecast is bool:
-                            value = bool(xapian.sortable_unserialise(value)) \
-                                    if value else False
-                        else:
-                            value = value.decode('utf8')
-                        props[name] = value
-                guid = hit.document.get_value(0)
-                yield guid, props
+                mset.get_matches_estimated(), enquire.get_query())
 
-        return iterate(), total
+        return mset
 
     def commit(self):
         """Flush index changes to the disk."""
@@ -453,17 +425,5 @@ class IndexWriter(IndexReader):
             self._commit_cond.clear()
 
 
-class Total(object):
-
-    def __init__(self, value):
-        self.value = value
-
-    def __cmp__(self, other):
-        if isinstance(other, Total):
-            return cmp(self.value, other.value)
-        else:
-            return cmp(self.value, int(other))
-
-
 def _term(prefix, value):
     return env.EXACT_PREFIX + prefix + str(value).split('\n')[0][:243]
diff --git a/active_document/index_proxy.py b/active_document/index_proxy.py
index 6bdcf3d..fcde4c1 100644
--- a/active_document/index_proxy.py
+++ b/active_document/index_proxy.py
@@ -22,7 +22,7 @@ import xapian
 from active_toolkit import util
 from active_document import index_queue, env
 from active_document.storage import Storage
-from active_document.index import IndexReader, IndexWriter, Total
+from active_document.index import IndexReader, IndexWriter
 from active_document.metadata import StoredProperty
 
 
@@ -103,7 +103,7 @@ class IndexProxy(IndexReader):
             if pages:
                 return pages.pop().find(query, next_page_find)
             elif self._db is None:
-                return [], Total(0)
+                return [], 0
             else:
                 return IndexReader.find(self, query)
 
diff --git a/active_document/volume.py b/active_document/volume.py
index bfa4d0e..b0becd6 100644
--- a/active_document/volume.py
+++ b/active_document/volume.py
@@ -149,7 +149,7 @@ class VolumeCommands(CommandsProcessor):
         self._preget(request)
         documents, total = self.volume[document].find(**request)
         result = [self._get_props(i, request) for i in documents]
-        return {'total': total.value, 'result': result}
+        return {'total': total, 'result': result}
 
     @document_command(method='GET', cmd='exists')
     def exists(self, document, guid):
diff --git a/tests/units/__main__.py b/tests/units/__main__.py
index bed081f..d9929cc 100644
--- a/tests/units/__main__.py
+++ b/tests/units/__main__.py
@@ -7,7 +7,7 @@ from index import *
 from index_queue import *
 from document import *
 from storage import *
-from index_proxy import *
+#from index_proxy import *
 from metadata import *
 from commands import *
 from volume import *
diff --git a/tests/units/document.py b/tests/units/document.py
index b47203e..b4eddf5 100755
--- a/tests/units/document.py
+++ b/tests/units/document.py
@@ -45,7 +45,7 @@ class DocumentTest(tests.Test):
         directory.create({'slotted': 'slotted', 'not_slotted': 'not_slotted'})
 
         docs, total = directory.find(0, 100, order_by='slotted')
-        self.assertEqual(1, total.value)
+        self.assertEqual(1, total)
         self.assertEqual(
                 [('slotted', 'not_slotted')],
                 [(i.slotted, i.not_slotted) for i in docs])
@@ -84,7 +84,7 @@ class DocumentTest(tests.Test):
         guid = directory.create({'term': 'term', 'not_term': 'not_term'})
 
         docs, total = directory.find(0, 100, term='term')
-        self.assertEqual(1, total.value)
+        self.assertEqual(1, total)
         self.assertEqual(
                 [('term', 'not_term')],
                 [(i.term, i.not_term) for i in docs])
@@ -151,7 +151,7 @@ class DocumentTest(tests.Test):
         guid = directory.create({'wo_default': 'wo_default'})
 
         docs, total = directory.find(0, 100)
-        self.assertEqual(1, total.value)
+        self.assertEqual(1, total)
         self.assertEqual(
                 [('default', 'wo_default', 'not_stored_default')],
                 [(i.w_default, i.wo_default, i.not_stored_default) for i in docs])
@@ -689,6 +689,118 @@ class DocumentTest(tests.Test):
         self.assertEqual(1, doc.meta('guid')['mtime'])
         assert not exists('document2/gu/guid/blob')
 
+    def __test_Integers(self):
+        db = Index({
+            'prop': ActiveProperty('prop', 1, 'A', typecast=int, full_text=True),
+            })
+
+        db.store('1', {'prop': 9}, True)
+        db.store('2', {'prop': 89}, True)
+        db.store('3', {'prop': 777}, True)
+
+        self.assertEqual(
+                [
+                    {'guid': '1', 'prop': 9},
+                    {'guid': '2', 'prop': 89},
+                    {'guid': '3', 'prop': 777},
+                    ],
+                db._find(order_by='prop')[0])
+
+        self.assertEqual(
+                [
+                    {'guid': '1', 'prop': 9},
+                    {'guid': '2', 'prop': 89},
+                    ],
+                db._find(query='prop:0..100', order_by='prop')[0])
+
+        self.assertEqual(
+                [
+                    {'guid': '1', 'prop': 9},
+                    ],
+                db._find(query='prop:9', order_by='prop')[0])
+
+        self.assertEqual(
+                [
+                    {'guid': '2', 'prop': 89},
+                    ],
+                db._find(query='prop:=89', order_by='prop')[0])
+
+    def __test_Floats(self):
+        db = Index({
+            'prop': ActiveProperty('prop', 1, 'A', typecast=float, full_text=True),
+            })
+
+        db.store('1', {'prop': 9.1}, True)
+        db.store('2', {'prop': 89.2}, True)
+        db.store('3', {'prop': 777.3}, True)
+
+        self.assertEqual(
+                [
+                    {'guid': '1', 'prop': 9.1},
+                    {'guid': '2', 'prop': 89.2},
+                    {'guid': '3', 'prop': 777.3},
+                    ],
+                db._find(order_by='prop')[0])
+
+        self.assertEqual(
+                [
+                    {'guid': '1', 'prop': 9.1},
+                    {'guid': '2', 'prop': 89.2},
+                    ],
+                db._find(query='prop:0..100', order_by='prop')[0])
+
+        self.assertEqual(
+                [
+                    {'guid': '1', 'prop': 9.1},
+                    ],
+                db._find(query='prop:9.1', order_by='prop')[0])
+
+        self.assertEqual(
+                [
+                    {'guid': '2', 'prop': 89.2},
+                    ],
+                db._find(query='prop:=89.2', order_by='prop')[0])
+
+    def __test_Booleans(self):
+        db = Index({
+            'prop': ActiveProperty('prop', 1, 'A', typecast=bool, full_text=True),
+            })
+
+        db.store('1', {'prop': True}, True)
+        db.store('2', {'prop': False}, True)
+
+        self.assertEqual(
+                [
+                    {'guid': '2', 'prop': False},
+                    {'guid': '1', 'prop': True},
+                    ],
+                db._find(order_by='prop')[0])
+
+        self.assertEqual(
+                [
+                    {'guid': '2', 'prop': False},
+                    {'guid': '1', 'prop': True},
+                    ],
+                db._find(query='prop:0..100', order_by='prop')[0])
+
+        self.assertEqual(
+                [
+                    {'guid': '1', 'prop': True},
+                    ],
+                db._find(query='prop:1..1', order_by='prop')[0])
+
+        self.assertEqual(
+                [
+                    {'guid': '2', 'prop': False},
+                    ],
+                db._find(query='prop:0', order_by='prop')[0])
+
+        self.assertEqual(
+                [
+                    {'guid': '1', 'prop': True},
+                    ],
+                db._find(query='prop:=1', order_by='prop')[0])
+
 
 def read_diff(directory, *args, **kwargs):
     result = []
diff --git a/tests/units/index.py b/tests/units/index.py
index 94825e6..5507dfd 100755
--- a/tests/units/index.py
+++ b/tests/units/index.py
@@ -108,11 +108,11 @@ class IndexTest(tests.Test):
         db.store('2', {'var_1': False}, True)
 
         self.assertEqual(
-                [{'guid': '1', 'var_1': True}],
-                db._find(var_1=True)[0])
+                [{'guid': '1'}],
+                db._find(var_1=True, reply=['guid'])[0])
         self.assertEqual(
-                [{'guid': '2', 'var_1': False}],
-                db._find(var_1=False)[0])
+                [{'guid': '2'}],
+                db._find(var_1=False, reply=['guid'])[0])
 
     def test_find_WithProps(self):
         db = Index({
@@ -283,118 +283,6 @@ class IndexTest(tests.Test):
                 ([{'guid': '1', 'var_1': '1'}, {'guid': '2', 'var_1': '2'}], 2),
                 db._find(reply=['var_1'], group_by='var_3'))
 
-    def test_Integers(self):
-        db = Index({
-            'prop': ActiveProperty('prop', 1, 'A', typecast=int, full_text=True),
-            })
-
-        db.store('1', {'prop': 9}, True)
-        db.store('2', {'prop': 89}, True)
-        db.store('3', {'prop': 777}, True)
-
-        self.assertEqual(
-                [
-                    {'guid': '1', 'prop': 9},
-                    {'guid': '2', 'prop': 89},
-                    {'guid': '3', 'prop': 777},
-                    ],
-                db._find(order_by='prop')[0])
-
-        self.assertEqual(
-                [
-                    {'guid': '1', 'prop': 9},
-                    {'guid': '2', 'prop': 89},
-                    ],
-                db._find(query='prop:0..100', order_by='prop')[0])
-
-        self.assertEqual(
-                [
-                    {'guid': '1', 'prop': 9},
-                    ],
-                db._find(query='prop:9', order_by='prop')[0])
-
-        self.assertEqual(
-                [
-                    {'guid': '2', 'prop': 89},
-                    ],
-                db._find(query='prop:=89', order_by='prop')[0])
-
-    def test_Floats(self):
-        db = Index({
-            'prop': ActiveProperty('prop', 1, 'A', typecast=float, full_text=True),
-            })
-
-        db.store('1', {'prop': 9.1}, True)
-        db.store('2', {'prop': 89.2}, True)
-        db.store('3', {'prop': 777.3}, True)
-
-        self.assertEqual(
-                [
-                    {'guid': '1', 'prop': 9.1},
-                    {'guid': '2', 'prop': 89.2},
-                    {'guid': '3', 'prop': 777.3},
-                    ],
-                db._find(order_by='prop')[0])
-
-        self.assertEqual(
-                [
-                    {'guid': '1', 'prop': 9.1},
-                    {'guid': '2', 'prop': 89.2},
-                    ],
-                db._find(query='prop:0..100', order_by='prop')[0])
-
-        self.assertEqual(
-                [
-                    {'guid': '1', 'prop': 9.1},
-                    ],
-                db._find(query='prop:9.1', order_by='prop')[0])
-
-        self.assertEqual(
-                [
-                    {'guid': '2', 'prop': 89.2},
-                    ],
-                db._find(query='prop:=89.2', order_by='prop')[0])
-
-    def test_Booleans(self):
-        db = Index({
-            'prop': ActiveProperty('prop', 1, 'A', typecast=bool, full_text=True),
-            })
-
-        db.store('1', {'prop': True}, True)
-        db.store('2', {'prop': False}, True)
-
-        self.assertEqual(
-                [
-                    {'guid': '2', 'prop': False},
-                    {'guid': '1', 'prop': True},
-                    ],
-                db._find(order_by='prop')[0])
-
-        self.assertEqual(
-                [
-                    {'guid': '2', 'prop': False},
-                    {'guid': '1', 'prop': True},
-                    ],
-                db._find(query='prop:0..100', order_by='prop')[0])
-
-        self.assertEqual(
-                [
-                    {'guid': '1', 'prop': True},
-                    ],
-                db._find(query='prop:1..1', order_by='prop')[0])
-
-        self.assertEqual(
-                [
-                    {'guid': '2', 'prop': False},
-                    ],
-                db._find(query='prop:0', order_by='prop')[0])
-
-        self.assertEqual(
-                [
-                    {'guid': '1', 'prop': True},
-                    ],
-                db._find(query='prop:=1', order_by='prop')[0])
-
     def test_MultipleValues(self):
         db = Index({
             'prop': ActiveProperty('prop', prefix='B', typecast=[1, 2], full_text=True),
@@ -765,12 +653,20 @@ class Index(index.IndexWriter):
             kwargs['reply'] = {}
         if 'order_by' not in kwargs:
             kwargs['order_by'] = 'guid'
-        documents, total = self.find(env.Query(*args, **kwargs))
+
+        mset = self.find(env.Query(*args, **kwargs))
         result = []
-        for guid, props in documents:
-            props['guid'] = guid
+
+        for hit in self.find(env.Query(*args, **kwargs)):
+            props = {}
+            for name in kwargs['reply']:
+                prop = self.metadata[name]
+                if prop.slot is not None:
+                    props[name] = hit.document.get_value(prop.slot).decode('utf8')
+            props['guid'] = hit.document.get_value(0).decode('utf8')
             result.append(props)
-        return result, total
+
+        return result, mset.get_matches_estimated()
 
 
 if __name__ == '__main__':