Return all indexed documents by default on low levelactivities_cache

8 files changed, 17 insertions, 23 deletions

diff --git a/misc/aslo-patch-authors b/misc/aslo-patch-authors
index a94ecfd..88df5a4 100755
--- a/misc/aslo-patch-authors
+++ b/misc/aslo-patch-authors
@@ -122,7 +122,7 @@ volume.populate()
 
 try:
     for document in ('context', 'review'):
-        documents, __ = volume[document].find(limit=db.MAX_LIMIT)
+        documents, __ = volume[document].find()
         for doc in documents:
             authors = new_authors(doc)
             if authors:
diff --git a/misc/aslo-patch-versions b/misc/aslo-patch-versions
index 04a7694..37d2710 100755
--- a/misc/aslo-patch-versions
+++ b/misc/aslo-patch-versions
@@ -41,7 +41,7 @@ volume = Volume(data_root.value)
 volume.populate()
 directory = volume['implementation']
 try:
-    items, __ = directory.find(limit=db.MAX_LIMIT)
+    items, __ = directory.find()
     for impl in items:
         data = impl['data']
         url = data.get('url')
diff --git a/misc/aslo-sync b/misc/aslo-sync
index 2e32b53..3a12c77 100755
--- a/misc/aslo-sync
+++ b/misc/aslo-sync
@@ -213,8 +213,7 @@ class Application(application.Application):
 
     def sync_activities(self, addon_id=None):
         directory = self.volume['context']
-        items, __ = directory.find(type='activity', layer='public',
-                limit=db.MAX_LIMIT)
+        items, __ = directory.find(type='activity', layer='public')
         existing_activities = set([i.guid for i in items])
 
         sql = """
@@ -247,8 +246,7 @@ class Application(application.Application):
 
     def sync_reviews(self, addon_id, bundle_id):
         directory = self.volume['review']
-        items, __ = directory.find(context=bundle_id, layer='public',
-                limit=db.MAX_LIMIT)
+        items, __ = directory.find(context=bundle_id, layer='public')
         existing_reviews = set([i.guid for i in items])
 
         sql = """
@@ -292,8 +290,7 @@ class Application(application.Application):
 
     def sync_versions(self, addon_id, bundle_id):
         directory = self.volume['implementation']
-        items, __ = directory.find(context=bundle_id, layer='public',
-                limit=db.MAX_LIMIT)
+        items, __ = directory.find(context=bundle_id, layer='public')
         existing_versions = set([i.guid for i in items])
 
         sql = """
@@ -468,9 +465,9 @@ class Application(application.Application):
     def sync_implementaiton(self, context, addon_id, filename,
             sugar_min, sugar_max, status, license, notes, date):
         bundle_path = join(ACTIVITIES_PATH, str(addon_id), filename)
-        with load_bundle(self.volume, bundle_path, {
-                'requires': 'sugar>=%s; sugar<=%s' % (sugar_min, sugar_max),
-                }) as impl:
+        with load_bundle(self.volume, ASLO_GUID, bundle_path,
+                requires='sugar>=%s; sugar<=%s' % (sugar_min, sugar_max),
+                ) as impl:
             if impl['license'] == spec.EMPTY_LICENSE:
                 if not license and context in LICENSES_MAP:
                     license = LICENSES_MAP[context]
diff --git a/sugar_network/client/clones.py b/sugar_network/client/clones.py
index b7c8ec8..bd8850f 100644
--- a/sugar_network/client/clones.py
+++ b/sugar_network/client/clones.py
@@ -21,7 +21,7 @@ import logging
 from os.path import join, exists, lexists, relpath, dirname, basename, isdir
 from os.path import abspath, islink
 
-from sugar_network import db, client, toolkit
+from sugar_network import client, toolkit
 from sugar_network.model.context import Context
 from sugar_network.toolkit.spec import Spec
 from sugar_network.toolkit.inotify import Inotify, \
@@ -96,7 +96,7 @@ class _Inotify(Inotify):
         if mtime <= self._contexts.mtime:
             return
 
-        docs, __ = self._contexts.find(limit=db.MAX_LIMIT, clone=[1, 2])
+        docs, __ = self._contexts.find(clone=[1, 2])
         for context in docs:
             root = _context_path(context.guid, '')
             found = False
diff --git a/sugar_network/db/__init__.py b/sugar_network/db/__init__.py
index 0a77a4d..b6bde81 100644
--- a/sugar_network/db/__init__.py
+++ b/sugar_network/db/__init__.py
@@ -355,6 +355,6 @@ from sugar_network.db.metadata import \
 from sugar_network.db.index import index_flush_timeout, \
         index_flush_threshold, index_write_queue
 from sugar_network.db.resource import Resource
-from sugar_network.db.directory import Directory, MAX_LIMIT
+from sugar_network.db.directory import Directory
 from sugar_network.db.volume import Volume
 from sugar_network.db.routes import Routes
diff --git a/sugar_network/db/directory.py b/sugar_network/db/directory.py
index b9fc02c..915e508 100644
--- a/sugar_network/db/directory.py
+++ b/sugar_network/db/directory.py
@@ -26,8 +26,6 @@ from sugar_network.db.metadata import IndexedProperty, StoredProperty
 from sugar_network.toolkit import http, exception, enforce
 
 
-MAX_LIMIT = 2147483648
-
 # To invalidate existed index on stcuture changes
 _LAYOUT_VERSION = 4
 
@@ -216,8 +214,6 @@ class Directory(object):
             params['order_by'] = '-seqno'
         else:
             params['order_by'] = 'seqno'
-        # TODO On big requests, xapian can raise an exception on edits
-        params['limit'] = MAX_LIMIT
         params['no_cache'] = True
 
         for start, end in seq:
diff --git a/sugar_network/db/index.py b/sugar_network/db/index.py
index 156ff76..708c609 100644
--- a/sugar_network/db/index.py
+++ b/sugar_network/db/index.py
@@ -128,7 +128,7 @@ class IndexReader(object):
         """
         raise NotImplementedError()
 
-    def find(self, offset=0, limit=64, query='', reply=('guid',),
+    def find(self, offset=0, limit=None, query='', reply=('guid',),
             order_by=None, no_cache=False, group_by=None, **request):
         """Search resources within the index.
 
@@ -139,8 +139,7 @@ class IndexReader(object):
             the resulting list should start with this offset;
             0 by default
         :param limit:
-            the resulting list will be at least `limit` size;
-            the `--find-limit` will be used by default
+            the resulting list will be at least `limit` size
         :param query:
             a string in Xapian serach format, empty to avoid text search
         :param reply:
@@ -163,6 +162,8 @@ class IndexReader(object):
         self.ensure_open()
 
         start_timestamp = time.time()
+        if limit is None:
+            limit = self._db.get_doccount()
         # This will assure that the results count is exact.
         check_at_least = offset + limit + 1
 
diff --git a/sugar_network/node/routes.py b/sugar_network/node/routes.py
index e2781e7..8b607cb 100644
--- a/sugar_network/node/routes.py
+++ b/sugar_network/node/routes.py
@@ -226,8 +226,8 @@ class NodeRoutes(db.Routes, model.Routes):
         implementations = self.volume['implementation']
         versions = []
 
-        impls, __ = implementations.find(limit=db.MAX_LIMIT,
-                context=context.guid, not_layer='deleted', **request)
+        impls, __ = implementations.find(context=context.guid,
+                not_layer='deleted', **request)
         for impl in impls:
             for arch, spec in impl.meta('data')['spec'].items():
                 spec['guid'] = impl.guid