From 6c16b4c366b33f8020cd414b88abcd8aa2e43ead Mon Sep 17 00:00:00 2001 From: Aleksey Lim Date: Tue, 13 Aug 2013 00:06:45 +0000 Subject: Return all indexed documents by default on low level --- diff --git a/misc/aslo-patch-authors b/misc/aslo-patch-authors index a94ecfd..88df5a4 100755 --- a/misc/aslo-patch-authors +++ b/misc/aslo-patch-authors @@ -122,7 +122,7 @@ volume.populate() try: for document in ('context', 'review'): - documents, __ = volume[document].find(limit=db.MAX_LIMIT) + documents, __ = volume[document].find() for doc in documents: authors = new_authors(doc) if authors: diff --git a/misc/aslo-patch-versions b/misc/aslo-patch-versions index 04a7694..37d2710 100755 --- a/misc/aslo-patch-versions +++ b/misc/aslo-patch-versions @@ -41,7 +41,7 @@ volume = Volume(data_root.value) volume.populate() directory = volume['implementation'] try: - items, __ = directory.find(limit=db.MAX_LIMIT) + items, __ = directory.find() for impl in items: data = impl['data'] url = data.get('url') diff --git a/misc/aslo-sync b/misc/aslo-sync index 2e32b53..3a12c77 100755 --- a/misc/aslo-sync +++ b/misc/aslo-sync @@ -213,8 +213,7 @@ class Application(application.Application): def sync_activities(self, addon_id=None): directory = self.volume['context'] - items, __ = directory.find(type='activity', layer='public', - limit=db.MAX_LIMIT) + items, __ = directory.find(type='activity', layer='public') existing_activities = set([i.guid for i in items]) sql = """ @@ -247,8 +246,7 @@ class Application(application.Application): def sync_reviews(self, addon_id, bundle_id): directory = self.volume['review'] - items, __ = directory.find(context=bundle_id, layer='public', - limit=db.MAX_LIMIT) + items, __ = directory.find(context=bundle_id, layer='public') existing_reviews = set([i.guid for i in items]) sql = """ @@ -292,8 +290,7 @@ class Application(application.Application): def sync_versions(self, addon_id, bundle_id): directory = self.volume['implementation'] - items, __ = directory.find(context=bundle_id, layer='public', - limit=db.MAX_LIMIT) + items, __ = directory.find(context=bundle_id, layer='public') existing_versions = set([i.guid for i in items]) sql = """ @@ -468,9 +465,9 @@ class Application(application.Application): def sync_implementaiton(self, context, addon_id, filename, sugar_min, sugar_max, status, license, notes, date): bundle_path = join(ACTIVITIES_PATH, str(addon_id), filename) - with load_bundle(self.volume, bundle_path, { - 'requires': 'sugar>=%s; sugar<=%s' % (sugar_min, sugar_max), - }) as impl: + with load_bundle(self.volume, ASLO_GUID, bundle_path, + requires='sugar>=%s; sugar<=%s' % (sugar_min, sugar_max), + ) as impl: if impl['license'] == spec.EMPTY_LICENSE: if not license and context in LICENSES_MAP: license = LICENSES_MAP[context] diff --git a/sugar_network/client/clones.py b/sugar_network/client/clones.py index b7c8ec8..bd8850f 100644 --- a/sugar_network/client/clones.py +++ b/sugar_network/client/clones.py @@ -21,7 +21,7 @@ import logging from os.path import join, exists, lexists, relpath, dirname, basename, isdir from os.path import abspath, islink -from sugar_network import db, client, toolkit +from sugar_network import client, toolkit from sugar_network.model.context import Context from sugar_network.toolkit.spec import Spec from sugar_network.toolkit.inotify import Inotify, \ @@ -96,7 +96,7 @@ class _Inotify(Inotify): if mtime <= self._contexts.mtime: return - docs, __ = self._contexts.find(limit=db.MAX_LIMIT, clone=[1, 2]) + docs, __ = self._contexts.find(clone=[1, 2]) for context in docs: root = _context_path(context.guid, '') found = False diff --git a/sugar_network/db/__init__.py b/sugar_network/db/__init__.py index 0a77a4d..b6bde81 100644 --- a/sugar_network/db/__init__.py +++ b/sugar_network/db/__init__.py @@ -355,6 +355,6 @@ from sugar_network.db.metadata import \ from sugar_network.db.index import index_flush_timeout, \ index_flush_threshold, index_write_queue from sugar_network.db.resource import Resource -from sugar_network.db.directory import Directory, MAX_LIMIT +from sugar_network.db.directory import Directory from sugar_network.db.volume import Volume from sugar_network.db.routes import Routes diff --git a/sugar_network/db/directory.py b/sugar_network/db/directory.py index b9fc02c..915e508 100644 --- a/sugar_network/db/directory.py +++ b/sugar_network/db/directory.py @@ -26,8 +26,6 @@ from sugar_network.db.metadata import IndexedProperty, StoredProperty from sugar_network.toolkit import http, exception, enforce -MAX_LIMIT = 2147483648 - # To invalidate existed index on stcuture changes _LAYOUT_VERSION = 4 @@ -216,8 +214,6 @@ class Directory(object): params['order_by'] = '-seqno' else: params['order_by'] = 'seqno' - # TODO On big requests, xapian can raise an exception on edits - params['limit'] = MAX_LIMIT params['no_cache'] = True for start, end in seq: diff --git a/sugar_network/db/index.py b/sugar_network/db/index.py index 156ff76..708c609 100644 --- a/sugar_network/db/index.py +++ b/sugar_network/db/index.py @@ -128,7 +128,7 @@ class IndexReader(object): """ raise NotImplementedError() - def find(self, offset=0, limit=64, query='', reply=('guid',), + def find(self, offset=0, limit=None, query='', reply=('guid',), order_by=None, no_cache=False, group_by=None, **request): """Search resources within the index. @@ -139,8 +139,7 @@ class IndexReader(object): the resulting list should start with this offset; 0 by default :param limit: - the resulting list will be at least `limit` size; - the `--find-limit` will be used by default + the resulting list will be at least `limit` size :param query: a string in Xapian serach format, empty to avoid text search :param reply: @@ -163,6 +162,8 @@ class IndexReader(object): self.ensure_open() start_timestamp = time.time() + if limit is None: + limit = self._db.get_doccount() # This will assure that the results count is exact. check_at_least = offset + limit + 1 diff --git a/sugar_network/node/routes.py b/sugar_network/node/routes.py index e2781e7..8b607cb 100644 --- a/sugar_network/node/routes.py +++ b/sugar_network/node/routes.py @@ -226,8 +226,8 @@ class NodeRoutes(db.Routes, model.Routes): implementations = self.volume['implementation'] versions = [] - impls, __ = implementations.find(limit=db.MAX_LIMIT, - context=context.guid, not_layer='deleted', **request) + impls, __ = implementations.find(context=context.guid, + not_layer='deleted', **request) for impl in impls: for arch, spec in impl.meta('data')['spec'].items(): spec['guid'] = impl.guid -- cgit v0.9.1