seems to work externalized fulltext indexer

4 files changed, 84 insertions, 22 deletions

diff --git a/bin/datastore b/bin/datastore
index 679df98..7dc8665 100755
--- a/bin/datastore
+++ b/bin/datastore
@@ -1,7 +1,10 @@
 #!/usr/bin/env python
 from ore.main import Application
 from olpc.datastore import DataStore
+from olpc.datastore.indexer import INDEX_SERVICE, INDEX_OBJECT_PATH
 import os, signal
+import dbus
+import dbus.mainloop.glib
 
 profile = os.environ.get('SUGAR_PROFILE', 'default')
 base_dir = os.path.join(os.path.expanduser('~'), '.sugar', profile)
@@ -11,7 +14,7 @@ db_dir   = "sqlite:///%s/datastore.db"  % repo_dir
 import logging
 
 def handle_shutdown(signum, frame):
-    ds.stop()
+    dsa.stop()
     raise SystemExit("Shutting down on signal %s" % signum)
 
 
@@ -29,6 +32,17 @@ class DataStoreApplication(Application):
                                action="store_true", default=False,
                                help="""Dump database work""")
 
+        self.parser.add_option("--olpc.fulltext.sync", dest="sync_index",
+                               action="store_true", default=False,
+                               help="""Force the indexer in-process in
+                               synchronous mode.""")
+
+        self.parser.add_option("--olpc.fulltext.debug_indexer",
+                               dest="start_indexer",
+                               action="store_false", default=True,
+                               help="""Don't make an attempt to start the indexer (use existing -- for debug)""")
+
+
     def main(self):
         # operate from the repo directory
         if not os.path.exists(repo_dir):
@@ -36,7 +50,13 @@ class DataStoreApplication(Application):
         os.chdir(repo_dir)
 
         self.ds = DataStore(self.options.repo_dir,
-                            self.options.md_db)
+                            self.options.md_db,
+                            querymanager_sync_index=self.options.sync_index)
+
+        if self.options.sync_index is False and self.options.start_indexer:
+            # trigger the index service
+            bus = dbus.SessionBus()
+            indexer = bus.get_object(INDEX_SERVICE, INDEX_OBJECT_PATH)
 
         if self.options.debug_db:
             logging.getLogger('sqlalchemy.orm.unitofwork').setLevel(logging.DEBUG)
diff --git a/bin/datastore-service b/bin/datastore-service
index 981e06d..aa85b9d 100755
--- a/bin/datastore-service
+++ b/bin/datastore-service
@@ -56,9 +56,9 @@ def main():
         logger.debug("Datastore shutdown with error",
                          exc_info=sys.exc_info())
      
-#main()
+main()
 
-import hotshot
-p = hotshot.Profile('hs.prof')
-p.run('main()')
+#import hotshot
+#p = hotshot.Profile('hs.prof')
+#p.run('main()')
 
diff --git a/bin/index-service b/bin/index-service
index 583c396..24a6b91 100644..100755
--- a/bin/index-service
+++ b/bin/index-service
@@ -8,26 +8,42 @@ access the Xapian repository in read only mode.
 """
 
 
-from ore.main import Application
+try: from ore.main import Application
+except ImportError: Application = object
+
 from olpc.datastore.datastore import DS_SERVICE, DS_OBJECT_PATH
 from olpc.datastore.datastore import DS_DBUS_INTERFACE
 from olpc.datastore.indexer import Indexer
 import dbus
 import dbus.mainloop.glib
-
+import gobject
+import logging
 import sys
 import os
+import signal
 
 profile = os.environ.get('SUGAR_PROFILE', 'default')
 base_dir = os.path.join(os.path.expanduser('~'), '.sugar', profile)
 repo_dir = os.path.join(base_dir, 'datastore')
+fulltext_dir = os.path.join(repo_dir, 'fulltext')
 os.chdir(repo_dir)
 
+logger = logging.getLogger('org.laptop.sugar.Indexer')
+
 class IndexService(Application):
+    def manage_options(self):
+        self.parser.add_option("--olpc.fulltext.repo",
+                               dest="fulltext_dir",
+                               action="store", default='fulltext',
+                               help="""Location of the FullText Repository""")
+
+
     def main(self):
         dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
         bus = dbus.SessionBus()
-        self.fulltext = Indexer()
+        self.fulltext = Indexer(self.options.fulltext_dir)
+        self.fulltext.use_fulltext = True
+        
         self.ds = bus.get_object(DS_SERVICE, DS_OBJECT_PATH)
 
         self.ds.connect_to_signal("Created", self.created,
@@ -52,7 +68,8 @@ class IndexService(Application):
         # the indexing on that
         filename = self.ds.get_filename(uid)
         if filename: self.fulltext.fulltext_index(uid, filename)
-
+        logger.debug("index creation of %s" % uid)
+        
     def updated(self, uid, props):
         """An object was updated on the bus and we want to index it"""
         # because the file isn't encoded anywhere accessible in the
@@ -60,25 +77,50 @@ class IndexService(Application):
         # the indexing on that
         filename = self.ds.get_filename(uid)
         if filename: self.fulltext.fulltext_index(uid, filename)
-
+        logger.debug("index update of %s" % uid)
+        
     def deleted(self, uid):
         """An object was updated on the bus and we want to index it"""
         # because the file isn't encoded anywhere accessible in the
         # create call we must actually get the filename and trigger
         # the indexing on that
-        try: self.fulltext.fulltext_unindex(uid)
+        try:
+            self.fulltext.fulltext_unindex(uid)
+            logger.debug("unindex deletion  of %s" % uid);
         except KeyError: pass
+
         
     def stopped(self):
         """Respond to the datastore being stopped by shutting down
         ourselves"""
-        self.index.stop()
+        self.fulltext.stop()
         sys.exit(0)
         
 
 
+
 if __name__ == "__main__":
+    def handle_shutdown(signum, frame):
+        idx.stopped()
+        print "shutdown cleanly"
+        raise SystemExit("Shutting down on signal %s" % signum)
+
+    signal.signal(signal.SIGHUP, handle_shutdown)
+    signal.signal(signal.SIGTERM, handle_shutdown)
+
     idx = IndexService()
-    idx.plugins.append('ore.main.profile_support.ProfileSupport')
-    idx()
+    #idx()
+
+    # w/o ore.main
+    idx.eventloop = gobject.MainLoop()
+    class options(object): pass
+    o = options()
+    o.fulltext_dir = 'fulltext'
+    idx.options = o
+    try:
+        idx.main()
+    except:
+        idx.stopped()
+
+
     
diff --git a/bin/sample-client.py b/bin/sample-client.py
index a642818..23945ec 100755
--- a/bin/sample-client.py
+++ b/bin/sample-client.py
@@ -11,19 +11,19 @@ def main():
     uid = datastore.create(dict(title="from dbus", author="Benjamin"), os.path.abspath('tests/test.pdf'))
     print "created uid", uid
     
-    #print "all", datastore.all()
-    #for u in datastore.all():
-    #    if u != uid:
-    #        datastore.delete(u)
+    #print "all", datastore.find()
+    #for u in datastore.find()[0]:
+    #    if u['uid'] != uid:
+    #        datastore.delete(u['uid'])
     print "find", datastore.find(dict(author="Benjamin", title="from"))
     
     print "bcsaller", datastore.find(dict(fulltext="bcsaller"))
     print "huh?", datastore.find(dict(fulltext="kfdshaksjd"))
 
-    datastore.update(uid, dict(title="updated title"), "/etc/passwd")
-    datastore.update(uid, dict(title="another updated title"), "/etc/passwd")
+    datastore.update(uid, dict(title="updated title"), title"), os.path.abspath('tests/test.doc'))
+    datastore.update(uid, dict(title="another updated title"), os.path.abspath('tests/test.odt'))
     print datastore.get_properties(uid)
-    datastore.delete(uid)
+    #datastore.delete(uid)
     
 if __name__ == '__main__':
     a = Application("client", main)