path: root/use_whoosh_index_to_check_links.diff

diff --git a/activity/activity.info.en b/activity/activity.info.en
index 2e85ec9..021d299 100644
--- a/activity/activity.info.en
+++ b/activity/activity.info.en
@@ -1,6 +1,6 @@
 [Activity]
 name = WikipediaEN
-activity_version = 33.6
+activity_version = 34
 bundle_id = org.laptop.WikipediaActivityEN
 icon = activity-wikipedia-en
 exec = sugar-activity activity_en.WikipediaActivityEN
diff --git a/activity/activity.info.es_lat b/activity/activity.info.es_lat
index f6a9348..023bc4f 100644
--- a/activity/activity.info.es_lat
+++ b/activity/activity.info.es_lat
@@ -1,6 +1,6 @@
 [Activity]
 name = Wikipedia
-activity_version = 33.6
+activity_version = 34
 bundle_id = org.laptop.WikipediaActivity
 icon = activity-wikipedia-es
 exec = sugar-activity activity_es.WikipediaActivityES
diff --git a/dataretriever.py b/dataretriever.py
index 838c6b0..9a9e109 100644
--- a/dataretriever.py
+++ b/dataretriever.py
@@ -2,42 +2,17 @@
 # -*- coding: utf-8 -*-
 # create index
 
-import codecs
 from subprocess import Popen, PIPE, STDOUT
 import re
+import os
 import logging
 
-def normalize_title(title):
-    return title.strip().replace(' ', '_').capitalize()
-
+from whoosh.qparser import QueryParser
+from whoosh.index import open_dir
 
-class RedirectParser:
 
-    def __init__(self, file_name):
-        self.link_re = re.compile('\[\[.*?\]\]')
-        # Load redirects
-        input_redirects = codecs.open('%s.redirects_used' % file_name,
-                encoding='utf-8', mode='r')
-
-        self.redirects = {}
-        for line in input_redirects.readlines():
-            links = self.link_re.findall(unicode(line))
-            if len(links) == 2:
-                origin = links[0][2:-2]
-                destination = links[1][2:-2]
-                self.redirects[origin] = destination
-            #print "Processing %s" % normalize_title(origin)
-        logging.debug("Loaded %d redirects" % len(self.redirects))
-        input_redirects.close()
-
-    def get_redirected(self, article_title):
-        try:
-            logging.debug("get_redirect %s" % article_title)
-            article_title = article_title.capitalize()
-            redirect = self.redirects[article_title]
-        except:
-            redirect = None
-        return redirect
+def normalize_title(title):
+    return title.strip().replace(' ', '_').capitalize()
 
 
 class DataRetriever():
@@ -46,41 +21,45 @@ class DataRetriever():
         self.system_id = system_id
         self._bzip_file_name = '%s.processed.bz2' % data_files_base
         self._bzip_table_file_name = '%s.processed.bz2t' % data_files_base
-        self._index_file_name = '%s.processed.idx' % data_files_base
         self.template_re = re.compile('({{.*?}})')
-        self.redirects_checker = RedirectParser(data_files_base)
+        base_path = os.path.dirname(data_files_base)
+        self.ix = open_dir(os.path.join(base_path, "index_dir"))
         # TODO: I need control cache size
         self.templates_cache = {}
 
-    def _get_article_position(self, article_title):
+    def check_existence(self, article_title):
         article_title = normalize_title(article_title)
-        index_file = codecs.open(self._index_file_name, encoding='utf-8',
-                mode='r')
-        #index_file = open(self._index_file_name, mode='r')
-
-        num_block = -1
-        position = -1
-        for index_line in  index_file.readlines():
-            words = index_line.split()
-            article = words[0]
-            if article == article_title:
-                num_block = int(words[1])
-                position = int(words[2])
-                break
-        index_file.close()
-
-        if num_block == -1:
-            # look at redirects
-            logging.debug("looking for '%s' at redirects" % article_title)
-            redirect = self.redirects_checker.get_redirected(article_title)
-            if redirect is not None:
-                if redirect == article_title:
-                    # to avoid infinite recursion
-                    return -1, -1
-                return self._get_article_position(redirect)
+        with self.ix.searcher() as searcher:
+            query = QueryParser("title",
+                    self.ix.schema).parse("'%s'" % article_title)
+            results = searcher.search(query, limit=1)
+            return results.scored_length() > 0
 
+    def _get_article_position(self, article_title):
+        article_title = normalize_title(article_title)
+        # look at the title in the index database
+        with self.ix.searcher() as searcher:
+            query = QueryParser("title",
+                            self.ix.schema).parse("'%s'" % article_title)
+            results = searcher.search(query, limit=1)
+            logging.error('Search article %s returns %s', article_title,
+                    results[0])
+            num_block = results[0]['block']
+            position = results[0]['position']
+            if num_block == 0 and position == 0:
+                # if block and position = 0 serach with the redirect_to value
+                return self._get_article_position(results[0]['redirect_to'])
         return num_block, position
 
+    def search(self, article_title):
+        with self.ix.searcher() as searcher:
+            query = QueryParser("title", self.ix.schema).parse(article_title)
+            results = searcher.search(query, limit=None)
+            articles = []
+            for n in range(results.scored_length()):
+                articles.append(results[n]['title'])
+            return articles
+
     def _get_block_start(self, num_block):
         bzip_table_file = open(self._bzip_table_file_name, mode='r')
         n = num_block
diff --git a/server.py b/server.py
index 930da64..c55d626 100755
--- a/server.py
+++ b/server.py
@@ -41,8 +41,6 @@ except ImportError:
     from md5 import md5
 
 import dataretriever
-from whoosh.qparser import QueryParser
-from whoosh.index import open_dir
 ##
 ## Libs we ship -- add lib path for
 ## shared objects
@@ -98,33 +96,6 @@ class LinkStats:
     pagetotal = 1
 
 
-class ArticleIndex:
-    # Prepare an in-memory index, using the already generated
-    # index file.
-
-    def __init__(self, path):
-        self.index_path = '%s.processed.idx' % path
-        self.redirect_parser = dataretriever.RedirectParser(path)
-        self.article_index = set()
-        with open(self.index_path, mode='r') as f:
-            for line in f.readlines():
-                m = re.search(r'(.*?)\s*\d+\s*\d+$', line)
-                if m is None:
-                    raise AssertionError("Match didn't work")
-                self.article_index.add(m.group(1))
-        print "INDEX HAVE %d articles" % len(self.article_index)
-
-    def __contains__(self, x):
-        found = dataretriever.normalize_title(x) in self.article_index
-        if not found:
-            redirect = self.redirect_parser.get_redirected(x)
-            if redirect is not None:
-                found = True
-
-        #print "TEST INDEX %s %s" % (dataretriever.normalize_title(x), found)
-        return found
-
-
 class WPWikiDB:
     """Retrieves article contents for mwlib."""
 
@@ -132,7 +103,7 @@ class WPWikiDB:
         self.lang = lang
         self.templateprefix = templateprefix
         self.templateblacklist = templateblacklist
-        self.data_retriever = dataretriever.DataRetriever(system_id, path)
+        self.dataretriever = dataretriever.DataRetriever(system_id, path)
         self.templates_cache = {'!' : '|'}  # a special case
 
     def getRawArticle(self, title, followRedirects=True):
@@ -142,7 +113,7 @@ class WPWikiDB:
             return ''
 
         article_text = \
-                self.data_retriever.get_text_article(title).decode('utf-8')
+                self.dataretriever.get_text_article(title).decode('utf-8')
 
         # Stripping leading & trailing whitespace fixes template expansion.
         article_text = article_text.lstrip()
@@ -151,6 +122,7 @@ class WPWikiDB:
         return article_text
 
     def getTemplate(self, title, followRedirects=False):
+        logging.error('getTemplate: %s', title)
         if title in self.templates_cache:
             return self.templates_cache[title]
         else:
@@ -287,8 +259,8 @@ class WPMathRenderer:
 class WPHTMLWriter(mwlib.htmlwriter.HTMLWriter):
     """Customizes HTML output from mwlib."""
 
-    def __init__(self, index, wfile, images=None, lang='en'):
-        self.index = index
+    def __init__(self, dataretriever, wfile, images=None, lang='en'):
+        self.dataretriever = dataretriever
         self.gallerylevel = 0
         self.lang = lang
         self.math_processed = False
@@ -315,7 +287,7 @@ class WPHTMLWriter(mwlib.htmlwriter.HTMLWriter):
             title = title[0].capitalize() + title[1:]
             title = title.replace("_", " ")
 
-            article_exists = title.encode('utf8') in self.index
+            article_exists = self.dataretriever.check_existence(article)
 
             if article_exists:
                 # Exact match.  Internal link.
@@ -337,7 +309,6 @@ class WPHTMLWriter(mwlib.htmlwriter.HTMLWriter):
             parts[0] = parts[0].replace(" ", "_")
             url = ("#".join([x for x in parts]))
 
-            #print "----> ""<a %s href='%s%s'>" % (link_attr, link_baseurl, url)
             self.out.write("<a %s href='%s%s'>" % (link_attr, link_baseurl,
                     url))
 
@@ -520,11 +491,10 @@ class WPHTMLWriter(mwlib.htmlwriter.HTMLWriter):
 
 
 class WikiRequestHandler(SimpleHTTPRequestHandler):
-    def __init__(self, wikidb, index, conf, request, client_address, server):
+    def __init__(self, wikidb, conf, request, client_address, server):
         # pullcord is currently offline
         # self.reporturl = 'pullcord.laptop.org:8000'
         self.reporturl = False
-        self.index = index
         self.port = conf['port']
         self.lang = conf['lang']
         self.templateprefix = conf['templateprefix']
@@ -532,6 +502,7 @@ class WikiRequestHandler(SimpleHTTPRequestHandler):
         self.wpheader = conf['wpheader']
         self.wpfooter = conf['wpfooter']
         self.resultstitle = conf['resultstitle']
+        self.base_path = os.path.dirname(conf['path'])
 
         if 'editdir' in conf:
             self.editdir = conf['editdir']
@@ -542,10 +513,6 @@ class WikiRequestHandler(SimpleHTTPRequestHandler):
         else:
             self.giturl = False
 
-        # init search index
-        self.base_path = os.path.dirname(conf['path'])
-        self.ix = open_dir(os.path.join(self.base_path, "index_dir"))
-
         self.wikidb = wikidb
 
         self.client_address = client_address
@@ -581,8 +548,8 @@ class WikiRequestHandler(SimpleHTTPRequestHandler):
         wiki_parsed.caption = title
 
         imagedb = WPImageDB(self.base_path + '/images/')
-        writer = WPHTMLWriter(self.index, htmlout, images=imagedb,
-                lang=self.lang)
+        writer = WPHTMLWriter(self.wikidb.dataretriever, htmlout,
+                images=imagedb, lang=self.lang)
         writer.write(wiki_parsed)
         return writer.math_processed
 
@@ -825,13 +792,7 @@ class WikiRequestHandler(SimpleHTTPRequestHandler):
         self.wfile.write("</body></html>")
 
     def search(self, article_title):
-        with self.ix.searcher() as searcher:
-            query = QueryParser("title", self.ix.schema).parse(article_title)
-            results = searcher.search(query, limit=None)
-            articles = []
-            for n in range(results.scored_length()):
-                articles.append(results[n]['title'])
-            return articles
+        return self.wikidb.dataretriever.search(article_title)
 
     def send_image(self, path):
         if os.path.exists(path.encode('utf8')[1:]):
@@ -911,7 +872,6 @@ class WikiRequestHandler(SimpleHTTPRequestHandler):
 
 
 def run_server(confvars):
-    index = ArticleIndex(confvars['path'])
 
     if 'editdir' in confvars:
         try:
@@ -941,7 +901,7 @@ def run_server(confvars):
             confvars['templateprefix'], confvars['templateblacklist'])
 
     httpd = MyHTTPServer(('', confvars['port']),
-        lambda *args: WikiRequestHandler(wikidb, index, confvars, *args))
+        lambda *args: WikiRequestHandler(wikidb, confvars, *args))
 
     if confvars['comandline']:
         httpd.serve_forever()
diff --git a/setup_new_wiki.py b/setup_new_wiki.py
index 0a5e829..4c94412 100755
--- a/setup_new_wiki.py
+++ b/setup_new_wiki.py
@@ -45,8 +45,7 @@ class WikiXOPackager(bundlebuilder.XOPackager):
 
         if self.data_file is not None:
             # Add the data files
-            needed_sufix = ['.processed.bz2', '.processed.bz2t',
-                            '.processed.idx', '.redirects_used']
+            needed_sufix = ['.processed.bz2', '.processed.bz2t']
             for sufix in needed_sufix:
                 data_file = self.data_file + sufix
                 print "Add %s" % data_file
diff --git a/tools2/create_index.py b/tools2/create_index.py
index f0e1957..2d90316 100755
--- a/tools2/create_index.py
+++ b/tools2/create_index.py
@@ -101,9 +101,10 @@ class RedirectParser:
 def create_search_index(input_xml_file_name, pages_blacklist):
     sys.path.append('..')
     from whoosh.index import create_in
-    from whoosh.fields import TEXT, Schema
+    from whoosh.fields import TEXT, NUMERIC, Schema
 
-    schema = Schema(title=TEXT(stored=True))
+    schema = Schema(title=TEXT(stored=True), block=NUMERIC(stored=True),
+                position=NUMERIC(stored=True), redirect_to=TEXT(stored=True))
     if not os.path.exists("index_dir"):
         os.mkdir("index_dir")
     ix = create_in("index_dir", schema)
@@ -115,9 +116,13 @@ def create_search_index(input_xml_file_name, pages_blacklist):
         parts = line.split()
         if len(parts) > 0:
             title_article = parts[0]
+            block_article = parts[1]
+            position_article = parts[2]
             title_article = normalize_title(title_article)
             if title_article not in pages_blacklist:
-                writer.add_document(title=unicode(title_article))
+                writer.add_document(title=unicode(title_article),
+                    block=int(block_article), position=int(position_article),
+                    redirect_to=unicode(''))
             else:
                 print "* Blacklisted %s " % title_article
         line = text_index_file.readline()
@@ -126,16 +131,20 @@ def create_search_index(input_xml_file_name, pages_blacklist):
     redirects_parser = RedirectParser(input_xml_file_name)
     for origin in redirects_parser.redirects.keys():
         origin = normalize_title(origin)
-        destination = normalize_title(redirects_parser.redirects[origin])
-        if origin not in pages_blacklist and \
-                destination not in pages_blacklist:
-            writer.add_document(title=unicode(origin))
-        else:
-            print "* Blacklisted %s " % origin
-
+        try:
+            destination = normalize_title(redirects_parser.redirects[origin])
+            if origin not in pages_blacklist and \
+                    destination not in pages_blacklist:
+                writer.add_document(title=unicode(origin), block=0, position=0,
+                                    redirect_to=unicode(destination))
+            else:
+                print "* Blacklisted %s " % origin
+        except:
+            print "ERROR: origin %s destination %s" % (origin, destination)
     writer.commit()
     text_index_file.close()
 
+
 def create_bzip_table():
     """
     ../seek-bzip2/seek-bzip2/bzip-table <