Encode title of the page before download it - SL #2855

If the word searched have a non ascii character, like a accent
could not be downloaded

1 files changed, 8 insertions, 1 deletions

diff --git a/infoslicer/processing/MediaWiki_Helper.py b/infoslicer/processing/MediaWiki_Helper.py
index a20c838..2dcc1ed 100644
--- a/infoslicer/processing/MediaWiki_Helper.py
+++ b/infoslicer/processing/MediaWiki_Helper.py
@@ -6,6 +6,8 @@ import logging
 

 import net

 

+import re

+

 logger = logging.getLogger('infoslicer')

 

 """

@@ -187,12 +189,17 @@ class MediaWiki_Helper:
         urllib._urlopener = NewURLopener()

         logger.debug("opening " + path)

         logger.debug("proxies: " + str(self.proxies))

-        doc = urllib.urlopen(path, proxies=self.proxies)

+        pathencoded = self.urlEncodeNonAscii(path)

+        logger.debug("pathencoded " + pathencoded)

+        doc = urllib.urlopen(pathencoded, proxies=self.proxies)

         output = doc.read()

         doc.close()

         logger.debug("url opened successfully")

         return output

     

+    def urlEncodeNonAscii(self, b):

+        return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)

+

     def stripTags(self, input, tag):

         """removes specified tag