verify indexing works with non-ascii files, all tests pass

3 files changed, 22 insertions, 2 deletions

diff --git a/tests/query.txt b/tests/query.txt
index 784e821..eef2735 100644
--- a/tests/query.txt
+++ b/tests/query.txt
@@ -226,6 +226,10 @@ True
 >>> qm.find(fulltext="amazed", author="Benjamin") == [a]
 True
 
+>>> qm.update(a, file="test.doc")
+>>> qm.find(fulltext="amazed", author="Benjamin") == [a]
+True
+
 
 Now for politeness we shut everything down
 >>> qm.stop()
diff --git a/tests/sugar_demo_may17.txt b/tests/sugar_demo_may17.txt
index 23fe1ef..0e35b91 100644
--- a/tests/sugar_demo_may17.txt
+++ b/tests/sugar_demo_may17.txt
@@ -54,4 +54,10 @@ Check content:
 'some other content\n'
 >>> fp.close()
 
+Set content as pdf:
+>>> ds.update(uid, dict(title="Same entry with some content in pdf"), 'test.pdf')
+>>> ds.update(uid, dict(title="Same entry with some content in doc"), 'test.doc')
+>>> ds.update(uid, dict(title="Same entry with some content in odt"), 'test.odt')
+
+
 >>> del ds
diff --git a/tests/test_conversion.py b/tests/test_conversion.py
index fbb589b..aae6fc0 100644
--- a/tests/test_conversion.py
+++ b/tests/test_conversion.py
@@ -1,6 +1,6 @@
 import unittest
 
-from olpc.datastore.converter import Purify
+from olpc.datastore.converter import Purify, converter
 from StringIO import StringIO
 
 class Test(unittest.TestCase):
@@ -20,7 +20,17 @@ class Test(unittest.TestCase):
             # and that we are getting string
             # (not unicode) back
             assert isinstance(line, str)
-            
+
+    def test_unicode(self):
+        # read each of the test files in doing conversion,
+        # there should be no unicode errors
+        fn_expectations = {
+            'test.pdf' : 'Don\'t',
+            'test.doc' : 'amazed.',
+            'test.odt' : 'amazed.'
+            }
+        for fn, expect in fn_expectations.iteritems():
+            assert expect in converter(fn).read()
 
         
 def test_suite():