#4235: Keep indexing an usb stick after an error in a single file. (tomeu)

2 files changed, 59 insertions, 52 deletions

diff --git a/NEWS b/NEWS
index 5f6df2d..1eb9f13 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,5 @@
+* #4235: Keep indexing an usb stick after an error in a single file. (tomeu)
+
 Snapshot 89ae26ced4
 
 * Improved mime type handling. (marco)
diff --git a/src/olpc/datastore/backingstore.py b/src/olpc/datastore/backingstore.py
index 93c735d..705b9a5 100644
--- a/src/olpc/datastore/backingstore.py
+++ b/src/olpc/datastore/backingstore.py
@@ -657,58 +657,63 @@ class InplaceFileBackingStore(FileBackingStore):
         # scan the uri for all non self.base files and update their
         # records in the db
         for dirpath, dirname, filenames in os.walk(self.uri):
-            # see if there is an entry for the filename
-            if self.base in dirpath: continue
-            if self.STORE_NAME in dirname:
-                dirname.remove(self.STORE_NAME)
-
-            # blacklist all the hidden directories
-            if '/.' in dirpath: continue
-
-            for fn in filenames:
-                # give the thread a chance to exit
-                if not self._runWalker: break
-                # blacklist files
-                #   ignore conventionally hidden files
-                if fn.startswith("."): continue
-                
-                source = os.path.join(dirpath, fn)
-                relative = source[len(self.uri)+1:]
-
-                result, count = self.indexmanager.search(dict(filename=relative))
-                mime_type = gnomevfs.get_mime_type(source)
-                stat = os.stat(source)
-                ctime = datetime.fromtimestamp(stat.st_ctime).isoformat()
-                mtime = datetime.fromtimestamp(stat.st_mtime).isoformat()
-                title = os.path.splitext(os.path.split(source)[1])[0]
-                metadata = dict(filename=relative,
-                                mime_type=mime_type,
-                                ctime=ctime,
-                                mtime=mtime,
-                                title=title)
-                if not count:
-                    # create a new record
-                    self.create(metadata, source)
-                else:
-                    # update the object with the new content iif the
-                    # checksum is different
-                    # XXX: what if there is more than one? (shouldn't
-                    # happen)
-
-                    # FIXME This is throwing away all the entry metadata.
-                    # Disabled for trial-3. We are not doing indexing
-                    # anyway so it would just update the mtime which is
-                    # not that useful. Also the journal is currently
-                    # setting the mime type before saving the file making
-                    # the mtime check useless.
-                    #
-                    # content = result.next()
-                    # uid = content.id
-                    # saved_mtime = content.get_property('mtime')
-                    # if mtime != saved_mtime:
-                    #     self.update(uid, metadata, source)
-                    pass
-
+            try:
+                # see if there is an entry for the filename
+                if self.base in dirpath: continue
+                if self.STORE_NAME in dirname:
+                    dirname.remove(self.STORE_NAME)
+
+                # blacklist all the hidden directories
+                if '/.' in dirpath: continue
+
+                for fn in filenames:
+                    try:
+                        # give the thread a chance to exit
+                        if not self._runWalker: break
+                        # blacklist files
+                        #   ignore conventionally hidden files
+                        if fn.startswith("."): continue
+                        
+                        source = os.path.join(dirpath, fn)
+                        relative = source[len(self.uri)+1:]
+
+                        result, count = self.indexmanager.search(dict(filename=relative))
+                        mime_type = gnomevfs.get_mime_type(source)
+                        stat = os.stat(source)
+                        ctime = datetime.fromtimestamp(stat.st_ctime).isoformat()
+                        mtime = datetime.fromtimestamp(stat.st_mtime).isoformat()
+                        title = os.path.splitext(os.path.split(source)[1])[0]
+                        metadata = dict(filename=relative,
+                                        mime_type=mime_type,
+                                        ctime=ctime,
+                                        mtime=mtime,
+                                        title=title)
+                        if not count:
+                            # create a new record
+                            self.create(metadata, source)
+                        else:
+                            # update the object with the new content iif the
+                            # checksum is different
+                            # XXX: what if there is more than one? (shouldn't
+                            # happen)
+
+                            # FIXME This is throwing away all the entry metadata.
+                            # Disabled for trial-3. We are not doing indexing
+                            # anyway so it would just update the mtime which is
+                            # not that useful. Also the journal is currently
+                            # setting the mime type before saving the file making
+                            # the mtime check useless.
+                            #
+                            # content = result.next()
+                            # uid = content.id
+                            # saved_mtime = content.get_property('mtime')
+                            # if mtime != saved_mtime:
+                            #     self.update(uid, metadata, source)
+                            pass
+                    except Exception, e:
+                        logging.exception('Error while processing %r: %r' % (fn, e))
+            except Exception, e:
+                logging.exception('Error while indexing mount point %r: %r' % (self.uri, e))
         self.indexmanager.flush()
         return