Ignore bad documents while populating

3 files changed, 61 insertions, 11 deletions

diff --git a/active_document/directory.py b/active_document/directory.py
index 9bf60cb..f03e401 100644
--- a/active_document/directory.py
+++ b/active_document/directory.py
@@ -229,17 +229,21 @@ class Directory(object):
             if migrate:
                 self._storage.migrate(guid)
 
-            props = {}
             record = self._storage.get(guid)
-            for name, prop in self.metadata.items():
-                if not isinstance(prop, StoredProperty):
-                    continue
-                meta = record.get(name)
-                if meta is not None:
-                    props[name] = meta['value']
-            self._index.store(guid, props, None, None, None)
-
-            yield
+            try:
+                props = {}
+                for name, prop in self.metadata.items():
+                    if not isinstance(prop, StoredProperty):
+                        continue
+                    meta = record.get(name)
+                    if meta is not None:
+                        props[name] = meta['value']
+                self._index.store(guid, props, None, None, None)
+                yield
+            except Exception:
+                util.exception('Cannot populate %r in %r, invalidate it',
+                        guid, self.metadata.name)
+                record.invalidate()
 
         if found:
             self._save_layout()
diff --git a/active_document/storage.py b/active_document/storage.py
index d9806f6..e24882b 100644
--- a/active_document/storage.py
+++ b/active_document/storage.py
@@ -171,6 +171,11 @@ class Record(object):
     def consistent(self):
         return exists(join(self._root, 'guid'))
 
+    def invalidate(self):
+        guid_path = join(self._root, 'guid')
+        if exists(guid_path):
+            os.unlink(guid_path)
+
     def get(self, prop):
         path = join(self._root, prop)
         if exists(path):
diff --git a/tests/units/document.py b/tests/units/document.py
index 9c99d5b..869c03e 100755
--- a/tests/units/document.py
+++ b/tests/units/document.py
@@ -246,7 +246,7 @@ class DocumentTest(tests.Test):
                 [],
                 [i.prop for i in directory.find(0, 1024)[0]])
 
-    def test_crawler(self):
+    def test_populate(self):
 
         class Document(document.Document):
 
@@ -290,6 +290,47 @@ class DocumentTest(tests.Test):
                     ],
                 [(i.ctime, i.mtime, i.prop) for i in directory.find(0, 10)[0]])
 
+    def test_populate_IgnoreBadDocuments(self):
+
+        class Document(document.Document):
+
+            @active_property(slot=1)
+            def prop(self, value):
+                return value
+
+        self.touch(
+                ('1/1/guid', '{"value": "1"}'),
+                ('1/1/ctime', '{"value": 1}'),
+                ('1/1/mtime', '{"value": 1}'),
+                ('1/1/prop', '{"value": "prop-1"}'),
+                ('1/1/seqno', '{"value": 0}'),
+
+                ('2/2/guid', '{"value": "2"}'),
+                ('2/2/ctime', ''),
+                ('2/2/mtime', '{"value": 2}'),
+                ('2/2/prop', '{"value": "prop-2"}'),
+                ('2/2/seqno', '{"value": 0}'),
+
+                ('3/3/guid', ''),
+                ('3/3/ctime', ''),
+                ('3/3/mtime', ''),
+                ('3/3/prop', ''),
+                ('3/3/seqno', ''),
+                )
+
+        directory = Directory(tests.tmpdir, Document, IndexWriter)
+
+        populated = 0
+        for i in directory.populate():
+            populated += 1
+        self.assertEqual(1, populated)
+        self.assertEqual(
+                ['1'],
+                [i.guid for i in directory.find(0, 10)[0]])
+        assert exists('1/1/guid')
+        assert not exists('2/2/guid')
+        assert not exists('3/3/guid')
+
     def test_create_with_guid(self):
 
         class Document(document.Document):