Web   ·   Wiki   ·   Activities   ·   Blog   ·   Lists   ·   Chat   ·   Meeting   ·   Bugs   ·   Git   ·   Translate   ·   Archive   ·   People   ·   Donate
summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJames Cameron <quozl@laptop.org>2010-11-30 05:14:14 (GMT)
committer James Cameron <quozl@laptop.org>2010-12-17 00:01:39 (GMT)
commitc4faafd51c63bf19e14f23b710d310269dff4b07 (patch)
treeb46cfd750d434d9753418f68b37808a0ded04e05
parent1d069eb3f58cb0eb03d5a31e3c4f6979cbcd8296 (diff)
fix journal scan of external media, dev.laptop.org #10140
Update the progress bar regularly and prevent the UI from stalling during a scan. Avoid following recursive symlinks, and symlinks that point outside the filesystem being scanned. Do not check for MIME type if file is excluded for other filter reasons. Do not report permission denied errors. Tested-By: Sascha Silbe <sascha-pgp@silbe.org> Reviewed-By: Sascha Silbe <sascha-pgp@silbe.org> Acked-By: Simon Schampijer <simon@laptop.org>
-rw-r--r--src/jarabe/journal/model.py137
1 files changed, 90 insertions, 47 deletions
diff --git a/src/jarabe/journal/model.py b/src/jarabe/journal/model.py
index 0773446..3d9b695 100644
--- a/src/jarabe/journal/model.py
+++ b/src/jarabe/journal/model.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2007-2008, One Laptop Per Child
+# Copyright (C) 2007-2010, One Laptop per Child
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -16,10 +16,11 @@
import logging
import os
+import errno
from datetime import datetime
import time
import shutil
-from stat import S_IFMT, S_IFDIR, S_IFREG
+from stat import S_IFLNK, S_IFMT, S_IFDIR, S_IFREG
import re
from operator import itemgetter
@@ -230,7 +231,9 @@ class InplaceResultSet(BaseResultSet):
BaseResultSet.__init__(self, query, page_size)
self._mount_point = mount_point
self._file_list = None
- self._pending_directories = 0
+ self._pending_directories = []
+ self._visited_directories = []
+ self._pending_files = []
self._stopped = False
query_text = query.get('query', '')
@@ -257,7 +260,10 @@ class InplaceResultSet(BaseResultSet):
def setup(self):
self._file_list = []
- self._recurse_dir(self._mount_point)
+ self._pending_directories = [self._mount_point]
+ self._visited_directories = []
+ self._pending_files = []
+ gobject.idle_add(self._scan)
def stop(self):
self._stopped = True
@@ -298,63 +304,100 @@ class InplaceResultSet(BaseResultSet):
return entries, total_count
- def _recurse_dir(self, dir_path):
- self._pending_directories += 1
- gobject.idle_add(self._idle_recurse_dir, dir_path)
+ def _scan(self):
+ if self._stopped:
+ return False
- def _idle_recurse_dir(self, dir_path):
- try:
- self._real_recurse_dir(dir_path)
- finally:
- self._pending_directories -= 1
- if self._pending_directories == 0:
- self.setup_ready()
+ self.progress.send(self)
- def _real_recurse_dir(self, dir_path):
- if self._stopped:
- return
+ if self._pending_files:
+ self._scan_a_file()
+ return True
+
+ if self._pending_directories:
+ self._scan_a_directory()
+ return True
+
+ self.setup_ready()
+ self._visited_directories = []
+ return False
+
+ def _scan_a_file(self):
+ full_path = self._pending_files.pop(0)
try:
- dirs = os.listdir(dir_path)
- except Exception:
- logging.exception('Error reading directory %r', dir_path)
- dirs = []
+ stat = os.lstat(full_path)
+ except OSError, e:
+ if e.errno != errno.ENOENT:
+ logging.exception(
+ 'Error reading metadata of file %r', full_path)
+ return
+
+ if S_IFMT(stat.st_mode) == S_IFLNK:
+ try:
+ link = os.readlink(full_path)
+ except OSError, e:
+ logging.exception(
+ 'Error reading target of link %r', full_path)
+ return
+
+ if not os.path.abspath(link).startswith(self._mount_point):
+ return
- for entry in dirs:
- if entry.startswith('.'):
- continue
- full_path = dir_path + '/' + entry
try:
stat = os.stat(full_path)
- if S_IFMT(stat.st_mode) == S_IFDIR:
- self._recurse_dir(full_path)
- elif S_IFMT(stat.st_mode) == S_IFREG:
- add_to_list = True
+ except OSError, e:
+ if e.errno != errno.ENOENT:
+ logging.exception(
+ 'Error reading metadata of linked file %r', full_path)
+ return
+
+ if S_IFMT(stat.st_mode) == S_IFDIR:
+ id_tuple = stat.st_ino, stat.st_dev
+ if not id_tuple in self._visited_directories:
+ self._visited_directories.append(id_tuple)
+ self._pending_directories.append(full_path)
+ return
- if self._regex is not None and \
- not self._regex.match(full_path):
- add_to_list = False
+ if S_IFMT(stat.st_mode) != S_IFREG:
+ return
+
+ if self._regex is not None and \
+ not self._regex.match(full_path):
+ return
+
+ if self._date_start is not None and stat.st_mtime < self._date_start:
+ return
+
+ if self._date_end is not None and stat.st_mtime > self._date_end:
+ return
- if None not in [self._date_start, self._date_end] and \
- (stat.st_mtime < self._date_start or
- stat.st_mtime > self._date_end):
- add_to_list = False
+ if self._mime_types:
+ mime_type = gio.content_type_guess(filename=full_path)
+ if mime_type not in self._mime_types:
+ return
- if self._mime_types:
- mime_type = gio.content_type_guess(filename=full_path)
- if mime_type not in self._mime_types:
- add_to_list = False
+ file_info = (full_path, stat, int(stat.st_mtime), stat.st_size)
+ self._file_list.append(file_info)
- if add_to_list:
- file_info = (full_path, stat, int(stat.st_mtime),
- stat.st_size)
- self._file_list.append(file_info)
+ return
- self.progress.send(self)
+ def _scan_a_directory(self):
+ dir_path = self._pending_directories.pop(0)
- except Exception:
- logging.exception('Error reading file %r', full_path)
+ try:
+ entries = os.listdir(dir_path)
+ except OSError, e:
+ if e.errno != errno.EACCES:
+ logging.exception('Error reading directory %r', dir_path)
+ return
+
+ for entry in entries:
+ if entry.startswith('.'):
+ continue
+ self._pending_files.append(dir_path + '/' + entry)
+ return
def _get_file_metadata(path, stat):