diff options
author | Sascha Silbe <sascha-pgp@silbe.org> | 2010-06-02 20:39:30 (GMT) |
---|---|---|
committer | Sascha Silbe <sascha-pgp@silbe.org> | 2010-06-02 20:39:30 (GMT) |
commit | c1959a1c2040f640ff86518a4ad992a0f2fafcff (patch) | |
tree | 73fdcd1ad7885288afdb632c9545b7665be0e920 | |
parent | 89fb3d7b4dbf8fea7b6f958cb0ff8a2d17f07a53 (diff) |
major restructuring, partially broken
-rwxr-xr-x | datastore-fuse.py | 673 |
1 files changed, 347 insertions, 326 deletions
diff --git a/datastore-fuse.py b/datastore-fuse.py index 19eed32..4638f5f 100755 --- a/datastore-fuse.py +++ b/datastore-fuse.py @@ -8,6 +8,7 @@ data store. import errno import fuse import logging +import operator import os import shutil import stat @@ -31,12 +32,15 @@ DS_DBUS_PATH = "/org/laptop/sugar/DataStore" class DataStoreObjectStat(fuse.Stat): # pylint: disable-msg=R0902,R0903 - def __init__(self, metadata, size): + def __init__(self, parent, metadata, size): fuse.Stat.__init__(self, st_mode=stat.S_IFREG | 0750, st_nlink=1, st_uid=os.getuid(), st_gid=os.getgid(), st_size=size, - st_mtime = self._parse_time(metadata.get('timestamp', ''))) + st_mtime=self._parse_time(metadata.get('timestamp', ''))) self.st_ctime = self.st_mtime self.st_atime = self.st_mtime + self.metadata = metadata + self.parent = parent + self.object_id = metadata['uid'] def _parse_time(self, timestamp): try: @@ -44,48 +48,294 @@ class DataStoreObjectStat(fuse.Stat): except ValueError: return 0 + def should_truncate(self): + return self.parent.should_truncate(self.object_id) -class DirectoryStat(fuse.Stat): + def reset_truncate(self): + return self.parent.reset_truncate(self.object_id) - # pylint: disable-msg=R0902,R0903 - def __init__(self, mode): - fuse.Stat.__init__(self, st_mode = stat.S_IFDIR | mode, st_nlink = 2, - st_uid = os.getuid(), st_gid = os.getgid(), st_size = 4096, - st_mtime = time.time()) + +class Symlink(fuse.Stat): + def __init__(self, filesystem, target): + self._filesystem = filesystem + self.target = target + fuse.Stat.__init__(self, st_mode=stat.S_IFLNK | 0777, st_nlink=1, + st_uid=os.getuid(), st_gid=os.getgid(), + st_mtime=time.time()) + self.st_ctime = self.st_mtime + self.st_atime = self.st_mtime + + +class Directory(fuse.Stat): + def __init__(self, filesystem, mode): + self._filesystem = filesystem + fuse.Stat.__init__(self, st_mode=stat.S_IFDIR | mode, st_nlink=2, + st_uid=os.getuid(), st_gid=os.getgid(), + st_mtime=time.time()) self.st_ctime = self.st_mtime self.st_atime = self.st_mtime + def getxattr(self, name_, attribute_): + # on Linux ENOATTR=ENODATA (Python errno doesn't contain ENOATTR) + raise IOError(errno.ENODATA, os.strerror(errno.ENODATA)) + + def listxattr(self, name_): + return [] + + def lookup(self, name_): + raise IOError(errno.ENOENT, os.strerror(errno.ENOENT)) + + def mkdir(self, name_): + raise IOError(errno.EACCES, os.strerror(errno.EACCES)) + + def mknod(self, name_): + raise IOError(errno.EACCES, os.strerror(errno.EACCES)) + + def readdir(self, offset_): + for name in ['.', '..']: + yield fuse.Direntry(name) + + def readlink(self, name): + entry = self.lookup(name) + if not isinstance(entry, Symlink): + raise IOError(errno.EINVAL, os.strerror(errno.EINVAL)) + + return entry.target + + def remove(self, name_): + raise IOError(errno.EACCES, os.strerror(errno.EACCES)) + + def truncate(self, name_): + raise IOError(errno.EACCES, os.strerror(errno.EACCES)) + + +class ByTitleDirectory(Directory): + def __init__(self, filesystem, root): + self.root = root + Directory.__init__(self, filesystem, 0750) + self._object_id_to_title_name = {} + self._title_name_to_object_id = {} + + def readdir(self, offset): + Directory.readdir(self, offset) + + for entry in self._filesystem.find({}, + {'metadata': ['title', 'uid', 'timestamp']}): + + name = self._object_id_to_title_name.get(entry['uid']) + if not name: + name = self._generate_title_name(entry) + self._add_title_name(name, entry) + + yield fuse.Direntry(name) + + def lookup(self, name): + object_id = self._resolve_title_name(name) + return Symlink(self._filesystem, 'by-id/' + str(object_id)) + + def mknod(self, name): + if name in self._title_name_to_object_id: + raise IOError(errno.EEXIST, os.strerror(errno.EEXIST)) + + metadata = {'title': name} + object_id = self._filesystem.create_new(metadata, '') + metadata['uid'] = object_id + self._add_title_name(name, metadata) + + def remove(self, name): + if name in ['.', '..']: + raise IOError(errno.EACCES, os.strerror(errno.EACCES)) + + object_id = self._resolve_title_name(name) + self.root.by_id_directory.remove(object_id) + self._remove_title_name_by_object_id(object_id) + + def _resolve_title_name(self, name): + try: + return self._title_name_to_object_id[name] + + except KeyError: + raise IOError(errno.ENOENT, os.strerror(errno.ENOENT)) + + def _add_title_name(self, name, metadata): + self._object_id_to_title_name[metadata['uid']] = name + self._title_name_to_object_id[name] = metadata['uid'] + return name + + @trace() + def _generate_title_name(self, metadata): + title = metadata.get('title') + try: + mtime = float(metadata['timestamp']) + except (KeyError, ValueError): + mtime = time.time() + + time_human = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(mtime)) + name = '%s - %s' % (title, time_human) + name = self._safe_name(name) + current_name = name + counter = 1 + while current_name in self._title_name_to_object_id: + counter += 1 + current_name = '%s %d' % (name, counter) + + return current_name + + def _remove_title_name_by_object_id(self, object_id): + name = self._object_id_to_title_name.pop(object_id, None) + if name: + del self._title_name_to_object_id[name] + + def _remove_title_name_by_name(self, name): + object_id = self._title_name_to_object_id.pop(name, None) + if object_id: + del self._object_id_to_title_name[object_id] + + def _safe_name(self, name): + return name.replace('/', '_') + + +class ByIdDirectory(Directory): + def __init__(self, filesystem): + Directory.__init__(self, filesystem, 0550) + self._truncate_object_ids = set() + + def getxattr(self, object_id, attribute): + metadata = self._filesystem.get_metadata(object_id) + if attribute in metadata: + return metadata[attribute] + + Directory.getxattr(self, object_id, attribute) + + def listxattr(self, object_id): + metadata = self._filesystem.get_metadata(object_id) + return [str(name) for name in metadata.keys()] + + def lookup(self, object_id): + metadata = self._filesystem.get_metadata(object_id) + size = self._get_size(object_id) + return DataStoreObjectStat(self, metadata, size) + + def readdir(self, offset): + Directory.readdir(self, offset) + + for entry in self._filesystem.find({}, {'metadata': ['uid']}): + yield fuse.Direntry(entry['uid']) + + def remove(self, object_id): + self._filesystem.remove_entry(object_id) + + def truncate(self, object_id): + self._truncate_object_ids.add(object_id) + + def should_truncate(self, object_id): + return object_id in self._truncate_object_ids + + def reset_truncate(self, object_id): + self._truncate_object_ids.discard(object_id) + + def _get_size(self, object_id): + file_name = self._filesystem.get_data(object_id) + if not file_name: + return 0 + + try: + return os.stat(file_name).st_size + finally: + os.remove(file_name) + + +# TODO +class ByTagsDirectory(Directory): + def __init__(self, filesystem): + Directory.__init__(self, filesystem, 0550) + + +class RootDirectory(ByTitleDirectory): + def __init__(self, filesystem): + ByTitleDirectory.__init__(self, filesystem, self) + self.by_id_directory = ByIdDirectory(filesystem) + self.by_tags_directory = ByTagsDirectory(filesystem) + self.by_title_directory = self + + def readdir(self, offset_): + for name in ['by-id', 'by-tags']: + yield fuse.Direntry(name) + + for name in ByTitleDirectory.readdir(self, offset_): + yield name + + def lookup(self, name): + if name == 'by-id': + return self.by_id_directory + elif name == 'by-tags': + return self.by_tags_directory + + return ByTitleDirectory.lookup(self, name) + + def remove(self, name): + if name in ['by-id', 'by-tags']: + raise IOError(errno.EACCES, os.strerror(errno.EACCES)) + + return ByTitleDirectory.remove(self, name) + class DataStoreFile(object): - _MODE_MASK = os.O_RDONLY | os.O_RDWR | os.O_WRONLY + _ACCESS_MASK = os.O_RDONLY | os.O_RDWR | os.O_WRONLY + direct_io = False + keep_cache = False - def __init__(self, filesystem, path, flags, mode_=None): + @trace() + def __init__(self, filesystem, path, flags, mode=None): self._filesystem = filesystem - self._path = path self._flags = flags self._read_only = False self._is_temporary = False - if flags & self._MODE_MASK == os.O_RDONLY: - self._read_only = True - self._file = self._checkout(path, False) + self._dirty = False + self._path = path - elif flags & os.O_EXCL: - self._filesystem.create_new(path) - self._file = self._create() + logging.debug('opening file %r with flags %r', path, flags) + # Contrary to what's documented in the wiki, we'll get passed O_CREAT + # and mknod() won't get called automatically, so we'll have to take + # care of all possible cases ourselves. + if flags & os.O_EXCL: + filesystem.mknod(path) + entry = filesystem.getattr(path) else: - self._file = self._checkout(path, flags & os.O_CREAT) - - if flags & os.O_TRUNC: - self.ftruncate(0) + try: + entry = filesystem.getattr(path) + + except IOError, exception: + if exception.errno != errno.ENOENT: + raise + + if not flags & os.O_CREAT: + raise + + filesystem.mknod(path, flags, mode) + entry = filesystem.getattr(path) + + # mknod() might have created a symlink at our path... + if isinstance(entry, Symlink): + entry = filesystem.getattr(entry.target) + + self._object_id = entry.object_id + self._read_only = flags & self._ACCESS_MASK == os.O_RDONLY + + if entry.should_truncate() or flags & os.O_TRUNC: + self._file = self._create() + entry.reset_truncate() + else: + self._file = self._checkout() def _create(self): self._is_temporary = True - return tempfile.NamedTemporaryFile() # TODO dir/prefix + return tempfile.NamedTemporaryFile(prefix='datastore-fuse') - def _checkout(self, path, allow_create): - name = self._filesystem.checkout(path, allow_create) + def _checkout(self): + name = self._filesystem.get_data(self._object_id) if not name: # existing, but empty entry return self._create() @@ -102,10 +352,12 @@ class DataStoreFile(object): finally: os.remove(name) + @trace() def read(self, length, offset): self._file.seek(offset) return self._file.read(length) + @trace() def write(self, buf, offset): if self._flags & os.O_APPEND: self._file.seek(0, os.SEEK_END) @@ -113,25 +365,35 @@ class DataStoreFile(object): self._file.seek(offset) self._file.write(buf) + self._dirty = True return len(buf) + @trace() def release(self, flags_): - self.fsync(False) + self.fsync() self._file.close() if not self._is_temporary: os.remove(self._file.name) - def fsync(self, isfsyncfile_): - self._file.flush() - if not self._read_only: - self._filesystem.save(self._path, self._file.name) + @trace() + def fsync(self, isfsyncfile_=None): + self.flush() + if self._read_only: + return + if self._dirty: + self._filesystem.write_data(self._object_id, self._file.name) + + @trace() def flush(self): self._file.flush() + @trace() def fgetattr(self): - return os.fstat(self._file.fileno()) +# return os.fstat(self._file.fileno()) + return self._filesystem.getattr(self._path) + @trace() def ftruncate(self, length): self._file.truncate(length) @@ -155,141 +417,41 @@ class DataStoreFS(fuse.Fuse): bus = dbus.SessionBus() self_fs._data_store = dbus.Interface(bus.get_object(DS_DBUS_SERVICE, DS_DBUS_PATH), DS_DBUS_INTERFACE) - self_fs._uid_to_title_name = {} - self_fs._title_name_to_uid = {} + self_fs._root = RootDirectory(self_fs) # TODO: listen to DS signals to update name mapping - # TODO: factor out name mapping code into separate class - - def getattr(self, path): - return self._distribute(path, 'getattr') @trace() - def _distribute(self, path, operation, *args): - components = path.lstrip('/').split('/') - logging.debug('components=%r', components) - if not components[0]: - path_name = 'root' - parameters = [] - - elif components[0] == 'by-tags': - path_name = 'by_tags' - parameters = [components[1:]] - - elif components[0] == 'by-id': - path_name = 'by_id' - if len(components) > 2: - self._throw_error(errno.ENOENT) - elif len(components) == 2: - parameters = [components[1]] - else: - parameters = [None] - - elif operation == 'readdir': - self._throw_error(errno.ENOTDIR) - - elif len(components) > 1: - self._throw_error(errno.ENOTDIR) - - else: - path_name = 'by_title' - parameters = [components[0] or None] - - if args: - parameters += list(args) - - logging.debug('parameters=%r', parameters) - # pylint: disable-msg=W0142 - return getattr(self, '_%s_%s' % (operation, path_name))(*parameters) - - def _getattr_root(self): - return DirectoryStat(0750) - - def _getattr_by_tags(self, tags): - if not tags: - return DirectoryStat(0550) - - # TODO - self._throw_error(errno.ENOENT) - - def _getattr_by_title(self, title): - return self._getattr_by_id(self._resolve_title_name(title)) - - def _getattr_by_id(self, object_id): - if not object_id: - return DirectoryStat(0550) - - entry = self._get_metadata(object_id) - size = self._get_size(object_id) - return DataStoreObjectStat(entry, size) - - def readdir(self, path, offset_): - return self._distribute(path, 'readdir') - - def _readdir_root(self): - # root directory contains by-* subdirectories and - # data store entries by title - for name in ['.', '..', 'by-id', 'by-tags']: - yield fuse.Direntry(name) - - for entry in self._find({}, {'metadata': ['title', 'uid']}): - name = self._uid_to_title_name.get(entry['uid']) - if not name: - name = self._generate_title_name(entry) - self._add_title_name(name, entry) + def getattr(self, path): + components = [name for name in path.lstrip('/').split('/') if name] + entry = self._root + while components: + entry = entry.lookup(components.pop(0)) - yield fuse.Direntry(name) + return entry - def _readdir_by_id(self, object_id): - if object_id: - self._throw_error(errno.ENOTDIR) + @trace() + def _delegate(self, path, action, *args): + directory_name, file_name = os.path.split(path.strip('/')) + directory = self.getattr(directory_name) + return getattr(directory, action)(file_name, *args) - for entry in self._find({}, {'metadata': ['uid']}): - yield fuse.Direntry(entry['uid']) + def readdir(self, path, offset): + return self.getattr(path).readdir(offset) - def _readdir_by_tags(self, tags_): - # TODO - return + def readlink(self, path): + return self._delegate(path, 'readlink') def mknod(self, path, mode_, dev_): - return self._distribute(path, 'mknod') - - def _mknod_root(self): - self._throw_error(errno.EEXIST) - - def _mknod_by_id(self, object_id): - if not object_id: - self._throw_error(errno.EEXIST) - - self._throw_error(errno.EACCES) + # called by FUSE for open(O_CREAT) before instantiating the file + return self._delegate(path, 'mknod') - def _mknod_by_tags(self, tags): - if not tags: - self._throw_error(errno.EEXIST) - - self._throw_error(errno.EACCES) - - def _mknod_by_title(self, title_): - #object_id_ = self._create({'title': title}, '') - # let's try out whether we really need mknod() support... - self._throw_error(errno.EACCES) - - @trace() - def truncate(self, path_, mode_=None, dev_=None): - # FIXME: apparently needed for all text editors :-/ - self._throw_error(errno.EACCES) + def truncate(self, path, mode_=None, dev_=None): + # Documented to be called by FUSE when opening files with O_TRUNC, + # unless -o o_trunc_atomic is passed as a CLI option + self._delegate(path, 'truncate') def unlink(self, path): - return self._distribute(path, 'unlink') - - def _unlink_by_tags(self, tags_): - self._throw_error(errno.EACCES) - - def _unlink_by_id(self, object_id): - self._remove(object_id) - self._remove_title_name_by_uid(object_id) - - def _unlink_by_title(self, title): - return self._unlink_by_id(self._resolve_title_name(title)) + self._delegate(path, 'remove') @trace() def utime(self, path_, times_): @@ -297,138 +459,55 @@ class DataStoreFS(fuse.Fuse): return def mkdir(self, path, mode_): - self._distribute(path, 'mkdir') - - def _mkdir_root(self): - self._throw_error(errno.EEXIST) - - def _mkdir_by_id(self, object_id): - if object_id: - self._throw_error(errno.EEXIST) - - self._throw_error(errno.EACCES) - - def _mkdir_by_title(self, title_): - self._throw_error(errno.EACCES) - - def _mkdir_by_tags(self, tags_): - # TODO - self._throw_error(errno.EACCES) + self._delegate(path, 'mkdir') @trace() def rmdir(self, path_): - self._throw_error(errno.EACCES) + raise IOError(errno.EACCES, os.strerror(errno.EACCES)) def rename(self, pathfrom, pathto): - return self._distribute(pathfrom, 'rename', pathto) - - def _rename_root(self, destination_): - self._throw_error(errno.EACCES) - - def _rename_by_id(self, destination_, object_id_): - self._throw_error(errno.EACCES) - - def _rename_by_title(self, destination_, title_): - # TODO - self._throw_error(errno.EACCES) - - def _rename_by_tags(self, destination_, tags_): - # TODO - self._throw_error(errno.EACCES) - - @trace() - def fsync(self, path_, isfsyncfile_): - return + self._delegate(pathfrom, 'rename', pathto) @trace() def symlink(self, destination_, path_): # TODO for tags? - self._throw_error(errno.EACCES) + raise IOError(errno.EACCES, os.strerror(errno.EACCES)) @trace() def link(self, destination_, path_): - self._throw_error(errno.EPERM) + raise IOError(errno.EACCES, os.strerror(errno.EACCES)) @trace() def chmod(self, path_, mode_): - self._throw_error(errno.EACCES) + raise IOError(errno.EACCES, os.strerror(errno.EACCES)) @trace() def chown(self, path_, user_, group_): - self._throw_error(errno.EACCES) - -# TODO -# def getxattr(self, path, name, size): -# val = name.swapcase() + '@' + path -# if size == 0: -# # We are asked for size of the value. -# return len(val) -# return val -# -# def listxattr(self, path, size): -# # We use the "user" namespace to please XFS utils -# aa = ["user." + a for a in ("foo", "bar")] -# if size == 0: -# # We are asked for size of the attr list, ie. joint size of attrs -# # plus null separators. -# return len("".join(aa)) + len(aa) -# return aa - - def checkout(self, path, allow_create): - return self._distribute(path, 'checkout', allow_create) - - def _checkout_root(self, allow_create): - self._throw_error(errno.EISDIR) - - def _checkout_by_id(self, object_id, allow_create): - return self._get_data(object_id) - - def _checkout_by_tags(self, tags_, allow_create): - self._throw_error(errno.ENOENT) - - def _checkout_by_title(self, title, allow_create): - return self._get_data(self._resolve_title_name(title, allow_create)) - - def create_new(self, path): - return self._distribute(path, 'create_new') - - def _create_new_root(self): - self._throw_error(errno.EISDIR) - - def _create_new_by_id(self, object_id): - self._throw_error(errno.EACCES) - - def _create_new_by_tags(self, tags_): - # TODO? - self._throw_error(errno.EACCES) - - def _create_new_by_title(self, title): - object_id = self._title_name_to_uid.get(title) - if object_id: - self._throw_error(errno.EEXIST) - - metadata = {'title': title} - object_id = self._create(metadata, '') - metadata['uid'] = object_id - return self._add_title_name(title, metadata) + raise IOError(errno.EACCES, os.strerror(errno.EACCES)) - def save(self, path, file_name): - return self._distribute(path, 'save', file_name) + def getxattr(self, path, name, size): + if not name.startswith('user.'): + raise IOError(errno.ENODATA, os.strerror(errno.ENODATA)) - def _save_root(self, path_, file_name_): - self._throw_error(errno.EISDIR) + name = name[5:] + value = self._delegate(path, 'getxattr', name) + if not size: + # We are asked for size of the value. + return len(value) - def _save_by_id(self, object_id, file_name): - self._write_data(object_id, file_name) + return str(value) - def _save_by_tags(self, tags_, file_name_): - self._throw_error(errno.EACCES) + def listxattr(self, path, size): + attribute_names = ['user.' + name + for name in self._delegate(path, 'listxattr')] + if not size: + # We are asked for the size of the \0-separated list. + return reduce(operator.add, + [len(name) + 1 for name in attribute_names], 0) - def _save_by_title(self, title, file_name): - self._write_data(self._resolve_title_name(title), file_name) + return attribute_names - @trace() - def _find(self, metadata, options): + def find(self, metadata, options): mess = metadata.copy() mess.update(options) properties = mess.pop('metadata', []) @@ -437,85 +516,27 @@ class DataStoreFS(fuse.Fuse): return self._data_store.find(mess, properties, timeout=-1, byte_arrays=True)[0] - def _get_metadata(self, object_id): + def get_metadata(self, object_id): return self._data_store.get_properties(object_id, timeout=-1, byte_arrays=True) - def _create(self, metadata, path): + def create_new(self, metadata, path): return self._data_store.create(metadata, path, False, timeout=-1, byte_arrays=True) - def _remove(self, object_id): + def remove_data(self, object_id): return self._data_store.delete(object_id) - def _get_data(self, object_id): + def get_data(self, object_id): return self._data_store.get_filename(object_id, timeout=-1, byte_arrays=True) - def _write_data(self, object_id, file_name): - metadata = self._get_metadata(object_id) + @trace() + def write_data(self, object_id, file_name): + metadata = self.get_metadata(object_id) return self._data_store.update(object_id, metadata, file_name, False, timeout=-1, byte_arrays=True) - def _get_size(self, object_id): - file_name = self._get_data(object_id) - if not file_name: - return 0 - - try: - return os.stat(file_name).st_size - finally: - os.remove(file_name) - - @trace() - def _resolve_title_name(self, title, allow_create=False): - object_id = self._title_name_to_uid.get(title) - if object_id: - return object_id - - if not allow_create: - self._throw_error(errno.ENOENT) - - metadata = {'title': title} - object_id = self._create(metadata, '') - metadata['uid'] = object_id - return self._add_title_name(title, metadata) - - def _add_title_name(self, name, metadata): - self._uid_to_title_name[metadata['uid']] = name - self._title_name_to_uid[name] = metadata['uid'] - return name - - def _generate_title_name(self, metadata): - title = metadata.get('title') - mtime = metadata.get('timestamp', time.time()) - time_human = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(mtime)) - name = '%s - %s' % (title, time_human) - name = self._safe_name(name) - current_name = name - counter = 1 - while current_name in self._title_name_to_uid: - counter += 1 - current_name = '%s %d' % (name, counter) - - return current_name - - def _remove_title_name_by_uid(self, uid): - name = self._uid_to_title_name.pop(uid, None) - if name: - del self._title_name_to_uid[name] - - def _remove_title_name_by_name(self, name): - uid = self._title_name_to_uid.pop(name, None) - if uid: - del self._uid_to_title_name[uid] - - def _safe_name(self, name): - return name.replace('/', '_') - - def _throw_error(self, number): - raise IOError(number, os.strerror(number)) - def main(): usage = __doc__ + fuse.Fuse.fusage |