path: root/src/jarabe/journal/model.py

# Copyright (C) 2007-2011, One Laptop per Child
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

import cPickle
import logging
import os
import subprocess
from datetime import datetime
import errno
import time
import shutil
import tempfile
import re
from operator import itemgetter
import simplejson
import urllib
from urlparse import urlsplit
import xapian
from gettext import gettext as _

import gobject
import dbus
import gio
import gconf

from sugar import dispatch
from sugar import env
from sugar import mime
from sugar import util
from sugar.logger import trace


DS_DBUS_SERVICE = 'org.laptop.sugar.DataStore'
DS_DBUS_INTERFACE = 'org.laptop.sugar.DataStore'
DS_DBUS_PATH = '/org/laptop/sugar/DataStore'

# Properties the journal cares about.
PROPERTIES = ['activity', 'activity_id', 'buddies', 'bundle_id',
              'creation_time', 'filesize', 'icon-color', 'keep', 'mime_type',
              'mtime', 'progress', 'timestamp', 'title', 'uid']

MIN_PAGES_TO_CACHE = 3
MAX_PAGES_TO_CACHE = 5

JOURNAL_0_METADATA_DIR = '.olpc.store'
JOURNAL_METADATA_DIR = '.Sugar-Metadata'

SUGAR_WEBDAV_NAMESPACE = 'http://people.sugarlabs.org/silbe/webdavns/sugar'
_SUGAR_WEBDAV_PREFIX = 'webdav::%s::' % (urllib.quote(SUGAR_WEBDAV_NAMESPACE,
                                                      safe=''), )
_QUERY_GIO_ATTRIBUTES = ','.join(['standard::*', 'webdav::*',
                                  gio.FILE_ATTRIBUTE_ID_FILE,
                                  gio.FILE_ATTRIBUTE_TIME_MODIFIED])

_datastore = None
created = dispatch.Signal()
updated = dispatch.Signal()
deleted = dispatch.Signal()

_documents_path = None


class _Cache(object):

    __gtype_name__ = 'model_Cache'

    def __init__(self, entries=None):
        self._array = []
        if entries is not None:
            self.append_all(entries)

    def prepend_all(self, entries):
        self._array[0:0] = entries

    def append_all(self, entries):
        self._array += entries

    def __len__(self):
        return len(self._array)

    def __getitem__(self, key):
        return self._array[key]

    def __delitem__(self, key):
        del self._array[key]


class BaseResultSet(object):
    """Encapsulates the result of a query
    """

    def __init__(self, query, page_size):
        self._total_count = -1
        self._position = -1
        self._query = query
        self._page_size = page_size

        self._offset = 0
        self._cache = _Cache()

        self.ready = dispatch.Signal()
        self.progress = dispatch.Signal()

    def setup(self):
        self.ready.send(self)

    def stop(self):
        pass

    def get_length(self):
        if self._total_count == -1:
            query = self._query.copy()
            query['limit'] = self._page_size * MIN_PAGES_TO_CACHE
            entries, self._total_count = self.find(query)
            self._cache.append_all(entries)
            self._offset = 0
        return self._total_count

    length = property(get_length)

    def find(self, query):
        raise NotImplementedError()

    def seek(self, position):
        self._position = position

    def read(self):
        if self._position == -1:
            self.seek(0)

        if self._position < self._offset:
            remaining_forward_entries = 0
        else:
            remaining_forward_entries = self._offset + len(self._cache) - \
                                        self._position

        if self._position > self._offset + len(self._cache):
            remaining_backwards_entries = 0
        else:
            remaining_backwards_entries = self._position - self._offset

        last_cached_entry = self._offset + len(self._cache)

        if remaining_forward_entries <= 0 and remaining_backwards_entries <= 0:

            # Total cache miss: remake it
            limit = self._page_size * MIN_PAGES_TO_CACHE
            offset = max(0, self._position - limit / 2)
            logging.debug('remaking cache, offset: %r limit: %r', offset,
                limit)
            query = self._query.copy()
            query['limit'] = limit
            query['offset'] = offset
            entries, self._total_count = self.find(query)

            del self._cache[:]
            self._cache.append_all(entries)
            self._offset = offset

        elif (remaining_forward_entries <= 0 and
              remaining_backwards_entries > 0):

            # Add one page to the end of cache
            logging.debug('appending one more page, offset: %r',
                last_cached_entry)
            query = self._query.copy()
            query['limit'] = self._page_size
            query['offset'] = last_cached_entry
            entries, self._total_count = self.find(query)

            # update cache
            self._cache.append_all(entries)

            # apply the cache limit
            cache_limit = self._page_size * MAX_PAGES_TO_CACHE
            objects_excess = len(self._cache) - cache_limit
            if objects_excess > 0:
                self._offset += objects_excess
                del self._cache[:objects_excess]

        elif remaining_forward_entries > 0 and \
                remaining_backwards_entries <= 0 and self._offset > 0:

            # Add one page to the beginning of cache
            limit = min(self._offset, self._page_size)
            self._offset = max(0, self._offset - limit)

            logging.debug('prepending one more page, offset: %r limit: %r',
                self._offset, limit)
            query = self._query.copy()
            query['limit'] = limit
            query['offset'] = self._offset
            entries, self._total_count = self.find(query)

            # update cache
            self._cache.prepend_all(entries)

            # apply the cache limit
            cache_limit = self._page_size * MAX_PAGES_TO_CACHE
            objects_excess = len(self._cache) - cache_limit
            if objects_excess > 0:
                del self._cache[-objects_excess:]

        return self._cache[self._position - self._offset]


class DatastoreResultSet(BaseResultSet):
    """Encapsulates the result of a query on the datastore
    """
    def __init__(self, query, page_size):

        if query.get('query', '') and not query['query'].startswith('"'):
            query_text = ''
            words = query['query'].split(' ')
            for word in words:
                if word:
                    if query_text:
                        query_text += ' '
                    query_text += word + '*'

            query['query'] = query_text

        BaseResultSet.__init__(self, query, page_size)

    def find(self, query):
        entries, total_count = _get_datastore().find(query, PROPERTIES,
                                                     byte_arrays=True)

        for entry in entries:
            entry['mount_uri'] = 'datastore:'

        return [('datastore:' + entry['uid'], entry)
                for entry in entries], total_count


class InplaceResultSet(BaseResultSet):
    """Encapsulates the result of a query on a mount point
    """

    _NUM_ENTRIES_PER_REQUEST = 100

    def __init__(self, query, page_size, uri):
        BaseResultSet.__init__(self, query, page_size)
        self._uri = uri
        self._file_list = None
        self._pending_directories = []
        self._visited_directories = set()
        self._pending_entries = []
        self._stopped = False

        query_text = query.get('query', '')
        if query_text.startswith('"') and query_text.endswith('"'):
            self._regex = re.compile('*%s*' % query_text.strip(['"']))
        elif query_text:
            expression = ''
            for word in query_text.split(' '):
                expression += '(?=.*%s.*)' % word
            self._regex = re.compile(expression, re.IGNORECASE)
        else:
            self._regex = None

        if query.get('stamp', ''):
            self._date_start = int(query['timestamp']['start'])
            self._date_end = int(query['timestamp']['end'])
        else:
            self._date_start = None
            self._date_end = None

        self._mime_types = query.get('mime_type', [])

        self._sort = query.get('order_by', ['+timestamp'])[0]

    def setup(self):
        self._file_list = []
        self._pending_directories = [gio.File(uri=self._uri)]
        self._visited_directories = set()
        self._pending_entries = []
        self._schedule_scan_iteration()

    def stop(self):
        self._stopped = True

    @trace()
    def setup_ready(self):
        if self._sort[1:] == 'filesize':
            keygetter = itemgetter(3)
        else:
            # timestamp
            keygetter = itemgetter(2)
        self._file_list.sort(lambda a, b: cmp(b, a),
                             key=keygetter,
                             reverse=(self._sort[0] == '-'))
        self.ready.send(self)

    def find(self, query):
        if self._file_list is None:
            raise ValueError('Need to call setup() first')

        if self._stopped:
            raise ValueError('InplaceResultSet already stopped')

        t = time.time()

        offset = int(query.get('offset', 0))
        limit = int(query.get('limit', len(self._file_list)))
        total_count = len(self._file_list)

        files = self._file_list[offset:offset + limit]

        entries = []
        for uri, mtime_, size_, metadata in files:
            metadata['mount_uri'] = self._uri
            entries.append((uri, metadata))

        logging.debug('InplaceResultSet.find took %f s.', time.time() - t)

        return entries, total_count

    @trace()
    def _scan(self):
        if self._stopped:
            return False

        self.progress.send(self)

        if self._pending_entries:
            self._scan_an_entry()
            return False

        if self._pending_directories:
            self._scan_a_directory()
            return False

        self.setup_ready()
        self._visited_directories = set()
        return False

    @trace()
    def _scan_an_entry(self):
        directory, entry = self._pending_entries.pop(0)
        logging.debug('Scanning entry %r / %r', directory.get_uri(),
                      entry.get_name())

        if entry.get_is_symlink():
            self._scan_a_symlink(directory, entry)
        elif entry.get_file_type() == gio.FILE_TYPE_DIRECTORY:
            self._scan_a_directory_entry(directory, entry)
        elif entry.get_file_type() == gio.FILE_TYPE_REGULAR:
            # FIXME: process file entries in bulk
            self._scan_a_file(directory, entry)

    @trace()
    def _scan_a_file(self, directory, entry):
        self._schedule_scan_iteration()
        metadata = None
        gfile = directory.get_child(entry.get_name())
        logging.debug('gfile.uri=%r, self._uri=%r', gfile.get_uri(), self._uri)
        path = urllib.unquote(urlsplit(gfile.get_uri())[2])

        if self._regex is not None and not self._regex.match(path):
            metadata = _get_file_metadata(gfile, entry, path,
                                          fetch_preview=False)
            if not metadata:
                return

            add_to_list = False
            for f in ['fulltext', 'title',
                      'description', 'tags']:
                if f in metadata and self._regex.match(metadata[f]):
                    add_to_list = True
                    break
            if not add_to_list:
                return

        mtime = entry.get_modification_time()
        if self._date_start is not None and mtime < self._date_start:
            return

        if self._date_end is not None and mtime > self._date_end:
            return

        if self._mime_types:
            if entry.get_content_type() not in self._mime_types:
                return

        if metadata is None:
            metadata = _get_file_metadata(gfile, entry, path,
                                          fetch_preview=False)

        self._add_a_file(directory, entry, gfile.get_uri(), metadata)

    @trace()
    def _add_a_file(self, directory, entry, uri, metadata):
        mtime = entry.get_modification_time()
        size = entry.get_size()
        file_info = (uri, int(mtime), size, metadata)
        self._file_list.append(file_info)

    @trace()
    def _scan_a_symlink(self, directory, entry):
        link = entry.get_symlink_target()
        directory_uri = directory.get_uri().rstrip('/') + '/'
        absolute_uri = urllib.basejoin(directory_uri, link)
        logging.debug('symlink %r in %r => %r', link, directory_uri,
                      absolute_uri)

        if not absolute_uri.startswith(self._uri):
            self._schedule_scan_iteration()
            return

        gfile = gio.File(uri=absolute_uri)
        gfile.query_info_async(_QUERY_GIO_ATTRIBUTES, self.__symlink_query_cb)

    @trace()
    def _scan_a_directory_entry(self, parent_directory, entry):
        self._schedule_scan_iteration()
        directory_id = entry.get_attribute_string(gio.FILE_ATTRIBUTE_ID_FILE)
        if directory_id in self._visited_directories:
            logging.debug('Skipping already visited directory %r (%r)',
                          entry.get_name(), directory_id)
            return

        logging.debug('Scheduling directory %r (%r) for scanning',
                      entry.get_name(), directory_id)
        directory = parent_directory.get_child(entry.get_name())
        self._visited_directories.add(directory_id)
        self._pending_directories.append(directory)

    @trace()
    def _scan_a_directory(self):
        directory = self._pending_directories.pop(0)
        logging.debug('Scanning directory %r', directory.get_uri())
        # TODO: pass a gio.Cancellable
        directory.enumerate_children_async(_QUERY_GIO_ATTRIBUTES,
                                           self.__enumerate_children_cb)

    @trace()
    def __symlink_query_cb(self, symlink, result):
        try:
            entry = symlink.query_info_finish(result)
        except gio.Error, exception:
            logging.error('Could not examine symlink %s: %s',
                          symlink.get_uri(), exception)
            self._schedule_scan_iteration()
            return

        self._pending_entries.append((symlink.get_parent(), entry))
        self._schedule_scan_iteration()

    @trace()
    def __enumerate_children_cb(self, directory, result):
        try:
            enumerator = directory.enumerate_children_finish(result)
        except gio.Error, exception:
            logging.error('Could not enumerate %s: %s', directory.get_uri(),
                          exception)
            self._schedule_scan_iteration()
            return

        enumerator.next_files_async(self._NUM_ENTRIES_PER_REQUEST,
                                    self.__next_files_cb)

    @trace()
    def __next_files_cb(self, enumerator, result):
        directory = enumerator.get_container()
        try:
            entries = enumerator.next_files_finish(result)
        except gio.Error, exception:
            logging.error('Error while enumerating %s: %s',
                          directory.get_uri(), exception)
            self._schedule_scan_iteration()
            return

        logging.debug('__next_files_cb: entries=%r', entries)
        logging.debug('__next_files_cb: names=%r',
                      [entry.get_name() for entry in entries])
        self._pending_entries += [(directory, entry) for entry in entries
                                  if not entry.get_name().startswith('.')]

        if len(entries) >= self._NUM_ENTRIES_PER_REQUEST:
            enumerator.next_files_async(self._NUM_ENTRIES_PER_REQUEST,
                                        self.__next_files_cb)
        else:
            self._schedule_scan_iteration()

    @trace()
    def _schedule_scan_iteration(self):
        gobject.idle_add(self._scan)


@trace()
def _get_file_metadata(gfile, info, path, fetch_preview=True):
    """Return the metadata from the corresponding file.

    Reads the metadata stored in the json file or create the
    metadata based on the file properties.

    """
    if gfile.is_native():
        filename = os.path.basename(path)
        dir_path = os.path.dirname(path)
        metadata = _get_file_metadata_from_json(dir_path, filename,
                                                fetch_preview)
        if metadata:
            metadata['filesize'] = info.get_size()
            #metadata['uid'] = gfile.get_uri()
            return metadata

    metadata = {#'uid': gfile.get_uri(),
                'title': info.get_display_name(),
                'timestamp': info.get_modification_time(),
                'filesize': info.get_size(),
                'mime_type': info.get_content_type(),
                'activity': '',
                'activity_id': '',
                'icon-color': '#000000,#ffffff',
                'description': path}

    for attr_name in info.list_attributes('webdav'):
        if not attr_name.startswith(_SUGAR_WEBDAV_PREFIX):
            continue

        attribute_type = info.get_attribute_type(attr_name)
        if attribute_type != gio.FILE_ATTRIBUTE_TYPE_STRING:
            logging.debug('%r is not a string: %s', attr_name, attribute_type)
            continue

        property_name = urllib.unquote(attr_name[len(_SUGAR_WEBDAV_PREFIX):])
        if property_name == 'filesize':
            continue

        metadata[property_name] = info.get_attribute_string(attr_name)

    return metadata


@trace()
def _get_file_metadata_from_json(dir_path, filename, fetch_preview):
    """Read the metadata from the json file and the preview
    stored on the external device.

    If the metadata is corrupted we do remove it and the preview as well.

    """
    metadata = None
    metadata_path = os.path.join(dir_path, JOURNAL_METADATA_DIR,
                                 filename + '.metadata')
    preview_path = os.path.join(dir_path, JOURNAL_METADATA_DIR,
                                filename + '.preview')

    if not os.path.exists(metadata_path):
        return None

    try:
        metadata = simplejson.load(open(metadata_path))
    except (ValueError, EnvironmentError):
        os.unlink(metadata_path)
        if os.path.exists(preview_path):
            os.unlink(preview_path)
        logging.error('Could not read metadata for file %r on '
                      'external device.', filename)
        return None

    if not fetch_preview:
        if 'preview' in metadata:
            del(metadata['preview'])
    else:
        if os.path.exists(preview_path):
            try:
                metadata['preview'] = dbus.ByteArray(open(preview_path).read())
            except EnvironmentError:
                logging.debug('Could not read preview for file %r on '
                              'external device.', filename)

    return metadata


def _get_datastore():
    global _datastore
    if _datastore is None:
        bus = dbus.SessionBus()
        remote_object = bus.get_object(DS_DBUS_SERVICE, DS_DBUS_PATH)
        _datastore = dbus.Interface(remote_object, DS_DBUS_INTERFACE)

        _datastore.connect_to_signal('Created', _datastore_created_cb)
        _datastore.connect_to_signal('Updated', _datastore_updated_cb)
        _datastore.connect_to_signal('Deleted', _datastore_deleted_cb)

    return _datastore


def _datastore_created_cb(object_id):
    created.send(None, object_uri='datastore:' + object_id)


def _datastore_updated_cb(object_id):
    updated.send(None, object_uri='datastore:' + object_id)


def _datastore_deleted_cb(object_id):
    deleted.send(None, object_uri='datastore:' + object_id)


def find(query_, page_size):
    """Returns a ResultSet
    """
    query = query_.copy()

    mount_uri = query.pop('mount_uri', None)
    if mount_uri != 'datastore:':
        return InplaceResultSet(query, page_size, mount_uri)

    return DatastoreResultSet(query, page_size)


def _get_mount_uri(gfile):
    try:
        mount = gfile.find_enclosing_mount()
    except gio.Error, exception:
        if exception.domain != gio.ERROR or \
           exception.code != gio.ERROR_NOT_FOUND:
            raise
    else:
        return mount.get_root().get_uri()

    # find_enclosing_mount() doesn't work for local "internal" mounts.
    # But since the XDG Documents folder is the only thing we show that
    # could contain paths on "internal" mounts, we can just "hardcode"
    # that directory.
    return 'file://' + get_documents_path()


def get(object_uri):
    """Returns the metadata for an object
    """
    logging.debug('get(%r)', object_uri)
    scheme, netloc_, quoted_path = urlsplit(object_uri)[:3]
    if scheme == 'datastore':
        object_id = quoted_path
        metadata = _get_datastore().get_properties(object_id, byte_arrays=True)
        metadata['mount_uri'] = 'datastore:'
    else:
        gfile = gio.File(uri=object_uri)
        info = gfile.query_info(_QUERY_GIO_ATTRIBUTES)
        path = urllib.unquote(quoted_path)
        metadata = _get_file_metadata(gfile, info, path)
        metadata['mount_uri'] = _get_mount_uri(gfile)
    return metadata


@trace()
def get_file(object_uri):
    """Returns the file for an object
    """
    # TODO: add async interface
    logging.debug('get_file(%r)', object_uri)
    scheme, netloc_, quoted_path = urlsplit(object_uri)[:3]
    if scheme == 'file':
        # We use local files as-is, so no pruning / book-keeping required.
        return gio.File(uri=object_uri).get_path()

    if scheme == 'datastore':
        object_id = quoted_path
        file_path = _get_datastore().get_filename(object_id)
    else:
        input_stream = gio.File(uri=object_uri).read()
        file_path = tempfile.mktemp(dir=env.get_profile_path('data'))
        output_stream = gio.File(file_path).create(gio.FILE_CREATE_PRIVATE)
        shutil.copyfileobj(input_stream, output_stream)
        input_stream.close()
        output_stream.close()

    if file_path:
        return util.TempFilePath(file_path)
    else:
        return None


def get_file_size(object_uri):
    """Return the file size for an object
    """
    logging.debug('get_file_size(%r)', object_uri)
    scheme, netloc_, quoted_path = urlsplit(object_uri)[:3]
    if scheme != 'datastore':
        gfile = gio.File(uri=object_uri)
        info = gfile.query_info(gio.FILE_ATTRIBUTE_STANDARD_SIZE)
        return info.get_attribute_uint64(gio.FILE_ATTRIBUTE_STANDARD_SIZE)

    object_id = quoted_path
    file_path = _get_datastore().get_filename(object_id)
    if file_path:
        size = os.stat(file_path).st_size
        os.remove(file_path)
        return size

    return 0


def get_unique_values(key):
    """Returns a list with the different values a property has taken
    """
    empty_dict = dbus.Dictionary({}, signature='ss')
    return _get_datastore().get_uniquevaluesfor(key, empty_dict)


def delete(object_uri):
    """Removes an object from persistent storage
    """
    scheme, netloc_, quoted_path = urlsplit(object_uri)[:3]
    if scheme == 'datastore':
        object_id = quoted_path
        _get_datastore().delete(object_id)
        return

    gfile = gio.File(uri=object_uri)
    gfile.delete()
    if gfile.get_uri_scheme() == 'file':
        _delete_metadata_files(gfile.get_path())

    deleted.send(None, object_uri=object_uri)


def _delete_metadata_files(path):
    """Delete Sugar metadata files associated with the given data file"""
    dir_path = os.path.dirname(path)
    filename = os.path.basename(path)
    old_files = [os.path.join(dir_path, JOURNAL_METADATA_DIR,
                              filename + '.metadata'),
                 os.path.join(dir_path, JOURNAL_METADATA_DIR,
                              filename + '.preview')]
    for old_file in old_files:
        if os.path.exists(old_file):
            try:
                os.unlink(old_file)
            except EnvironmentError:
                logging.error('Could not remove metadata=%s '
                              'for file=%s', old_file, filename)


def copy(object_uri, mount_uri):
    """Copies an object to another mount point
    """
    metadata = get(object_uri)
    color = metadata.get('icon-color')
    if mount_uri == 'datastore:' and color == '#000000,#ffffff':
        client = gconf.client_get_default()
        metadata['icon-color'] = client.get_string('/desktop/sugar/user/color')
    file_path = get_file(object_uri)
    if file_path is None:
        file_path = ''

    metadata['mount_uri'] = mount_uri
    metadata.pop('uid', None)

    return write(metadata, mount_uri, None, file_path,
                 transfer_ownership=False)


# FIXME: pass target uri (mount_uri) as parameter
@trace()
def write(metadata, mount_uri, object_uri=None, file_path='',
          update_mtime=True, transfer_ownership=True):
    """Create or update an entry on mounted object storage

    If object_uri is None, create a new entry, otherwise update an
    existing entry.

    If update_mtime is True (default), update the time of last
    modification in metadata (in-place).

    If transfer_ownership is True (default), move file_path into place.
    If that is not possible, remove file_path after copying.

    For objects on POSIX file systems, the title property may be
    updated in metadata (in-place).

    Return the URI of the written entry. Even for updates this may be
    different from the original value.
    """
    logging.debug('model.write %r %r %r', object_uri, file_path,
        update_mtime)

    assert mount_uri
    if object_uri and not object_uri.startswith(mount_uri):
        raise ValueError('Object to be updated not located on given mount.')

    if update_mtime:
        metadata['mtime'] = datetime.now().isoformat()
        metadata['timestamp'] = int(time.time())

    if mount_uri == 'datastore:':
        return _write_entry_to_datastore(object_uri, metadata, file_path,
                                         transfer_ownership)
    elif mount_uri.startswith('dav'):
        return _write_entry_on_webdav_share(mount_uri, object_uri, metadata,
                                            file_path, transfer_ownership)
    elif mount_uri.startswith('file:'):
        return _write_entry_on_external_device(mount_uri, object_uri,
                                               metadata, file_path,
                                               transfer_ownership)
    else:
        raise NotImplementedError("Don't know how to write to"
                                  " %r" % (mount_uri, ))


def _write_entry_to_datastore(object_uri, metadata, file_path,
                              transfer_ownership):
    if object_uri:
        object_id = urlsplit(object_uri)[2]
        _get_datastore().update(object_id, dbus.Dictionary(metadata),
                                file_path, transfer_ownership)
        return object_uri
    else:
        object_id = _get_datastore().create(dbus.Dictionary(metadata),
                                            file_path, transfer_ownership)
        return 'datastore:' + object_id


def _write_entry_on_webdav_share(mount_uri, object_uri, metadata, file_path,
                                 transfer_ownership):
    title = metadata.get('title') or _('Untitled')
    proposed_name = get_file_name(title, metadata.get('mime_type', ''))
    output_stream, gfile = create_unique_file(mount_uri, proposed_name)
    if file_path:
        shutil.copyfileobj(file(file_path), output_stream)

    output_stream.close()

    info = gio.FileInfo()
    attr_infos = gfile.query_settable_attributes() or []
    settable_attrs = [attr_info.name for attr_info in attr_infos]
    if gio.FILE_ATTRIBUTE_TIME_MODIFIED in settable_attrs:
        info.set_modification_time(metadata.get('timestamp', time.time()))
    if gio.FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE in settable_attrs:
        info.set_content_type(metadata.get('mime_type',
                                           'application/octet-stream'))

    if 'webdav' in gfile.query_writable_namespaces():
        for name, value in metadata.items():
            attr_name = _SUGAR_WEBDAV_PREFIX + urllib.quote(name)
            if isinstance(value, basestring) and '\0' in value:
                # strings with NUL bytes are not supported by gio
                continue

            info.set_attribute_string(attr_name, str(value))

    gfile.set_attributes_from_info(info)


def _rename_entry_on_external_device(file_path, destination_path,
                                     metadata_dir_path):
    """Rename an entry with the associated metadata on an external device."""
    old_file_path = file_path
    if old_file_path != destination_path:
        os.rename(file_path, destination_path)
        old_fname = os.path.basename(file_path)
        old_files = [os.path.join(metadata_dir_path,
                                  old_fname + '.metadata'),
                     os.path.join(metadata_dir_path,
                                  old_fname + '.preview')]
        for ofile in old_files:
            if os.path.exists(ofile):
                try:
                    os.unlink(ofile)
                except EnvironmentError:
                    logging.error('Could not remove metadata=%s '
                                  'for file=%s', ofile, old_fname)


def _write_entry_on_external_device(mount_uri, object_uri, metadata,
                                    file_path, transfer_ownership):
    """Create or update an entry on a mounted POSIX file system

    Besides copying the associated file a file for the preview
    and one for the metadata are stored in the hidden directory
    .Sugar-Metadata.

    This function handles renames of an entry on the
    external device and avoids name collisions. Renames are
    handled failsafe.

    """
    if not file_path or not os.path.exists(file_path):
        raise ValueError('Entries without a file cannot be copied to '
                         'removable devices')

    if not metadata.get('title'):
        metadata['title'] = _('Untitled')
    destination_name = get_file_name(metadata['title'], metadata['mime_type'])

    mount_point = gio.File(mount_uri).get_path()
    logging.debug('_write_entry_on_external_device: mount_point=%r,'
                  ' destination_name=%r', mount_point, destination_name)
    destination_path = os.path.join(mount_point, destination_name)
    if destination_path != file_path:
        destination_name = get_unique_file_name(mount_point, destination_name)
        destination_path = os.path.join(mount_point, destination_name)
        clean_name, extension_ = os.path.splitext(destination_name)
        metadata['title'] = clean_name

    metadata_copy = metadata.copy()
    metadata_copy.pop('mount_uri')
    metadata_copy.pop('uid', None)
    metadata_copy.pop('filesize', None)

    metadata_dir_path = os.path.join(mount_point,
                                     JOURNAL_METADATA_DIR)
    if not os.path.exists(metadata_dir_path):
        os.mkdir(metadata_dir_path)

    preview = metadata_copy.pop('preview', None)
    if preview:
        preview_fname = destination_name + '.preview'

    try:
        metadata_json = simplejson.dumps(metadata_copy)
    except (UnicodeDecodeError, EnvironmentError):
        logging.error('Could not convert metadata to json.')
    else:
        (fh, fn) = tempfile.mkstemp(dir=mount_point)
        os.write(fh, metadata_json)
        os.close(fh)
        os.rename(fn, os.path.join(metadata_dir_path,
                                   destination_name + '.metadata'))

        if preview:
            (fh, fn) = tempfile.mkstemp(dir=mount_point)
            os.write(fh, preview)
            os.close(fh)
            os.rename(fn, os.path.join(metadata_dir_path, preview_fname))

    if not transfer_ownership:
        shutil.copy(file_path, destination_path)
    elif gio.File(object_uri).get_path() == file_path:
        _rename_entry_on_external_device(file_path, destination_path,
                                         metadata_dir_path)
    else:
        shutil.move(file_path, destination_path)

    object_uri = gio.File(path=destination_path).get_uri()
    created.send(None, object_uri=object_uri)
    return object_uri


def get_file_name(title, mime_type):
    file_name = title

    extension = mime.get_primary_extension(mime_type)
    if extension is not None and extension:
        extension = '.' + extension
        if not file_name.endswith(extension):
            file_name += extension

    # Invalid characters in VFAT filenames. From
    # http://en.wikipedia.org/wiki/File_Allocation_Table
    invalid_chars = ['/', '\\', ':', '*', '?', '"', '<', '>', '|', '\x7F']
    invalid_chars.extend([chr(x) for x in range(0, 32)])
    for char in invalid_chars:
        file_name = file_name.replace(char, '_')

    # FAT limit is 255, leave some space for uniqueness
    max_len = 250
    if len(file_name) > max_len:
        name, extension = os.path.splitext(file_name)
        file_name = name[0:max_len - len(extension)] + extension

    return file_name


def get_unique_file_name(mount_point, file_name):
    if os.path.exists(os.path.join(mount_point, file_name)):
        i = 1
        name, extension = os.path.splitext(file_name)
        while len(file_name) <= 255:
            file_name = name + '_' + str(i) + extension
            if not os.path.exists(os.path.join(mount_point, file_name)):
                break
            i += 1

    return file_name


@trace()
def create_unique_file(mount_uri, file_name):
    """Create a new file with a unique name on given mount

    Create a new file on the mount identified by mount_uri. Use
    file_name as the name on the mount if possible, otherwise use it
    as a template for the name of the new file, inserting a decimal
    number as necessary. Return both the gio.FileOutputStream instance
    and the gio.File instance of the new object (in this order).
    """
    gdir = gio.File(mount_uri)
    name, extension = os.path.splitext(file_name)
    last_error = None
    for try_nr in range(255):
        if not try_nr:
            try_name = file_name
        else:
            try_name = '%s_%d%s' % (name, try_nr, extension)

        gfile = gdir.get_child(try_name)
        try:
            output_stream = gfile.create()
        except gio.Error, exception:
            last_error = exception
            continue

        return output_stream, gfile

    raise last_error

@trace()
def is_editable(object_uri):
    scheme, netloc_, quoted_path = urlsplit(object_uri)[:3]
    if scheme == 'file':
        return os.access(urllib.unquote(quoted_path), os.W_OK)
    return scheme in ['datastore', 'dav', 'davs']


def get_documents_path():
    """Gets the path of the DOCUMENTS folder

    If xdg-user-dir can not find the DOCUMENTS folder it returns
    $HOME, which we omit. xdg-user-dir handles localization
    (i.e. translation) of the filenames.

    Returns: Path to $HOME/DOCUMENTS or None if an error occurs
    """
    global _documents_path
    if _documents_path is not None:
        return _documents_path

    try:
        pipe = subprocess.Popen(['xdg-user-dir', 'DOCUMENTS'],
                                stdout=subprocess.PIPE)
        documents_path = os.path.normpath(pipe.communicate()[0].strip())
        if os.path.exists(documents_path) and \
                os.environ.get('HOME') != documents_path:
            _documents_path = documents_path
            return _documents_path
    except OSError, exception:
        if exception.errno != errno.ENOENT:
            logging.exception('Could not run xdg-user-dir')
    return None


def _get_id(document):
    """Get the ID for the document in the xapian database."""
    tl = document.termlist()
    try:
        term = tl.skip_to('Q').term
        if len(term) == 0 or term[0] != 'Q':
            return None
        return term[1:]
    except StopIteration:
        return None


def convert_datastore_0_entries(root):
    """Convert entries written by the datastore version 0.

    The metadata and the preview will be written using the new
    scheme for writing Journal entries to removable storage
    devices.

    - entries that do not have an associated file are not
    converted.
    - if an entry has no title we set it to Untitled and rename
    the file accordingly, taking care of creating a unique
    filename

    """
    try:
        database = xapian.Database(os.path.join(root, JOURNAL_0_METADATA_DIR,
                                                'index'))
    except xapian.DatabaseError:
        logging.exception('Convert DS-0 Journal entries: error reading db: %s',
                      os.path.join(root, JOURNAL_0_METADATA_DIR, 'index'))
        return

    metadata_dir_path = os.path.join(root, JOURNAL_METADATA_DIR)
    if not os.path.exists(metadata_dir_path):
        try:
            os.mkdir(metadata_dir_path)
        except EnvironmentError:
            logging.error('Convert DS-0 Journal entries: '
                          'error creating the Journal metadata directory.')
            return

    for posting_item in database.postlist(''):
        try:
            document = database.get_document(posting_item.docid)
        except xapian.DocNotFoundError, e:
            logging.debug('Convert DS-0 Journal entries: error getting '
                          'document %s: %s', posting_item.docid, e)
            continue
        _convert_entry(root, document)


def _convert_entry(root, document):
    try:
        metadata_loaded = cPickle.loads(document.get_data())
    except cPickle.PickleError, e:
        logging.debug('Convert DS-0 Journal entries: '
                      'error converting metadata: %s', e)
        return

    if not ('activity_id' in metadata_loaded and
            'mime_type' in metadata_loaded and
            'title' in metadata_loaded):
        return

    metadata = {}

    uid = _get_id(document)
    if uid is None:
        return

    for key, value in metadata_loaded.items():
        metadata[str(key)] = str(value[0])

    if 'uid' not in metadata:
        metadata['uid'] = uid

    filename = metadata.pop('filename', None)
    if not filename:
        return
    if not os.path.exists(os.path.join(root, filename)):
        return

    if not metadata.get('title'):
        metadata['title'] = _('Untitled')
        fn = get_file_name(metadata['title'], metadata['mime_type'])
        new_filename = get_unique_file_name(root, fn)
        os.rename(os.path.join(root, filename),
                  os.path.join(root, new_filename))
        filename = new_filename

    preview_path = os.path.join(root, JOURNAL_0_METADATA_DIR,
                                'preview', uid)
    if os.path.exists(preview_path):
        preview_fname = filename + '.preview'
        new_preview_path = os.path.join(root, JOURNAL_METADATA_DIR,
                                        preview_fname)
        if not os.path.exists(new_preview_path):
            shutil.copy(preview_path, new_preview_path)

    metadata_fname = filename + '.metadata'
    metadata_path = os.path.join(root, JOURNAL_METADATA_DIR,
                                 metadata_fname)
    if not os.path.exists(metadata_path):
        (fh, fn) = tempfile.mkstemp(dir=root)
        os.write(fh, simplejson.dumps(metadata))
        os.close(fh)
        os.rename(fn, metadata_path)

        logging.debug('Convert DS-0 Journal entries: entry converted: '
                      'file=%s metadata=%s',
                      os.path.join(root, filename), metadata)