path: root/src/carquinyol/datastore.py

# Copyright (C) 2008, One Laptop Per Child
# Based on code Copyright (C) 2007, ObjectRealms, LLC
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

# pylint fails on @debian's arguments
# pylint: disable=C0322

import logging
import uuid
import time
import os

import dbus
import dbus.service
import gconf
import gobject

from sugar import mime
from sugar.logger import trace

from carquinyol import layoutmanager
from carquinyol import migration
from carquinyol.layoutmanager import MAX_QUERY_LIMIT
from carquinyol.metadatastore import MetadataStore
from carquinyol.indexstore import IndexStore
from carquinyol.filestore import FileStore
from carquinyol.optimizer import Optimizer

# the name used by the logger
DS_LOG_CHANNEL = 'org.laptop.sugar.DataStore'

DS_SERVICE = "org.laptop.sugar.DataStore"
DS_DBUS_INTERFACE1 = "org.laptop.sugar.DataStore"
DS_OBJECT_PATH1 = "/org/laptop/sugar/DataStore"
DS_DBUS_INTERFACE2 = "org.laptop.sugar.DataStore2"
DS_OBJECT_PATH2 = "/org/laptop/sugar/DataStore2"

logger = logging.getLogger(DS_LOG_CHANNEL)


class DBusAPIv1(dbus.service.Object):
    """Compatibility layer for old (unversioned) D-Bus API
    """

    # pylint: disable-msg=W0212

    def __init__(self, data_store):
        self._data_store = data_store
        bus_name = dbus.service.BusName(DS_SERVICE, bus=dbus.SessionBus(),
            replace_existing=False, allow_replacement=False)
        dbus.service.Object.__init__(self, bus_name, DS_OBJECT_PATH1)

    @dbus.service.method(DS_DBUS_INTERFACE1,
        in_signature='a{sv}sb', out_signature='s',
        async_callbacks=('async_cb', 'async_err_cb'),
        byte_arrays=True)
    def create(self, props, file_path, transfer_ownership,
               async_cb, async_err_cb):

        def success_cb(tree_id, child_id):
            self.Created(tree_id)
            async_cb(tree_id)

        self._data_store._save(tree_id='', parent_id='', metadata=props,
            path=file_path, delete_after=transfer_ownership,
            async_cb=success_cb, async_err_cb=async_err_cb)

    @dbus.service.signal(DS_DBUS_INTERFACE1, signature="s")
    def Created(self, uid):
        pass

    @dbus.service.method(DS_DBUS_INTERFACE1,
             in_signature='sa{sv}sb',
             out_signature='',
             async_callbacks=('async_cb', 'async_err_cb'),
             byte_arrays=True)
    def update(self, uid, props, file_path, transfer_ownership,
               async_cb, async_err_cb):

        def success_cb(tree_id, child_id):
            self.Updated(tree_id)
            async_cb()

        latest_versions = self._get_latest(uid)
        if not latest_versions:
            raise ValueError('Trying to update non-existant entry %s - wanted'
                ' to use create()?' % (uid, ))

        parent = latest_versions[0]
        if self._compare_checksums(parent, file_path):
            self._data_store._change_metadata(parent['tree_id'],
                parent['version_id'], props)
            return success_cb(uid, None)

        self._data_store._save(tree_id=uid,
            parent_id=parent['version_id'], metadata=props,
            path=file_path, delete_after=transfer_ownership,
            async_cb=success_cb, async_err_cb=async_err_cb)

    @dbus.service.signal(DS_DBUS_INTERFACE1, signature="s")
    def Updated(self, uid):
        pass

    @dbus.service.method(DS_DBUS_INTERFACE1,
             in_signature='a{sv}as',
             out_signature='aa{sv}u')
    def find(self, query, properties):
        if 'uid' in properties:
            properties.append('tree_id')
            properties.remove('uid')

        options = {'metadata': properties}
        for name in ['offset', 'limit', 'order_by']:
            if name in query:
                options[name] = query.pop(name)

        if 'uid' in query:
            query['tree_id'] = query.pop('uid')

        results, count = self._data_store._find(query, options,
            query.get('query'))

        if 'tree_id' in properties:
            for entry in results:
                entry['uid'] = entry.pop('tree_id')

        return results, count

    @dbus.service.method(DS_DBUS_INTERFACE1,
             in_signature='s',
             out_signature='s',
             sender_keyword='sender')
    def get_filename(self, uid, sender=None):
        latest_versions = self._get_latest(uid)
        if not latest_versions:
            raise ValueError('Entry %s does not exist' % (uid, ))

        return self._data_store._get_data(uid,
            latest_versions[0]['version_id'], sender=sender)

    @dbus.service.method(DS_DBUS_INTERFACE1,
                         in_signature='s',
                         out_signature='a{sv}')
    @trace(logger=logger)
    def get_properties(self, uid):
        latest_versions = self._get_latest(uid)
        if not latest_versions:
            raise ValueError('Entry %s does not exist' % (uid, ))

        latest_versions[0]['uid'] = latest_versions[0].pop('tree_id')
        del latest_versions[0]['version_id']
        return latest_versions[0]

    @dbus.service.method(DS_DBUS_INTERFACE1,
                         in_signature='sa{sv}',
                         out_signature='as')
    def get_uniquevaluesfor(self, propertyname, query=None):
        return self._data_store._find_unique_values(query, propertyname)

    @dbus.service.method(DS_DBUS_INTERFACE1,
             in_signature='s',
             out_signature='')
    def delete(self, uid):
        latest_versions = self._get_latest(uid)
        if not latest_versions:
            raise ValueError('Entry %s does not exist' % (uid, ))

        self._data_store._delete(tree_id=uid,
            version_id=latest_versions[0]['version_id'])
        self.Deleted(uid)

    @dbus.service.signal(DS_DBUS_INTERFACE1, signature="s")
    def Deleted(self, uid):
        pass

    def stop(self):
        """shutdown the service"""
        self._data_store._stop()
        self.Stopped()

    @dbus.service.signal(DS_DBUS_INTERFACE1)
    def Stopped(self):
        pass

    @dbus.service.method(DS_DBUS_INTERFACE1,
                         in_signature="sa{sv}",
                         out_signature='s')
    def mount(self, uri, options=None):
        return ''

    @dbus.service.method(DS_DBUS_INTERFACE1,
                         in_signature="",
                         out_signature="aa{sv}")
    def mounts(self):
        return [{'id': 1}]

    @dbus.service.method(DS_DBUS_INTERFACE1,
                         in_signature="s",
                         out_signature="")
    def unmount(self, mountpoint_id):
        pass

    @dbus.service.signal(DS_DBUS_INTERFACE1, signature="a{sv}")
    def Mounted(self, descriptior):
        pass

    @dbus.service.signal(DS_DBUS_INTERFACE1, signature="a{sv}")
    def Unmounted(self, descriptor):
        pass

    def _get_latest(self, uid):
        return self._data_store._find({'tree_id': uid},
            {'limit': 1, 'order_by': ['+timestamp']})[0]

    @trace()
    def _compare_checksums(self, parent, child_data):
        parent_data = self._data_store._file_store.get_file_path(
            (parent['tree_id'], parent['version_id']))
        if bool(child_data) ^ os.path.exists(parent_data):
            return False
        elif not child_data:
            return True

        if 'checksum' in parent:
            parent_checksum = parent['checksum']
        else:
            parent_checksum = self._data_store._optimizer.calculate_md5sum(
                parent_data)

        child_checksum = self._data_store._optimizer.calculate_md5sum(
            child_data)
        return parent_checksum == child_checksum


class DataStore(dbus.service.Object):
    """D-Bus API and logic for connecting all the other components.
    """

    def __init__(self, **options):
        bus_name = dbus.service.BusName(DS_SERVICE,
                                        bus=dbus.SessionBus(),
                                        replace_existing=False,
                                        allow_replacement=False)
        dbus.service.Object.__init__(self, bus_name, DS_OBJECT_PATH2)
        self._api_v1 = DBusAPIv1(self)
        gconf_client = gconf.client_get_default()
        self._max_versions = gconf_client.get_int(
            '/desktop/sugar/datastore/max_versions')
        logging.debug('max_versions=%r', self._max_versions)

        migrated, initiated = self._open_layout()

        self._metadata_store = MetadataStore()
        self._file_store = FileStore()
        self._optimizer = Optimizer(self._file_store, self._metadata_store)
        self._index_store = IndexStore()
        self._index_updating = False

        if migrated:
            self._rebuild_index()
            return

        try:
            self._index_store.open_index()
        except Exception:
            logging.exception('Failed to open index, will rebuild')
            self._rebuild_index()
            return

        if initiated:
            logging.debug('Initiate datastore')
            self._index_store.flush()
        elif not self._index_store.index_updated:
            logging.debug('Index is not up-to-date, will update')
            self._update_index()

    def _open_layout(self):
        """Open layout manager, check version of data store on disk and
        migrate if necessary.

        Returns a pair of booleans. For the first, True if migration was done
        and an index rebuild is required. For the second, True if datastore was
        just initiated.
        """
        layout_manager = layoutmanager.get_instance()

        if layout_manager.is_empty():
            layout_manager.set_version(layoutmanager.CURRENT_LAYOUT_VERSION)
            return False, True

        old_version = layout_manager.get_version()
        if old_version == layoutmanager.CURRENT_LAYOUT_VERSION:
            return False, False

        if old_version == 0:
            migration.migrate_from_0()
        elif old_version in [1, 2, 4]:
            migration.migrate_from_1()

        layout_manager.set_version(layoutmanager.CURRENT_LAYOUT_VERSION)
        return True, False

    def _rebuild_index(self):
        """Remove and recreate index."""
        self._index_store.close_index()
        self._index_store.remove_index()
        self._index_store.open_index()
        self._update_index()

    def _update_index(self):
        """Find entries that are not yet in the index and add them."""
        object_ids = layoutmanager.get_instance().find_all()
        logging.debug('Going to update the index with object_ids %r',
            object_ids)
        self._index_updating = True
        gobject.idle_add(lambda: self.__update_index_cb(object_ids),
                            priority=gobject.PRIORITY_LOW)

    def __update_index_cb(self, object_ids):
        if object_ids:
            object_id = object_ids.pop()

            logging.debug('Updating entry %r in index. %d to go.', object_id,
                len(object_ids))

            if not self._index_store.contains(object_id):
                try:
                    update_metadata = False
                    props = self._metadata_store.retrieve(object_id)
                    if 'filesize' not in props:
                        path = self._file_store.get_file_path(object_id)
                        if os.path.exists(path):
                            props['filesize'] = os.stat(path).st_size
                            update_metadata = True
                    if 'creation_time' not in props:
                        if 'ctime' in props:
                            try:
                                props['creation_time'] = time.mktime(
                                        time.strptime(props['ctime'],
                                            migration.DATE_FORMAT))
                            except (TypeError, ValueError):
                                pass
                        if 'creation_time' not in props:
                            props['creation_time'] = props['timestamp']
                        update_metadata = True
                    if update_metadata:
                        self._metadata_store.store(object_id, props)
                    self._index_store.store(object_id, props)
                except Exception:
                    logging.exception('Error processing %r', object_id)

        if not object_ids:
            self._index_store.flush()
            self._index_updating = False
            logging.debug('Finished updating index.')
            return False
        else:
            return True

    @dbus.service.method(DS_DBUS_INTERFACE2,
             in_signature='ssa{sv}sb',
             out_signature='ss',
             async_callbacks=('async_cb', 'async_err_cb'),
             byte_arrays=True)
    def save(self, tree_id, parent_id, metadata, path, delete_after, async_cb,
        async_err_cb):
        """Store new (version of an) object and return (tree_id, child_id).

        Submit new version (data and/or metadata) of the given object,
        returning (tree_id, child_id). parent_id and child_id are the
        version_id of the predecessor resp. the to-be-saved entry.

        If tree_id is empty parent_id must be empty as well and both will be
        newly allocated (equivalent to create() in previous API). If only
        parent_id is empty there must not be any entry with the same tree_id
        already in datastore.

        File storage happens synchronously, emitting the signal
        Saved(tree_id, parent_id, child_id) on completion (i.e. before
        returning from save()).

        If path is non-empty it designates a file or directory containing the
        data and the parent must contain of the same type (file / directory;
        not metadata-only), if a parent exists at all. If path is empty and
        the parent contained data then a metadata-only update is performed,
        reusing the data of the parent. If both parent_id and path are empty
        a metadata-only entry is created.

        If delete_after is True the contents of path are deleted after they
        have been committed, but before sending the Saved() signal.

        metadata may contain a version_id field that will be used for the new
        entry. In that case tree_id must be non-empty and there must not be an
        existing entry in the data store with the same tree_id and version_id.
        This functionality is intended for restoring from backups and importing
        entries from other data stores; most API clients should leave it up to
        the data store to choose a unique version_id.
        """
        self._save(tree_id, parent_id, metadata, path, delete_after, async_cb,
            async_err_cb)

    @trace(logger=logger)
    def _save(self, tree_id, parent_id, metadata, path, delete_after, async_cb,
              async_err_cb):
        if path:
            if not os.access(path, os.R_OK):
                raise ValueError('Invalid path given.')

            if delete_after and not os.access(os.path.dirname(path), os.W_OK):
                raise ValueError('Deletion requested for read-only directory')

        if (not tree_id) and parent_id:
            raise ValueError('tree_id is empty but parent_id is not')

        if tree_id and not parent_id:
            if self._find({'tree_id': tree_id}, {'limit': 1})[1]:
                raise ValueError('No parent_id given but tree_id already '
                    'exists')

        elif parent_id:
            if not self._find({'tree_id': tree_id, 'version_id': parent_id},
                {'limit': 1})[1]:
                raise ValueError('Given parent_id does not exist')

        if not tree_id:
            tree_id = self._gen_uuid()

        child_id = metadata.get('version_id')
        if not child_id:
            child_id = self._gen_uuid()
        elif not tree_id:
            raise ValueError('No tree_id given but metadata contains version_id')
        elif self._find({'tree_id': tree_id, 'version_id': child_id},
            {'limit': 1})[1]:

            raise ValueError('There is an existing entry with the same tree_id'
                ' and version_id')

        if not tree_id:
            tree_id = self._gen_uuid()

        metadata.setdefault('timestamp', time.time())
        metadata['parent_id'] = parent_id

        # FIXME: Support for the deprecated ctime property. Remove in 0.92.
        if 'ctime' in metadata:
            try:
                metadata['creation_time'] = time.mktime(time.strptime(
                    migration.DATE_FORMAT, metadata['ctime']))
            except (TypeError, ValueError):
                pass

        if 'creation_time' not in metadata:
            metadata['creation_time'] = metadata['timestamp']

        # FIXME: how to handle creation_time vs. versions?

        if os.path.exists(path):
            stat = os.stat(path)
            metadata['filesize'] = stat.st_size
        else:
            metadata['filesize'] = 0

        object_id = (tree_id, child_id)
        self._metadata_store.store(object_id, metadata)
        self._index_store.store(object_id, metadata)

        if (not path) and self._file_store.has_data((tree_id, parent_id)):
            # metadata-only update, reuse parent data
            path = self._file_store.retrieve((tree_id, parent_id),
                os.getuid(), 'update')
            delete_after = True

        if path:
            self._file_store.store(object_id, path, delete_after,
                lambda *args, **kwargs: self._save_completion_cb(
                tree_id, parent_id, child_id, metadata,
                async_cb, async_err_cb, **kwargs))
        else:
            self._save_completion_cb(tree_id, parent_id, child_id, metadata,
                async_cb, async_err_cb)

    @trace(logger=logger)
    def _save_completion_cb(self, tree_id, parent_id, child_id, metadata,
        async_cb, async_err_cb, exc=None):
        object_id = (tree_id, child_id)
        if exc is not None:
            async_err_cb(exc)
            return

        self._check_max_versions(tree_id)
        self.Saved(tree_id, parent_id, child_id, metadata)
        self._optimizer.optimize(object_id)
        async_cb(tree_id, child_id)

    def _check_max_versions(self, tree_id):
        if not self._max_versions:
            return

        old_versions = self._find({'tree_id': tree_id},
            {'all_versions': True, 'offset': self._max_versions,
            'metadata': ['tree_id', 'version_id', 'timestamp'],
            'order_by': ['+timestamp']})[0]
        logging.info('Deleting old versions: %r', old_versions)
        for entry in old_versions:
            self._delete(entry['tree_id'], entry['version_id'])

    @dbus.service.signal(DS_DBUS_INTERFACE2, signature='sssa{sv}')
    def Saved(self, tree_id, parent_id, child_id, metadata):
        """Entry created or updated.

        Entry identified by object_id has been saved to permanent storage.
        """
        pass

    @dbus.service.method(DS_DBUS_INTERFACE2,
             in_signature='ssa{sv}',
             out_signature='',
             byte_arrays=True)
    def change_metadata(self, tree_id, version_id, metadata):
        """Change metadata of existing entry.

        Changes the metadata of the object identified by object_id to match
        metadata. Fully synchronous, no return value.
        Emits signal ChangedMetadata(tree_id, version_id, metadata).
        """
        self._change_metadata(tree_id, version_id, metadata)

    @trace(logger=logger)
    def _change_metadata(self, tree_id, version_id, metadata):
        object_id = (tree_id, version_id)
        metadata.setdefault('timestamp', time.time())
        if 'creation_time' not in metadata:
            old_metadata = self._metadata_store.retrieve(object_id,
                                                         ['creation_time'])
            metadata['creation_time'] = old_metadata['creation_time']

        self._metadata_store.store(object_id, metadata)
        self._index_store.store(object_id, metadata)
        self.ChangedMetadata(tree_id, version_id, metadata)

    @dbus.service.signal(DS_DBUS_INTERFACE2, signature='ssa{sv}')
    def ChangedMetadata(self, tree_id, version_id, metadata):
        """Metadata of entry changed."""
        pass

    @dbus.service.method(DS_DBUS_INTERFACE2,
             in_signature='a{sv}a{sv}',
             out_signature='aa{sv}u',
             byte_arrays=True)
    def find(self, query, options):
        return self._find(query, options)

    @dbus.service.method(DS_DBUS_INTERFACE2,
             in_signature='sa{sv}a{sv}',
             out_signature='aa{sv}u',
             byte_arrays=True)
    def textsearch(self, querystring, query, options):
        return self._find(query, options, querystring)

    @trace(logger=logger)
    def _find(self, query, options, querystring=None):
        if not self._index_updating:
            try:
                object_ids, count = self._index_store.find(query, querystring,
                    options)
            except Exception:
                logging.exception('Failed to query index, will rebuild')
                self._rebuild_index()

        if self._index_updating:
            logging.warning('Index updating, returning all entries')
            return self._find_all(query, options)

        entries = []
        for object_id in object_ids:
            entry_path = layoutmanager.get_instance().get_entry_path(object_id)
            if not os.path.exists(entry_path):
                logging.warning(
                    'Inconsistency detected, returning all entries')
                self._rebuild_index()
                return self._find_all(query, options)

            metadata = self._metadata_store.retrieve(object_id,
                options.get('metadata'))
            entries.append(metadata)

        return entries, count

    def _find_all(self, query, options):
        object_ids = layoutmanager.get_instance().find_all()

        if not options.get('all_versions', False):
            tids_vtime = {}
            for tree_id, version_id in object_ids:
                tids_vtime.setdefault(tree_id, []).append((version_id,
                    self._metadata_store.retrieve((tree_id, version_id),
                    'timestamp')))

            object_ids = [(tree_id, sorted(candidates,
                key=lambda candidate_id: candidate_id[1], reverse=True)[0][0])
                for (tree_id, candidates) in tids_vtime.items()]

        count = len(object_ids)

        offset = options.get('offset', 0)
        limit = options.get('limit', MAX_QUERY_LIMIT)
        object_ids = object_ids[offset:offset + limit]

        entries = []
        for object_id in object_ids:
            metadata = self._metadata_store.retrieve(object_id,
                options.get('metadata'))
            entries.append(metadata)

        return entries, count

    @dbus.service.method(DS_DBUS_INTERFACE2,
             in_signature='ss',
             out_signature='s',
             sender_keyword='sender',
             byte_arrays=True)
    @trace(logger=logger)
    def get_data(self, tree_id, version_id, sender=None):
        """Make given entry readable and return path.

        Hardlinks the given object into a global directory, making it readable
        to the caller. Fully synchronous.
        Signal GotData(sender, object_id, path) is emitted when data
        is available. Sender is the bus name of the sender of the get_data()
        request. GotData() may be sent before get_data() returns.
        """
        return self._get_data(tree_id, version_id, sender)

    def _get_data(self, tree_id, version_id, sender=None):
        object_id = (tree_id, version_id)
        user_id = dbus.Bus().get_unix_user(sender)
        extension = self._get_extension(object_id)
        path = self._file_store.retrieve(object_id, user_id, extension)
        self.GotData(sender, tree_id, version_id, path)
        return path

    @dbus.service.signal(DS_DBUS_INTERFACE2, signature='ssss')
    def GotData(self, sender, tree_id, version_id, path):
        """Data is available.

        Data of entry identified by (tree_id, version_id) as requested by
        sender is available at path now. Permissions are matching the sender,
        so path may not be accessible to all listeners.
        """
        pass

    def _get_extension(self, object_id):
        mime_type = self._metadata_store.get_property(object_id, 'mime_type')
        if not mime_type:
            return ''
        return mime.get_primary_extension(mime_type)

    @dbus.service.method(DS_DBUS_INTERFACE2,
                         in_signature='a{sv}s',
                         out_signature='as')
    def find_unique_values(self, query, name):
        """Collect unique values of property.

        Similar to find(), but returns the set of unique values for the
        requested property.
        """
        return self._find_unique_values(query, name)

    def _find_unique_values(self, query, name):
        if name != 'activity':
            raise ValueError('Only "activity" is a supported property name')
        if query:
            raise ValueError('The query parameter is not supported')
        if not self._index_updating:
            return self._index_store.get_activities()
        else:
            logging.warning('Index updating, returning an empty list')
            return []

    @dbus.service.method(DS_DBUS_INTERFACE2,
             in_signature='ss',
             out_signature='',
             byte_arrays=True)
    def delete(self, tree_id, version_id):
        """Delete entry.

        Remove given object from data store. Fully synchronous. Doesn't
        return anything, emits signal Deleted(tree_id, version_id).
        """
        return self._delete(tree_id, version_id)

    def _delete(self, tree_id, version_id):
        object_id = (tree_id, version_id)
        self._optimizer.remove(object_id)

        self._index_store.delete(object_id)
        self._file_store.delete(object_id)
        self._metadata_store.delete(object_id)

        entry_path = layoutmanager.get_instance().get_entry_path(object_id)
        os.removedirs(entry_path)

        self.Deleted(*object_id)
        logger.debug('deleted %r', object_id)

    @dbus.service.signal(DS_DBUS_INTERFACE2, signature='ss')
    def Deleted(self, tree_id, version_id):
        """Entry has been deleted."""
        pass

    def stop(self):
        """shutdown the service"""
        self._stop()

    def _stop(self):
        self._index_store.close_index()
        self.Stopped()

    @dbus.service.signal(DS_DBUS_INTERFACE2)
    def Stopped(self):
        pass

    def _gen_uuid(self):
        return str(uuid.uuid4())