Keep sync and database in the mount root directory

7 files changed, 94 insertions, 175 deletions

diff --git a/sugar-network-server b/sugar-network-server
index 11b23cb..0c4eebd 100755
--- a/sugar-network-server
+++ b/sugar-network-server
@@ -15,8 +15,10 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
 import locale
 import logging
+from os.path import exists
 
 import active_document as ad
 import sugar_network_webui as webui
@@ -40,6 +42,8 @@ class Application(application.Daemon):
     jobs = coroutine.Pool()
 
     def run(self):
+        if not exists(node.tmpdir.value):
+            os.makedirs(node.tmpdir.value)
         sneakernet.TMPDIR = node.tmpdir.value
 
         ssl_args = {}
diff --git a/sugar-network-service b/sugar-network-service
index 2292ea7..39c258d 100755
--- a/sugar-network-service
+++ b/sugar-network-service
@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
 import locale
 import logging
 from contextlib import contextmanager
@@ -77,6 +78,8 @@ class Application(application.Daemon):
             application.logdir.value = sugar.profile_path('logs')
         application.rundir.value = join(local.local_root.value, 'run')
 
+        if not exists(local.tmpdir.value):
+            os.makedirs(local.tmpdir.value)
         sneakernet.TMPDIR = local.tmpdir.value
 
     @application.command(
diff --git a/sugar-network-sync b/sugar-network-sync
index 2c39617..bcabffc 100755
--- a/sugar-network-sync
+++ b/sugar-network-sync
@@ -158,8 +158,12 @@ disk_limit=$(expr 1024 \* 1024 \* 10)
 mkdir -p "${sync_path}" || abort "Cannot create ${sync_path} sync directory"
 cd "${sync_path}" || abort "Cannot switch to ${sync_path} sync directory"
 
+mountpoint="$(stat --printf %m .)"
+[ "${mountpoint}" = "$PWD" ] || info "NOTICE To make $PWD capable for further auto synchronization on a node side, place its content to the mount's root"
+touch .sugar-network-sync
+
 if [ "${clone_url}" ]; then
-    info "Clone full dump"
+    info "Clone master"
     pull "${clone_url}"
     exit 0
 fi
diff --git a/sugar_network/local/mountset.py b/sugar_network/local/mountset.py
index 2ef982e..62e1351 100644
--- a/sugar_network/local/mountset.py
+++ b/sugar_network/local/mountset.py
@@ -17,7 +17,7 @@ import os
 import locale
 import socket
 import logging
-from os.path import join, isdir, exists
+from os.path import join, exists
 
 import active_document as ad
 
@@ -34,8 +34,8 @@ from sugar_network.resources.volume import Volume
 from active_toolkit import util, coroutine, enforce
 
 
-_DB_DIRNAME = 'sugar-network'
-_SYNC_DIRNAME = 'sugar-network-sync'
+_DB_DIRNAME = '.sugar-network'
+_SYNC_DIRNAME = '.sugar-network-sync'
 
 _COMPLETE_MOUNT_TIMEOUT = 3
 
@@ -171,6 +171,12 @@ class Mountset(dict, ad.CommandsProcessor):
         try:
             mounts_root = local.mounts_root.value
             if mounts_root:
+                for filename in os.listdir(mounts_root):
+                    self._found_mount(join(mounts_root, filename))
+                # In case if sync mounts processed before server mounts
+                # TODO More obvious code
+                for filename in os.listdir(mounts_root):
+                    self._found_mount(join(mounts_root, filename))
                 self._jobs.spawn(self._mounts_monitor)
 
             if '/' in self:
@@ -208,45 +214,59 @@ class Mountset(dict, ad.CommandsProcessor):
                     break
 
     def _mounts_monitor(self):
-        roots = []
-        with _Inotify(self._found_mount, self._lost_mount) as monitor:
-            roots.append(_MountRoot(monitor, local.mounts_root.value))
-            while True:
+        root = local.mounts_root.value
+        _logger.info('Start monitoring %r for mounts', root)
+
+        with Inotify() as monitor:
+            monitor.add_watch(root, IN_DELETE_SELF | IN_CREATE | \
+                    IN_DELETE | IN_MOVED_TO | IN_MOVED_FROM)
+            while not monitor.closed:
                 coroutine.select([monitor.fileno()], [], [])
-                if monitor.closed:
-                    break
-                for filename, event, cb in monitor.read():
+                for filename, event, __ in monitor.read():
+                    path = join(root, filename)
                     try:
-                        cb(filename, event)
+                        if event & IN_DELETE_SELF:
+                            _logger.warning('Lost %r, cannot monitor anymore',
+                                    root)
+                            monitor.close()
+                            break
+                        elif event & (IN_DELETE | IN_MOVED_FROM):
+                            self._lost_mount(path)
+                        elif event & (IN_CREATE | IN_MOVED_TO):
+                            # Right after moutning, access to directory
+                            # might be restricted; let system enough time
+                            # to complete mounting
+                            coroutine.sleep(_COMPLETE_MOUNT_TIMEOUT)
+                            self._found_mount(path)
                     except Exception:
-                        util.exception('Cannot dispatch 0x%X event ' \
-                                'for %r mount', event, filename)
+                        util.exception(_logger, 'Mount %r failed', path)
 
-    def _found_mount(self, root, dirnames):
-        if _DB_DIRNAME in dirnames and root not in self:
-            _logger.debug('Found %r server mount', root)
-            volume, server_mode = self._mount_volume(join(root, _DB_DIRNAME))
+    def _found_mount(self, path):
+        if exists(join(path, _DB_DIRNAME)) and path not in self:
+            _logger.debug('Found %r server mount', path)
+            volume, server_mode = self._mount_volume(path)
             if server_mode:
-                self[root] = NodeMount(volume, self.home_volume)
+                self[path] = NodeMount(volume, self.home_volume)
             else:
-                self[root] = LocalMount(volume)
+                self[path] = LocalMount(volume)
 
-        if _SYNC_DIRNAME in dirnames:
-            _logger.debug('Found %r sync mount', root)
-            self._sync_dirs.add(join(root, _SYNC_DIRNAME))
+        if exists(join(path, _SYNC_DIRNAME)):
+            self._sync_dirs.add(path)
             if self._servers:
+                _logger.debug('Found %r sync mount', path)
                 self.start_sync()
+            else:
+                _logger.debug('Found %r sync mount but no servers', path)
 
-    def _lost_mount(self, root, dirnames):
-        if _SYNC_DIRNAME in dirnames:
-            _logger.debug('Lost %r sync mount', root)
-            self._sync_dirs.remove(join(root, _SYNC_DIRNAME))
-            if not self._sync_dirs:
-                self.break_sync()
+    def _lost_mount(self, path):
+        _logger.debug('Lost %r mount', path)
 
-        if _DB_DIRNAME in dirnames and root in self:
-            _logger.debug('Lost %r server mount', root)
-            del self[root]
+        self._sync_dirs.remove(join(path, _SYNC_DIRNAME))
+        if not self._sync_dirs:
+            self.break_sync()
+
+        if path in self:
+            del self[path]
 
     def _mount_volume(self, path):
         lazy_open = local.lazy_open.value
@@ -281,89 +301,3 @@ class Mountset(dict, ad.CommandsProcessor):
             coroutine.dispatch()
 
         return volume, server_mode
-
-
-class _Inotify(Inotify):
-
-    def __init__(self, found_cb, lost_cb):
-        Inotify.__init__(self)
-        self.found_cb = found_cb
-        self.lost_cb = lost_cb
-
-
-class _MountRoot(object):
-
-    def __init__(self, monitor, path):
-        self.path = path
-        self._monitor = monitor
-        self._nodes = {}
-
-        _logger.info('Start monitoring %r for mounts', self.path)
-
-        monitor.add_watch(self.path,
-                IN_DELETE_SELF | IN_CREATE | IN_DELETE | \
-                        IN_MOVED_TO | IN_MOVED_FROM,
-                self.__watch_cb)
-
-        for filename in os.listdir(self.path):
-            path = join(self.path, filename)
-            if isdir(path):
-                self._nodes[filename] = _MountDir(monitor, path)
-
-    def __watch_cb(self, filename, event):
-        if event & IN_DELETE_SELF:
-            _logger.warning('Lost ourselves, cannot monitor anymore')
-            self._nodes.clear()
-
-        elif event & (IN_CREATE | IN_MOVED_TO):
-            path = join(self.path, filename)
-            if isdir(path):
-                # Right after moutning, access to directory might be
-                # restricted; let system enough time to complete mounting
-                coroutine.sleep(_COMPLETE_MOUNT_TIMEOUT)
-                self._nodes[filename] = _MountDir(self._monitor, path)
-
-        elif event & (IN_DELETE | IN_MOVED_FROM):
-            item = self._nodes.get(filename)
-            if item is not None:
-                item.unlink()
-                del self._nodes[filename]
-
-
-class _MountDir(object):
-
-    def __init__(self, monitor, root):
-        self._root = root
-        self._monitor = monitor
-        self._found = set([i for i in os.listdir(root) \
-                if isdir(join(root, i))])
-
-        _logger.debug('Start monitoring %r mount', root)
-
-        self._wd = monitor.add_watch(root,
-                IN_CREATE | IN_DELETE | IN_MOVED_TO | IN_MOVED_FROM,
-                self.__watch_cb)
-        if self._found:
-            monitor.found_cb(self._root, self._found)
-
-    def unlink(self):
-        if self._found:
-            self._monitor.lost_cb(self._root, self._found)
-            self._found.clear()
-        _logger.debug('Stop monitoring %r mount', self._root)
-        self._monitor.rm_watch(self._wd)
-
-    def __watch_cb(self, filename, event):
-        path = join(self._root, filename)
-
-        if event & (IN_CREATE | IN_MOVED_TO) and isdir(path):
-            if filename not in self._found:
-                _logger.debug('Found %r mount directory', path)
-                self._found.add(filename)
-                self._monitor.found_cb(self._root, [filename])
-
-        elif event & (IN_DELETE | IN_MOVED_FROM):
-            if filename in self._found:
-                _logger.debug('Lost %r mount directory', path)
-                self._found.remove(filename)
-                self._monitor.lost_cb(self._root, [filename])
diff --git a/tests/integration/sync.py b/tests/integration/sync.py
index 8f1d920..7183d68 100755
--- a/tests/integration/sync.py
+++ b/tests/integration/sync.py
@@ -63,13 +63,13 @@ class SyncTest(tests.Test):
             context.upload_blob('preview', 'preview_2')
 
         # Clone initial dump
-        os.makedirs('mnt_1/sugar-network-sync')
-        pid = self.popen('V=1 sugar-network-sync mnt_1/sugar-network-sync http://localhost:8100', shell=True)
+        pid = self.popen('V=1 sugar-network-sync mnt_1 http://localhost:8100', shell=True)
         self.waitpid(pid, 0)
 
         # Start node and import cloned data
-        self.touch(('mnt_1/sugar-network/node', 'node'))
-        self.touch(('mnt_1/sugar-network/master', 'master'))
+        self.touch('mnt_1/.sugar-network')
+        self.touch(('mnt_1/node', 'node'))
+        self.touch(('mnt_1/master', 'master'))
         os.rename('mnt_1', 'mnt/mnt_1')
         self.wait_for_events({'event': 'sync_complete'})
         mountpoint = tests.tmpdir + '/mnt/mnt_1'
@@ -90,13 +90,13 @@ class SyncTest(tests.Test):
             context.upload_blob('preview', 'preview_4')
 
         # Create node push packets with newly create data
-        os.makedirs('mnt_2/sugar-network-sync')
+        self.touch('mnt_2/.sugar-network-sync')
         os.rename('mnt_2', 'mnt/mnt_2')
         self.wait_for_events({'event': 'sync_complete'})
 
         # Upload node data to master
         shutil.copytree('mnt/mnt_2', 'mnt_3')
-        pid = self.popen('V=1 sugar-network-sync mnt_3/sugar-network-sync', shell=True)
+        pid = self.popen('V=1 sugar-network-sync mnt_3', shell=True)
         self.waitpid(pid, 0)
 
         # Process master's reply
diff --git a/tests/units/mountset.py b/tests/units/mountset.py
index 21e5e55..ba00908 100755
--- a/tests/units/mountset.py
+++ b/tests/units/mountset.py
@@ -53,8 +53,8 @@ class MountsetTest(tests.Test):
         return mounts
 
     def test_Populate(self):
-        os.makedirs('1/sugar-network')
-        os.makedirs('2/sugar-network')
+        os.makedirs('1/.sugar-network')
+        os.makedirs('2/.sugar-network')
 
         self.mountset()
 
@@ -87,7 +87,7 @@ class MountsetTest(tests.Test):
     def test_Mount(self):
         self.mountset()
 
-        os.makedirs('tmp/1/sugar-network')
+        os.makedirs('tmp/1/.sugar-network')
         shutil.move('tmp/1', '.')
 
         self.mounted.wait()
@@ -106,7 +106,7 @@ class MountsetTest(tests.Test):
                 summary='summary',
                 description='description').post()
 
-        os.makedirs('tmp/2/sugar-network')
+        os.makedirs('tmp/2/.sugar-network')
         shutil.move('tmp/2', '.')
 
         self.mounted.wait()
@@ -126,31 +126,9 @@ class MountsetTest(tests.Test):
                 summary='summary',
                 description='description').post()
 
-        os.makedirs('3')
-        coroutine.sleep(.5)
-        os.makedirs('3/sugar-network')
-
-        self.mounted.wait()
-        self.mounted.clear()
-        self.assertEqual(
-                [('mount', tests.tmpdir + '/3')],
-                self.mount_events[2:])
-        self.assertEqual(
-                sorted([
-                    {'mountpoint': tests.tmpdir + '/1', 'name': '1', 'private': True},
-                    {'mountpoint': tests.tmpdir + '/2', 'name': '2', 'private': True},
-                    {'mountpoint': tests.tmpdir + '/3', 'name': '3', 'private': True},
-                    ]),
-                sorted(Client.mounts()))
-        Client(tests.tmpdir + '/2').Context(
-                type='activity',
-                title='remote',
-                summary='summary',
-                description='description').post()
-
     def test_Unmount(self):
-        os.makedirs('1/sugar-network')
-        os.makedirs('2/sugar-network')
+        os.makedirs('1/.sugar-network')
+        os.makedirs('2/.sugar-network')
 
         self.mountset()
         self.mounted.wait()
@@ -175,23 +153,12 @@ class MountsetTest(tests.Test):
                     ]),
                 sorted(Client.mounts()))
 
-        del self.mount_events[:]
-        shutil.rmtree('2/sugar-network')
-        self.mounted.wait()
-        self.mounted.clear()
-        self.assertEqual(
-                [('unmount', tests.tmpdir + '/2')],
-                self.mount_events)
-        self.assertEqual(
-                sorted([
-                    ]),
-                sorted(Client.mounts()))
-
     def test_MountNode(self):
         local.server_mode.value = True
         self.mountset()
 
-        self.touch(('tmp/mnt/sugar-network/node', 'node'))
+        self.touch('tmp/mnt/.sugar-network')
+        self.touch(('tmp/mnt/node', 'node'))
         shutil.move('tmp/mnt', '.')
 
         self.mounted.wait()
diff --git a/tests/units/node_mount.py b/tests/units/node_mount.py
index 9e99a73..957e9a0 100755
--- a/tests/units/node_mount.py
+++ b/tests/units/node_mount.py
@@ -60,7 +60,8 @@ class NodeMountTest(tests.Test):
         return mounts
 
     def test_GetKeep(self):
-        self.touch(('mnt/sugar-network/node', 'node'))
+        self.touch('mnt/.sugar-network')
+        self.touch(('mnt/node', 'node'))
         mounts = self.start_server()
         self.got_event.wait()
 
@@ -92,7 +93,8 @@ class NodeMountTest(tests.Test):
                 [(i['guid'], i['keep'], i['keep_impl']) for i in remote.Context.cursor(reply=['keep', 'keep_impl'])])
 
     def test_SetKeep(self):
-        self.touch(('mnt/sugar-network/node', 'node'))
+        self.touch('mnt/.sugar-network')
+        self.touch(('mnt/node', 'node'))
         mounts = self.start_server()
         mounts['~'] = HomeMount(mounts.home_volume)
         self.got_event.wait()
@@ -174,7 +176,8 @@ class NodeMountTest(tests.Test):
                 'sugar_network.resources.implementation',
                 ]
 
-        self.touch(('mnt/sugar-network/node', 'node'))
+        self.touch('mnt/.sugar-network')
+        self.touch(('mnt/node', 'node'))
         mounts = self.start_server(ipc=True)
         mounts['~'] = HomeMount(mounts.home_volume)
         self.got_event.wait()
@@ -194,13 +197,13 @@ class NodeMountTest(tests.Test):
                 date=0,
                 stability='stable',
                 notes='').post()
-        with file('mnt/sugar-network/context/%s/%s/feed' % (context[:2], context), 'w') as f:
+        with file('mnt/context/%s/%s/feed' % (context[:2], context), 'w') as f:
             json.dump({
                 'seqno': 0,
                 'mime_type': 'application/octet-stream',
                 'digest': 'digest',
                 }, f)
-        with file('mnt/sugar-network/context/%s/%s/feed.blob' % (context[:2], context), 'w') as f:
+        with file('mnt/context/%s/%s/feed.blob' % (context[:2], context), 'w') as f:
             json.dump({
                 '1': {
                     '*-*': {
@@ -239,7 +242,8 @@ class NodeMountTest(tests.Test):
         assert exists('Activities/TestActivitry/activity/activity.info')
 
     def test_Events(self):
-        self.touch(('mnt/sugar-network/node', 'node'))
+        self.touch('mnt/.sugar-network')
+        self.touch(('mnt/node', 'node'))
         self.start_server()
         self.got_event.wait()
         self.got_event.clear()
@@ -278,7 +282,7 @@ class NodeMountTest(tests.Test):
             events)
         del events[:]
 
-        guid_path = 'mnt/sugar-network/context/%s/%s' % (guid[:2], guid)
+        guid_path = 'mnt/context/%s/%s' % (guid[:2], guid)
         assert exists(guid_path)
         client.Context.delete(guid)
         assert not exists(guid_path)
@@ -291,7 +295,8 @@ class NodeMountTest(tests.Test):
         del events[:]
 
     def test_upload_blob(self):
-        self.touch(('mnt/sugar-network/node', 'node'))
+        self.touch('mnt/.sugar-network')
+        self.touch(('mnt/node', 'node'))
         self.start_server()
         self.got_event.wait()
         remote = Client(tests.tmpdir + '/mnt')
@@ -312,7 +317,8 @@ class NodeMountTest(tests.Test):
         assert not exists('file2')
 
     def test_GetAbsetnBLOB(self):
-        self.touch(('mnt/sugar-network/node', 'node'))
+        self.touch('mnt/.sugar-network')
+        self.touch(('mnt/node', 'node'))
         self.start_server()
         self.got_event.wait()
         client = Client(tests.tmpdir + '/mnt')
@@ -333,7 +339,8 @@ class NodeMountTest(tests.Test):
                 'sugar_network.resources.implementation',
                 ]
 
-        self.touch(('mnt/sugar-network/node', 'node'))
+        self.touch('mnt/.sugar-network')
+        self.touch(('mnt/node', 'node'))
         self.start_server()
         self.got_event.wait()
         remote = Client(tests.tmpdir + '/mnt')