1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
|
# Copyright (C) 2012-2013 Aleksey Lim
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import time
import json
import shutil
from os.path import exists, join, isdir, basename
from sugar_network import toolkit
from sugar_network.toolkit.router import Blob
_BLOB_SUFFIX = '.blob'
class Storage(object):
"""Get access to documents' data storage."""
def __init__(self, root, metadata):
self._root = root
self.metadata = metadata
def get(self, guid):
"""Get access to particular document's properties.
:param guid:
document GUID to get access to
:returns:
`Record` object
"""
return Record(self._path(guid))
def delete(self, guid):
"""Remove document properties from the storage.
:param guid:
document to remove
"""
path = self._path(guid)
if not exists(path):
return
try:
shutil.rmtree(path)
except Exception, error:
toolkit.exception()
raise RuntimeError('Cannot delete %r document from %r: %s' %
(guid, self.metadata.name, error))
def walk(self, mtime):
"""Generator function to enumerate all existing documents.
:param mtime:
return entities that were modified after `mtime`
:returns:
generator returns (guid, properties) typle for all found
documents; the properties dictionary will contain only
`StoredProperty` properties
"""
if not exists(self._root):
return
for guids_dirname in os.listdir(self._root):
guids_dir = join(self._root, guids_dirname)
if not isdir(guids_dir) or \
mtime and os.stat(guids_dir).st_mtime < mtime:
continue
for guid in os.listdir(guids_dir):
path = join(guids_dir, guid, 'guid')
if exists(path) and os.stat(path).st_mtime > mtime:
yield guid
def migrate(self, guid):
pass
def _path(self, guid, *args):
return join(self._root, guid[:2], guid, *args)
class Record(object):
"""Interface to document data."""
def __init__(self, root):
self._root = root
@property
def guid(self):
return basename(self._root)
@property
def exists(self):
return exists(self._root)
@property
def consistent(self):
return exists(join(self._root, 'guid'))
def path(self, *args):
return join(self._root, *args)
def blob_path(self, prop, *args):
return join(self._root, prop + _BLOB_SUFFIX, *args)
def invalidate(self):
guid_path = join(self._root, 'guid')
if exists(guid_path):
os.unlink(guid_path)
def get(self, prop):
path = join(self._root, prop)
if not exists(path):
return None
with file(path) as f:
meta = Blob(json.load(f))
blob_path = path + _BLOB_SUFFIX
if exists(blob_path):
meta['blob'] = blob_path
if 'blob_size' not in meta:
meta['blob_size'] = os.stat(blob_path).st_size
meta['mtime'] = int(os.stat(path).st_mtime)
return meta
def set(self, prop, mtime=None, **meta):
if not exists(self._root):
os.makedirs(self._root)
meta_path = join(self._root, prop)
if 'blob' in meta:
dst_blob_path = meta_path + _BLOB_SUFFIX
blob = meta.pop('blob')
if hasattr(blob, 'read'):
with toolkit.new_file(dst_blob_path) as f:
shutil.copyfileobj(blob, f)
elif blob is not None:
os.rename(blob, dst_blob_path)
elif exists(dst_blob_path):
os.unlink(dst_blob_path)
with toolkit.new_file(meta_path) as f:
json.dump(meta, f)
if mtime:
os.utime(meta_path, (mtime, mtime))
if prop == 'guid':
if not mtime:
mtime = time.time()
# Touch directory to let it possible to crawl it on startup
# when index was not previously closed properly
os.utime(join(self._root, '..'), (mtime, mtime))
|
