1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
|
"""
olpc.datastore.model
~~~~~~~~~~~~~~~~~~~~
The datamodel for the metadata
"""
__author__ = 'Benjamin Saller <bcsaller@objectrealms.net>'
__docformat__ = 'restructuredtext'
__copyright__ = 'Copyright ObjectRealms, LLC, 2007'
__license__ = 'The GNU Public License V2+'
import datetime
import mimetypes
import os
import re
import time
import warnings
from olpc.datastore.utils import timeparse
import dbus
# XXX: Open issues
# list properties - Contributors (a, b, c)
# difficult to index now
# content state - searches don't include content deletion flag
# - not recording if content is on other storage yet
propertyTypes = {}
_marker = object()
# used in cases where we convert title to a filename
titleSanitizer = re.compile("[\W]+")
def registerPropertyType(kind, get, set, xapian_sort_type=None,
defaults=None, for_xapian=None, from_xapain=None):
propertyTypes[kind] = PropertyImpl(get, set, xapian_sort_type,
defaults, for_xapian=for_xapian, from_xapain=from_xapain)
def propertyByKind(kind): return propertyTypes[kind]
class PropertyImpl(object):
__slots__ = ('_get', '_set', 'xapian_sort_type', 'defaults', '_for_xapian', '_from_xapian')
def __init__(self, get, set, xapian_sort_type=None, defaults=None,
for_xapian=None, from_xapain=None):
self._get, self._set = get, set
self.xapian_sort_type = xapian_sort_type
self.defaults = defaults
if not for_xapian: for_xapian = self._get
self._for_xapian = for_xapian
if not from_xapain: from_xapain = self._set
self._from_xapian = from_xapain
def get(self, value): return self._get(value)
def set(self, value): return self._set(value)
def for_xapian(self, value): return self._for_xapian(value)
def from_xapian(self, value): return self._from_xapian(value)
class Property(object):
"""Light-weight property implementation.
Handles typed properties via a global registry of type->callbacks
>>> p = Property(key, value, 'string')
>>> b = Property(key, value, 'binary')
"""
def __init__(self, key, value, kind=None):
self.kind = kind
if kind not in propertyTypes:
warnings.warn("Unknown property type: %s on key %s" % \
(kind, key), RuntimeWarning)
else: self._impl = propertyTypes[kind]
self.key = key
self.value = value
@classmethod
def fromstring(cls, key, value=''):
kind = 'string'
if ':' in key:
key, kind = key.split(':', 1)
# now resolve the kind to a property class
return cls(key, value, kind)
def __repr__(self):
return "<%s(%s) %s:%r>" % (self.__class__.__name__,
self.kind,
self.key, self.value)
def get_value(self): return self._impl.get(self._value)
def set_value(self, value): self._value = self._impl.set(value)
value = property(get_value, set_value)
@property
def for_xapian(self): return self._impl.for_xapian(self._value)
def __str__(self): return str(self.value)
class Model(object):
"""Object containing the field/property model used by the
system"""
def __init__(self):
self.fields = {}
self.fieldnames = []
def copy(self):
m = Model()
m.fields = self.fields.copy()
m.fieldnames = self.fieldnames[:]
return m
def fromstring(self, key, value, allowAddition=False):
"""create a property from the key name by looking it up in the
model."""
kind = None
if ':' in key: key, kind = key.split(':', 1)
added = False
field = self.fields.get(key)
if field: mkind = field[1]
elif allowAddition:
# create a new field, this will force a change in the
# model
# and in turn should add a new field action
if not kind: kind = "string"
self.addField(key,kind)
mkind = kind
added = True
else:
raise KeyError("no field specification for %s" % key)
if kind and mkind:
if kind != mkind: raise ValueError("""Specified wire
encoding for property %s was %s, expected %s""" %(key, kind, mkind))
kind = mkind
return Property(key, value, kind), added
def addField(self, key, kind, overrides=None):
""" Add a field to the model.
key -- field name
kind -- type by name (registered with registerPropertyType)
kwargs -- overrides and additional values to the default
arguments supplied by kind
"""
if key in self.fields:
raise KeyError("""Another source tried to add %s field to the model""" % key)
impl = propertyByKind(kind)
options = impl.defaults.copy()
if overrides: options.update(overrides)
if impl.xapian_sort_type:
if 'type' not in options:
options['type'] = impl.xapian_sort_type
self.fields[key] = (key, kind, options)
self.fieldnames.append(key)
return self
def addFields(self, *args):
""" List of arguments to addField """
for arg in args: self.addField(*arg)
return self
def apply(self, indexmanager):
addField = indexmanager.addField
for fn in self.fieldnames:
args = self.fields[fn]
addField(args[0], **args[2])
# Properties we don't automatically include in properties dict
EXCLUDED_PROPERTIES = ['fulltext', 'changeset', 'file_rev']
class Content(object):
"""A light weight proxy around Xapian Documents from secore.
This provides additional methods which are used in the
backingstore to assist in storage
"""
__slots__ = ('_doc', '_backingstore', '_file', '_model')
def __init__(self, xapdoc, backingstore=None, model=None):
self._doc = xapdoc
self._backingstore = backingstore
self._file = None
self._model = model
self._file = None
def __repr__(self):
return "<%s %s>" %(self.__class__.__name__,
self.properties)
def get_property(self, key, default=_marker):
result = self._doc.data.get(key, default)
if result is _marker: raise KeyError(key)
if isinstance(result, list) and len(result) == 1:
result = result[0]
field = self._model.fields.get(key)
kind = propertyByKind(field[1])
# Errors here usually property request for a missing field
return kind.from_xapian(result)
@property
def properties(self):
d = {}
for k, v in self.data.iteritems():
if k in EXCLUDED_PROPERTIES: continue
if isinstance(v, list) and len(v) == 1:
v = v[0]
field = self._model.fields.get(k)
if field:
kind = propertyByKind(field[1])
v = kind.from_xapian(v)
else:
# do some generic property handling
if v: v = str(v)
else: v = ''
d[k] = v
return d
def suggestName(self):
# we look for certain known property names
# - filename
# - ext
# and create a base file name that will be used for the
# checkout name
filename = self.get_property('filename', None)
ext = self.get_property('ext', '')
title = self.get_property('title', '')
title = titleSanitizer.sub("_", title)
if filename:
# some backingstores keep the full relative path
filename = os.path.split(filename)[1]
f, e = os.path.splitext(filename)
if e: return filename, None
if ext: return "%s.%s" % (filename, ext), None
elif ext: return None, ext
else:
# try to get an extension from the mimetype if available
mt = self.get_property('mime_type', None)
if mt:
ext = mimetypes.guess_extension(mt)
# .ksh is a strange ext for plain text
if ext and ext == '.ksh': ext = '.txt'
if title: return title, ext
if ext: return None, ext
if title: return title, None
return None, None
def get_file(self):
if not self._file or self._file.closed is True:
target, ext = self.suggestName()
try:
targetfile = self.backingstore._targetFile(self.id, target, ext)
if not targetfile:
self._file = None
return self._file
self._file = open(targetfile, "rw")
except OSError:
self._file = None
return self._file
def set_file(self, fileobj):
self._file = fileobj
file = property(get_file, set_file)
@property
def filename(self):
if self.file:
return os.path.abspath(self.file.name)
return ''
@property
def contents(self): return self.file.read()
@property
def backingstore(self): return self._backingstore
# id of this record (unique in index)
@property
def id(self):
return self._doc.id
@property
def data(self): return self._doc.data
def noop(value): return value
import re
base64hack = re.compile("(\S{212})")
def base64enc(value):
if isinstance(value, list):
# its a bytearray
value = ''.join((str(v) for v in value))
import pdb;pdb.set_trace()
return ' '.join(base64hack.split(value.encode('base64')))
def base64dec(value):
return value.replace(' ', '').decode('base64')
DATEFORMAT = "%Y-%m-%dT%H:%M:%S"
def date2string(value): return value.replace(microsecond=0).isoformat()
def string2date(value): return timeparse(value, DATEFORMAT)
def encode_datetime(value):
# encode datetime to timestamp (float)
# parse the typelib form to a datetime first
if isinstance(value, basestring): value = string2date(value)
return str(time.mktime(value.timetuple()))
def decode_datetime(value):
# convert a float to a local datetime
if not value: return None
return datetime.datetime.fromtimestamp(float(value)).isoformat()
def datedec(value, dateformat=DATEFORMAT):
return timeparse(value, DATEFORMAT)
def dateenc(value, dateformat=DATEFORMAT):
if isinstance(value, basestring):
# XXX: there is an issue with microseconds not getting parsed
value = timeparse(value, DATEFORMAT)
value = value.replace(microsecond=0)
return value.isoformat()
# type, get, set, xapian sort type [string|float|date], defaults
# defaults are the default options to addField in IndexManager
# these can be overridden on model assignment
registerPropertyType('string', noop, noop, 'string', {'store' : True,
'exact' : True,
'sortable' : True})
registerPropertyType('text', noop, noop, 'string', {'store' : True,
'exact' : False,
'sortable' : False,
'collapse' : True,
})
registerPropertyType('tag', noop, str.lower, 'string', {'store' : True,
'exact' : True,
})
# Now the convention is to directly use DBus.ByteArray
registerPropertyType('binary', str, dbus.ByteArray, None, {'store' : True,
'exact' : False,
'sortable' : False})
registerPropertyType('int', str, int, 'float', {'store' : True,
'exact' : True,
'sortable' : True},
for_xapian=str)
registerPropertyType('number', str, float, 'float', {'store' : True,
'exact' : True,
'sortable' : True})
registerPropertyType('date', dateenc, datedec, 'float', {'store' : True,
'exact' : True,
'sortable' : True
},
for_xapian=encode_datetime,
from_xapain=decode_datetime)
defaultModel = Model().addFields(
('fulltext', 'text'),
# unique to the object through out its history
('uid', 'string'),
# vid is version id
# linear version id's don't imply direct lineage, only the relative merge order
('vid', 'string'),
# unique token per change to help disambiguate for merges
('changeset', 'string'),
('file_rev', 'int'), # this is the revision of the file pointed to
# by this revision of the object
('checksum', 'string'),
('filename', 'string'),
('ext', 'string'), # its possible we don't store a filename, but
# only an extension we are interested in
# Title has additional weight
('title', 'text', {'weight' : 2 }),
('url', 'string'),
('mime_type', 'string'),
('author', 'string'),
('language', 'string'),
('ctime', 'date'),
('mtime', 'date'),
# this will just be a space delimited list of tags
# tags are case-insensitive and managed via the tags() API call
# while they can be set as normal properties with checkin that
# will create new versions and the tags() call will not.
('tags', 'tag'),
# olpc specific
('activity', 'string'),
('activity_id', 'string'),
('title_set_by_user', 'text'),
('keep', 'int'),
('icon-color', 'string'),
('preview', 'binary'),
('buddies', 'text'),
)
|