diff options
Diffstat (limited to 'websdk/hatta/storage.py')
| -rw-r--r-- | websdk/hatta/storage.py | 586 |
1 files changed, 586 insertions, 0 deletions
diff --git a/websdk/hatta/storage.py b/websdk/hatta/storage.py new file mode 100644 index 0000000..f1f2ee8 --- /dev/null +++ b/websdk/hatta/storage.py @@ -0,0 +1,586 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import tempfile +import thread +import re +import werkzeug +import datetime +import errno + +# Note: we have to set these before importing Mercurial +os.environ['HGENCODING'] = 'utf-8' +os.environ['HGMERGE'] = "internal:merge" + +import mercurial.hg +import mercurial.ui +import mercurial.revlog +import mercurial.util +import mercurial.hgweb +import mercurial.commands +import mercurial.merge + +import error +import page + + +def locked_repo(func): + """A decorator for locking the repository when calling a method.""" + + def new_func(self, *args, **kwargs): + """Wrap the original function in locks.""" + + wlock = self.repo.wlock() + lock = self.repo.lock() + try: + func(self, *args, **kwargs) + finally: + lock.release() + wlock.release() + + return new_func + + +def _find_repo_path(path): + """Go up the directory tree looking for a repository.""" + + while not os.path.isdir(os.path.join(path, ".hg")): + old_path, path = path, os.path.dirname(path) + if path == old_path: + return None + return path + + +def _get_ui(): + try: + ui = mercurial.ui.ui(report_untrusted=False, + interactive=False, quiet=True) + except TypeError: + # Mercurial 1.3 changed the way we setup the ui object. + ui = mercurial.ui.ui() + ui.quiet = True + ui._report_untrusted = False + ui.setconfig('ui', 'interactive', False) + return ui + + +class WikiStorage(object): + """ + Provides means of storing wiki pages and keeping track of their + change history, using Mercurial repository as the storage method. + """ + + def __init__(self, path, charset=None, _=lambda x: x, unix_eol=False, + extension=None): + """ + Takes the path to the directory where the pages are to be kept. + If the directory doesn't exist, it will be created. If it's inside + a Mercurial repository, that repository will be used, otherwise + a new repository will be created in it. + """ + + self._ = _ + self.charset = charset or 'utf-8' + self.unix_eol = unix_eol + self.extension = extension + self.path = os.path.abspath(path) + if not os.path.exists(self.path): + os.makedirs(self.path) + self.ui = _get_ui() + self.repo_path = _find_repo_path(self.path) + if self.repo_path is None: + # Create the repository if needed. + self.repo_path = self.path + mercurial.hg.repository(self.ui, self.repo_path, create=True) + self.repo_prefix = self.path[len(self.repo_path):].strip('/') + self._repos = {} + + def reopen(self): + """Close and reopen the repo, to make sure we are up to date.""" + + self._repos = {} + + @property + def repo(self): + """Keep one open repository per thread.""" + + thread_id = thread.get_ident() + try: + return self._repos[thread_id] + except KeyError: + repo = mercurial.hg.repository(self.ui, self.repo_path) + self._repos[thread_id] = repo + return repo + + def _check_path(self, path): + """ + Ensure that the path is within allowed bounds. + """ + + _ = self._ + abspath = os.path.abspath(path) + if os.path.islink(path) or os.path.isdir(path): + raise error.ForbiddenErr( + _(u"Can't use symbolic links or directories as pages")) + if not abspath.startswith(self.path): + raise error.ForbiddenErr( + _(u"Can't read or write outside of the pages repository")) + + def _file_path(self, title): + return os.path.join(self.repo_path, self._title_to_file(title)) + + def _title_to_file(self, title): + title = unicode(title).strip() + filename = werkzeug.url_quote(title, safe='') + # Escape special windows filenames and dot files + _windows_device_files = ('CON', 'AUX', 'COM1', 'COM2', 'COM3', + 'COM4', 'LPT1', 'LPT2', 'LPT3', 'PRN', + 'NUL') + if (filename.split('.')[0].upper() in _windows_device_files or + filename.startswith('_') or filename.startswith('.')): + filename = '_' + filename + if page.page_mime(title) == 'text/x-wiki' and self.extension: + filename += self.extension + return os.path.join(self.repo_prefix, filename) + + def _file_to_title(self, filepath): + _ = self._ + if not filepath.startswith(self.repo_prefix): + raise error.ForbiddenErr( + _(u"Can't read or write outside of the pages repository")) + name = filepath[len(self.repo_prefix):].strip('/') + # Un-escape special windows filenames and dot files + if name.startswith('_') and len(name) > 1: + name = name[1:] + if self.extension and name.endswith(self.extension): + name = name[:-len(self.extension)] + return werkzeug.url_unquote(name) + + def __contains__(self, title): + if title: + file_path = self._file_path(title) + return os.path.isfile(file_path) and not os.path.islink(file_path) + + def __iter__(self): + return self.all_pages() + + def merge_changes(self, changectx, repo_file, text, user, parent): + """Commits and merges conflicting changes in the repository.""" + + _ = self._ + tip_node = changectx.node() + filectx = changectx[repo_file].filectx(parent) + parent_node = filectx.changectx().node() + + self.repo.dirstate.setparents(parent_node) + node = self._commit([repo_file], text, user) + + partial = lambda filename: repo_file == filename + try: + mercurial.merge.update(self.repo, tip_node, True, True, partial) + msg = _(u'merge of edit conflict') + except mercurial.util.Abort: + msg = _(u'failed merge of edit conflict') + self.repo.dirstate.setparents(tip_node, node) + # Mercurial 1.1 and later need updating the merge state + try: + mergestate = mercurial.merge.mergestate + except AttributeError: + pass + else: + state = mergestate(self.repo) + try: + state.mark(repo_file, "r") + except KeyError: + # There were no conflicts to mark + pass + else: + # Mercurial 1.7+ needs a commit + try: + commit = state.commit + except AttributeError: + pass + else: + commit() + return msg + + @locked_repo + def save_file(self, title, file_name, author=u'', comment=u'', + parent=None): + """Save an existing file as specified page.""" + + _ = self._ + user = author.encode('utf-8') or _(u'anon').encode('utf-8') + text = comment.encode('utf-8') or _(u'comment').encode('utf-8') + repo_file = self._title_to_file(title) + file_path = self._file_path(title) + self._check_path(file_path) + try: + mercurial.util.rename(file_name, file_path) + except OSError, e: + if e.errno == errno.ENAMETOOLONG: + # "File name too long" + raise error.RequestURITooLarge() + else: + raise + changectx = self._changectx() + try: + # Mercurial 1.5 and earlier have .add() on the repo + add = self.repo.add + except AttributeError: + # Mercurial 1.6 + add = self.repo[None].add + try: + filectx_tip = changectx[repo_file] + current_page_rev = filectx_tip.filerev() + except mercurial.revlog.LookupError: + add([repo_file]) + current_page_rev = -1 + if parent is not None and current_page_rev != parent: + msg = self.merge_changes(changectx, repo_file, text, user, parent) + user = '<wiki>' + text = msg.encode('utf-8') + self._commit([repo_file], text, user) + + def _commit(self, files, text, user): + try: + return self.repo.commit(files=files, text=text, user=user, + force=True, empty_ok=True) + except TypeError: + # Mercurial 1.3 doesn't accept empty_ok or files parameter + match = mercurial.match.exact(self.repo_path, '', list(files)) + return self.repo.commit(match=match, text=text, user=user, + force=True) + + def save_data(self, title, data, author=u'', comment=u'', parent=None): + """Save data as specified page.""" + + try: + temp_path = tempfile.mkdtemp(dir=self.path) + file_path = os.path.join(temp_path, 'saved') + f = open(file_path, "wb") + f.write(data) + f.close() + self.save_file(title, file_path, author, comment, parent) + finally: + try: + os.unlink(file_path) + except OSError: + pass + try: + os.rmdir(temp_path) + except OSError: + pass + + def save_text(self, title, text, author=u'', comment=u'', parent=None): + """Save text as specified page, encoded to charset.""" + + data = text.encode(self.charset) + if self.unix_eol: + data = data.replace('\r\n', '\n') + self.save_data(title, data, author, comment, parent) + + def page_text(self, title): + """Read unicode text of a page.""" + + data = self.open_page(title).read() + text = unicode(data, self.charset, 'replace') + return text + + def page_lines(self, page): + for data in page.xreadlines(): + yield unicode(data, self.charset, 'replace') + + @locked_repo + def delete_page(self, title, author=u'', comment=u''): + user = author.encode('utf-8') or 'anon' + text = comment.encode('utf-8') or 'deleted' + repo_file = self._title_to_file(title) + file_path = self._file_path(title) + self._check_path(file_path) + try: + # Mercurial 1.5 and earlier have .remove() on the repo + remove = self.repo.remove + except AttributeError: + # Mercurial 1.6 + remove = self.repo[None].remove + try: + os.unlink(file_path) + except OSError: + pass + remove([repo_file]) + self._commit([repo_file], text, user) + + def open_page(self, title): + """Open the page and return a file-like object with its contents.""" + + file_path = self._file_path(title) + self._check_path(file_path) + try: + return open(file_path, "rb") + except IOError: + raise error.NotFoundErr() + + def page_file_meta(self, title): + """Get page's inode number, size and last modification time.""" + + try: + (st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, + st_atime, st_mtime, st_ctime) = os.stat(self._file_path(title)) + except OSError: + return 0, 0, 0 + return st_ino, st_size, st_mtime + + def page_meta(self, title): + """Get page's revision, date, last editor and his edit comment.""" + + filectx_tip = self._find_filectx(title) + if filectx_tip is None: + raise error.NotFoundErr() + #return -1, None, u'', u'' + rev = filectx_tip.filerev() + filectx = filectx_tip.filectx(rev) + date = datetime.datetime.fromtimestamp(filectx.date()[0]) + author = unicode(filectx.user(), "utf-8", + 'replace').split('<')[0].strip() + comment = unicode(filectx.description(), "utf-8", 'replace') + return rev, date, author, comment + + def repo_revision(self): + """Give the latest revision of the repository.""" + + return self._changectx().rev() + + def _changectx(self): + """Get the changectx of the tip.""" + + try: + # This is for Mercurial 1.0 + return self.repo.changectx() + except TypeError: + # Mercurial 1.3 (and possibly earlier) needs an argument + return self.repo.changectx('tip') + + def _find_filectx(self, title): + """Find the last revision in which the file existed.""" + + repo_file = self._title_to_file(title) + stack = [self._changectx()] + while stack: + changectx = stack.pop() + if repo_file in changectx: + return changectx[repo_file] + if changectx.rev() == 0: + return None + for parent in changectx.parents(): + if parent != changectx: + stack.append(parent) + return None + + def page_history(self, title): + """Iterate over the page's history.""" + + filectx_tip = self._find_filectx(title) + if filectx_tip is None: + return + maxrev = filectx_tip.filerev() + minrev = 0 + for rev in range(maxrev, minrev - 1, -1): + filectx = filectx_tip.filectx(rev) + date = datetime.datetime.fromtimestamp(filectx.date()[0]) + author = unicode(filectx.user(), "utf-8", + 'replace').split('<')[0].strip() + comment = unicode(filectx.description(), "utf-8", 'replace') + yield rev, date, author, comment + + def page_revision(self, title, rev): + """Get binary content of the specified revision of the page.""" + + filectx_tip = self._find_filectx(title) + if filectx_tip is None: + raise error.NotFoundErr() + try: + data = filectx_tip.filectx(rev).data() + except IndexError: + raise error.NotFoundErr() + return data + + def revision_text(self, title, rev): + """Get unicode text of the specified revision of the page.""" + + data = self.page_revision(title, rev) + text = unicode(data, self.charset, 'replace') + return text + + def history(self): + """Iterate over the history of entire wiki.""" + + changectx = self._changectx() + maxrev = changectx.rev() + minrev = 0 + for wiki_rev in range(maxrev, minrev - 1, -1): + change = self.repo.changectx(wiki_rev) + date = datetime.datetime.fromtimestamp(change.date()[0]) + author = unicode(change.user(), "utf-8", + 'replace').split('<')[0].strip() + comment = unicode(change.description(), "utf-8", 'replace') + for repo_file in change.files(): + if repo_file.startswith(self.repo_prefix): + title = self._file_to_title(repo_file) + try: + rev = change[repo_file].filerev() + except mercurial.revlog.LookupError: + rev = -1 + yield title, rev, date, author, comment + + def all_pages(self): + """Iterate over the titles of all pages in the wiki.""" + + for filename in os.listdir(self.path): + file_path = os.path.join(self.path, filename) + file_repopath = os.path.join(self.repo_prefix, filename) + if (os.path.isfile(file_path) + and not os.path.islink(file_path) + and not filename.startswith('.')): + yield self._file_to_title(file_repopath) + + def changed_since(self, rev): + """ + Return all pages that changed since specified repository revision. + """ + + try: + last = self.repo.lookup(int(rev)) + except IndexError: + for page in self.all_pages(): + yield page + return + current = self.repo.lookup('tip') + status = self.repo.status(current, last) + modified, added, removed, deleted, unknown, ignored, clean = status + for filename in modified + added + removed + deleted: + if filename.startswith(self.repo_prefix): + yield self._file_to_title(filename) + + +class WikiSubdirectoryStorage(WikiStorage): + """ + A version of WikiStorage that keeps the subpages in real subdirectories in + the filesystem. Indexes supported. + + """ + + periods_re = re.compile(r'^[.]|(?<=/)[.]') + slashes_re = re.compile(r'^[/]|(?<=/)[/]') + + # TODO: make them configurable + index = "Index" + + def _title_to_file(self, title): + """ + Modified escaping allowing (some) slashes and spaces. + If the entry is a directory, use an index file. + """ + + title = unicode(title).strip() + escaped = werkzeug.url_quote(title, safe='/ ') + escaped = self.periods_re.sub('%2E', escaped) + escaped = self.slashes_re.sub('%2F', escaped) + path = os.path.join(self.repo_prefix, escaped) + if os.path.isdir(os.path.join(self.repo_path, path)): + path = os.path.join(path, self.index) + if page.page_mime(title) == 'text/x-wiki' and self.extension: + path += self.extension + return path + + def _file_to_title(self, filepath): + """If the path points to an index file, use the directory.""" + + if os.path.basename(filepath) == self.index: + filepath = os.path.dirname(filepath) + return super(WikiSubdirectoryStorage, self)._file_to_title(filepath) + + def turn_into_subdirectory(self, path): + """Turn a single-file page into an index page inside a subdirectory.""" + + _ = self._ + self._check_path(path) + dir_path = os.path.dirname(path) + if not os.path.isdir(dir_path): + self.turn_into_subdirectory(dir_path) + if not os.path.exists(path): + os.mkdir(path) + return + try: + temp_dir = tempfile.mkdtemp(dir=self.path) + temp_path = os.path.join(temp_dir, 'saved') + mercurial.commands.rename(self.ui, self.repo, path, temp_path) + os.makedirs(path) + index_path = os.path.join(path, self.index) + mercurial.commands.rename(self.ui, self.repo, temp_path, + index_path) + finally: + try: + os.rmdir(temp_dir) + except OSError: + pass + def repo_path(path): + return path[len(self.repo_path)+1:] + files = [repo_path(index_path), repo_path(path)] + self._commit(files, _(u"made subdirectory page"), "<wiki>") + + @locked_repo + def save_file(self, title, file_name, author=u'', comment=u'', + parent=None): + """Save the file and make the subdirectories if needed.""" + + path = self._file_path(title) + dir_path = os.path.dirname(path) + if not os.path.isdir(dir_path): + self.turn_into_subdirectory(dir_path) + try: + os.makedirs(os.path.join(self.repo_path, dir_path)) + except OSError, e: + if e.errno != errno.EEXIST: + # "File exists" + raise + super(WikiSubdirectoryStorage, self).save_file(title, file_name, + author, comment, parent) + + @locked_repo + def delete_page(self, title, author=u'', comment=u''): + """ + Remove empty directories after deleting a page. + + Note that Mercurial doesn't track directories, so we don't have to + commit after removing empty directories. + """ + + super(WikiSubdirectoryStorage, self).delete_page(title, author, + comment) + file_path = self._file_path(title) + self._check_path(file_path) + dir_path = os.path.dirname(file_path) + if dir_path != self.repo_path: + try: + os.removedirs(dir_path) + except OSError, e: + if e.errno != errno.ENOTEMPTY: + # "Directory not empty" + raise + + def all_pages(self): + """ + Iterate over the titles of all pages in the wiki. + Include subdirectories but skip over index. + """ + + for (dirpath, dirnames, filenames) in os.walk(self.path): + path = dirpath[len(self.path) + 1:] + for name in filenames: + filepath = os.path.join(dirpath, name) + repopath = os.path.join(self.repo_prefix, path, name) + if (os.path.isfile(filepath) + and not name.startswith('.')): + yield self._file_to_title(repopath) |