diff options
Diffstat (limited to 'buildbot/buildbot/changes/bonsaipoller.py')
| -rw-r--r-- | buildbot/buildbot/changes/bonsaipoller.py | 320 |
1 files changed, 320 insertions, 0 deletions
diff --git a/buildbot/buildbot/changes/bonsaipoller.py b/buildbot/buildbot/changes/bonsaipoller.py new file mode 100644 index 0000000..2e319bb --- /dev/null +++ b/buildbot/buildbot/changes/bonsaipoller.py @@ -0,0 +1,320 @@ +import time +from xml.dom import minidom + +from twisted.python import log, failure +from twisted.internet import reactor +from twisted.internet.task import LoopingCall +from twisted.web.client import getPage + +from buildbot.changes import base, changes + +class InvalidResultError(Exception): + def __init__(self, value="InvalidResultError"): + self.value = value + def __str__(self): + return repr(self.value) + +class EmptyResult(Exception): + pass + +class NoMoreCiNodes(Exception): + pass + +class NoMoreFileNodes(Exception): + pass + +class BonsaiResult: + """I hold a list of CiNodes""" + def __init__(self, nodes=[]): + self.nodes = nodes + + def __cmp__(self, other): + if len(self.nodes) != len(other.nodes): + return False + for i in range(len(self.nodes)): + if self.nodes[i].log != other.nodes[i].log \ + or self.nodes[i].who != other.nodes[i].who \ + or self.nodes[i].date != other.nodes[i].date \ + or len(self.nodes[i].files) != len(other.nodes[i].files): + return -1 + + for j in range(len(self.nodes[i].files)): + if self.nodes[i].files[j].revision \ + != other.nodes[i].files[j].revision \ + or self.nodes[i].files[j].filename \ + != other.nodes[i].files[j].filename: + return -1 + + return 0 + +class CiNode: + """I hold information baout one <ci> node, including a list of files""" + def __init__(self, log="", who="", date=0, files=[]): + self.log = log + self.who = who + self.date = date + self.files = files + +class FileNode: + """I hold information about one <f> node""" + def __init__(self, revision="", filename=""): + self.revision = revision + self.filename = filename + +class BonsaiParser: + """I parse the XML result from a bonsai cvsquery.""" + + def __init__(self, data): + try: + # this is a fix for non-ascii characters + # because bonsai does not give us an encoding to work with + # it impossible to be 100% sure what to decode it as but latin1 covers + # the broadest base + data = data.decode("latin1") + data = data.encode("ascii", "replace") + self.dom = minidom.parseString(data) + log.msg(data) + except: + raise InvalidResultError("Malformed XML in result") + + self.ciNodes = self.dom.getElementsByTagName("ci") + self.currentCiNode = None # filled in by _nextCiNode() + self.fileNodes = None # filled in by _nextCiNode() + self.currentFileNode = None # filled in by _nextFileNode() + self.bonsaiResult = self._parseData() + + def getData(self): + return self.bonsaiResult + + def _parseData(self): + """Returns data from a Bonsai cvsquery in a BonsaiResult object""" + nodes = [] + try: + while self._nextCiNode(): + files = [] + try: + while self._nextFileNode(): + files.append(FileNode(self._getRevision(), + self._getFilename())) + except NoMoreFileNodes: + pass + except InvalidResultError: + raise + cinode = CiNode(self._getLog(), self._getWho(), + self._getDate(), files) + # hack around bonsai xml output bug for empty check-in comments + if not cinode.log and nodes and \ + not nodes[-1].log and \ + cinode.who == nodes[-1].who and \ + cinode.date == nodes[-1].date: + nodes[-1].files += cinode.files + else: + nodes.append(cinode) + + except NoMoreCiNodes: + pass + except InvalidResultError, EmptyResult: + raise + + return BonsaiResult(nodes) + + + def _nextCiNode(self): + """Iterates to the next <ci> node and fills self.fileNodes with + child <f> nodes""" + try: + self.currentCiNode = self.ciNodes.pop(0) + if len(self.currentCiNode.getElementsByTagName("files")) > 1: + raise InvalidResultError("Multiple <files> for one <ci>") + + self.fileNodes = self.currentCiNode.getElementsByTagName("f") + except IndexError: + # if there was zero <ci> nodes in the result + if not self.currentCiNode: + raise EmptyResult + else: + raise NoMoreCiNodes + + return True + + def _nextFileNode(self): + """Iterates to the next <f> node""" + try: + self.currentFileNode = self.fileNodes.pop(0) + except IndexError: + raise NoMoreFileNodes + + return True + + def _getLog(self): + """Returns the log of the current <ci> node""" + logs = self.currentCiNode.getElementsByTagName("log") + if len(logs) < 1: + raise InvalidResultError("No log present") + elif len(logs) > 1: + raise InvalidResultError("Multiple logs present") + + # catch empty check-in comments + if logs[0].firstChild: + return logs[0].firstChild.data + return '' + + def _getWho(self): + """Returns the e-mail address of the commiter""" + # convert unicode string to regular string + return str(self.currentCiNode.getAttribute("who")) + + def _getDate(self): + """Returns the date (unix time) of the commit""" + # convert unicode number to regular one + try: + commitDate = int(self.currentCiNode.getAttribute("date")) + except ValueError: + raise InvalidResultError + + return commitDate + + def _getFilename(self): + """Returns the filename of the current <f> node""" + try: + filename = self.currentFileNode.firstChild.data + except AttributeError: + raise InvalidResultError("Missing filename") + + return filename + + def _getRevision(self): + return self.currentFileNode.getAttribute("rev") + + +class BonsaiPoller(base.ChangeSource): + """This source will poll a bonsai server for changes and submit + them to the change master.""" + + compare_attrs = ["bonsaiURL", "pollInterval", "tree", + "module", "branch", "cvsroot"] + + parent = None # filled in when we're added + loop = None + volatile = ['loop'] + working = False + + def __init__(self, bonsaiURL, module, branch, tree="default", + cvsroot="/cvsroot", pollInterval=30): + """ + @type bonsaiURL: string + @param bonsaiURL: The base URL of the Bonsai server + (ie. http://bonsai.mozilla.org) + @type module: string + @param module: The module to look for changes in. Commonly + this is 'all' + @type branch: string + @param branch: The branch to look for changes in. This must + match the + 'branch' option for the Scheduler. + @type tree: string + @param tree: The tree to look for changes in. Commonly this + is 'all' + @type cvsroot: string + @param cvsroot: The cvsroot of the repository. Usually this is + '/cvsroot' + @type pollInterval: int + @param pollInterval: The time (in seconds) between queries for + changes + """ + + self.bonsaiURL = bonsaiURL + self.module = module + self.branch = branch + self.tree = tree + self.cvsroot = cvsroot + self.pollInterval = pollInterval + self.lastChange = time.time() + self.lastPoll = time.time() + + def startService(self): + self.loop = LoopingCall(self.poll) + base.ChangeSource.startService(self) + + reactor.callLater(0, self.loop.start, self.pollInterval) + + def stopService(self): + self.loop.stop() + return base.ChangeSource.stopService(self) + + def describe(self): + str = "" + str += "Getting changes from the Bonsai service running at %s " \ + % self.bonsaiURL + str += "<br>Using tree: %s, branch: %s, and module: %s" % (self.tree, \ + self.branch, self.module) + return str + + def poll(self): + if self.working: + log.msg("Not polling Bonsai because last poll is still working") + else: + self.working = True + d = self._get_changes() + d.addCallback(self._process_changes) + d.addCallbacks(self._finished_ok, self._finished_failure) + return + + def _finished_ok(self, res): + assert self.working + self.working = False + + # check for failure -- this is probably never hit but the twisted docs + # are not clear enough to be sure. it is being kept "just in case" + if isinstance(res, failure.Failure): + log.msg("Bonsai poll failed: %s" % res) + return res + + def _finished_failure(self, res): + log.msg("Bonsai poll failed: %s" % res) + assert self.working + self.working = False + return None # eat the failure + + def _make_url(self): + args = ["treeid=%s" % self.tree, "module=%s" % self.module, + "branch=%s" % self.branch, "branchtype=match", + "sortby=Date", "date=explicit", + "mindate=%d" % self.lastChange, + "maxdate=%d" % int(time.time()), + "cvsroot=%s" % self.cvsroot, "xml=1"] + # build the bonsai URL + url = self.bonsaiURL + url += "/cvsquery.cgi?" + url += "&".join(args) + + return url + + def _get_changes(self): + url = self._make_url() + log.msg("Polling Bonsai tree at %s" % url) + + self.lastPoll = time.time() + # get the page, in XML format + return getPage(url, timeout=self.pollInterval) + + def _process_changes(self, query): + try: + bp = BonsaiParser(query) + result = bp.getData() + except InvalidResultError, e: + log.msg("Could not process Bonsai query: " + e.value) + return + except EmptyResult: + return + + for cinode in result.nodes: + files = [file.filename + ' (revision '+file.revision+')' + for file in cinode.files] + c = changes.Change(who = cinode.who, + files = files, + comments = cinode.log, + when = cinode.date, + branch = self.branch) + self.parent.addChange(c) + self.lastChange = self.lastPoll |