path: root/buildbot/buildbot/changes/svnpoller.py

# -*- test-case-name: buildbot.test.test_svnpoller -*-

# Based on the work of Dave Peticolas for the P4poll
# Changed to svn (using xml.dom.minidom) by Niklaus Giger
# Hacked beyond recognition by Brian Warner

from twisted.python import log
from twisted.internet import defer, reactor, utils
from twisted.internet.task import LoopingCall

from buildbot import util
from buildbot.changes import base
from buildbot.changes.changes import Change

import xml.dom.minidom

def _assert(condition, msg):
    if condition:
        return True
    raise AssertionError(msg)

def dbgMsg(myString):
    log.msg(myString)
    return 1

# these split_file_* functions are available for use as values to the
# split_file= argument.
def split_file_alwaystrunk(path):
    return (None, path)

def split_file_branches(path):
    # turn trunk/subdir/file.c into (None, "subdir/file.c")
    # and branches/1.5.x/subdir/file.c into ("branches/1.5.x", "subdir/file.c")
    pieces = path.split('/')
    if pieces[0] == 'trunk':
        return (None, '/'.join(pieces[1:]))
    elif pieces[0] == 'branches':
        return ('/'.join(pieces[0:2]), '/'.join(pieces[2:]))
    else:
        return None


class SVNPoller(base.ChangeSource, util.ComparableMixin):
    """This source will poll a Subversion repository for changes and submit
    them to the change master."""

    compare_attrs = ["svnurl", "split_file_function",
                     "svnuser", "svnpasswd",
                     "pollinterval", "histmax",
                     "svnbin"]

    parent = None # filled in when we're added
    last_change = None
    loop = None
    working = False

    def __init__(self, svnurl, split_file=None,
                 svnuser=None, svnpasswd=None,
                 pollinterval=10*60, histmax=100,
                 svnbin='svn'):
        """
        @type  svnurl: string
        @param svnurl: the SVN URL that describes the repository and
                       subdirectory to watch. If this ChangeSource should
                       only pay attention to a single branch, this should
                       point at the repository for that branch, like
                       svn://svn.twistedmatrix.com/svn/Twisted/trunk . If it
                       should follow multiple branches, point it at the
                       repository directory that contains all the branches
                       like svn://svn.twistedmatrix.com/svn/Twisted and also
                       provide a branch-determining function.

                       Each file in the repository has a SVN URL in the form
                       (SVNURL)/(BRANCH)/(FILEPATH), where (BRANCH) could be
                       empty or not, depending upon your branch-determining
                       function. Only files that start with (SVNURL)/(BRANCH)
                       will be monitored. The Change objects that are sent to
                       the Schedulers will see (FILEPATH) for each modified
                       file.

        @type  split_file: callable or None
        @param split_file: a function that is called with a string of the
                           form (BRANCH)/(FILEPATH) and should return a tuple
                           (BRANCH, FILEPATH). This function should match
                           your repository's branch-naming policy. Each
                           changed file has a fully-qualified URL that can be
                           split into a prefix (which equals the value of the
                           'svnurl' argument) and a suffix; it is this suffix
                           which is passed to the split_file function.

                           If the function returns None, the file is ignored.
                           Use this to indicate that the file is not a part
                           of this project.
                           
                           For example, if your repository puts the trunk in
                           trunk/... and branches are in places like
                           branches/1.5/..., your split_file function could
                           look like the following (this function is
                           available as svnpoller.split_file_branches)::

                            pieces = path.split('/')
                            if pieces[0] == 'trunk':
                                return (None, '/'.join(pieces[1:]))
                            elif pieces[0] == 'branches':
                                return ('/'.join(pieces[0:2]),
                                        '/'.join(pieces[2:]))
                            else:
                                return None

                           If instead your repository layout puts the trunk
                           for ProjectA in trunk/ProjectA/... and the 1.5
                           branch in branches/1.5/ProjectA/..., your
                           split_file function could look like::

                            pieces = path.split('/')
                            if pieces[0] == 'trunk':
                                branch = None
                                pieces.pop(0) # remove 'trunk'
                            elif pieces[0] == 'branches':
                                pieces.pop(0) # remove 'branches'
                                # grab branch name
                                branch = 'branches/' + pieces.pop(0)
                            else:
                                return None # something weird
                            projectname = pieces.pop(0)
                            if projectname != 'ProjectA':
                                return None # wrong project
                            return (branch, '/'.join(pieces))

                           The default of split_file= is None, which
                           indicates that no splitting should be done. This
                           is equivalent to the following function::

                            return (None, path)

                           If you wish, you can override the split_file
                           method with the same sort of function instead of
                           passing in a split_file= argument.


        @type  svnuser:      string
        @param svnuser:      If set, the --username option will be added to
                             the 'svn log' command. You may need this to get
                             access to a private repository.
        @type  svnpasswd:    string
        @param svnpasswd:    If set, the --password option will be added.

        @type  pollinterval: int
        @param pollinterval: interval in seconds between polls. The default
                             is 600 seconds (10 minutes). Smaller values
                             decrease the latency between the time a change
                             is recorded and the time the buildbot notices
                             it, but it also increases the system load.

        @type  histmax:      int
        @param histmax:      maximum number of changes to look back through.
                             The default is 100. Smaller values decrease
                             system load, but if more than histmax changes
                             are recorded between polls, the extra ones will
                             be silently lost.

        @type  svnbin:       string
        @param svnbin:       path to svn binary, defaults to just 'svn'. Use
                             this if your subversion command lives in an
                             unusual location.
        """

        if svnurl.endswith("/"):
            svnurl = svnurl[:-1] # strip the trailing slash
        self.svnurl = svnurl
        self.split_file_function = split_file or split_file_alwaystrunk
        self.svnuser = svnuser
        self.svnpasswd = svnpasswd

        self.svnbin = svnbin
        self.pollinterval = pollinterval
        self.histmax = histmax
        self._prefix = None
        self.overrun_counter = 0
        self.loop = LoopingCall(self.checksvn)

    def split_file(self, path):
        # use getattr() to avoid turning this function into a bound method,
        # which would require it to have an extra 'self' argument
        f = getattr(self, "split_file_function")
        return f(path)

    def startService(self):
        log.msg("SVNPoller(%s) starting" % self.svnurl)
        base.ChangeSource.startService(self)
        # Don't start the loop just yet because the reactor isn't running.
        # Give it a chance to go and install our SIGCHLD handler before
        # spawning processes.
        reactor.callLater(0, self.loop.start, self.pollinterval)

    def stopService(self):
        log.msg("SVNPoller(%s) shutting down" % self.svnurl)
        self.loop.stop()
        return base.ChangeSource.stopService(self)

    def describe(self):
        return "SVNPoller watching %s" % self.svnurl

    def checksvn(self):
        # Our return value is only used for unit testing.

        # we need to figure out the repository root, so we can figure out
        # repository-relative pathnames later. Each SVNURL is in the form
        # (ROOT)/(PROJECT)/(BRANCH)/(FILEPATH), where (ROOT) is something
        # like svn://svn.twistedmatrix.com/svn/Twisted (i.e. there is a
        # physical repository at /svn/Twisted on that host), (PROJECT) is
        # something like Projects/Twisted (i.e. within the repository's
        # internal namespace, everything under Projects/Twisted/ has
        # something to do with Twisted, but these directory names do not
        # actually appear on the repository host), (BRANCH) is something like
        # "trunk" or "branches/2.0.x", and (FILEPATH) is a tree-relative
        # filename like "twisted/internet/defer.py".

        # our self.svnurl attribute contains (ROOT)/(PROJECT) combined
        # together in a way that we can't separate without svn's help. If the
        # user is not using the split_file= argument, then self.svnurl might
        # be (ROOT)/(PROJECT)/(BRANCH) . In any case, the filenames we will
        # get back from 'svn log' will be of the form
        # (PROJECT)/(BRANCH)/(FILEPATH), but we want to be able to remove
        # that (PROJECT) prefix from them. To do this without requiring the
        # user to tell us how svnurl is split into ROOT and PROJECT, we do an
        # 'svn info --xml' command at startup. This command will include a
        # <root> element that tells us ROOT. We then strip this prefix from
        # self.svnurl to determine PROJECT, and then later we strip the
        # PROJECT prefix from the filenames reported by 'svn log --xml' to
        # get a (BRANCH)/(FILEPATH) that can be passed to split_file() to
        # turn into separate BRANCH and FILEPATH values.

        # whew.

        if self.working:
            log.msg("SVNPoller(%s) overrun: timer fired but the previous "
                    "poll had not yet finished." % self.svnurl)
            self.overrun_counter += 1
            return defer.succeed(None)
        self.working = True

        log.msg("SVNPoller polling")
        if not self._prefix:
            # this sets self._prefix when it finishes. It fires with
            # self._prefix as well, because that makes the unit tests easier
            # to write.
            d = self.get_root()
            d.addCallback(self.determine_prefix)
        else:
            d = defer.succeed(self._prefix)

        d.addCallback(self.get_logs)
        d.addCallback(self.parse_logs)
        d.addCallback(self.get_new_logentries)
        d.addCallback(self.create_changes)
        d.addCallback(self.submit_changes)
        d.addCallbacks(self.finished_ok, self.finished_failure)
        return d

    def getProcessOutput(self, args):
        # this exists so we can override it during the unit tests
        d = utils.getProcessOutput(self.svnbin, args, {})
        return d

    def get_root(self):
        args = ["info", "--xml", "--non-interactive", self.svnurl]
        if self.svnuser:
            args.extend(["--username=%s" % self.svnuser])
        if self.svnpasswd:
            args.extend(["--password=%s" % self.svnpasswd])
        d = self.getProcessOutput(args)
        return d

    def determine_prefix(self, output):
        try:
            doc = xml.dom.minidom.parseString(output)
        except xml.parsers.expat.ExpatError:
            dbgMsg("_process_changes: ExpatError in %s" % output)
            log.msg("SVNPoller._determine_prefix_2: ExpatError in '%s'"
                    % output)
            raise
        rootnodes = doc.getElementsByTagName("root")
        if not rootnodes:
            # this happens if the URL we gave was already the root. In this
            # case, our prefix is empty.
            self._prefix = ""
            return self._prefix
        rootnode = rootnodes[0]
        root = "".join([c.data for c in rootnode.childNodes])
        # root will be a unicode string
        _assert(self.svnurl.startswith(root),
                "svnurl='%s' doesn't start with <root>='%s'" %
                (self.svnurl, root))
        self._prefix = self.svnurl[len(root):]
        if self._prefix.startswith("/"):
            self._prefix = self._prefix[1:]
        log.msg("SVNPoller: svnurl=%s, root=%s, so prefix=%s" %
                (self.svnurl, root, self._prefix))
        return self._prefix

    def get_logs(self, ignored_prefix=None):
        args = []
        args.extend(["log", "--xml", "--verbose", "--non-interactive"])
        if self.svnuser:
            args.extend(["--username=%s" % self.svnuser])
        if self.svnpasswd:
            args.extend(["--password=%s" % self.svnpasswd])
        args.extend(["--limit=%d" % (self.histmax), self.svnurl])
        d = self.getProcessOutput(args)
        return d

    def parse_logs(self, output):
        # parse the XML output, return a list of <logentry> nodes
        try:
            doc = xml.dom.minidom.parseString(output)
        except xml.parsers.expat.ExpatError:
            dbgMsg("_process_changes: ExpatError in %s" % output)
            log.msg("SVNPoller._parse_changes: ExpatError in '%s'" % output)
            raise
        logentries = doc.getElementsByTagName("logentry")
        return logentries


    def _filter_new_logentries(self, logentries, last_change):
        # given a list of logentries, return a tuple of (new_last_change,
        # new_logentries), where new_logentries contains only the ones after
        # last_change
        if not logentries:
            # no entries, so last_change must stay at None
            return (None, [])

        mostRecent = int(logentries[0].getAttribute("revision"))

        if last_change is None:
            # if this is the first time we've been run, ignore any changes
            # that occurred before now. This prevents a build at every
            # startup.
            log.msg('svnPoller: starting at change %s' % mostRecent)
            return (mostRecent, [])

        if last_change == mostRecent:
            # an unmodified repository will hit this case
            log.msg('svnPoller: _process_changes last %s mostRecent %s' % (
                      last_change, mostRecent))
            return (mostRecent, [])

        new_logentries = []
        for el in logentries:
            if last_change == int(el.getAttribute("revision")):
                break
            new_logentries.append(el)
        new_logentries.reverse() # return oldest first
        return (mostRecent, new_logentries)

    def get_new_logentries(self, logentries):
        last_change = self.last_change
        (new_last_change,
         new_logentries) = self._filter_new_logentries(logentries,
                                                       self.last_change)
        self.last_change = new_last_change
        log.msg('svnPoller: _process_changes %s .. %s' %
                (last_change, new_last_change))
        return new_logentries


    def _get_text(self, element, tag_name):
        try:
            child_nodes = element.getElementsByTagName(tag_name)[0].childNodes
            text = "".join([t.data for t in child_nodes])
        except:
            text = "<unknown>"
        return text

    def _transform_path(self, path):
        _assert(path.startswith(self._prefix),
                "filepath '%s' should start with prefix '%s'" %
                (path, self._prefix))
        relative_path = path[len(self._prefix):]
        if relative_path.startswith("/"):
            relative_path = relative_path[1:]
        where = self.split_file(relative_path)
        # 'where' is either None or (branch, final_path)
        return where

    def create_changes(self, new_logentries):
        changes = []

        for el in new_logentries:
            branch_files = [] # get oldest change first
            revision = str(el.getAttribute("revision"))
            dbgMsg("Adding change revision %s" % (revision,))
            # TODO: the rest of buildbot may not be ready for unicode 'who'
            # values
            author   = self._get_text(el, "author")
            comments = self._get_text(el, "msg")
            # there is a "date" field, but it provides localtime in the
            # repository's timezone, whereas we care about buildmaster's
            # localtime (since this will get used to position the boxes on
            # the Waterfall display, etc). So ignore the date field and use
            # our local clock instead.
            #when     = self._get_text(el, "date")
            #when     = time.mktime(time.strptime("%.19s" % when,
            #                                     "%Y-%m-%dT%H:%M:%S"))
            branches = {}
            pathlist = el.getElementsByTagName("paths")[0]
            for p in pathlist.getElementsByTagName("path"):
                action = p.getAttribute("action")
                path = "".join([t.data for t in p.childNodes])
                # the rest of buildbot is certaily not yet ready to handle
                # unicode filenames, because they get put in RemoteCommands
                # which get sent via PB to the buildslave, and PB doesn't
                # handle unicode.
                path = path.encode("ascii")
                if path.startswith("/"):
                    path = path[1:]
                where = self._transform_path(path)

                # if 'where' is None, the file was outside any project that
                # we care about and we should ignore it
                if where:
                    branch, filename = where
                    if not branch in branches:
                        branches[branch] = { 'files': []}
                    branches[branch]['files'].append(filename)

                    if not branches[branch].has_key('action'):
                        branches[branch]['action'] = action

            for branch in branches.keys():
                action = branches[branch]['action']
                files  = branches[branch]['files']
                number_of_files_changed = len(files)

                if action == u'D' and number_of_files_changed == 1 and files[0] == '':
                    log.msg("Ignoring deletion of branch '%s'" % branch)
                else:
                    c = Change(who=author,
                               files=files,
                               comments=comments,
                               revision=revision,
                               branch=branch)
                    changes.append(c)

        return changes

    def submit_changes(self, changes):
        for c in changes:
            self.parent.addChange(c)

    def finished_ok(self, res):
        log.msg("SVNPoller finished polling")
        dbgMsg('_finished : %s' % res)
        assert self.working
        self.working = False
        return res

    def finished_failure(self, f):
        log.msg("SVNPoller failed")
        dbgMsg('_finished : %s' % f)
        assert self.working
        self.working = False
        return None # eat the failure