diff options
Diffstat (limited to 'buildbot/buildbot/steps/shell.py')
| -rw-r--r-- | buildbot/buildbot/steps/shell.py | 487 |
1 files changed, 0 insertions, 487 deletions
diff --git a/buildbot/buildbot/steps/shell.py b/buildbot/buildbot/steps/shell.py deleted file mode 100644 index e979f04..0000000 --- a/buildbot/buildbot/steps/shell.py +++ /dev/null @@ -1,487 +0,0 @@ -# -*- test-case-name: buildbot.test.test_steps,buildbot.test.test_properties -*- - -import re -from twisted.python import log -from buildbot.process.buildstep import LoggingBuildStep, RemoteShellCommand -from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, STDOUT, STDERR - -# for existing configurations that import WithProperties from here. We like -# to move this class around just to keep our readers guessing. -from buildbot.process.properties import WithProperties -_hush_pyflakes = [WithProperties] -del _hush_pyflakes - -class ShellCommand(LoggingBuildStep): - """I run a single shell command on the buildslave. I return FAILURE if - the exit code of that command is non-zero, SUCCESS otherwise. To change - this behavior, override my .evaluateCommand method. - - By default, a failure of this step will mark the whole build as FAILURE. - To override this, give me an argument of flunkOnFailure=False . - - I create a single Log named 'log' which contains the output of the - command. To create additional summary Logs, override my .createSummary - method. - - The shell command I run (a list of argv strings) can be provided in - several ways: - - a class-level .command attribute - - a command= parameter to my constructor (overrides .command) - - set explicitly with my .setCommand() method (overrides both) - - @ivar command: a list of renderable objects (typically strings or - WithProperties instances). This will be used by start() - to create a RemoteShellCommand instance. - - @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs - of their corresponding logfiles. The contents of the file - named FILENAME will be put into a LogFile named NAME, ina - something approximating real-time. (note that logfiles= - is actually handled by our parent class LoggingBuildStep) - - """ - - name = "shell" - description = None # set this to a list of short strings to override - descriptionDone = None # alternate description when the step is complete - command = None # set this to a command, or set in kwargs - # logfiles={} # you can also set 'logfiles' to a dictionary, and it - # will be merged with any logfiles= argument passed in - # to __init__ - - # override this on a specific ShellCommand if you want to let it fail - # without dooming the entire build to a status of FAILURE - flunkOnFailure = True - - def __init__(self, workdir=None, - description=None, descriptionDone=None, - command=None, - usePTY="slave-config", - **kwargs): - # most of our arguments get passed through to the RemoteShellCommand - # that we create, but first strip out the ones that we pass to - # BuildStep (like haltOnFailure and friends), and a couple that we - # consume ourselves. - - if description: - self.description = description - if isinstance(self.description, str): - self.description = [self.description] - if descriptionDone: - self.descriptionDone = descriptionDone - if isinstance(self.descriptionDone, str): - self.descriptionDone = [self.descriptionDone] - if command: - self.setCommand(command) - - # pull out the ones that LoggingBuildStep wants, then upcall - buildstep_kwargs = {} - for k in kwargs.keys()[:]: - if k in self.__class__.parms: - buildstep_kwargs[k] = kwargs[k] - del kwargs[k] - LoggingBuildStep.__init__(self, **buildstep_kwargs) - self.addFactoryArguments(workdir=workdir, - description=description, - descriptionDone=descriptionDone, - command=command) - - # everything left over goes to the RemoteShellCommand - kwargs['workdir'] = workdir # including a copy of 'workdir' - kwargs['usePTY'] = usePTY - self.remote_kwargs = kwargs - # we need to stash the RemoteShellCommand's args too - self.addFactoryArguments(**kwargs) - - def setDefaultWorkdir(self, workdir): - rkw = self.remote_kwargs - rkw['workdir'] = rkw['workdir'] or workdir - - def setCommand(self, command): - self.command = command - - def describe(self, done=False): - """Return a list of short strings to describe this step, for the - status display. This uses the first few words of the shell command. - You can replace this by setting .description in your subclass, or by - overriding this method to describe the step better. - - @type done: boolean - @param done: whether the command is complete or not, to improve the - way the command is described. C{done=False} is used - while the command is still running, so a single - imperfect-tense verb is appropriate ('compiling', - 'testing', ...) C{done=True} is used when the command - has finished, and the default getText() method adds some - text, so a simple noun is appropriate ('compile', - 'tests' ...) - """ - - if done and self.descriptionDone is not None: - return list(self.descriptionDone) - if self.description is not None: - return list(self.description) - - properties = self.build.getProperties() - words = self.command - if isinstance(words, (str, unicode)): - words = words.split() - # render() each word to handle WithProperties objects - words = properties.render(words) - if len(words) < 1: - return ["???"] - if len(words) == 1: - return ["'%s'" % words[0]] - if len(words) == 2: - return ["'%s" % words[0], "%s'" % words[1]] - return ["'%s" % words[0], "%s" % words[1], "...'"] - - def setupEnvironment(self, cmd): - # merge in anything from Build.slaveEnvironment - # This can be set from a Builder-level environment, or from earlier - # BuildSteps. The latter method is deprecated and superceded by - # BuildProperties. - # Environment variables passed in by a BuildStep override - # those passed in at the Builder level. - properties = self.build.getProperties() - slaveEnv = self.build.slaveEnvironment - if slaveEnv: - if cmd.args['env'] is None: - cmd.args['env'] = {} - fullSlaveEnv = slaveEnv.copy() - fullSlaveEnv.update(cmd.args['env']) - cmd.args['env'] = properties.render(fullSlaveEnv) - # note that each RemoteShellCommand gets its own copy of the - # dictionary, so we shouldn't be affecting anyone but ourselves. - - def checkForOldSlaveAndLogfiles(self): - if not self.logfiles: - return # doesn't matter - if not self.slaveVersionIsOlderThan("shell", "2.1"): - return # slave is new enough - # this buildslave is too old and will ignore the 'logfiles' - # argument. You'll either have to pull the logfiles manually - # (say, by using 'cat' in a separate RemoteShellCommand) or - # upgrade the buildslave. - msg1 = ("Warning: buildslave %s is too old " - "to understand logfiles=, ignoring it." - % self.getSlaveName()) - msg2 = "You will have to pull this logfile (%s) manually." - log.msg(msg1) - for logname,remotefilename in self.logfiles.items(): - newlog = self.addLog(logname) - newlog.addHeader(msg1 + "\n") - newlog.addHeader(msg2 % remotefilename + "\n") - newlog.finish() - # now prevent setupLogfiles() from adding them - self.logfiles = {} - - def start(self): - # this block is specific to ShellCommands. subclasses that don't need - # to set up an argv array, an environment, or extra logfiles= (like - # the Source subclasses) can just skip straight to startCommand() - properties = self.build.getProperties() - - warnings = [] - - # create the actual RemoteShellCommand instance now - kwargs = properties.render(self.remote_kwargs) - kwargs['command'] = properties.render(self.command) - kwargs['logfiles'] = self.logfiles - - # check for the usePTY flag - if kwargs.has_key('usePTY') and kwargs['usePTY'] != 'slave-config': - slavever = self.slaveVersion("shell", "old") - if self.slaveVersionIsOlderThan("svn", "2.7"): - warnings.append("NOTE: slave does not allow master to override usePTY\n") - - cmd = RemoteShellCommand(**kwargs) - self.setupEnvironment(cmd) - self.checkForOldSlaveAndLogfiles() - - self.startCommand(cmd, warnings) - - - -class TreeSize(ShellCommand): - name = "treesize" - command = ["du", "-s", "-k", "."] - kib = None - - def commandComplete(self, cmd): - out = cmd.logs['stdio'].getText() - m = re.search(r'^(\d+)', out) - if m: - self.kib = int(m.group(1)) - self.setProperty("tree-size-KiB", self.kib, "treesize") - - def evaluateCommand(self, cmd): - if cmd.rc != 0: - return FAILURE - if self.kib is None: - return WARNINGS # not sure how 'du' could fail, but whatever - return SUCCESS - - def getText(self, cmd, results): - if self.kib is not None: - return ["treesize", "%d KiB" % self.kib] - return ["treesize", "unknown"] - -class SetProperty(ShellCommand): - name = "setproperty" - - def __init__(self, **kwargs): - self.property = None - self.extract_fn = None - self.strip = True - - if kwargs.has_key('property'): - self.property = kwargs['property'] - del kwargs['property'] - if kwargs.has_key('extract_fn'): - self.extract_fn = kwargs['extract_fn'] - del kwargs['extract_fn'] - if kwargs.has_key('strip'): - self.strip = kwargs['strip'] - del kwargs['strip'] - - ShellCommand.__init__(self, **kwargs) - - self.addFactoryArguments(property=self.property) - self.addFactoryArguments(extract_fn=self.extract_fn) - self.addFactoryArguments(strip=self.strip) - - assert self.property or self.extract_fn, \ - "SetProperty step needs either property= or extract_fn=" - - self.property_changes = {} - - def commandComplete(self, cmd): - if self.property: - result = cmd.logs['stdio'].getText() - if self.strip: result = result.strip() - propname = self.build.getProperties().render(self.property) - self.setProperty(propname, result, "SetProperty Step") - self.property_changes[propname] = result - else: - log = cmd.logs['stdio'] - new_props = self.extract_fn(cmd.rc, - ''.join(log.getChunks([STDOUT], onlyText=True)), - ''.join(log.getChunks([STDERR], onlyText=True))) - for k,v in new_props.items(): - self.setProperty(k, v, "SetProperty Step") - self.property_changes = new_props - - def createSummary(self, log): - props_set = [ "%s: %r" % (k,v) for k,v in self.property_changes.items() ] - self.addCompleteLog('property changes', "\n".join(props_set)) - - def getText(self, cmd, results): - if self.property_changes: - return [ "set props:" ] + self.property_changes.keys() - else: - return [ "no change" ] - -class Configure(ShellCommand): - - name = "configure" - haltOnFailure = 1 - flunkOnFailure = 1 - description = ["configuring"] - descriptionDone = ["configure"] - command = ["./configure"] - -class WarningCountingShellCommand(ShellCommand): - warnCount = 0 - warningPattern = '.*warning[: ].*' - - def __init__(self, warningPattern=None, **kwargs): - # See if we've been given a regular expression to use to match - # warnings. If not, use a default that assumes any line with "warning" - # present is a warning. This may lead to false positives in some cases. - if warningPattern: - self.warningPattern = warningPattern - - # And upcall to let the base class do its work - ShellCommand.__init__(self, **kwargs) - - self.addFactoryArguments(warningPattern=warningPattern) - - def createSummary(self, log): - self.warnCount = 0 - - # Now compile a regular expression from whichever warning pattern we're - # using - if not self.warningPattern: - return - - wre = self.warningPattern - if isinstance(wre, str): - wre = re.compile(wre) - - # Check if each line in the output from this command matched our - # warnings regular expressions. If did, bump the warnings count and - # add the line to the collection of lines with warnings - warnings = [] - # TODO: use log.readlines(), except we need to decide about stdout vs - # stderr - for line in log.getText().split("\n"): - if wre.match(line): - warnings.append(line) - self.warnCount += 1 - - # If there were any warnings, make the log if lines with warnings - # available - if self.warnCount: - self.addCompleteLog("warnings", "\n".join(warnings) + "\n") - - warnings_stat = self.step_status.getStatistic('warnings', 0) - self.step_status.setStatistic('warnings', warnings_stat + self.warnCount) - - try: - old_count = self.getProperty("warnings-count") - except KeyError: - old_count = 0 - self.setProperty("warnings-count", old_count + self.warnCount, "WarningCountingShellCommand") - - - def evaluateCommand(self, cmd): - if cmd.rc != 0: - return FAILURE - if self.warnCount: - return WARNINGS - return SUCCESS - - -class Compile(WarningCountingShellCommand): - - name = "compile" - haltOnFailure = 1 - flunkOnFailure = 1 - description = ["compiling"] - descriptionDone = ["compile"] - command = ["make", "all"] - - OFFprogressMetrics = ('output',) - # things to track: number of files compiled, number of directories - # traversed (assuming 'make' is being used) - - def createSummary(self, log): - # TODO: grep for the characteristic GCC error lines and - # assemble them into a pair of buffers - WarningCountingShellCommand.createSummary(self, log) - -class Test(WarningCountingShellCommand): - - name = "test" - warnOnFailure = 1 - description = ["testing"] - descriptionDone = ["test"] - command = ["make", "test"] - - def setTestResults(self, total=0, failed=0, passed=0, warnings=0): - """ - Called by subclasses to set the relevant statistics; this actually - adds to any statistics already present - """ - total += self.step_status.getStatistic('tests-total', 0) - self.step_status.setStatistic('tests-total', total) - failed += self.step_status.getStatistic('tests-failed', 0) - self.step_status.setStatistic('tests-failed', failed) - warnings += self.step_status.getStatistic('tests-warnings', 0) - self.step_status.setStatistic('tests-warnings', warnings) - passed += self.step_status.getStatistic('tests-passed', 0) - self.step_status.setStatistic('tests-passed', passed) - - def describe(self, done=False): - description = WarningCountingShellCommand.describe(self, done) - if done: - if self.step_status.hasStatistic('tests-total'): - total = self.step_status.getStatistic("tests-total", 0) - failed = self.step_status.getStatistic("tests-failed", 0) - passed = self.step_status.getStatistic("tests-passed", 0) - warnings = self.step_status.getStatistic("tests-warnings", 0) - if not total: - total = failed + passed + warnings - - if total: - description.append('%d tests' % total) - if passed: - description.append('%d passed' % passed) - if warnings: - description.append('%d warnings' % warnings) - if failed: - description.append('%d failed' % failed) - return description - -class PerlModuleTest(Test): - command=["prove", "--lib", "lib", "-r", "t"] - total = 0 - - def evaluateCommand(self, cmd): - # Get stdio, stripping pesky newlines etc. - lines = map( - lambda line : line.replace('\r\n','').replace('\r','').replace('\n',''), - self.getLog('stdio').readlines() - ) - - total = 0 - passed = 0 - failed = 0 - rc = cmd.rc - - # New version of Test::Harness? - try: - test_summary_report_index = lines.index("Test Summary Report") - - del lines[0:test_summary_report_index + 2] - - re_test_result = re.compile("^Result: (PASS|FAIL)$|Tests: \d+ Failed: (\d+)\)|Files=\d+, Tests=(\d+)") - - mos = map(lambda line: re_test_result.search(line), lines) - test_result_lines = [mo.groups() for mo in mos if mo] - - for line in test_result_lines: - if line[0] == 'PASS': - rc = SUCCESS - elif line[0] == 'FAIL': - rc = FAILURE - elif line[1]: - failed += int(line[1]) - elif line[2]: - total = int(line[2]) - - except ValueError: # Nope, it's the old version - re_test_result = re.compile("^(All tests successful)|(\d+)/(\d+) subtests failed|Files=\d+, Tests=(\d+),") - - mos = map(lambda line: re_test_result.search(line), lines) - test_result_lines = [mo.groups() for mo in mos if mo] - - if test_result_lines: - test_result_line = test_result_lines[0] - - success = test_result_line[0] - - if success: - failed = 0 - - test_totals_line = test_result_lines[1] - total_str = test_totals_line[3] - - rc = SUCCESS - else: - failed_str = test_result_line[1] - failed = int(failed_str) - - total_str = test_result_line[2] - - rc = FAILURE - - total = int(total_str) - - if total: - passed = total - failed - - self.setTestResults(total=total, failed=failed, passed=passed) - - return rc |