Index: README =================================================================== RCS file: /space/cvs/marcuscom-cvs/portstools/tinderbox/README,v retrieving revision 1.97 diff -u -r1.97 README --- README 20 Jun 2012 20:47:42 -0000 1.97 +++ README 15 Jul 2012 18:44:42 -0000 @@ -456,7 +456,7 @@ -noduds packages which are forbidden or ignored will be built) -noclean does not clean up the Build hierarchy after the - port build completes + port build completes successfull -plistcheck makes any plist verification problems (e.g. leftover files) fatal -cleandistfiles removes all files and directories in the distfile @@ -468,6 +468,9 @@ Phase 1) -norebuild do not force a rebuild of packages specified on the command line + -uets As soon a portbuild has successfully finished or + the port has not to rebuild this option updates + all dependencies from status ENQUEUED to SUCCESS -------------------------------------------------------------- Index: lib/portbuild =================================================================== RCS file: /space/cvs/marcuscom-cvs/portstools/tinderbox/lib/portbuild,v retrieving revision 1.143 diff -u -r1.143 portbuild --- lib/portbuild 11 Jun 2012 08:04:17 -0000 1.143 +++ lib/portbuild 15 Jul 2012 18:44:42 -0000 @@ -5,7 +5,7 @@ . ${pb}/scripts/lib/tinderlib.sh usage () { - echo "usage: $0 [-noclean] [-plistcheck] [-nullfs] [-fetch-original] [-nolog] ED PD FD BD RD TD PKGD pkgname dirname" + echo "usage: $0 [-noclean] [-plistcheck] [-nullfs] [-fetch-original] [-nolog] [-uets] ED PD FD BD RD TD PKGD pkgname dirname" exit 1 } @@ -80,6 +80,7 @@ logdir="" docopy=0 compress_logs=0 +uets=0 # check parameter count if [ $# -lt 10 ]; then @@ -127,6 +128,9 @@ x-compress-logs) compress_logs=1 shift;; + x-uets) + uets=1 + shift;; x-*) echo "portbuild: unknown argument: $1" exit 1;; @@ -460,6 +464,28 @@ if [ ${status} = "LEFTOVERS" ]; then ${tc} sendBuildErrorMail -d ${portdir} -b ${build} -p ${pkgname} -x ${lext} -l fi + if [ ${status} = "SUCCESS" -a ${uets} -eq 1 ]; then + uets_debug=0 + + # generate a list of all ports in this build with status ENQUEUED + ${tc} listBuildPortsQueue -b ${build} -r -s ENQUEUED | cut -d\: -f 4 | sort -u > ${chroot}/tmp/_ENQUEUED 2>&1 + + if [ -s ${chroot}/tmp/_ENQUEUED ]; then + # build a list of all ports this ports depends on + chroot ${chroot} /uets ${dirname} /tmp/_DEPENDS ${uets_debug} 2>&1 + # make sure this files exists + touch ${chroot}/tmp/_ENQUEUED ${chroot}/tmp/_DEPENDS + # generate list of ports we can set to success + comm -12 ${chroot}/tmp/_ENQUEUED ${chroot}/tmp/_DEPENDS > ${chroot}/tmp/_FDEPS + + # set status of all dependencies to SUCCESS + if [ -s ${chroot}/tmp/_FDEPS ]; then + while read DEPEND; do + ${tc} updateEnqueuedToSuccess -b ${build} -d ${DEPEND} + done < ${chroot}/tmp/_FDEPS + fi + fi + fi else ln -f ${logsDir}/${pkgname}.log${lext} ${errorsDir}/${pkgname}.log${lext} reason="" Index: lib/tc_command.pl =================================================================== RCS file: /space/cvs/marcuscom-cvs/portstools/tinderbox/lib/tc_command.pl,v retrieving revision 1.181 diff -u -r1.181 tc_command.pl --- lib/tc_command.pl 4 Mar 2012 12:53:08 -0000 1.181 +++ lib/tc_command.pl 15 Jul 2012 18:44:42 -0000 @@ -423,6 +423,13 @@ usage => "-i id -s ", optstr => 'i:s:', }, + "updateEnqueuedToSuccess" => { + func => \&updateEnqueuedToSuccess, + help => + "Update the current status for all ports in specific build from ENQUEUED to SUCCESS", + usage => "-d -b ", + optstr => 'd:b:', + }, "getPortLastBuiltVersion" => { func => \&getPortLastBuiltVersion, help => @@ -606,7 +613,7 @@ "tinderbuild" => { help => "Generate packages from an installed Build", usage => - "-b [-init] [-cleanpackages] [-updateports] [-skipmake] [-noclean] [-noduds] [-plistcheck] [-nullfs] [-cleandistfiles] [-fetch-original] [-onceonly] [portdir/portname [...]]", + "-b [-init] [-cleanpackages] [-updateports] [-skipmake] [-noclean] [-noduds] [-plistcheck] [-nullfs] [-cleandistfiles] [-fetch-original] [-onceonly] [-uets] [portdir/portname [...]]", optstr => 'b:', }, @@ -2120,6 +2127,27 @@ } } +# uets +sub updateEnqueuedToSuccess { + if (!$opts->{'b'} || !$opts->{'d'}) { + usage("updateEnqueuedToSuccess"); + } + + if (!$ds->isValidBuild($opts->{'b'})) { + cleanup($ds, 1, "Unknown build, " . $opts->{'b'} . "\n"); + } + + my $build = $ds->getBuildByName($opts->{'b'}); + my $rc = $ds->updateEnqueuedToSuccess($build, $opts->{'d'}); + + if (!$rc) { + cleanup($ds, 1, "Failed to update EnqueuedToSuccess " + . $opts->{'b'} . ": " . $opts->{'d'} . ": " + . $ds->getError() + . "\n"); + } +} + sub rmBuildPortsQueue { my $rc = $ds->removeBuildPortsQueue(); Index: lib/tc_command.sh =================================================================== RCS file: /space/cvs/marcuscom-cvs/portstools/tinderbox/lib/tc_command.sh,v retrieving revision 1.155 diff -u -r1.155 tc_command.sh --- lib/tc_command.sh 20 Jun 2012 20:47:42 -0000 1.155 +++ lib/tc_command.sh 15 Jul 2012 18:44:43 -0000 @@ -1602,6 +1602,10 @@ tinderbuild_cleanup 1 fi + if ! cp -p $(tinderLoc scripts lib/uets) ${buildRoot}; then + echo "tinderbuild: ${build}: cannot copy uets" + tinderbuild_cleanup 1 + fi if ! cc -o ${buildRoot}/pnohang -static \ $(tinderLoc scripts lib/pnohang.c); then echo "tinderbuild: ${build}: cannot compile pnohang" @@ -1695,6 +1699,41 @@ fi fi +# If we are in phase 0 and there is nothing to build then we can assume +# the port and all dependencies are up to date. +# Conclusion: If the port has dependencies we update there status +# from ENQUEUED to SUCCESS + if [ ${uets} -eq 1 -a ${remake_cnt} -eq 0 -a ${num} -eq 0 ]; then + ENTRY=$(${tc} listBuildPortsQueue -s PROCESSING -r | head -1) + BUILD=$(echo ${ENTRY} | cut -d: -f3) + PORT=$(echo ${ENTRY} | cut -d: -f4) + + uets_debug=0 + + # generate a list of all ports in this build with status ENQUEUED + ${tc} listBuildPortsQueue -b ${BUILD} -r -s ENQUEUED | cut -d\: -f 4 | sort -u > ${buildRoot}/tmp/_ENQUEUED 2>&1 + + if [ -s ${buildRoot}/tmp/_ENQUEUED ]; then + echo 0 > ${buildRoot}/tmp/status + # build a list of all ports this ports depends on + chroot ${buildRoot} /uets /usr/ports/${PORT} /tmp/_DEPENDS ${uets_debug} 2>&1 + # make sure this files exists + touch ${buildRoot}/tmp/_ENQUEUED ${buildRoot}/tmp/_DEPENDS + # generate list of ports we can set to success + comm -12 ${buildRoot}/tmp/_ENQUEUED ${buildRoot}/tmp/_DEPENDS > ${buildRoot}/tmp/_FDEPS + + # set status of all dependencies to SUCCESS + if [ -s ${buildRoot}/tmp/_FDEPS ]; then + while read DEPEND; do + if [ -d ${buildRoot}/usr/ports/${DEPEND} ]; then + ${tc} updateEnqueuedToSuccess -b ${BUILD} -d ${DEPEND} + fi + done < ${buildRoot}/tmp/_FDEPS + fi + rm -f ${buildRoot}/tmp/_ENQUEUED ${buildRoot}/tmp/_DEPENDS ${buildRoot}/tmp/_FDEPS + fi + fi + echo "ended at $(date)" end=$(date +%s) @@ -1754,6 +1793,7 @@ norebuild=0 date="" logdir="" + uets=0 # argument processing while [ $# -gt 0 ]; do @@ -1797,6 +1837,7 @@ x-nullfs) pbargs="${pbargs} -nullfs"; nullfs="-n";; x-plistcheck) pbargs="${pbargs} -plistcheck";; x-onceonly) onceonly=1;; + x-uets) pbargs="${pbargs} -uets"; uets=1;; -*) return 1;; *) ports="${ports} $1";; Index: lib/Tinderbox/TinderboxDS.pm =================================================================== RCS file: /space/cvs/marcuscom-cvs/portstools/tinderbox/lib/Tinderbox/TinderboxDS.pm,v retrieving revision 1.101 diff -u -r1.101 TinderboxDS.pm --- lib/Tinderbox/TinderboxDS.pm 20 Jun 2012 20:47:42 -0000 1.101 +++ lib/Tinderbox/TinderboxDS.pm 15 Jul 2012 18:44:43 -0000 @@ -293,6 +293,26 @@ return $rc; } +sub updateEnqueuedToSuccess { + my $self = shift; + my $build = shift; + my $port_directory = shift; + + if (!defined($build)) { + croak "ERROR: no build given\n"; + } + + if (!defined($port_directory)) { + croak "ERROR: no port_directory given\n"; + } + + my $rc = $self->_doQuery( + "UPDATE build_ports_queue SET status='SUCCESS',completion_date=$self->{'now_fn'} WHERE port_directory=? and build_id=? and status='ENQUEUED'", + [$port_directory, $build->getId() ] + ); + return $rc; +} + sub moveBuildPortsQueueFromUserToUser { my $self = shift; my $old_id = shift; --- /dev/null 2012-07-15 20:40:27.000000000 +0200 +++ lib/uets 2012-03-11 17:02:37.508128226 +0100 @@ -0,0 +1,115 @@ +#!/bin/sh + +# ======================================================================= +# Copyright (c) 2011-2012 Olli Hauer +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $MCom$ +# + + +# ======================================================================= +# Description: +# This script extends FreeBSD ports-mgmt/tinderbox. +# +# To build ports for my production / test machines I periodically +# inject a big num of ports into the tinderbox build queue. +# Even $tb/$build_root is located in a ram disk there is room +# to speed up the build of a view hundred/thousands ports if +# the flag -uets is set. +# +# Given the following buildPortsQueue list (in this order) +# 1) devel/viewvc +# 2) devel/subversion +# 3) lang/perl +# 4) editors/vim-lite +# +# Without this extension tinderbox will start a build for all +# three ports even 2) and 3) are rebuild during the build of 1). +# +# With this patch (and parameter -uets): +# As soon a port is finished this extension calculates recursive +# all dependencies and set them also to SUCCESS in the +# buildPortsQueue. For example after building 1) we can set +# also 2) and 3) but not 4) to SUCCESS. +# +# Some values of recursive (EXTRACT|FETCH|LIB|BUILD|RUN)_DEPENDS +# viewvc (with svn): 25+ +# kde4 : 400+ +# +# My build times for 1000+ ports are in duration now 30-50% faster. + +# ======================================================================= +# usage: $0 DIRNAME DEPENDS_LOG + + +dir=${1:-"/nonexistent/$$"} +depend_list=${2:-"/tmp/_DEPENDS"} +debug=${3:-"0"} + +# some defaults +path_ports=/usr/ports +tmp=${depend_list}.$$ +me=$(basename ${0}) + +# ======================================================================= +usage () { + echo "${0} usage: PORT_DIRECTORY DEPENDS_LOG" + exit 64 +} + +cleanup () { + if [ ${debug} -ne 0 -a -s ${depend_list} ]; then + echo "==================================================" + echo "===============< all-depends-list >===============" + cat ${depend_list} + echo "==================================================" + fi + rm -f ${tmp} +} + +trap cleanup 0 1 2 3 9 10 11 15 + +# ======================================================================= +[ $# -lt 1 ] && usage +[ -f /tmp/leftovers ] && exit 78 + +if [ ! -s ${dir}/Makefile ]; then + echo "${me}: cannot find ${dir}/Makefile" 1>&2 + exit 72 +fi + +if [ -s /tmp/status ]; then + error=$(cat ${chroot}/tmp/status) + [ "${error}" = "0" ] || exit 65 +else + error=255 + exit 66 +fi + +# ======================================================================= +rm -f ${depend_list} ${tmp} +make all-depends-list -C ${dir} > ${tmp} +[ -s ${tmp} ] && ( sort -u ${tmp} | sed "s;^${path_ports}/;;" > ${depend_list} ) +exit 0