From ceb6ac54df5b2cec4f1a7e69b124c8242ae3fde6 Mon Sep 17 00:00:00 2001 From: Florent Thoumie Date: Thu, 21 Apr 2011 23:15:31 +0100 Subject: [PATCH] portbuild: remove hardcoded /var/portbuild. - Don't hardcode /var/portbuild or /var/portbuild/scripts. - Create per-build scripts/ and sources/ directories. - Scripts which aren't tied to a specific build are uploaded first. This change makes it effectively possible to checkout portbuild anywhere and test changes without affecting any other builds. --- qmanager/dumpdb.py | 12 ++++-- qmanager/packagebuild | 31 +++++++++-------- qmanager/qmanager | 12 ++++-- qmanager/qmanager.py | 14 +++++--- scripts/allgohans | 46 +++++++++++++++++++------ scripts/build | 38 ++++++++++---------- scripts/buildenv | 8 ++-- scripts/buildfailure | 19 +++++----- scripts/buildproxy | 12 ++++-- scripts/buildproxy-client | 10 ++++-- scripts/buildsuccess | 21 ++++++----- scripts/claim-chroot | 14 ++++---- scripts/clean-chroot | 10 +++--- scripts/cleanup-chroots | 8 ++-- scripts/client-metrics | 10 +++--- scripts/cpdistfiles | 8 +++-- scripts/cppackages | 6 ++-- scripts/do-cleanup-chroots | 5 ++- scripts/docppackages | 6 +++- scripts/dodistfiles | 5 ++- scripts/dologs | 15 ++++---- scripts/dopackages | 79 ++++++++++++++++++++++--------------------- scripts/dopackages.wrapper | 15 ++++---- scripts/dopackagestats | 14 ++++--- scripts/dosetupnode | 53 +++++++++++++++-------------- scripts/flushsquid | 10 +++-- scripts/keeprestr | 11 +++--- scripts/makeduds | 16 +++++---- scripts/makeindex | 15 ++++---- scripts/makerestr | 16 +++++---- scripts/makeworld | 13 ++++--- scripts/mkbindist | 12 ++++--- scripts/pdispatch | 43 ++++++++++++++---------- scripts/pollmachine | 25 ++++++++----- scripts/portbuild | 24 +++++++------- scripts/processfail | 14 ++++--- scripts/processlogs | 8 +++-- scripts/prunefailure | 27 ++++++++------- scripts/reportload | 13 ++++--- scripts/setupnode | 18 +++++----- scripts/stats | 8 +++-- scripts/updatesnap | 5 ++- scripts/updatesnap.ports | 5 ++- 43 files changed, 427 insertions(+), 327 deletions(-) diff --git a/qmanager/dumpdb.py b/qmanager/dumpdb.py index 47f053d..71ed662 100755 --- a/qmanager/dumpdb.py +++ b/qmanager/dumpdb.py @@ -4,17 +4,21 @@ import sys, os, threading, socket, Queue +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" + +sys.path.insert(0, '%s/lib/python' % pbc) + from signal import * from sys import exc_info from itertools import chain -sys.path.insert(0, '/var/portbuild/lib/python') - from freebsd_config import * from qmanagerobj import * -CONFIG_DIR="/var/portbuild" CONFIG_SUBDIR="conf" CONFIG_FILENAME="server.conf" @@ -127,7 +131,7 @@ def show_machines_for_arch( engine, arch ): if __name__ == '__main__': print "acquiring engine and session" - config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) + config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) QMANAGER_PATH = config.get( 'QMANAGER_PATH' ) QMANAGER_DATABASE_FILE = config.get( 'QMANAGER_DATABASE_FILE' ) (engine, session) = obj_startup( \ diff --git a/qmanager/packagebuild b/qmanager/packagebuild index 65c48c2..aa47ad7 100755 --- a/qmanager/packagebuild +++ b/qmanager/packagebuild @@ -32,31 +32,33 @@ # * check mtime for package staleness (cf make) # * option to skip phase 2 +import os import sys -sys.path.insert(0, '/var/portbuild/lib/python') +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" -from qmanagerclient import * +sys.path.insert(0, '%s/lib/python' % pbc) +from qmanagerclient import * from freebsd_config import * -import os, string, threading, time, subprocess -#import random +import string, threading, time, subprocess from itertools import chain -#import gc from stat import * from Queue import Queue from heapq import * -CONFIG_DIR="/var/portbuild" CONFIG_SUBDIR="conf" CONFIG_FILENAME="server.conf" -config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) +config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) QMANAGER_MAX_JOB_ATTEMPTS = int( \ config.get( 'QMANAGER_MAX_JOB_ATTEMPTS' ) ) -QMANAGER_PRIORITY_PACKAGES = string.split( \ +QMANAGER_PRIORITY_PACKAGES = string.split( \ config.get( 'QMANAGER_PRIORITY_PACKAGES' ) ) QMANAGER_RUNAWAY_PERCENTAGE = float( \ config.get( 'QMANAGER_RUNAWAY_PERCENTAGE' ) ) @@ -323,8 +325,8 @@ they still need to know about us as dependencies etc """ def packagename(self, arch, branch, buildid): """ Return the path where a package may be found""" - return "/var/portbuild/%s/%s/builds/%s/packages/All/%s.tbz" \ - % (arch, branch, buildid, self.name) + return "%s/%s/%s/builds/%s/packages/All/%s.tbz" \ + % (pbd, arch, branch, buildid, self.name) def is_stale(self, arch, branch, buildid): """ Does a package need to be (re)-built? @@ -409,6 +411,7 @@ class worker(threading.Thread): self.branch = branch self.buildid = buildid self.queue = queue + self.basedir = pbd+"/"+arch+"/"+branch+"/builds/"+buildid self.setDaemon(True) @@ -422,7 +425,7 @@ class worker(threading.Thread): print try: runenv={'HOME':"/root", - 'PATH':'/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:/var/portbuild/scripts', + 'PATH':'/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:%s/scripts' + pbc, 'FD':" ".join(["%s.tbz" % p.name for p in pkg.fdep]), 'ED':" ".join(["%s.tbz" % p.name for p in pkg.edep]), 'PD':" ".join(["%s.tbz" % p.name for p in pkg.pdep]), @@ -432,9 +435,9 @@ class worker(threading.Thread): if var in os.environ: runenv[var] = os.environ.get(var) build = subprocess.Popen( - ["/bin/sh", "/var/portbuild/scripts/pdispatch", + ["/bin/sh", "%s/scripts/pdispatch" % pbc, self.arch, self.branch, self.buildid, self.machine, - "/var/portbuild/scripts/portbuild", "%s.tbz" % pkg.name, + "%s/scripts/portbuild" % self.basedir, "%s.tbz" % pkg.name, pkg.path], env=runenv, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, bufsize=0) @@ -506,7 +509,7 @@ class worker(threading.Thread): def main(arch, branch, buildid, args): global index - basedir="/var/portbuild/"+arch+"/"+branch+"/builds/"+buildid + basedir=pbd+"/"+arch+"/"+branch+"/builds/"+buildid portsdir=basedir+"/ports" # get the major branch number. diff --git a/qmanager/qmanager b/qmanager/qmanager index a42734e..8f907c6 100755 --- a/qmanager/qmanager +++ b/qmanager/qmanager @@ -49,11 +49,15 @@ # - OR, NOT job description entries # - query jobs.machine properties +import os import sys -sys.path.insert(0, '/var/portbuild/lib/python') +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" -from freebsd_config import * +sys.path.insert(0, '%s/lib/python' % pbc) import os, threading, socket, Queue @@ -61,12 +65,12 @@ from signal import * from itertools import chain from qmanagerobj import * +from freebsd_config import * -CONFIG_DIR="/var/portbuild" CONFIG_SUBDIR="conf" CONFIG_FILENAME="server.conf" -config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) +config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) QMANAGER_SOCKET_FILE = config.get( 'QMANAGER_SOCKET_FILE' ) class Worker(object): diff --git a/qmanager/qmanager.py b/qmanager/qmanager.py index 4c9207d..e5bf18e 100755 --- a/qmanager/qmanager.py +++ b/qmanager/qmanager.py @@ -49,24 +49,28 @@ # - OR, NOT job description entries # - query jobs.machine properties +import os import sys -sys.path.insert(0, '/var/portbuild/lib/python') +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" -from freebsd_config import * +sys.path.insert(0, '%s/lib/python' % pbc) -import os, socket, threading, time, Queue +import socket, threading, time, Queue from signal import * from itertools import chain from qmanagerobj import * +from freebsd_config import * -CONFIG_DIR="/var/portbuild" CONFIG_SUBDIR="conf" CONFIG_FILENAME="server.conf" -config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) +config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) QMANAGER_SOCKET_FILE = config.get( 'QMANAGER_SOCKET_FILE' ) DEBUG = True diff --git a/scripts/allgohans b/scripts/allgohans index 4d6d100..f225547 100755 --- a/scripts/allgohans +++ b/scripts/allgohans @@ -1,30 +1,46 @@ #!/bin/sh +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} + +. ${pbc}/conf/server.conf + doarch() { arch=$1 shift - if [ -f "${pb}/${arch}/portbuild.conf" ]; then - . ${pb}/${arch}/portbuild.conf + if [ -f "${pbd}/${arch}/portbuild.conf" ]; then + . ${pbd}/${arch}/portbuild.conf else echo "Invalid arch" exit 1 fi - machines=$(awk '{print $1}' ${pb}/${arch}/mlist) + machines=$(awk '{print $1}' ${pbd}/${arch}/mlist) for i in ${machines}; do - . ${pb}/${arch}/portbuild.conf - if [ -f "${pb}/${arch}/portbuild.${i}" ]; then - . ${pb}/${arch}/portbuild.${i} + . ${pbd}/${arch}/portbuild.conf + if [ -f "${pbd}/${arch}/portbuild.${i}" ]; then + . ${pbd}/${arch}/portbuild.${i} fi if [ ${quiet} -eq 0 ]; then echo "[$i]" fi - lockf -t 60 ${pb}/${arch}/lockfiles/lock.$i su ports-${arch} -c "ssh ${client_user}@$i ${sudo_cmd} $@" + cmdpath=${cmd} + if [ ${scpfirst} -ne 0 ]; then + # If we need to scp first, the command given in the parameters is the local + # path, which we need to upload first. The command will return the remote + # temporary file, which we can subsequently execute. So this isn't really scp. + cmdpath=$(su ports-${arch} -c "cat ${cmd} | ssh ${client_user}@$i 't=$(mktemp -t portbuild); cat >\$t; echo \$t; chmod 755 \$t'") + case ${cmdpath} in /tmp/*) ;; *) echo "Failed to scp ${cmd} to $i."; return 1;; esac + fi + lockf -t 60 ${pbd}/${arch}/lockfiles/lock.$i su ports-${arch} -c "ssh ${client_user}@$i ${sudo_cmd} ${cmdpath} $@" result=$? if [ $result -ne 0 ]; then - echo "could not execute command $@ on $i: $result" + echo "could not execute command ${cmdpath} $@ on $i: $result" + fi + if [ ${scpfirst} -ne 0 ]; then + su ports-${arch} -c "ssh ${client_user}@$i 'rm -f ${cmdpath}'" fi done } @@ -39,6 +55,7 @@ else root=0 fi +# XXX - Have to convert this to getopt(). if [ "$1" = "-q" ]; then quiet=1 shift @@ -46,11 +63,18 @@ else quiet=0 fi -pb=/var/portbuild -. ${pb}/conf/server.conf +if [ "$1" = "-scp" ]; then + scpfirst=1 + shift +else + scpfirst=0 +fi + +cmd=$1 +shift if [ "${arch}" = "all" ]; then - arches=$(find ${pb}/*/portbuild.conf) + arches=$(find ${pbd}/*/portbuild.conf) for i in ${arches}; do arch=$(basename $(dirname $i)) doarch $arch "$@" diff --git a/scripts/build b/scripts/build index 554ce6c..4f0b930 100755 --- a/scripts/build +++ b/scripts/build @@ -4,7 +4,8 @@ # server-side script to handle various commands common to builds # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} # subdirectories to process. yes, this is a hack, but it saves code # duplication. @@ -25,7 +26,7 @@ do_list() { arch=$1 branch=$2 - buildpar=/var/portbuild/${arch}/${branch}/builds + buildpar=${pbd}/${arch}/${branch}/builds if [ -d ${buildpar} ]; then snaps=$(cd ${buildpar}; ls -1d 2* 2> /dev/null) @@ -51,7 +52,7 @@ do_create() { buildid=$3 shift 3 - archivedir=${pb}/${arch}/archive + archivedir=${pbd}/${arch}/archive # create directory for all build logs buildlogsdir=${archivedir}/buildlogs @@ -93,8 +94,7 @@ do_create() { ln -sf ${builddir} ${pbab}/builds/latest # create the links for the webserver - # XXX MCL hardcoding - errorlogs=/var/portbuild/errorlogs + errorlogs=${pbd}/errorlogs ln -s ${buildsdir}/latest/bak/errors ${errorlogs}/${arch}-${branch}-previous ln -s ${buildsdir}/latest/bak/logs ${errorlogs}/${arch}-${branch}-previous-logs @@ -148,7 +148,7 @@ do_clone() { if [ -d ${newbuilddir} ]; then if [ ! -f ${pbab}/builds/previous/.keep ]; then - /var/portbuild/scripts/build destroy ${arch} ${branch} previous + ${pbc}/scripts/build destroy ${arch} ${branch} previous fi rm -f ${pbab}/builds/previous mv ${pbab}/builds/latest ${pbab}/builds/previous @@ -251,10 +251,10 @@ cleanup_client() { echo "Started cleaning up ${arch}/${branch} build ID ${buildid} on ${mach}" - test -f ${pb}/${arch}/portbuild.${mach} && . ${pb}/${arch}/portbuild.${mach} + test -f ${pbd}/${arch}/portbuild.${mach} && . ${pbd}/${arch}/portbuild.${mach} # Kill off builds and clean up chroot - ${pb}/scripts/dosetupnode ${arch} ${branch} ${buildid} ${mach} -nocopy -queue -full + ${pbc}/scripts/dosetupnode ${arch} ${branch} ${buildid} ${mach} -nocopy -queue -full echo "Finished cleaning up ${arch}/${branch} build ID ${buildid} on ${mach}" } @@ -267,7 +267,7 @@ do_cleanup() { arg=$5 shift 5 - for i in `cat ${pb}/${arch}/mlist`; do + for i in `cat ${pbd}/${arch}/mlist`; do cleanup_client ${arch} ${branch} ${buildid} ${i} ${arg} & done wait @@ -394,7 +394,7 @@ do_destroy() { shift 4 echo - buildid=$(resolve ${pb} ${arch} ${branch} ${buildid}) + buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${buildid}" ]; then echo "Invalid build ID ${buildid}" exit 1 @@ -435,7 +435,7 @@ proxy_root() { id=$(id -u) if [ ${id} != "0" ]; then - /var/portbuild/scripts/buildproxy-client "build ${cmd} ${arch} ${branch} ${buildid} ${args}" + ${pbc}/scripts/buildproxy-client "build ${cmd} ${arch} ${branch} ${buildid} ${args}" error=$? if [ ${error} -eq 254 ]; then echo "Proxy error" @@ -463,7 +463,7 @@ proxy_user() { eval "do_${cmd} ${arch} ${branch} ${buildid} \"${builddir}\" ${args}" error=$? else - su ports-${arch} -c "/var/portbuild/scripts/build ${cmd} ${arch} ${branch} ${buildid} \"${builddir}\" ${args}" + su ports-${arch} -c "${pbc}/scripts/build ${cmd} ${arch} ${branch} ${buildid} \"${builddir}\" ${args}" error=$? fi @@ -486,11 +486,11 @@ arch=$2 branch=$3 shift 3 -. ${pb}/conf/server.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbc}/scripts/buildenv -pbab=${pb}/${arch}/${branch} +pbab=${pbd}/${arch}/${branch} validate_env ${arch} ${branch} || exit 1 @@ -505,7 +505,7 @@ if [ $# -ge 1 ]; then case "$cmd" in cleanup) # Resolve symlinks but don't bail if the build doesn't exist. - newbuildid=$(resolve ${pb} ${arch} ${branch} ${buildid}) + newbuildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ ! -z "${newbuildid}" -a "${newbuildid}" != "${buildid}" ]; then echo "Resolved ${buildid} to ${newbuildid}" buildid=${newbuildid} @@ -527,7 +527,7 @@ if [ $# -ge 1 ]; then # We can't rely on buildenv for this code path ;; *) - newbuildid=$(resolve ${pb} ${arch} ${branch} ${buildid}) + newbuildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${newbuildid}" ]; then echo "Build ID ${buildid} does not exist" exit 1 @@ -539,7 +539,7 @@ if [ $# -ge 1 ]; then builddir=$(realpath ${pbab}/builds/${buildid}/) - buildenv ${pb} ${arch} ${branch} ${builddir} + buildenv ${pbd} ${arch} ${branch} ${builddir} ;; esac fi diff --git a/scripts/buildenv b/scripts/buildenv index fe15eae..edbc1b8 100644 --- a/scripts/buildenv +++ b/scripts/buildenv @@ -40,7 +40,7 @@ validate_env() { # only used on server side resolve() { - pb=$1 + pbd=$1 arch=$2 branch=$3 buildid=$4 @@ -48,7 +48,7 @@ resolve() { # Resolve a possibly symlinked buildid (e.g. "latest") to the # underlying physical directory - pbab=${pb}/${arch}/${branch} + pbab=${pbd}/${arch}/${branch} builddir=${pbab}/builds/${buildid}/ if [ ! -d ${builddir} ]; then @@ -83,7 +83,7 @@ export_src_metadata() { # establish commonly-used environment variables (server-side) # buildenv () { - pb=$1 + pbd=$1 arch=$2 branch=$3 builddir=$4 @@ -139,7 +139,7 @@ buildenv () { fi # probably only used in mkbindist - export __MAKE_CONF=${pb}/${arch}/make.conf + export __MAKE_CONF=${pbd}/${arch}/make.conf } # diff --git a/scripts/buildfailure b/scripts/buildfailure index e5d95d2..ecfcf9a 100755 --- a/scripts/buildfailure +++ b/scripts/buildfailure @@ -9,7 +9,8 @@ cleanup() { } # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} usage () { echo "usage: buildfailure arch branch buildid pkgname" @@ -26,14 +27,14 @@ buildid=$3 pkgname=$4 shift 4 -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} -. ${pb}/conf/server.conf -. ${pb}/conf/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/conf/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbc}/scripts/buildenv -buildenv ${pb} ${arch} ${branch} ${builddir} +buildenv ${pbd} ${arch} ${branch} ${builddir} # Don't pick up installed packages from the host export LOCALBASE=/nonexistentlocal @@ -43,7 +44,7 @@ index=${PORTSDIR}/${INDEXFILE} portloc=$(grep "^$pkgname|" ${index} | cut -f 2 -d \| | sed s,/usr/ports/,,) pkgbase=$(cd ${PORTSDIR}/${portloc}/ && make -V PKGBASE) -cd ${pb}/${arch}/${branch} +cd ${pbd}/${arch}/${branch} entry=$(grep "^${portloc}|" failure) date=$(date +%s) IFS='|' @@ -58,7 +59,7 @@ else (echo "${portloc}|${pkgbase}|${pkgname}|${date}|${date}|1" >> failure) || cleanup fi -link=${pb}/${arch}/${branch}/latest/${portloc} +link=${pbd}/${arch}/${branch}/latest/${portloc} mkdir -p $(dirname ${link}) errorloc=$(realpath ${builddir}/errors/${pkgname}.log) ln -sf ${errorloc} ${link} diff --git a/scripts/buildproxy b/scripts/buildproxy index 0d664ca..d4e892f 100755 --- a/scripts/buildproxy +++ b/scripts/buildproxy @@ -6,12 +6,16 @@ import sys, socket, os, commands -sys.path.insert(0, '/var/portbuild/lib/python') +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" + +sys.path.insert(0, '%s/lib/python' % pbc) from freebsd import * from freebsd_config import * -CONFIG_DIR="/var/portbuild" CONFIG_SUBDIR="conf" CONFIG_FILENAME="server.conf" @@ -48,11 +52,11 @@ def process(cmd, sockfile): if not j.isalnum() and not j in "-_.": return (254, "Illegal characters in input") - (status, out) = commands.getstatusoutput("/var/portbuild/scripts/build %s" % " ".join(cmd[1:])) + (status, out) = commands.getstatusoutput("%s/scripts/build %s" % (pbc, " ".join(cmd[1:]))) return (status, out) -config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) +config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) BUILDPROXY_SOCKET_FILE = config.get( 'BUILDPROXY_SOCKET_FILE' ) if os.path.exists(BUILDPROXY_SOCKET_FILE): diff --git a/scripts/buildproxy-client b/scripts/buildproxy-client index ee1a33d..2e9898d 100755 --- a/scripts/buildproxy-client +++ b/scripts/buildproxy-client @@ -5,16 +5,20 @@ import sys, socket, os, commands -sys.path.insert(0, '/var/portbuild/lib/python') +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" + +sys.path.insert(0, '%s/lib/python' % pbc) from freebsd import * from freebsd_config import * -CONFIG_DIR="/var/portbuild" CONFIG_SUBDIR="conf" CONFIG_FILENAME="server.conf" -config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) +config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) BUILDPROXY_SOCKET_FILE = config.get( 'BUILDPROXY_SOCKET_FILE' ) try: diff --git a/scripts/buildsuccess b/scripts/buildsuccess index 53a84e7..58bff9b 100755 --- a/scripts/buildsuccess +++ b/scripts/buildsuccess @@ -3,7 +3,8 @@ # buildsuccess # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} usage () { echo "usage: buildsuccess arch branch buildid pkgname" @@ -20,14 +21,14 @@ buildid=$3 pkgname=$4 shift 4 -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} -. ${pb}/conf/server.conf -. ${pb}/conf/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/conf/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbc}/scripts/buildenv -buildenv ${pb} ${arch} ${branch} ${builddir} +buildenv ${pbd} ${arch} ${branch} ${builddir} # Don't pick up installed packages from the host export LOCALBASE=/nonexistentlocal @@ -36,13 +37,13 @@ index=${PORTSDIR}/${INDEXFILE} portloc=$(grep "^$pkgname|" ${index} | cut -f 2 -d \| | sed s,/usr/ports/,,) -cd ${pb}/${arch}/${branch} +cd ${pbd}/${arch}/${branch} if grep -q "^${portloc}|" failure; then echo | mail -s "${pkgname} now builds on ${arch} ${branch}" ${mailto} grep -v "^${portloc}|" failure > failure.new mv failure.new failure fi -if [ -L ${pb}/${arch}/${branch}/latest/${portloc} ]; then - rm -f ${pb}/${arch}/${branch}/latest/${portloc} +if [ -L ${pbd}/${arch}/${branch}/latest/${portloc} ]; then + rm -f ${pbd}/${arch}/${branch}/latest/${portloc} fi diff --git a/scripts/claim-chroot b/scripts/claim-chroot index 349db63..5f99e06 100755 --- a/scripts/claim-chroot +++ b/scripts/claim-chroot @@ -18,7 +18,7 @@ # could use a lockf lock, and be able to tell if the setup process was # still running or died prematurely -pb=/var/portbuild +pbd=${PORTBUILD_DATA:-/var/portbuild} usage () { echo "usage: claim-chroot arch branch buildid" @@ -48,7 +48,7 @@ fi # provide mutual exclusion against other callers, since we only want # one of them to perform setup -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} # Is the build environment populated? Again we only want a single # instance to gain setup rights if not. @@ -64,10 +64,10 @@ if [ ! -f ${builddir}/.ready ]; then exit 1 fi -. ${pb}/${arch}/client.conf -. ${pb}/${arch}/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/${arch}/portbuild.$(hostname) +. ${pbd}/${arch}/client.conf +. ${pbd}/${arch}/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbd}/${arch}/portbuild.$(hostname) buildroot=${scratchdir} pkgname=${pkgname%.${PKGSUFFIX}} @@ -109,7 +109,7 @@ for dir in ${chrootdir}/*; do mkdir ${dir}/used 2>/dev/null || continue touch ${dir}/used/${pkgname} if [ -f ${dir}/.dirty ]; then - ${pb}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${dir} 2 >/dev/null 2>/dev/null & + ${builddir}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${dir} 2 >/dev/null 2>/dev/null & continue fi found=1 diff --git a/scripts/clean-chroot b/scripts/clean-chroot index a645f47..feaeed9 100755 --- a/scripts/clean-chroot +++ b/scripts/clean-chroot @@ -42,12 +42,12 @@ buildid=$3 chroot=$4 clean=$5 -pb=/var/portbuild +pbd=${PORTBUILD_DATA:-/var/portbuild} -. ${pb}/${arch}/client.conf -. ${pb}/${arch}/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/${arch}/portbuild.$(hostname) +. ${pbd}/${arch}/client.conf +. ${pbd}/${arch}/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbd}/${arch}/portbuild.$(hostname) # directories to clean cleandirs="${LOCALBASE} /compat /var/db/pkg" diff --git a/scripts/cleanup-chroots b/scripts/cleanup-chroots index 3a22c15..b86eb2d 100755 --- a/scripts/cleanup-chroots +++ b/scripts/cleanup-chroots @@ -7,7 +7,7 @@ # port builds that have timed out or shut down uncleanly) and prunes # them to reclaim space. -pb=/var/portbuild +pbd=${PORTBUILD_DATA:-/var/portbuild} kill_procs() { @@ -44,9 +44,9 @@ cleanup_mount() { # note: uname is not being overridden (should not need client.conf here) arch=$(uname -m) -. ${pb}/${arch}/client.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/${arch}/portbuild.$(hostname) +. ${pbd}/${arch}/client.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbd}/${arch}/portbuild.$(hostname) if [ "${use_zfs}" = "1" ]; then old=$(find ${scratchdir}/*/*/* -prune -mmin +60 2> /dev/null) diff --git a/scripts/client-metrics b/scripts/client-metrics index f68f94d..5ec72c9 100755 --- a/scripts/client-metrics +++ b/scripts/client-metrics @@ -6,7 +6,7 @@ # - max vnodes # - number of packages built in the past hour -pb=/var/portbuild +pbd=${PORTBUILD_DATA:-/var/portbuild} arch=$(uname -m) me=$(hostname) @@ -16,10 +16,10 @@ export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin gmetric --name="maxvnodes" --value=`sysctl -n kern.maxvnodes` --tmax=120 --dmax=0 --type=uint32 --units="# vnodes" gmetric --name="vnodes" --value=`sysctl -n vfs.numvnodes` --tmax=120 --dmax=0 --type=uint32 --units="# vnodes" -if [ -f ${pb}/${arch}/portbuild.conf -a -f ${pb}/${arch}/portbuild.${me} ]; then - . ${pb}/${arch}/client.conf - . ${pb}/${arch}/portbuild.conf - . ${pb}/${arch}/portbuild.${me} +if [ -f ${pbd}/${arch}/portbuild.conf -a -f ${pbd}/${arch}/portbuild.${me} ]; then + . ${pbd}/${arch}/client.conf + . ${pbd}/${arch}/portbuild.conf + . ${pbd}/${arch}/portbuild.${me} else exit 1 fi diff --git a/scripts/cpdistfiles b/scripts/cpdistfiles index 08bfbc0..1bf1e88 100755 --- a/scripts/cpdistfiles +++ b/scripts/cpdistfiles @@ -6,14 +6,16 @@ if [ $# -lt 3 ]; then fi # configurable variables -pb=/var/portbuild -. ${pb}/conf/server.conf +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} + +. ${pbc}/conf/server.conf arch=$1 branch=$2 buildid=$3 -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} yesreally=0 dryrun=-n diff --git a/scripts/cppackages b/scripts/cppackages index 7174afe..ec9facd 100755 --- a/scripts/cppackages +++ b/scripts/cppackages @@ -1,11 +1,11 @@ #!/bin/sh # configurable variables -pb=/var/portbuild +pbd=${PORTBUILD_DATA:-/var/portbuild} -. ${pb}/portbuild.conf +. ${pbd}/portbuild.conf -lock=${pb}/cppackages.lock +lock=${pbd}/cppackages.lock unset DISPLAY diff --git a/scripts/do-cleanup-chroots b/scripts/do-cleanup-chroots index 8fefc1a..5d3f5a1 100755 --- a/scripts/do-cleanup-chroots +++ b/scripts/do-cleanup-chroots @@ -2,4 +2,7 @@ # # clean up stale chroots on all client machines. MCL 20081216. # -/var/portbuild/scripts/allgohans all -q /var/portbuild/scripts/cleanup-chroots + +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} + +${pbc}/scripts/allgohans all -q -scp ${pbc}/scripts/cleanup-chroots diff --git a/scripts/docppackages b/scripts/docppackages index 5d3187a..689894c 100755 --- a/scripts/docppackages +++ b/scripts/docppackages @@ -1,7 +1,11 @@ #!/bin/sh + if [ $# != 1 ]; then echo "usage: $0 branch" exit 1 fi -echo /var/portbuild/scripts/cppackages $1 | at + 1 minute + +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} + +echo ${pbc}/scripts/cppackages $1 | at + 1 minute atq diff --git a/scripts/dodistfiles b/scripts/dodistfiles index 2f46562..8fbe622 100755 --- a/scripts/dodistfiles +++ b/scripts/dodistfiles @@ -30,8 +30,9 @@ arch=$1 branch=$2 buildid=$3 -pb=/var/portbuild -builddir=${pb}/${arch}/${branch}/builds/${buildid} +pbd=${PORTBUILD_DATA:-/var/portbuild} + +builddir=${pbd}/${arch}/${branch}/builds/${buildid} distdir=${builddir}/distfiles diff --git a/scripts/dologs b/scripts/dologs index da04e97..93e1a26 100755 --- a/scripts/dologs +++ b/scripts/dologs @@ -4,21 +4,22 @@ arch=$1 versions="latest full" -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} -. ${pb}/conf/server.conf -. ${pb}/${arch}/portbuild.conf +. ${pbc}/conf/server.conf +. ${pbd}/${arch}/portbuild.conf -home=${pb}/errorlogs -scripts=${pb}/scripts +home=${pbd}/errorlogs +scripts=${pbc}/scripts -branches=`cd ${pb}/${arch} && ls -d [1-9]* 2> /dev/null` +branches=`cd ${pbd}/${arch} && ls -d [1-9]* 2> /dev/null` for version in ${versions}; do for branch in ${branches}; do dir=$home/$arch-$branch-$version test -d $dir && cd $dir && ${scripts}/processlogs ${arch} - dir=${pb}/${arch}/${branch} + dir=${pbd}/${arch}/${branch} test -d $dir && cd $dir && ${scripts}/processfail ${arch} ${branch} done for branch in ${branches}; do diff --git a/scripts/dopackages b/scripts/dopackages index c463b52..900425d 100755 --- a/scripts/dopackages +++ b/scripts/dopackages @@ -4,9 +4,10 @@ # main server-side script to run a package build # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} -PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${pb}/scripts +PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${pbc}/scripts # writable by portmgr umask 002 @@ -52,26 +53,26 @@ buildid=$3 datestamp=$4 shift 4 -. ${pb}/conf/server.conf -. ${pb}/conf/common.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/conf/common.conf +. ${pbc}/scripts/buildenv validate_env ${arch} ${branch} || usage # XXX MCL too early to do this here. -buildid=$(resolve ${pb} ${arch} ${branch} ${buildid}) +buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${buildid}" ]; then echo "Invalid build ID ${buildid}" exit 1 fi -if [ -f ${pb}/${arch}/portbuild.conf ]; then - . ${pb}/conf/server.conf - . ${pb}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/portbuild.conf ]; then + . ${pbc}/conf/server.conf + . ${pbd}/${arch}/portbuild.conf else usage fi -pbab=${pb}/${arch}/${branch} +pbab=${pbd}/${arch}/${branch} trap "exit 1" 1 2 3 9 10 11 15 @@ -97,7 +98,7 @@ portstar() { # usage: makeindex pb arch branch builddir [target] # note: can take ~24 minutes! makeindex () { - pb=$1 + pbc=$1 arch=$2 branch=$3 buildid=$4 @@ -109,7 +110,7 @@ makeindex () { echo "generating index" echo "================================================" echo "index generation started at $(date)" - ${pb}/scripts/makeindex ${arch} ${branch} ${buildid} ${target} || return 1 + ${pbc}/scripts/makeindex ${arch} ${branch} ${buildid} ${target} || return 1 echo "index generation ended at $(date)" echo $(wc -l ${INDEXFILE} | awk '{print $1}') "lines in INDEX" @@ -142,7 +143,7 @@ checkindex () { # usage: makeduds pb arch branch builddir [target] # note: can take ~21 minutes! makeduds () { - pb=$1 + pbc=$1 arch=$2 branch=$3 buildid=$4 @@ -157,7 +158,7 @@ makeduds () { if [ -e ${builddir}/duds ]; then cp -p ${builddir}/duds ${builddir}/duds.old fi - if ! ${pb}/scripts/makeduds ${arch} ${branch} ${buildid} ${target}; then + if ! ${pbc}/scripts/makeduds ${arch} ${branch} ${buildid} ${target}; then echo "error(s) detected, exiting script at $(date). Failed duds list was:" cat ${builddir}/duds mailexit 1 @@ -176,7 +177,7 @@ makeduds () { # usage: restrictedlist pb arch branch builddir [target] # note: can take ~25 minutes! restrictedlist () { - pb=$1 + pbc=$1 arch=$2 branch=$3 buildid=$4 @@ -188,7 +189,7 @@ restrictedlist () { echo "creating restricted list" echo "================================================" echo "restricted list generation started at $(date)" - ${pb}/scripts/makerestr ${arch} ${branch} ${buildid} ${target} || return 1 + ${pbc}/scripts/makerestr ${arch} ${branch} ${buildid} ${target} || return 1 echo "restricted list generation ended at $(date)" echo $(grep -c '^#' ${builddir}/restricted.sh) "ports in ${builddir}/restricted.sh" } @@ -196,7 +197,7 @@ restrictedlist () { # usage: cdromlist pb arch branch builddir # note: can take ~48 minutes! cdromlist () { - pb=$1 + pbc=$1 arch=$2 branch=$3 builddir=$4 @@ -218,7 +219,7 @@ cdromlist () { # XXX Should generate these as the packages are copied in, instead of all at once at the end # usage: generatemd5 pb arch branch builddir generatemd5 () { - pb=$1 + pbc=$1 arch=$2 branch=$3 builddir=$4 @@ -231,7 +232,7 @@ generatemd5 () { dobuild() { - pb=$1 + pbc=$1 arch=$2 branch=$3 builddir=$4 @@ -241,7 +242,7 @@ dobuild() { echo "================================================" echo "started at $(date)" start=$(date +%s) - ${QMANAGER_PATH}/packagebuild ${arch} ${branch} ${buildid} > ${builddir}/${journalname} 2>&1 < /dev/null + ${pbc}/qmanager/packagebuild ${arch} ${branch} ${buildid} > ${builddir}/${journalname} 2>&1 < /dev/null result=$? if [ $result -ne 0 ]; then echo "ERROR: packagebuild ${arch} ${branch} ${buildid} failed: see ${builddir}/${journalname} for details" @@ -422,7 +423,7 @@ if [ ! -d ${builddir} ]; then fi # Set up our environment variables -buildenv ${pb} ${arch} ${branch} ${builddir} +buildenv ${pbd} ${arch} ${branch} ${builddir} # XXX MCL might not return 'latest' ??? echo | mail -s "$(basename $0) started for ${arch}-${branch} ${buildid} at $(date)" ${mailto} @@ -439,7 +440,7 @@ fi touch ${builddir}/.active # Update link to current logfile created by dopackages.wrapper -ln -sf ${pb}/${arch}/archive/buildlogs/log.${branch}.${datestamp} \ +ln -sf ${pbd}/${arch}/archive/buildlogs/log.${branch}.${datestamp} \ ${builddir}/build.log if [ "$skipstart" = 0 ]; then @@ -518,11 +519,11 @@ if [ "$skipstart" = 0 ]; then # XXX MCL could background these? # not run in background to check return status if [ "$noindex" = 0 ]; then - makeindex ${pb} ${arch} ${branch} ${buildid} ${builddir} ${target} || mailexit 1 + makeindex ${pbc} ${arch} ${branch} ${buildid} ${builddir} ${target} || mailexit 1 fi checkindex ${builddir} || mailexit 1 if [ "$noduds" = 0 ]; then - makeduds ${pb} ${arch} ${branch} ${buildid} ${builddir} ${target} || mailexit 1 + makeduds ${pbc} ${arch} ${branch} ${buildid} ${builddir} ${target} || mailexit 1 fi wait # for tar creation @@ -542,17 +543,17 @@ if [ "$skipstart" = 0 ]; then if [ -f ${pbab}/newfailure ]; then cp ${pbab}/newfailure ${builddir}/bak/ fi - lockf -k ${pbab}/failure.lock ${pb}/scripts/prunefailure ${arch} ${branch} ${builddir} + lockf -k ${pbab}/failure.lock ${pbc}/scripts/prunefailure ${arch} ${branch} ${builddir} fi # XXX These can happen after build start if [ "$norestr" = 0 ]; then - restrictedlist ${pb} ${arch} ${branch} ${buildid} ${builddir} ${target} & + restrictedlist ${pbc} ${arch} ${branch} ${buildid} ${builddir} ${target} & job_restrictedlist=$! fi if [ "$cdrom" = 1 ]; then - cdromlist ${pb} ${arch} ${branch} ${builddir} & + cdromlist ${pbc} ${arch} ${branch} ${builddir} & job_cdromlist=$! fi @@ -567,8 +568,8 @@ if [ "$skipstart" = 0 ]; then oldlogs=$(readlink ${builddir}/logs) # XXX MCL hardcoding of archive/errorlogs - newerrors=${pb}/${arch}/archive/errorlogs/e.${branch}.${buildid} - newlogs=${pb}/${arch}/archive/errorlogs/a.${branch}.${buildid} + newerrors=${pbd}/${arch}/archive/errorlogs/e.${branch}.${buildid} + newlogs=${pbd}/${arch}/archive/errorlogs/a.${branch}.${buildid} # Cycle out the previous symlinks rm -f bak/errors @@ -637,7 +638,7 @@ if [ "$skipstart" = 0 ]; then sed "s,$,${PKGSUFFIX}," ${builddir}/.oldports | xargs rm -f # XXX MCL takes an unknown period of time. # XXX MCL return value not checked. - ${pb}/scripts/prunepkgs ${PORTSDIR}/${INDEXFILE} ${PACKAGES} + ${pbc}/scripts/prunepkgs ${PORTSDIR}/${INDEXFILE} ${PACKAGES} cd ${builddir}/errors/ sed "s,\$,.log," ${builddir}/.oldports | xargs rm -f @@ -678,9 +679,9 @@ if [ "$nobuild" = 0 ]; then fi # Compile ptimeout. - /usr/bin/gcc -o ${builddir}/ptimeout -Wall ${pb}/sources/ptimeout.c + /usr/bin/gcc -o ${builddir}/ptimeout -Wall ${pbc}/sources/ptimeout.c - dobuild ${pb} ${arch} ${branch} ${builddir} + dobuild ${pbc} ${arch} ${branch} ${builddir} fi @@ -696,7 +697,7 @@ if [ "$nofinish" = 0 ]; then if [ "$norestr" = 0 ]; then # Before deleting restricted packages, save a copy so we don't # have to rebuild them next time - ${pb}/scripts/keeprestr ${arch} ${branch} ${buildid} + ${pbc}/scripts/keeprestr ${arch} ${branch} ${buildid} else rm -rf ${builddir}/bak/restricted/ fi @@ -712,7 +713,7 @@ if [ "$nofinish" = 0 ]; then fi # Remove packages not listed in INDEX - ${pb}/scripts/prunepkgs ${builddir}/ports/${INDEXFILE} ${builddir}/packages + ${pbc}/scripts/prunepkgs ${builddir}/ports/${INDEXFILE} ${builddir}/packages fi # XXX Checking for bad packages should be done after the package is uploaded @@ -729,10 +730,10 @@ fi #done if [ "$nofinish" = 0 ]; then - generatemd5 ${pb} ${arch} ${branch} ${builddir} & + generatemd5 ${pbc} ${arch} ${branch} ${builddir} & # Remove INDEX entries for packages that do not exist - ${pb}/scripts/chopindex ${builddir}/ports/${INDEXFILE} ${builddir}/packages > ${builddir}/packages/INDEX + ${pbc}/scripts/chopindex ${builddir}/ports/${INDEXFILE} ${builddir}/packages > ${builddir}/packages/INDEX # Copy UPDATING and MOVED into the packages folder cp ${builddir}/ports/UPDATING ${builddir}/packages/UPDATING @@ -751,7 +752,7 @@ if [ "$nofinish" = 0 ]; then echo "================================================" echo "started at $(date)" cd ${builddir} - ${pb}/scripts/dodistfiles ${arch} ${branch} ${buildid} + ${pbc}/scripts/dodistfiles ${arch} ${branch} ${buildid} # Always delete restricted distfiles echo "deleting restricted distfiles" @@ -767,9 +768,9 @@ fi if [ "${nocleanup}" -eq 1 ]; then echo "Not cleaning up build, when you are finished be sure to run:" - echo " ${pb}/scripts/build cleanup ${arch} ${branch} ${buildid} -full" + echo " ${pbc}/scripts/build cleanup ${arch} ${branch} ${buildid} -full" else - ${pb}/scripts/build cleanup ${arch} ${branch} ${buildid} -full + ${pbc}/scripts/build cleanup ${arch} ${branch} ${buildid} -full fi endtime=$(date +%s) diff --git a/scripts/dopackages.wrapper b/scripts/dopackages.wrapper index 16e7377..fea1dd3 100755 --- a/scripts/dopackages.wrapper +++ b/scripts/dopackages.wrapper @@ -6,7 +6,8 @@ # NOTE: rearranged 20100615 to not have to be linked to dopackages.N # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} if [ $# -lt 3 ]; then echo "usage: $0 [ ...]" @@ -18,27 +19,27 @@ branch=$2 buildid=$3 shift 3 -. ${pb}/conf/server.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/scripts/buildenv if ! validate_env ${arch} ${branch} ; then echo "dopackages.wrapper: invalid build environment: \"${arch}/${branch}\"" exit 1 fi -buildid2=$(resolve ${pb} ${arch} ${branch} ${buildid}) +buildid2=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${buildid2}" ]; then echo "dopackages.wrapper: build ID \"${buildid}\" does not exist for \"${arch}/${branch}\"" exit 1 fi -lock=${pb}/${arch}/${branch}/${buildid2}.lock +lock=${pbd}/${arch}/${branch}/${buildid2}.lock # datestamp will be used as buildid iff the latest/previous dance # is needed, but in any case, it distinguishes all logfile names # (e.g. for restarted and continued builds, there will be unique # lognames for each attempt.) datestamp=$(date '+%Y%m%d%H%M%S')"."`hostname -s` -logdir=${pb}/${arch}/archive/buildlogs +logdir=${pbd}/${arch}/archive/buildlogs if [ ! -d ${logdir} ]; then echo "use 'build create' to create the ${logdir} directory before trying dopackages" exit 1 @@ -47,7 +48,7 @@ logname=${logdir}/log.${branch}.${datestamp} dorun() { - lockf -k -t 0 ${lock} ${pb}/scripts/dopackages ${arch} ${branch} ${buildid2} ${datestamp} $@ 2>&1 \ + lockf -k -t 0 ${lock} ${pbc}/scripts/dopackages ${arch} ${branch} ${buildid2} ${datestamp} $@ 2>&1 \ | tee ${logname} } diff --git a/scripts/dopackagestats b/scripts/dopackagestats index eb909ee..19a258f 100755 --- a/scripts/dopackagestats +++ b/scripts/dopackagestats @@ -5,8 +5,10 @@ # accessible to the web server. # -pb=/var/portbuild -. ${pb}/conf/server.conf +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} + +. ${pbc}/conf/server.conf here=`pwd` tmp=`basename $0 | sed -e "s/^do//"`".html" @@ -59,7 +61,7 @@ write_row () { arch=$1 build=$2 - directory=${pb}/${arch}/${build}/builds/latest + directory=${pbd}/${arch}/${build}/builds/latest journal=${directory}/${journalname} branch=`echo $build | awk -F '-' '{print $1}'` @@ -176,7 +178,7 @@ write_row () { running_processes_for_build=`ps axww | \ grep "pdispatch $arch $build " | \ grep -v grep | \ - sed -e "s@.*pdispatch @@;s@ /var/portbuild/scripts/.*@@;s@ @-@g" | \ + sed -e "s@.*pdispatch @@;s@ ${pbc}/scripts/.*@@;s@ @-@g" | \ grep "^$arch-$build-2"` if [ ! -z "$running_processes_for_build" ]; then running_flag="Y" @@ -335,7 +337,7 @@ write_header # (i.e. where build = branch, e.g. "7", "10") for arch in ${SUPPORTED_ARCHS}; do - cd ${pb}/${arch} + cd ${pbd}/${arch} builds=`ls | \ grep "${SRC_BRANCHES_PATTERN}$" | \ sort -n` @@ -355,7 +357,7 @@ done # examples: 8.1; 8-exp; 8-exp-gettext; 8.1R for arch in ${SUPPORTED_ARCHS}; do - cd ${pb}/${arch} + cd ${pbd}/${arch} branches=`ls | \ grep "${SRC_BRANCHES_PATTERN}[-\.]" | \ sed -e "s@[-\.].*@@" | \ diff --git a/scripts/dosetupnode b/scripts/dosetupnode index d62fada..a3ed206 100755 --- a/scripts/dosetupnode +++ b/scripts/dosetupnode @@ -15,21 +15,23 @@ # scripts/ and ${arch}/ directories (e.g. after client reboot) # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} + arch=$1 branch=$2 buildid=$3 nodelist=$4 shift 4 -. ${pb}/conf/server.conf -if [ -f ${pb}/${arch}/portbuild.conf ]; then - . ${pb}/${arch}/portbuild.conf +. ${pbc}/conf/server.conf +if [ -f ${pbd}/${arch}/portbuild.conf ]; then + . ${pbd}/${arch}/portbuild.conf else echo "Invalid arch ${arch}" exit 1 fi -. ${pb}/scripts/buildenv +. ${pbc}/scripts/buildenv # Check for non-fatal rsync errors checkerror() { @@ -55,41 +57,42 @@ setup() { echo "setting up of $node started at $(date)" - . ${pb}/${arch}/portbuild.conf - . ${pb}/${arch}/portbuild.${node} + . ${pbd}/${arch}/portbuild.conf + . ${pbd}/${arch}/portbuild.${node} - ${scp_cmd} -q -p ${pb}/scripts/setupnode ${client_user}@${node}:/tmp || return 1 + cmdpath=$(cat ${pbc}/scripts/setupnode | ssh -a ${client_user}@${node} 't=$(mktemp -t portbuild); cat >$t; echo $t; chmod 755 $t') + case ${cmdpath} in /tmp/*) ;; *) echo "Failed to scp claim-chroot to ${host}."; return 1;; esac - client_setup="${ssh_cmd} -n ${client_user}@${node} sh /tmp/setupnode ${pb} ${arch} ${branch} ${buildid} ${scratchdir} \"${portsmd5}\" \"${srcmd5}\" \"${bindistmd5}\"" + client_setup="${ssh_cmd} -n ${client_user}@${node} ${cmdpath} ${pbd} ${arch} ${branch} ${buildid} ${scratchdir} \"${portsmd5}\" \"${srcmd5}\" \"${bindistmd5}\"" args="${nocopy} ${force}" ${client_setup} pre-copy ${args} || (echo "pre-copy for ${node} failed"; return 1) if [ "${norsync}" -eq 0 ]; then - rsync ${rsync_gzip} -e "${ssh_cmd}" -r -l -p --delete ${pb}/scripts ${pb}/sources \ - ${client_user}@${node}:${pb}/ + rsync ${rsync_gzip} -e "${ssh_cmd}" -r -l -p --delete ${pbc}/scripts ${pbc}/sources \ + ${client_user}@${node}:${builddir}/ checkerror $? || (echo "Copying scripts to ${node} failed"; return 1) # client.conf and common.conf can be symlinks outside this dir, so # copy the actual files - rsync ${rsync_gzip} -e "${ssh_cmd}" -r -L -p --delete ${pb}/${arch}/*.conf \ - ${client_user}@${node}:${pb}/${arch} + rsync ${rsync_gzip} -e "${ssh_cmd}" -r -L -p --delete ${pbd}/${arch}/*.conf \ + ${client_user}@${node}:${pbd}/${arch} checkerror $? || (echo "copying *.conf to ${node} failed"; return 1) # portbuild.* can be symlinks outside this dir, so copy the actual # files - rsync ${rsync_gzip} -e "${ssh_cmd}" -r -L -p --delete ${pb}/${arch}/portbuild* \ - ${client_user}@${node}:${pb}/${arch} + rsync ${rsync_gzip} -e "${ssh_cmd}" -r -L -p --delete ${pbd}/${arch}/portbuild* \ + ${client_user}@${node}:${pbd}/${arch} checkerror $? || (echo "copying portbuild* files to ${node} failed"; return 1) - if [ -f "${pb}/${arch}/clients/bindist-${node}.tar" ]; then + if [ -f "${pbd}/${arch}/clients/bindist-${node}.tar" ]; then rsync ${rsync_gzip} -e "${ssh_cmd}" -r -L -p --delete \ - ${pb}/${arch}/clients/bindist-${node}.tar \ - ${client_user}@${node}:${pb}/${arch}/clients/ + ${pbd}/${arch}/clients/bindist-${node}.tar \ + ${client_user}@${node}:${pbd}/${arch}/clients/ checkerror $? || (echo "Copying bindist-${node}.tar to ${node} failed"; return 1) else - echo "Host customization file not found: ${pb}/${arch}/clients/bindist-${node}.tar" + echo "Host customization file not found: ${pbd}/${arch}/clients/bindist-${node}.tar" return 1 fi @@ -109,20 +112,20 @@ setup() { ${client_setup} post-copy ${args} || (echo "post-copy for ${node} failed"; return 1) if [ "${queue}" -eq 1 ]; then - jobs=$(python /var/portbuild/qmanager/qclient jobs | grep "${node}" | grep "${arch}/${branch}/${buildid} package" | awk '{print $1}' | tail +1) + jobs=$(python ${pbc}/qmanager/qclient jobs | grep "${node}" | grep "${arch}/${branch}/${buildid} package" | awk '{print $1}' | tail +1) for j in ${jobs}; do - python /var/portbuild/qmanager/qclient release $j + python ${pbc}/qmanager/qclient release $j done fi if [ "${full}" -eq 1 ]; then - ${ssh_cmd} ${client_user}@${node} ${sudo_cmd} rm -rf ${pb}/${arch}/${branch}/builds/${buildid}/.ready ${pb}/${arch}/${branch}/builds/${buildid} /tmp/.setup-${buildid} + ${ssh_cmd} ${client_user}@${node} ${sudo_cmd} rm -rf ${pbd}/${arch}/${branch}/builds/${buildid}/.ready ${pbd}/${arch}/${branch}/builds/${buildid} /tmp/.setup-${buildid} fi echo "setting up of $node ended at $(date)" } -pbab=${pb}/${arch}/${branch} +pbab=${pbd}/${arch}/${branch} norsync=0 queue=0 @@ -149,7 +152,7 @@ done if [ "${norsync}" -eq 0 ]; then if [ "${branch}" != "-" -a "${buildid}" != "-" ]; then - buildid=$(resolve ${pb} ${arch} ${branch} ${buildid}) + buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${buildid}" ]; then echo "Invalid build ID ${buildid}" @@ -182,7 +185,7 @@ if [ "${norsync}" -eq 0 ]; then fi if [ "${nodelist}" = "all" ]; then - nodelist=$(cat ${pb}/${arch}/mlist) + nodelist=$(cat ${pbd}/${arch}/mlist) fi for node in ${nodelist}; do diff --git a/scripts/flushsquid b/scripts/flushsquid index 094d604..b097582 100755 --- a/scripts/flushsquid +++ b/scripts/flushsquid @@ -5,12 +5,14 @@ # note: uname is not being overridden arch=$(uname -m) -pb=/var/portbuild -. ${pb}/${arch}/client.conf -. ${pb}/${arch}/portbuild.conf + +pbd=${PORTBUILD_DATA:-/var/portbuild} + +. ${pbd}/${arch}/client.conf +. ${pbd}/${arch}/portbuild.conf host=$(hostname) -test -f ${pb}/${arch}/portbuild.${host} && . ${pb}/${arch}/portbuild.${host} +test -f ${pbd}/${arch}/portbuild.${host} && . ${pbd}/${arch}/portbuild.${host} if [ ! -z "${squid_dir}" ] ; then /usr/local/etc/rc.d/squid stop diff --git a/scripts/keeprestr b/scripts/keeprestr index 12e14a4..a99d63e 100755 --- a/scripts/keeprestr +++ b/scripts/keeprestr @@ -16,18 +16,19 @@ arch=$1 branch=$2 buildid=$3 -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} -. ${pb}/conf/server.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/scripts/buildenv -buildid=$(resolve ${pb} ${arch} ${branch} ${buildid}) +buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${buildid}" ]; then echo "Invalid build ID ${buildid}" exit 1 fi -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} cd ${builddir} rm -rf bak/restricted diff --git a/scripts/makeduds b/scripts/makeduds index 70c3873..b2dba80 100755 --- a/scripts/makeduds +++ b/scripts/makeduds @@ -10,23 +10,25 @@ if [ $# -lt 3 ]; then fi # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} + arch=$1 branch=$2 buildid=$3 target=$4 -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} -. ${pb}/conf/server.conf -. ${pb}/conf/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/conf/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbc}/scripts/buildenv # -j# to make duds DUDSJOBS=4 -buildenv ${pb} ${arch} ${branch} ${builddir} +buildenv ${pbd} ${arch} ${branch} ${builddir} duds=${builddir}/duds index=${PORTSDIR}/${INDEXFILE} diff --git a/scripts/makeindex b/scripts/makeindex index c7fa986..886ab91 100755 --- a/scripts/makeindex +++ b/scripts/makeindex @@ -13,7 +13,8 @@ export INDEX_JOBS=6 # For debugging purposes only #export INDEX_VERBOSE=1 -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} usage () { echo "usage: makeindex arch branch buildid [target]" @@ -29,15 +30,15 @@ branch=$2 buildid=$3 target=$4 -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} -. ${pb}/conf/server.conf -. ${pb}/conf/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/conf/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbc}/scripts/buildenv # Set up the build env variables -buildenv ${pb} ${arch} ${branch} ${builddir} +buildenv ${pbd} ${arch} ${branch} ${builddir} unset DISPLAY diff --git a/scripts/makerestr b/scripts/makerestr index cc3217a..474cf26 100755 --- a/scripts/makerestr +++ b/scripts/makerestr @@ -9,20 +9,22 @@ if [ $# -lt 3 ]; then fi # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} + arch=$1 branch=$2 buildid=$3 target=$4 -. ${pb}/conf/server.conf -. ${pb}/conf/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/conf/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbc}/scripts/buildenv -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} -buildenv ${pb} ${arch} ${branch} ${builddir} +buildenv ${pbd} ${arch} ${branch} ${builddir} unset DISPLAY diff --git a/scripts/makeworld b/scripts/makeworld index 7b25212..6e0b0eb 100755 --- a/scripts/makeworld +++ b/scripts/makeworld @@ -14,13 +14,14 @@ branch=$2 buildid=$3 shift 3 -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} -. ${pb}/conf/server.conf -. ${pb}/conf/common.conf -. ${pb}/${arch}/portbuild.conf +. ${pbc}/conf/server.conf +. ${pbc}/conf/common.conf +. ${pbd}/${arch}/portbuild.conf # NB: we can't use buildenv because it sets ARCH and MACHINE_ARCH that # confuses cross-builds @@ -49,7 +50,7 @@ done # XXX MCL I don't know what this is supposed to do. if [ "$client" = "1" ]; then - SRC_BASE=${pb}/${arch}/src-client + SRC_BASE=${pbd}/${arch}/src-client shift 1 else SRC_BASE=${builddir}/src diff --git a/scripts/mkbindist b/scripts/mkbindist index d2d70ba..cdc0970 100755 --- a/scripts/mkbindist +++ b/scripts/mkbindist @@ -25,18 +25,20 @@ branch=$2 buildid=$3 shift 3 -pb=/var/portbuild -. ${pb}/conf/server.conf -. ${pb}/${arch}/portbuild.conf +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbd}/${arch}/portbuild.conf + +. ${pbc}/scripts/buildenv if ! validate_env ${arch} ${branch}; then echo "Invalid build environment ${arch}/${branch}" exit 1 fi -here=${pb}/${arch}/${branch}/builds/${buildid} +here=${pbd}/${arch}/${branch}/builds/${buildid} if [ ! -d ${here} ]; then echo "Invalid build ID ${buildid}" exit 1 diff --git a/scripts/pdispatch b/scripts/pdispatch index 2a4dc40..e8d3aa5 100755 --- a/scripts/pdispatch +++ b/scripts/pdispatch @@ -6,7 +6,8 @@ # # server-side script to dispatch the job to a host via the ptimeout script. -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} arch=$1 branch=$2 @@ -15,25 +16,25 @@ host=$4 command=$5 shift 5 -pbab=${pb}/${arch}/${branch} +pbab=${pbd}/${arch}/${branch} -. ${pb}/conf/server.conf -. ${pb}/conf/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/conf/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbc}/scripts/buildenv timeout=${PDISPATCH_TIMEOUT} loglength=${PDISPATCH_LOGLENGTH} hdrlength=${PDISPATCH_HDRLENGTH} -buildid=$(resolve ${pb} ${arch} ${branch} ${buildid}) +buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${buildid}" ]; then echo "Invalid build ID ${buildid}" exit 1 fi builddir=${pbab}/builds/${buildid} -buildenv ${pb} ${arch} ${branch} ${builddir} +buildenv ${pbd} ${arch} ${branch} ${builddir} # XXX needed still? unset DISPLAY @@ -77,16 +78,22 @@ if [ "x$TRYBROKEN" != "x" ]; then fi chroot= -. ${pb}/${arch}/portbuild.conf -test -f ${pb}/${arch}/portbuild.${host} && . ${pb}/${arch}/portbuild.${host} -chrootdata=$(${ssh_cmd} -a -n ${client_user}@${host} ${sudo_cmd} ${pb}/scripts/claim-chroot ${arch} ${branch} ${buildid} ${pkgname} 2>&1) +. ${pbd}/${arch}/portbuild.conf +test -f ${pbd}/${arch}/portbuild.${host} && . ${pbd}/${arch}/portbuild.${host} + +# Upload scripts/claim-chroot as per-build scripts aren't in place yet. +cmdpath=$(cat ${pbc}/scripts/claim-chroot | ssh -a ${client_user}@${host} 't=$(mktemp -t portbuild); cat >$t; echo $t; chmod 755 $t') +case ${cmdpath} in /tmp/*) ;; *) echo "Failed to scp claim-chroot to ${host}."; exit 254;; esac +chrootdata=$(${ssh_cmd} -a -n ${client_user}@${host} ${sudo_cmd} ${cmdpath} ${arch} ${branch} ${buildid} ${pkgname} 2>&1) +${ssh_cmd} -a ${client_user}@${host} "rm -f ${cmdpath}" + if [ -z "${chrootdata}" ]; then echo "Failed to claim chroot on ${host}" exit 254 fi case "${chrootdata}" in - */var/portbuild/scripts/claim-chroot*) + *${cmdpath}*) # Error executing script, assume system is booting chrootdata="wait boot" ;; @@ -111,7 +118,7 @@ if [ $# -ge 2 ]; then # Make sure to close stdin/stderr in the child # or make will hang until the child process # exits - ${pb}/scripts/dosetupnode ${arch} ${branch} ${buildid} ${host} > /tmp/setupnode.$$ 2>&1 & + ${pbc}/scripts/dosetupnode ${arch} ${branch} ${buildid} ${host} > /tmp/setupnode.$$ 2>&1 & exit 253 ;; error) @@ -128,8 +135,8 @@ if [ -z "${chroot}" ]; then exit 254 fi -. ${pb}/${arch}/portbuild.conf -test -f ${pb}/${arch}/portbuild.${host} && . ${pb}/${arch}/portbuild.${host} +. ${pbd}/${arch}/portbuild.conf +test -f ${pbd}/${arch}/portbuild.${host} && . ${pbd}/${arch}/portbuild.${host} rm -f ${builddir}/logs/${pkgname}.log ${builddir}/logs/${pkgname}.log.bz2 rm -f ${builddir}/errors/${pkgname}.log ${builddir}/errors/${pkgname}.log.bz2 @@ -162,7 +169,7 @@ if [ "${error}" = 0 ]; then # Force rebuild of html page to remove this package from list touch ${builddir}/errors/.force fi - lockf -k ${pbab}/failure.lock ${pb}/scripts/buildsuccess ${arch} ${branch} ${buildid} ${pkgname} + lockf -k ${pbab}/failure.lock ${pbc}/scripts/buildsuccess ${arch} ${branch} ${buildid} ${pkgname} log=${builddir}/logs/$pkgname.log if grep -q "even though it is marked BROKEN" ${log}; then echo | mail -s "${pkgname} BROKEN but built on ${arch} ${branch}" ${mailto} @@ -188,11 +195,11 @@ else (echo "Excerpt from the build log at"; echo; echo " http://${MASTER_URL}/errorlogs/${arch}-errorlogs/${baselogdir}/$(basename $log)";echo;sed -e '/^build started/,$d' $log;echo;echo " [... lines trimmed ...]";echo;tail -${loglength} ${log}) | mail -s "${pkgname} failed on ${arch} ${branch}" ${mailto} fi fi - lockf -k ${pbab}/failure.lock ${pb}/scripts/buildfailure ${arch} ${branch} ${buildid} ${pkgname} + lockf -k ${pbab}/failure.lock ${pbc}/scripts/buildfailure ${arch} ${branch} ${buildid} ${pkgname} fi fi -${ssh_cmd} -a -n ${client_user}@${host} ${sudo_cmd} ${pb}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${chroot} ${clean} +${ssh_cmd} -a -n ${client_user}@${host} ${sudo_cmd} ${builddir}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${chroot} ${clean} # XXX Set a dirty variable earlier and check here if grep -q "^build of .*ended at" ${builddir}/logs/${pkgname}.log; then diff --git a/scripts/pollmachine b/scripts/pollmachine index ddc7100..8e35bc9 100755 --- a/scripts/pollmachine +++ b/scripts/pollmachine @@ -28,6 +28,13 @@ import sys, threading, socket from time import sleep import os, subprocess, logging +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" + +sys.path.insert(0, '%s/lib/python' % pbc) + if len(sys.argv) < 1: print "Usage: %s [ ...]" % sys.argv[0] sys.exit(1) @@ -52,8 +59,6 @@ for i in sys.argv[1:]: else: arches.add(i) -pb="/var/portbuild" - # set of machines for each arch machines={} for i in arches: @@ -181,8 +186,8 @@ class MachinePoll(threading.Thread): envs = self.vars['buildenvs'] for e in envs.split(): (arch, branch, buildid) = e.split("/") - f = "/var/portbuild/%s/%s/builds/%s/.active" % \ - (arch, branch, buildid) + f = "%s/%s/%s/builds/%s/.active" % \ + (pbd, arch, branch, buildid) if os.path.exists(f): continue # Clean up a stale buildenv @@ -219,7 +224,7 @@ class MachinePoll(threading.Thread): # Record current system load try: - f = file("%s/%s/loads/%s" % (pb, self.arch, self.mach), "w") + f = file("%s/%s/loads/%s" % (pbd, self.arch, self.mach), "w") except: return try: @@ -229,8 +234,8 @@ class MachinePoll(threading.Thread): f.close() def setup(self, branch, buildid, args = ""): - cmd = "su ports-%s -c \"/var/portbuild/scripts/dosetupnode %s %s %s %s %s\""\ - % (self.arch, self.arch, branch, buildid, self.mach, args) + cmd = "su ports-%s -c \"%s/scripts/dosetupnode %s %s %s %s %s\""\ + % (pbc, self.arch, self.arch, branch, buildid, self.mach, args) child = subprocess.Popen(cmd, shell=True, stderr = subprocess.STDOUT, stdout = subprocess.PIPE) err = child.wait() @@ -254,7 +259,7 @@ while True: try: now = mlist[arch] except KeyError: - mlistfile="%s/%s/mlist" % (pb, arch) + mlistfile="%s/%s/mlist" % (pbd, arch) try: f = file(mlistfile, "r") except OSError, error: @@ -278,8 +283,8 @@ while True: logging.info("Adding machine %s/%s" % (arch, mach)) # XXX set up qmgr - pc="%s/%s/portbuild.conf" % (pb, arch) - pch="%s/%s/portbuild.%s" % (pb, arch, mach) + pc="%s/%s/portbuild.conf" % (pbd, arch) + pch="%s/%s/portbuild.%s" % (pbd, arch, mach) cmd = "test -f %s && . %s; test -f %s && . %s; echo $infoseek_host; echo $infoseek_port" % (pc, pc, pch, pch) config = subprocess.Popen(cmd, shell = True, stdout = subprocess.PIPE) diff --git a/scripts/portbuild b/scripts/portbuild index 19d13ee..0ca2109 100755 --- a/scripts/portbuild +++ b/scripts/portbuild @@ -8,7 +8,7 @@ # usage: $0 ARCH BRANCH BUILDID CHROOT [-noclean] [-norestr] [-noplistcheck] [-distfiles] [-fetch-original] [-trybroken] PKGNAME.tgz DIRNAME [DEPENDENCY.tgz ...] -pb=/var/portbuild +pbd=${PORTBUILD_DATA:-/var/portbuild} mount_fs() { @@ -25,14 +25,14 @@ mount_fs() copypkg() { - pb=$1 + pbd=$1 host=$2 from=$3 to=$4 http_proxy=$5 if [ ${host} = $(hostname) ]; then - cp ${pb}/${arch}/${branch}/packages/All/${from} ${to} + cp ${pbd}/${arch}/${branch}/packages/All/${from} ${to} else if [ ! -z "${http_proxy}" ]; then env HTTP_PROXY=${http_proxy} fetch -m -o ${to} http://${host}/errorlogs/${arch}-${branch}-packages-latest/All/${from} @@ -64,12 +64,12 @@ shift 4 # Default niceness value nice=0 -. ${pb}/${arch}/client.conf -. ${pb}/${arch}/common.conf +. ${pbd}/${arch}/client.conf +. ${pbd}/${arch}/common.conf # note: should NOT need anything from server.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/${arch}/portbuild.$(hostname) -. ${pb}/scripts/buildenv +. ${pbd}/${arch}/portbuild.conf +. ${pbd}/${arch}/portbuild.$(hostname) +. ${pbd}/scripts/buildenv buildroot=${scratchdir} error=0 @@ -117,7 +117,7 @@ FD=$3 BD=$4 RD=$5 -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} buildenv.common @@ -286,7 +286,7 @@ while [ $# -gt 0 ]; do # XXX MCL more hard-coding if [ ! -f ${chroot}/tmp/depends/$1 ]; then echo "copying package $1 for ${pkgname}" - copypkg ${pb} ${CLIENT_UPLOAD_HOST} $1 ${chroot}/tmp/depends "${http_proxy}" + copypkg ${pbd} ${CLIENT_UPLOAD_HOST} $1 ${chroot}/tmp/depends "${http_proxy}" # Test for copy failure and bail # XXX MCL more hard-coding @@ -298,8 +298,8 @@ while [ $# -gt 0 ]; do shift done -cp -p ${pb}/scripts/buildscript ${chroot} -cp -p ${pb}/sources/pnohang.c ${chroot} +cp -p ${builddir}/scripts/buildscript ${chroot} +cp -p ${builddir}/sources/pnohang.c ${chroot} # phase 0, compile pnohang chroot ${chroot} /usr/bin/gcc -o /pnohang -Wall /pnohang.c 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log diff --git a/scripts/processfail b/scripts/processfail index 5c41d09..f5bce00 100755 --- a/scripts/processfail +++ b/scripts/processfail @@ -5,19 +5,21 @@ arch=$1 branch=$2 -pb=/var/portbuild -. ${pb}/conf/server.conf +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} + +. ${pbc}/conf/server.conf ERRORLOGS_DIRECTORY="${WWW_DIRECTORY}/errorlogs" of=${ERRORLOGS_DIRECTORY}/.${arch}-${branch}-failure.html -cd ${pb}/${arch}/${branch} +cd ${pbd}/${arch}/${branch} if [ -e .newfailure.stamp -a $(echo $(find . -maxdepth 1 -newer .newfailure.stamp -name newfailure 2>&1 /dev/null | wc -l)) = "0" ]; then exit; fi touch .newfailure.stamp -newfailure=${pb}/${arch}/${branch}/newfailure +newfailure=${pbd}/${arch}/${branch}/newfailure num=0 if [ -e ${newfailure} ]; then num=$(wc -l ${newfailure} | awk '{print $1}') @@ -59,8 +61,8 @@ IFS='|' while read dir name ver date last count; do echo "" >> $of echo "$dir" >> $of - if [ -L ${pb}/${arch}/${branch}/latest/${dir} ]; then - err=$(readlink ${pb}/${arch}/${branch}/latest/${dir}) + if [ -L ${pbd}/${arch}/${branch}/latest/${dir} ]; then + err=$(readlink ${pbd}/${arch}/${branch}/latest/${dir}) echo "$ver" >> $of else echo "$ver" >> $of diff --git a/scripts/processlogs b/scripts/processlogs index b79673c..9b6a15f 100755 --- a/scripts/processlogs +++ b/scripts/processlogs @@ -39,9 +39,11 @@ scriptdir=$(dirname $0) errorscript=$scriptdir/processonelog # Figure out which arch we're building for -pb=/var/portbuild -. ${pb}/conf/server.conf -. ${pb}/${arch}/portbuild.conf +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} + +. ${pbc}/conf/server.conf +. ${pbd}/${arch}/portbuild.conf # get the list of buildlogs. diff --git a/scripts/prunefailure b/scripts/prunefailure index e8bc09e..20ed1b7 100755 --- a/scripts/prunefailure +++ b/scripts/prunefailure @@ -4,13 +4,14 @@ # # This must be called via: # -# lockf -k ${pb}/${arch}/${branch}/failure.lock ${pb}/scripts/prunefailure ${arch} ${branch} ${buildid} +# lockf -k ${pbd}/${arch}/${branch}/failure.lock ${pbc}/scripts/prunefailure ${arch} ${branch} ${buildid} # # to avoid racing with any package builds in progress that might try to append to # these files. # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} cleanup() { echo "Problem writing new failure file!" @@ -28,16 +29,16 @@ branch=$2 buildid=$3 shift 3 -. ${pb}/conf/server.conf -. ${pb}/conf/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/conf/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbc}/scripts/buildenv -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} -buildenv ${pb} ${arch} ${branch} ${builddir} +buildenv ${pbd} ${arch} ${branch} ${builddir} -home=${pb}/${arch}/${branch} +home=${pbd}/${arch}/${branch} cd $home pkgdir=${builddir}/packages/All @@ -57,14 +58,14 @@ while read dir name ver olddate date count; do if [ -z "$dir" -o -z "$name" -o -z "$ver" -o -z "$olddate" -o -z "$date" -o -z "$count" ]; then echo Malformed entry "$dir|$name|$ver|$olddate|$date|$count" # Clean up the 'latest error log' symlink - rm -f ${pb}/${arch}/${branch}/latest/${dir} + rm -f ${pbd}/${arch}/${branch}/latest/${dir} continue fi entry=$(grep "|/usr/ports/$dir|" $index) if [ -z "$entry" ]; then echo $dir not in index - rm -f ${pb}/${arch}/${branch}/latest/${dir} + rm -f ${pbd}/${arch}/${branch}/latest/${dir} continue fi @@ -72,13 +73,13 @@ while read dir name ver olddate date count; do if [ -e "${builddir}/packages/All/$newver${PKGSUFFIX}" ]; then echo "$newver package exists, should not still be here!" - rm -f ${pb}/${arch}/${branch}/latest/${dir} + rm -f ${pbd}/${arch}/${branch}/latest/${dir} continue fi if grep -qxF $newver ${builddir}/duds.full; then echo "$newver listed in duds, should not be here" - rm -f ${pb}/${arch}/${branch}/latest/${dir} + rm -f ${pbd}/${arch}/${branch}/latest/${dir} continue fi diff --git a/scripts/reportload b/scripts/reportload index cfeaaac..7bc7d05 100755 --- a/scripts/reportload +++ b/scripts/reportload @@ -3,15 +3,16 @@ # client script to report load to the server. -pb=/var/portbuild +pbd=${PORTBUILD_DATA:-/var/portbuild} + # note: uname is not being overridden arch=$(uname -m) osver=$(sysctl -n kern.osreldate) -. ${pb}/${arch}/client.conf -. ${pb}/${arch}/portbuild.conf -if [ -f ${pb}/${arch}/portbuild.$(hostname) ]; then - . ${pb}/${arch}/portbuild.$(hostname) +. ${pbd}/${arch}/client.conf +. ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/portbuild.$(hostname) ]; then + . ${pbd}/${arch}/portbuild.$(hostname) fi # Look for exceptional conditions @@ -35,7 +36,7 @@ echo "arch=${arch}" echo "osversion=${osver}" echo "jobs=${num}" -cd /var/portbuild/${arch} +cd ${pbd}/${arch} for i in */builds/*; do buildenv=${arch}/${i%%/*}/${i##*/} if [ -f ${i}/.ready ]; then diff --git a/scripts/setupnode b/scripts/setupnode index f5c995c..96d6e83 100755 --- a/scripts/setupnode +++ b/scripts/setupnode @@ -10,7 +10,7 @@ if [ $# -lt 8 ]; then exit 1 fi -pb=$1 +pbd=$1 arch=$2 branch=$3 buildid=$4 @@ -29,7 +29,7 @@ precopy() { # Prepare all directories, they will be populated by a rsync # push from the master - mkdir -p ${pb}/scripts ${pb}/${arch}/clients/ + mkdir -p ${pbd}/${arch}/clients/ if [ "${buildid}" != "-" -a "${branch}" != "-" ]; then if [ ${nocopy} -eq 0 ]; then @@ -96,10 +96,10 @@ postcopy() { # Clean up the tmpdir # By now the portbuild.conf files are in place so we can source them - . ${pb}/${arch}/portbuild.conf + . ${pbd}/${arch}/portbuild.conf me=$(hostname) - if [ -f ${pb}/${arch}/portbuild.${me} ] ; then - . ${pb}/${arch}/portbuild.${me} + if [ -f ${pbd}/${arch}/portbuild.${me} ] ; then + . ${pbd}/${arch}/portbuild.${me} fi if [ "${buildid}" != "-" -a "${branch}" != "-" ]; then @@ -111,20 +111,20 @@ postcopy() { zfs create ${zbase}/${branch}/${buildid}/world || true zfs create ${zbase}/${branch}/${buildid}/chroot || true tar xfpC ${builddir}/bindist.tar ${scratchdir}/${branch}/${buildid}/world - tar xfpC ${pb}/${arch}/clients/bindist-$(hostname).tar ${scratchdir}/${branch}/${buildid}/world + tar xfpC ${pbd}/${arch}/clients/bindist-$(hostname).tar ${scratchdir}/${branch}/${buildid}/world zfs snapshot ${zbase}/${branch}/${buildid}/world@base else mkdir -p ${wrkdir}/tarballs if [ ${nocopy} -eq 0 ]; then ln -sf ${pbab}/builds/${buildid}/bindist.tar ${wrkdir}/tarballs - ln -sf ${pb}/${arch}/clients/bindist-$(hostname).tar ${wrkdir}/tarballs + ln -sf ${pbd}/${arch}/clients/bindist-$(hostname).tar ${wrkdir}/tarballs fi fi fi for i in ${wrkdir}/chroot/*; do - ${sudo_cmd} ${pb}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${i} 2 + ${sudo_cmd} ${builddir}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${i} 2 done wait else @@ -136,7 +136,7 @@ postcopy() { } if [ "${branch}" != "-" -a "${buildid}" != "-" ]; then - pbab=${pb}/${arch}/${branch} + pbab=${pbd}/${arch}/${branch} builddir=${pbab}/builds/${buildid} wrkdir=${tmpdir}/${branch}/${buildid} diff --git a/scripts/stats b/scripts/stats index 3e84e57..443266a 100755 --- a/scripts/stats +++ b/scripts/stats @@ -1,7 +1,9 @@ #!/bin/sh -pb=/var/portbuild -. ${pb}/conf/server.conf +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} + +. ${pbc}/conf/server.conf if [ $# -ne 1 ]; then echo "usage: " @@ -11,7 +13,7 @@ fi branch=$1 for i in ${SUPPORTED_ARCHS}; do - all=${pb}/$i/${branch}/builds/latest/packages/All + all=${pbd}/$i/${branch}/builds/latest/packages/All if [ -d ${all} ]; then count=$(find ${all} -name \*.tbz -o -name \*.tgz |wc -l) echo -n "$i: ${count} " diff --git a/scripts/updatesnap b/scripts/updatesnap index 2fc9cce..918c149 100755 --- a/scripts/updatesnap +++ b/scripts/updatesnap @@ -3,8 +3,9 @@ # Update the master source trees that are used by package builds # and other consumers -pb=/var/portbuild -. ${pb}/conf/server.conf +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} + +. ${pbc}/conf/server.conf base=${ZFS_MOUNTPOINT}/${SNAP_SRC_DIRECTORY_PREFIX} zbase=${ZFS_VOLUME}/${SNAP_SRC_DIRECTORY_PREFIX} diff --git a/scripts/updatesnap.ports b/scripts/updatesnap.ports index 81a928d..2a0c479 100755 --- a/scripts/updatesnap.ports +++ b/scripts/updatesnap.ports @@ -3,8 +3,9 @@ # Update the master ports tree that is used by package builds # and other consumers -pb=/var/portbuild -. ${pb}/conf/server.conf +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} + +. ${pbc}/conf/server.conf base=${ZFS_MOUNTPOINT}/${SNAP_PORTS_DIRECTORY} zbase=${ZFS_VOLUME}/${SNAP_PORTS_DIRECTORY} -- 1.7.3.5