Index: portbuild
===================================================================
--- portbuild	(revision 234662)
+++ portbuild	(working copy)
@@ -46,11 +46,12 @@
 
 bailout()
 {
-  chroot=$1
+  jname=$1
   clean=$2
   error=$3
   pkgname=$4
 
+  jail -r $jname
   echo -n "$pkgname failed unexpectedly on $(hostname) at "
   date
 
@@ -61,13 +62,14 @@
 branch=$2
 buildid=$3
 chroot=$4
+jname=${branch}-${chroot##*/}
 shift 4
 
 # Default niceness value
 nice=0
 
 if [ ! -d ${pbd}/${arch}/ ]; then
-  bailout ${chroot} 0 255 ${pkgname}
+  bailout ${jname} 0 255 ${pkgname}
 fi
 . ${pbd}/${arch}/client.conf
 . ${pbd}/${arch}/common.conf
@@ -81,7 +83,7 @@
 # if buildenv isn't there, then ${pbd} is probably the default, which for
 # some reason you have not populated.
 if [ ! -f ${pbd}/scripts/buildenv ]; then
-  bailout ${chroot} 0 255 ${pkgname}
+  bailout ${jname} 0 255 ${pkgname}
 fi
 . ${pbd}/scripts/buildenv
 
@@ -186,7 +188,7 @@
 bindist=${buildroot}/${branch}/${buildid}/tarballs/bindist.tar
 bindistlocal=${buildroot}/${branch}/${buildid}/tarballs/bindist-$(hostname).tar
 if [ ! -f $bindist ]; then
-  bailout ${chroot} ${clean} 255 ${pkgname}
+  bailout ${jname} ${clean} 255 ${pkgname}
 fi
 
 if [ -f ${chroot}/.notready ]; then
@@ -194,6 +196,16 @@
   if [ -f ${bindistlocal} ]; then
     tar -C ${chroot} -xpf ${bindistlocal}
   fi
+  # jexec -U root will have the right arch in uname -m and uname -p
+  LOGIN_ENV=",UNAME_p=${arch},UNAME_m=${arch}"
+  sed -i "" -e "s/:\(setenv.*\):/:\1${LOGIN_ENV}:/" ${chroot}/etc/login.conf
+  cap_mkdb ${chroot}/etc/login.conf
+
+  cat >> ${chroot}/etc/make.conf << EOF
+ARCH=${arch}
+MACHINE=${arch}
+MACHINE_ARCH=${arch}
+EOF
   # allow additional (not replacement!) knobs for make.conf
   if [ -f ${builddir}/make.conf.client ]; then
       cat ${builddir}/make.conf.client >> ${chroot}/etc/make.conf
@@ -208,20 +220,11 @@
   touch ${chroot}/.ready
 fi
 
-if [ "${use_jail}" = "1" ]; then
-  # Figure out jail IP addr
-  chrootpid=$(basename ${chroot})
-  ipbase=$((${chrootpid}+2))
-  ip1=$(($ipbase /(256*256)))
-  ip2=$((($ipbase - ($ip1*256*256)) /256))
-  ip3=$((($ipbase - ($ip1*256*256) - ($ip2*256))))
-fi
+trap "bailout ${jname} ${clean} ${error} ${pkgname}" 1 2 3 9 10 11 15
 
-trap "bailout ${chroot} ${clean} ${error} ${pkgname}" 1 2 3 9 10 11 15
-
 rm -rf ${chroot}/tmp/*
-cd ${chroot}/tmp || bailout ${chroot} ${clean} 255 ${pkgname}
-mkdir -p depends distfiles packages || bailout ${chroot} ${clean} 255 ${pkgname}
+cd ${chroot}/tmp || bailout ${jname} ${clean} 255 ${pkgname}
+mkdir -p depends distfiles packages || bailout ${jname} ${clean} 255 ${pkgname}
 
 echo "building ${pkgname} on $(hostname)" | tee ${chroot}/tmp/${pkgname}.log
 echo "in directory ${chroot}" | tee -a ${chroot}/tmp/${pkgname}.log
@@ -247,12 +250,12 @@
   fi
 fi
 
-mount_fs ${builddir}/ports ${chroot}/a/ports ${CLIENT_NFS_MASTER} || bailout ${chroot} ${clean} 255 ${pkgname}
+mount_fs ${builddir}/ports ${chroot}/a/ports ${CLIENT_NFS_MASTER} || bailout ${jname} ${clean} 255 ${pkgname}
 ln -sf ../a/ports ${chroot}/usr/ports
 
-mkdir -p ${chroot}/usr/src || bailout ${chroot} ${clean} 255 ${pkgname}
+mkdir -p ${chroot}/usr/src || bailout ${jname} ${clean} 255 ${pkgname}
 
-mount_fs ${builddir}/src ${chroot}${CLIENT_SRCBASE} ${CLIENT_NFS_MASTER} || bailout ${chroot} ${clean} 255 ${pkgname}
+mount_fs ${builddir}/src ${chroot}${CLIENT_SRCBASE} ${CLIENT_NFS_MASTER} || bailout ${jname} ${clean} 255 ${pkgname}
 
 # set overrides for uname
 buildenv.client ${arch} ${chroot}${CLIENT_SRCBASE}
@@ -283,27 +286,21 @@
 for i in ${ARCHS_REQUIRING_LINPROCFS}; do
   if [ ${i} = ${arch} ]; then
     # JDK ports need linprocfs :(
-    mkdir -p ${chroot}/compat/linux/proc || bailout ${chroot} ${clean} 255 ${pkgname}
-    mount -t linprocfs linprocfs ${chroot}/compat/linux/proc || bailout ${chroot} ${clean} 255 ${pkgname}
+    mkdir -p ${chroot}/compat/linux/proc || bailout ${jname} ${clean} 255 ${pkgname}
+    mount -t linprocfs linprocfs ${chroot}/compat/linux/proc || bailout ${jname} ${clean} 255 ${pkgname}
     break
   fi
 done
 
-_ldconfig_dirs="/lib /usr/lib /usr/lib/compat"
-ldconfig_dirs=""
-for i in ${_ldconfig_dirs}; do
-  if [ -d ${chroot}/${i} ]; then
-    ldconfig_dirs="${ldconfig_dirs} ${i}"
-  fi
-done
-chroot ${chroot} /sbin/ldconfig ${ldconfig_dirs}
-for i in ${ARCHS_REQUIRING_AOUT_COMPAT}; do
-  if [ ${i} = ${arch} ]; then
-	chroot ${chroot} /sbin/ldconfig -aout /usr/lib/aout /usr/lib/compat/aout
-	break
-  fi
-done
+# Start a jail 
+# XXX bapt: the network 127.0.0.1 allows ports like postfix do not fail querying a network
+# device
+jail -c persist name=${jname} ip4.addr=127.0.0.1 ip6=disable path=${chroot} \
+	host.hostname=${jname} allow.sysvipc allow.socket_af allow.raw_sockets \
+	allow.enforce_statfs allow.chflags_allowed
 
+jexec -U root ${jname} /usr/sbin/service ldconfig start
+
 set x $ED $FD $PD $BD $RD $PKGD
 shift 1
 while [ $# -gt 0 ]; do
@@ -316,7 +313,7 @@
     # XXX MCL more hard-coding
     if [ ! -f ${chroot}/tmp/depends/$1 ]; then
       echo "ERROR: Couldn't copy $1" | tee -a ${chroot}/tmp/${pkgname}.log
-      bailout ${chroot} ${clean} 255 ${pkgname}
+      bailout ${jname} ${clean} 255 ${pkgname}
     fi
   fi
   shift
@@ -327,7 +324,7 @@
 # phase 0, compile pnohang if necessary
 if [ ! -f /tmp/${buildid}/sources/pnohang ]; then
   cp -p /tmp/${buildid}/sources/pnohang.c ${chroot}
-  chroot ${chroot} /usr/bin/gcc -o /pnohang -Wall /pnohang.c 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log
+  jexec -U root ${jname} /usr/bin/gcc -o /pnohang -Wall /pnohang.c 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log
   if [ $? -ne 0 ]; then
     error=255
   else
@@ -340,8 +337,16 @@
 
 if [ "${error}" = 0 ]; then
 # phase 1, make checksum
-# Needs to be chroot not jail so that port can be fetched
-  chroot ${chroot} /buildscript ${dirname} 1 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log
+# restart the jail with network for the fetch part
+  jail -r ${jname}
+  jail -c persist name=${jname} ip4=inherit ip6=inherit path=${chroot} \
+	host.hostname=${jname} allow.sysvipc allow.socket_af allow.raw_sockets \
+	allow.enforce_statfs allow.chflags_allowed
+  jexec -U root ${jname} /buildscript ${dirname} 1 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log
+  jail -r ${jname}
+  jail -c persist name=${jname} ip4.addr=127.0.0.1 ip6=disable path=${chroot} \
+	host.hostname=${jname} allow.sysvipc allow.socket_af allow.raw_sockets \
+	allow.enforce_statfs allow.chflags_allowed
   if [ -f ${chroot}/tmp/status ]; then
     error=$(cat ${chroot}/tmp/status)
   else
@@ -354,13 +359,7 @@
 # phase 2, make package
   ln -sf ${pkgname}.log2 ${chroot}/tmp/make.log
 
-  if [ "${use_jail}" = 1 ]; then
-    ifconfig lo0 alias 127.${ip1}.${ip2}.${ip3}/32
-    jail -J ${chroot}/tmp/jail.id ${chroot} jail-${chrootpid} 127.${ip1}.${ip2}.${ip3} /usr/bin/env JAIL_ADDR=127.${ip1}.${ip2}.${ip3} HTTP_PROXY=${http_proxy} /usr/bin/nice -n $nice /buildscript ${dirname} 2 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" > ${chroot}/tmp/${pkgname}.log2 2>&1
-    ifconfig lo0 delete 127.${ip1}.${ip2}.${ip3}
-  else
-    chroot ${chroot} /usr/bin/nice -n ${nice} /buildscript ${dirname} 2 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" > ${chroot}/tmp/${pkgname}.log2 2>&1
-  fi
+  jexec -U root ${jname} /usr/bin/nice -n ${nice} /buildscript ${dirname} 2 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" > ${chroot}/tmp/${pkgname}.log2 2>&1
 
   grep pnohang ${chroot}/tmp/${pkgname}.log2
   cat ${chroot}/tmp/${pkgname}.log2 >> ${chroot}/tmp/${pkgname}.log
@@ -368,6 +367,7 @@
   error=$(cat ${chroot}/tmp/status)
 fi
 
+jail -r ${jname}
 rm -rf ${chroot}/${WRKDIRPREFIX}
 
 # Record build completion time for ganglia
Index: clean-chroot
===================================================================
--- clean-chroot	(revision 234662)
+++ clean-chroot	(working copy)
@@ -3,37 +3,26 @@
 
 # client-side script to clean up a chroot
 
-kill_procs()
-{
-    dir=$1
-    mount=$2
-
-    pids="XXX"
-    while [ ! -z "${pids}" ]; do
-	pids=$(fstat -f "${dir}${mount}" | tail +2 | awk '{print $3}' | sort -u)
-	if [ ! -z "${pids}" ]; then
-	    echo "clean-chroot: Killing off pids in ${dir}"
-	    ps -p $pids
-	    kill -KILL ${pids} 2> /dev/null
-	    sleep 2
-	fi
-    done
-}
-
 cleanup_mount() {
     chroot=$1
     mount=$2
+
+    chrootnb=${chroot##*/}
+    #chroot dir
+    branch=${chroot%/*}
+    #buildid dir
+    branch=${branch%/*}
+    #branch dir
+    branch=${branch%/*}
+    #branch name
+    branch=${branch##*/}
+
+    jname=${branch}-${chrootnb}
+
+    # Stop the jail if running
+    jls -qj ${jname} > /dev/null 2>&1 && jail -r ${jname}
     
-    if [ -d ${chroot}${mount} ]; then
-	mdir=$(fstat -f ${chroot}${mount} | head -2 | tail -1 | awk '{print $5}')
-	if [ "${mdir}" = "MOUNT" ]; then
-	    umount -f ${chroot}${mount} || echo "clean-chroot: Cleanup of ${chroot}${mount} on $(hostname) failed!"
-	fi
-	if [ "${mdir}" = "${chroot}${mount}" ]; then
-	    kill_procs ${chroot} ${mount}
-	    umount -f ${chroot}${mount} || echo "clean-chroot: Cleanup of ${chroot}${mount} on $(hostname) failed!" 
-	fi
-    fi
+    umount -f ${chroot}${mount} || echo "clean-chroot: Cleanup of ${chroot}${mount} on $(hostname) failed!"
 }
 
 arch=$1
@@ -78,9 +67,12 @@
     fi
 done
 
-for i in /a/ports /usr/src /dev /root/.ccache; do
+for i in /a/ports /usr/src /dev; do
     cleanup_mount ${chroot} ${i}
 done
+if [ ! -z "${ccache_dir}" ]; then
+    cleanup_mount ${chroot} /root/.ccache
+fi
 
 if [ "${use_zfs}" = "1" ]; then
     cleanup_mount ${chroot} ""
Index: cleanup-chroots
===================================================================
--- cleanup-chroots	(revision 234662)
+++ cleanup-chroots	(working copy)
@@ -17,62 +17,34 @@
 
 #VERBOSE=1
 
-kill_procs()
-{
-    dir=$1
-
-    pids="XXX"
-    while [ ! -z "${pids}" ]; do
-	pids=$(fstat -f "$dir" | tail +2 | awk '{print $3}' | sort -u)
-	if [ ! -z "${pids}" ]; then
-	    echo "cleanup-chroots: Killing off pids in ${dir} on $(hostname)"
-	    ps -p $pids
-	    kill -KILL ${pids} 2> /dev/null
-	    sleep 2
-	    echo
-	fi
-    done
-}
-
-# clean up a single mount
-cleanup_mount() {
-    chroot=$1
-    mount=$2
-
-    if [ -d ${chroot}${mount} ]; then
-	mdir=$(fstat -f ${chroot}${mount} | head -2 | tail -1 | awk '{print $5}')
-	if [ "${mdir}" = "MOUNT" ]; then
-	    umount ${chroot}${mount} || echo "cleanup-chroots: Cleanup of ${chroot}${mount} on $(hostname) failed!"
-	fi
-	if [ "${mdir}" = "${chroot}${mount}" ]; then
-	    kill_procs ${chroot}${mount}
-	    umount ${chroot}${mount} || echo "cleanup-chroots: Cleanup of ${chroot}${mount} on $(hostname) failed!"
-	fi
-    fi
-}
-
 # cleanup all mounts (old NFS, devfs, linux), as well as mds
 cleanup_all() {
     chroot=$1
 
-    mounts=$(mount | grep ${chroot} | awk '{print $3}')
-    if [ ! -z "${mounts}" ]; then
-	if [ $VERBOSE ]; then
-	    echo "cleanup-chroots: trying to kill off mounts:"
-	    echo ${mounts}
-	    echo
-	fi
-	for j in ${mounts}; do
-	    umount ${j} || cleanup_mount ${j}
-	done
-	# XXX MCL is this redundant with the above?
-	umount ${chroot}/compat/linux/proc || cleanup_mount ${chroot}/compat/linux/proc
-    fi
-    if [ "${use_zfs}" != "1" -a "${use_md_swap}" = "1" ]; then
-	chrootnum=$(basename ${chroot})
-	umount -f /dev/md${chroot}
-	mdconfig -d -u ${chrootnum}
-    fi
+    chrootnb=${chroot##*/}
+    #chroot dir
+    branch=${chroot%/*}
+    #buildid dir
+    branch=${branch%/*}
+    #branch dir
+    branch=${branch%/*}
+    #branch name
+    branch=${branch##*/}
+
+    jname=${branch}-${chrootnb}
+
+    # Stop the jail if running
+    jls -qj ${jname} > /dev/null 2>&1 && jail -r ${jname}
+
+    # umounting normal device
+    for MNT in $( mount | awk -v mnt="${chroot}/" 'BEGIN{ gsub(/\//, "\\\/", mnt); } { if ($3 ~ mnt && $1 !~ /\/dev\/md/ ) { print $3 }}' |  sort -r ); do
+	umount -f ${MNT}
+    done
+    # umount md device if any
+    for unit in $(mount | awk -v mnt="${chroot}" 'BEGIN{ gsub(/\//, "\\\/", mnt); } { if ($3 ~ mnt && $1 ~ /\/dev\/md/ ) { sub(/\/dev\/md/, "", $1); print $1 }}' | sort -r); do
+	umount -f /dev/md${unit}
+	mdconfig -d -u ${unit}
+    done
 }
 
 # note: uname is not being overridden (should not need client.conf here)