Index: src/poudriere.d/clean.sh ================================================================== --- src/poudriere.d/clean.sh +++ src/poudriere.d/clean.sh @@ -1,10 +1,16 @@ #!/bin/sh JAILMNT=$1 PKGNAME=$2 CLEAN_RDEPENDS=${3:-0} + +sql() { + sqlite3 --init ${JAILMNT}/poudriere/sql \ + ${JAILMNT}/poudriere/queue "$@" || return 1 + return 0 +} case "${JAILMNT}" in /?*) ;; *) @@ -24,21 +30,17 @@ local dep_dir dep_pkgname fulldep # Determine everything that depends on the given package # Recursively cleanup anything that depends on this port. if [ ${clean_rdepends} -eq 1 ]; then - find ${JAILMNT}/poudriere/pool -name "${pkgname}" -type f | while read fulldep; do - dep_dir=${fulldep%/*} - dep_pkgname=${dep_dir##*/} - + for d in $(sql "select origin from deps where port='${pkgname}';") do # clean_pool() in common.sh will pick this up and add to SKIPPED - echo "${dep_pkgname}" - - clean_pool ${dep_pkgname} ${clean_rdepends} + echo "${d}" + clean_pool ${d} ${clean_rdepends} done fi - rm -rf "${JAILMNT}/poudriere/pool/${pkgname}" - find ${JAILMNT}/poudriere/pool -name "${pkgname}" -type f -delete + sql "delete from port where name='${pkgname}';" + sql "delete from deps where port='${pkgname}';'" } clean_pool "${PKGNAME}" ${CLEAN_RDEPENDS} Index: src/poudriere.d/common.sh ================================================================== --- src/poudriere.d/common.sh +++ src/poudriere.d/common.sh @@ -31,10 +31,16 @@ 0) err 1 "No arguments expected" ;; 1) err 1 "1 argument expected: $1" ;; *) err 1 "$# arguments expected: $*" ;; esac } + +sql() { + sqlite3 --init ${JAILMNT}/poudriere/sql \ + ${JAILMNT}/poudriere/queue "$@" || return 1 + return 0 +} log_start() { local logfile=$1 # Make sure directory exists @@ -859,11 +865,11 @@ [ ${read_queue} -eq 0 ] && continue pkgname=$(next_in_queue) if [ -z "${pkgname}" ]; then # pool empty ? - [ -z "$(ls -A ${JAILMNT}/poudriere/pool)" ] && return + [ $(sql "select count(name) from port;") -eq 0 ] && return # Pool is waiting on dep, wait until a build # is done before checking the queue again read_queue=0 else @@ -926,12 +932,11 @@ local port skipped_origin [ ${clean_rdepends} -eq 1 ] && port=$(cache_get_origin "${pkgname}") # Cleaning queue (pool is cleaned here) - lockf -s -k ${MASTERMNT:-${JAILMNT}}/.lock sh ${SCRIPTPREFIX}/clean.sh "${MASTERMNT:-${JAILMNT}}" "${pkgname}" ${clean_rdepends} | sort -u | while read skipped_pkgname; do - skipped_origin=$(cache_get_origin "${skipped_pkgname}") + lockf -s -k ${MASTERMNT:-${JAILMNT}}/.lock sh ${SCRIPTPREFIX}/clean.sh "${MASTERMNT:-${JAILMNT}}" "${pkgname}" ${clean_rdepends} | sort -u | while read skipped_origin; do echo "${skipped_origin} ${pkgname}" >> ${MASTERMNT:-${JAILMNT}}/poudriere/ports.skipped job_msg "Skipping build of ${skipped_origin}: Dependent port ${port} failed" done } @@ -1206,14 +1211,13 @@ done } next_in_queue() { local p - [ ! -d ${JAILMNT}/poudriere/pool ] && err 1 "Build pool is missing" - p=$(lockf -k -t 60 ${JAILMNT}/.lock find ${JAILMNT}/poudriere/pool -type d -depth 1 -empty -print -quit || :) + p=$(sql "select origin from port where running == 0 and name not in (select port from deps) limit 1;") [ -n "$p" ] || return 0 - touch ${p}/.building + sql "update port set running=1 where name='${p}';" # pkgname echo ${p##*/} } lock_acquire() { @@ -1272,20 +1276,20 @@ [ $# -lt 1 ] && eargs port [ $# -gt 2 ] && eargs port pkgnme local port=$1 local pkgname="${2:-$(cache_get_pkgname ${port} 1)}" local dep_pkgname dep_port - local pkg_pooldir="${JAILMNT}/poudriere/pool/${pkgname}" - mkdir "${pkg_pooldir}" 2>/dev/null || return 0 + sql "insert into port (name, origin) values ('${pkgname}', '${port}');" \ + 2>/dev/null || return 0 msg "Computing deps for ${port}" for dep_port in `list_deps ${port}`; do debug "${port} depends on ${dep_port}" dep_pkgname=$(cache_get_pkgname ${dep_port} 1) compute_deps "${dep_port}" "${dep_pkgname}" - touch "${pkg_pooldir}/${dep_pkgname}" + sql "insert or ignore into deps (origin, port) values ('${dep_port}', '${pkgname}');" done } listed_ports() { if [ ${ALL:-0} -eq 1 ]; then @@ -1362,11 +1366,26 @@ local pkg msg "Calculating ports order and dependencies" mkdir -p "${JAILMNT}/poudriere" [ -n "${TMPFS_DATA}" ] && mount -t tmpfs tmpfs "${JAILMNT}/poudriere" - mkdir -p "${JAILMNT}/poudriere/pool" "${JAILMNT}/poudriere/var/run" "${JAILMNT}/poudriere/var/cache" + + echo ".timeout 180 +CREATE TABLE IF NOT EXISTS port ( + name TEXT NOT NULL UNIQUE, + origin TEXT NOT NULL UNIQUE, + running INTEGER default 0 +); + +CREATE TABLE IF NOT EXISTS deps ( + origin TEXT NOT NULL, + port TEXT NOT NULL, + UNIQUE(origin, port) +); +" > ${JAILMNT}/poudriere/sql + + mkdir -p "${JAILMNT}/poudriere/var/run" "${JAILMNT}/poudriere/var/cache" touch "${JAILMNT}/poudriere/var/cache/origin-pkgname" lock_release origin-pkgname || : zset stats_queued "0" :> ${JAILMNT}/poudriere/ports.built @@ -1404,20 +1423,19 @@ find -L ${PKGDIR} -type l -exec rm -vf {} + zset status "cleaning:" msg "Cleaning the build queue" export LOCALBASE=${MYBASE:-/usr/local} - find ${JAILMNT}/poudriere/pool -type d -depth 1 | while read p; do - pn=${p##*/} + for p in $(sql "select name from port"); do if [ -f "${PKGDIR}/All/${pn}.${PKG_EXT}" ]; then rm -rf ${p} - find ${JAILMNT}/poudriere/pool -name "${pn}" -type f -delete + sql "delete from deps where port="${p}""; fi done local nbq=0 - nbq=$(find ${JAILMNT}/poudriere/pool -type d -depth 1 | wc -l) + npq=$(sql "select count(name) from port;") zset stats_queued "${nbq##* }" # Minimize PARALLEL_JOBS to queue size if [ ${PARALLEL_JOBS} -gt ${nbq} ]; then PARALLEL_JOBS=${nbq##* }