From d7f48ae5af136930d6334055f8dcf7e32c0cb243 Mon Sep 17 00:00:00 2001 From: Silvan Calarco Date: Sat, 4 May 2013 15:40:10 +0200 Subject: [PATCH] autodist/autoport: improve logging support for multiple packages targets; fix some checks with delayed repositories --- autodist | 71 ++++++++++++++++++++++++++-------------------------- autodist-cgi | 20 ++++++++------- autoport | 45 ++++++++++++++++++++------------- 3 files changed, 74 insertions(+), 62 deletions(-) diff --git a/autodist b/autodist index 1bf0060..183b9e4 100755 --- a/autodist +++ b/autodist @@ -56,6 +56,25 @@ ${me[0]} ${me[1]} } +function fetch_repository_list() { + local REPOLIST=$1 + + if [ -r ${LOCAL_REPS_BASE_DIR}/$REPOLIST ]; then + echo ${LOCAL_REPS_BASE_DIR}/$REPOLIST + return 0 + else + if [ ! -r $USERCONFDIR/$REPOLIST -o "`find $USERCONFDIR/$REPOLIST -mmin +60 2>/dev/null`" ]; then + mkdir -p `dirname $USERCONFDIR/$REPOLIST` + curl -s $REPS_BASE_URL/$REPOLIST -o $USERCONFDIR/$REPOLIST || { + echo "Error: unable to fetch $REPS_BASE_DIR/$REPOLIST" + exit 1 + } + fi + echo $USERCONFDIR/$REPOLIST + return 0 + fi +} + [ $# -gt 0 ] || { usage ; exit 1; } DATE_NOW=`LANG=C date +%s` @@ -201,36 +220,6 @@ AUTOUPDATEDIR=${LOCAL_REPS_BASE_DIR}/$AUTODIST_REPOSITORY/autoupdate/ mkdir -p $AUTOUPDATEDIR } -SOURCESDIR=$AUTOUPDATEDIR/sources/ -SUCCESSLISTDIR=$AUTOUPDATEDIR -SKIPPEDLISTDIR=$AUTOUPDATEDIR - -SRCPKGLIST=${LOCAL_REPS_BASE_DIR}/$AUTODIST_REPOSITORY/srcpkglist -[ -r $SRCPKGLIST ] || { - SRCPKGLIST=$USERCONFDIR/$AUTODIST_REPOSITORY/srcpkglist - if [ ! -r $SRCPKGLIST -o "`find $SRCPKGLIST -mmin +60`" ]; then - mkdir -p $USERCONFDIR/$AUTODIST_REPOSITORY - curl -s $REPS_BASE_URL/$AUTODIST_REPOSITORY/srcpkglist -o $USERCONFDIR/$AUTODIST_REPOSITORY/srcpkglist || { - echo "Error: unable to fetch $REPS_BASE_DIR/$AUTODIST_REPOSITORY/srcpkglist" - exit 1 - } - fi -} - -[ "$AUTODIST_DELAYED_REPOSITORY" ] && { - SRCPKGLIST_DELAYED=${LOCAL_REPS_BASE_DIR}/$AUTODIST_DELAYED_REPOSITORY/srcpkglist - [ -r $SRCPKGLIST_DELAYED ] || { - SRCPKGLIST_DELAYED=$USERCONFDIR/$AUTODIST_DELAYED_REPOSITORY/srcpkglist - if [ ! -r $SRCPKGLIST_DELAYED -o "`find $SRCPKGLIST -mmin +60`" ]; then - mkdir -p $USERCONFDIR/$AUTODIST_DELAYED_REPOSITORY - curl -s $REPS_BASE_URL/$AUTODIST_DELAYED_REPOSITORY/srcpkglist -o $USERCONFDIR/$AUTODIST_DELAYED_REPOSITORY/srcpkglist || { - echo "Error: unable to fetch $REPS_BASE_DIR/$AUTODIST_DELAYED_REPOSITORY/srcpkglist" - exit 1 - } - fi - } -} - # perform arch names conversions case $BUILDARCH in i386|i486|i686) BUILDARCH=i586 ;; @@ -240,6 +229,16 @@ case $TARGETARCH in i386|i486|i686) TARGETARCH=i586 ;; esac +SOURCESDIR=$AUTOUPDATEDIR/sources/ +SUCCESSLISTDIR=$AUTOUPDATEDIR +SKIPPEDLISTDIR=$AUTOUPDATEDIR + +SRCPKGLIST=`fetch_repository_list $AUTODIST_REPOSITORY/srcpkglist` +[ "$AUTODIST_DELAYED_REPOSITORY" ] && SRCPKGLIST_DELAYED=`fetch_repository_list $AUTODIST_DELAYED_REPOSITORY/srcpkglist` + +BUILDSLIST=`fetch_repository_list distromatic/$AUTODIST_REPOSITORY/builds-$TARGETARCH` +[ "$AUTODIST_DELAYED_REPOSITORY" ] && BUILDSLIST_DELAYED=`fetch_repository_list distromatic/$AUTODIST_DELAYED_REPOSITORY/builds-$TARGETARCH` + [ -e "$DISTDB" ] || { echo "ERROR: missing distdb file $DISTDB; aborting." exit 1 @@ -588,7 +587,9 @@ function launch_pkgs_loop() { [ -e "$SRCPKGLIST" ] && PKGLINE=`grep "^${pkg} " $SRCPKGLIST` || PKGLINE= + [ "$PKGLINE" -a -e "$BUILDSLIST" ] && BUILDSLINE=`grep "^${pkg}:" $BUILDSLIST` || BUILDSLINE= [ -e "$SRCPKGLIST_DELAYED" ] && PKGLINE_DELAYED=`grep "^${pkg} " $SRCPKGLIST_DELAYED` || PKGLINE_DELAYED= + [ "$PKGLINE_DELAYED" -a -e "$BUILDSLIST_DELAYED" ] && BUILDSLINE_DELAYED=`grep "^${pkg} " $SRCPKGLIST_DELAYED` || BUILDSLINE_DELAYED= if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then set -- $PKGLINE @@ -647,7 +648,7 @@ function launch_pkgs_loop() { command_opts="-a3:4" ;; build) # skip package in job if it is in the delayed repository - if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then + if [ "$PKGLINE_DELAYED" -a "$BUILDSLINE_DELAYED" -a "$do_autobuild" ]; then set -- $PKGLINE_DELAYED if [ "$SPEC_VERSION-$SPEC_RELEASE" = "$2-$6" ]; then echo "!! Warning: skipping ${pkg} package build because already present in delayed repository." @@ -660,7 +661,7 @@ function launch_pkgs_loop() { command_opts="-a5:6" ;; buildinstall) # skip package in job if up to date - if [ "$PKGLINE" ]; then + if [ "$PKGLINE" -a "$BUILDSLINE" ]; then set -- $PKGLINE pkglinever=$2 pkglinerel=$6 @@ -675,7 +676,7 @@ function launch_pkgs_loop() { } fi # skip package in job if it is in the delayed repository - if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then + if [ "$PKGLINE_DELAYED" -a "$BUILDSLINE_DELAYED" -a "$do_autobuild" ]; then set -- $PKGLINE_DELAYED if [ "$SPEC_VERSION-$SPEC_RELEASE" = "$2-$6" ]; then echo "!! Warning: skipping ${pkg} package build because already present in delayed repository." @@ -685,7 +686,7 @@ function launch_pkgs_loop() { command_opts="-a5,6,10,11 --force-install" ;; send) # skip package in job if up to date - if [ "$PKGLINE" -a "$do_autobuild" ]; then + if [ "$PKGLINE" -a "$BUILDSLINE" -a "$do_autobuild" ]; then set -- $PKGLINE pkglinever=$2 pkglinerel=$6 @@ -700,7 +701,7 @@ function launch_pkgs_loop() { } fi # skip package in job if it is in the delayed repository - if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then + if [ "$PKGLINE_DELAYED" -a "$BUILDSLINE_DELAYED" -a "$do_autobuild" ]; then set -- $PKGLINE_DELAYED if [ "$SPEC_VERSION-$SPEC_RELEASE" = "$2-$6" ]; then echo "!! Warning: skipping ${pkg} package send because already present in delayed repository." diff --git a/autodist-cgi b/autodist-cgi index 147ecc6..75ccf94 100755 --- a/autodist-cgi +++ b/autodist-cgi @@ -153,11 +153,7 @@ _EOF # Security check [ "${SHOWLOG/\/\.}" != "${SHOWLOG}" ] && continue if [ "${AUTOPORT_CHROOT[$i]}" ]; then - if [ "${AUTOPORT_CHROOT_USER[$i]}" = "autodist" ]; then - BUILDLOGDIR="/var/autoport/${AUTOPORT_CHROOT[$i]}/var/autodist/log/" - else - BUILDLOGDIR="/var/autoport/${AUTOPORT_CHROOT[$i]}/home/${AUTOPORT_CHROOT_USER[$i]}/.autodist/log/" - fi + BUILDLOGDIR="$LOGDIR" elif [ "${AUTOPORT_NATIVE[$i]}" -o "${AUTOPORT_UPDATE[$i]}" ]; then BUILDLOGDIR="/var/autodist/log/" fi @@ -175,6 +171,8 @@ _EOF fi if [ -r ${BUILDLOGDIR}${SHOWLOG} ]; then LOGFILESIZE=`stat -c %s ${BUILDLOGDIR}${SHOWLOG}` + LOGDATE=`stat -c %y ${BUILDLOGDIR}${SHOWLOG}` + echo -n "${SHOWLOG} ($LOGDATE; $LOGFILESIZE):" if [ $LOGFILESIZE -lt 131072 ]; then cat ${BUILDLOGDIR}${SHOWLOG} | parse_build_output else @@ -257,12 +255,16 @@ _EOF fi done BUILDNOW[$BUILDNOWIDX]="$1" - if [ "$2" = "port" ]; then - CURRLOGFILE=build/$3/$1.${AUTOPORT_ARCH[$i]} + if [ "$4" ]; then + BUILDLOG[$BUILDNOWIDX]="${BUILDLOG[$BUILDNOWIDX]} `echo $4 | sed "s|.*/log/||"`" else - CURRLOGFILE=$2/$3/$1.${AUTOPORT_ARCH[$i]} + if [ "$2" = "port" ]; then + CURRLOGFILE=build/$3/$1.${AUTOPORT_ARCH[$i]} + else + CURRLOGFILE=$2/$3/$1.${AUTOPORT_ARCH[$i]} + fi + BUILDLOG[$BUILDNOWIDX]="${BUILDLOG[$BUILDNOWIDX]} $CURRLOGFILE" fi - BUILDLOG[$BUILDNOWIDX]="${BUILDLOG[$BUILDNOWIDX]} $CURRLOGFILE" # BUILDLOG[$BUILDNOWIDX]="${BUILDLOG[$BUILDNOWIDX]} $2/$3/$1.i586" BUILDNOWSTATUS[$BUILDNOWIDX]= BUILDNOWIDX=`expr $BUILDNOWIDX + 1` diff --git a/autoport b/autoport index 32618ae..3aff99d 100755 --- a/autoport +++ b/autoport @@ -48,10 +48,14 @@ function autoport_log() { local pkg=$1 local operation=$2 local result=$3 - local logfile=$4 + local loggrep=`grep "?= See" $4 | sed "s|?= See ||"` + local logfiles="" + for f in $loggrep; do + logfiles="$logfiles $f" + done [ "$pkg" -a "$operation" -a "$result" -a "$BATCH_MODE" ] || return - echo "$pkg $operation $result $logfile" >> $DATADIR/autoport-$PORT_REPOSITORY-current + echo "$pkg $operation $result $logfiles" >> $DATADIR/autoport-$PORT_REPOSITORY-current } # for webbuild message @@ -442,7 +446,8 @@ for TARGET_ARCH in ${TARGET_ARCHS}; do echo -n "prepare" [ "$VERBOSE_MODE" ] && echo " %% COMMAND: LANG=C LC_ALL=C autodist -p ${JOB_CURRENT} --server $PORT_REPOSITORY --repository $PORT_REPOSITORY --severity 2 -- $STAGEOPTS" - LANG=C LC_ALL=C autodist -p ${JOB_CURRENT} --server $PORT_REPOSITORY --repository $PORT_REPOSITORY --severity 2 -- $STAGEOPTS &>>$logfile + LANG=C LC_ALL=C autodist -p ${JOB_CURRENT} --server $PORT_REPOSITORY --repository $PORT_REPOSITORY --severity 2 -- $STAGEOPTS &>$tmpfile + cat $tmpfile >> $logfile [ $? -gt 0 ] && { RES=`find_source_by_provide $PORT_REPOSITORY $JOB_CURRENT` for j in ${RES}; do @@ -455,37 +460,39 @@ for TARGET_ARCH in ${TARGET_ARCHS}; do echo "(FAILED) " fi JOB_CANTPREPARE=(${JOB_CANTPREPARE[*]} $JOB_CURRENT) - autoport_log ${JOB_CURRENT} prepare failed + autoport_log ${JOB_CURRENT} prepare failed $tmpfile continue } || { echo $JOB_CURRENT >> $DATAARCHDIR/preparedjobs echo -n "(OK) " - autoport_log ${JOB_CURRENT} prepare ok + autoport_log ${JOB_CURRENT} prepare ok $tmpfile } if [ "$REBUILD_MODE" = "1" ]; then echo -n "update" [ "$VERBOSE_MODE" ] && echo " %% COMMAND: LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"$REBUILD_CHANGELOG\"" - LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"$REBUILD_CHANGELOG\" &>>$logfile + LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"$REBUILD_CHANGELOG\" &>$tmpfile + cat $tmpfile >> $logfile [ $? -gt 0 ] && { echo "(FAILED) " - autoport_log ${JOB_CURRENT} update failed + autoport_log ${JOB_CURRENT} update failed $tmpfile continue } || { - autoport_log ${JOB_CURRENT} update ok + autoport_log ${JOB_CURRENT} update ok $tmpfile echo -n "(OK) " } elif [ "$PORT_REPOSITORY" != "$DEST_REPOSITORY" ]; then echo -n "update" [ "$VERBOSE_MODE" ] && echo " %% COMMAND: LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"automatic port from $PORT_REPOSITORY\"" - LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"automatic port from $PORT_REPOSITORY\" &>>$logfile + LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"automatic port from $PORT_REPOSITORY\" &>$tmpfile + cat $tmpfile >> $logfile [ $? -gt 0 ] && { echo "(FAILED) " - autoport_log ${JOB_CURRENT} update failed + autoport_log ${JOB_CURRENT} update failed $tmpfile continue } || { - autoport_log ${JOB_CURRENT} update ok + autoport_log ${JOB_CURRENT} update ok $tmpfile echo -n "(OK) " } elif [ "$REBUILD_MILESTONE" -a "$pkg_milestone" != "$REBUILD_MILESTONE" ]; then @@ -511,12 +518,12 @@ for TARGET_ARCH in ${TARGET_ARCHS}; do echo "(FAILED)" JOB_LOG=`grep "^?= See " $tmpfile | awk '{ print $3 }'` [ "$JOB_LOG" ] || JOB_LOG=$tmpfile - autoport_log ${JOB_CURRENT} update failed + autoport_log ${JOB_CURRENT} update failed $tmpfile continue } || { cat $tmpfile >> $logfile echo -n "(OK) " - autoport_log ${JOB_CURRENT} update ok + autoport_log ${JOB_CURRENT} update ok $tmpfile } fi } @@ -532,6 +539,8 @@ for TARGET_ARCH in ${TARGET_ARCHS}; do fi [ "$PORT_REPOSITORY" != "$DEST_REPOSITORY" ] && pkg_repository="$DEST_REPOSITORY" [ ! "$DONT_PREPARE_THIS" -a "$PORT_REPOSITORY" = "$DEST_REPOSITORY" -a ! "$REBUILD_MILESTONE" ] && NOSRPM_OPTS="--nosrpm" || NOSRPM_OPTS="--force" + [ "$VERBOSE_MODE" ] && echo " +%% COMMAND: LANG=C LC_ALL=C autodist -b -s ${JOB_CURRENT} --arch $TARGET_ARCH --server $pkg_repository --repository $PORT_REPOSITORY --severity 2 $FORCE_FLAG -- $STAGEOPTS $NOSRPM_OPTS" LANG=C LC_ALL=C autodist -b -s ${JOB_CURRENT} --arch $TARGET_ARCH --server $pkg_repository --repository $PORT_REPOSITORY --severity 2 $FORCE_FLAG -- $STAGEOPTS $NOSRPM_OPTS &>$tmpfile [ $? -gt 0 ] && { if [ "$VERBOSE_MODE" ]; then @@ -548,14 +557,14 @@ for TARGET_ARCH in ${TARGET_ARCHS}; do JOB_LOG=`grep "^?= See " $tmpfile | awk '{ print $3 }'` [ "$JOB_LOG" ] || JOB_LOG=$tmpfile [ ! "$REBUILD_MILESTONE" ] && find_requirements $JOB_LOG - autoport_log ${JOB_CURRENT} port failed + autoport_log ${JOB_CURRENT} port failed $tmpfile continue } || { cat $tmpfile >> $logfile JOB_SENT=(${JOB_SENT[*]} $JOB_CURRENT) echo $JOB_CURRENT >> $DATAARCHDIR/sentjobs echo -n "(OK) " - autoport_log ${JOB_CURRENT} port ok + autoport_log ${JOB_CURRENT} port ok $tmpfile if [ "$WEBBUILD_URL" -a "$WEBBUILD_USER" ]; then curl -s "$WEBBUILD_URL?REQUEST=message&USER=$WEBBUILD_USER&SECRET=$WEBBUILD_SECRET&USER_EMAIL=$WEBBUILD_EMAIL&\ MESSAGE=`cgi_encodevar \"ported $JOB_CURRENT to $TARGET_ARCH and sent it to $pkg_repository\"`" >/dev/null @@ -584,7 +593,7 @@ MESSAGE=`cgi_encodevar \"ported $JOB_CURRENT to $TARGET_ARCH and s JOB_LOG=`grep "^?= See " $tmpfile | awk '{ print $3 }'` [ "$JOB_LOG" ] || JOB_LOG=$tmpfile [ ! "$REBUILD_MILESTONE" ] && find_requirements $JOB_LOG - autoport_log ${JOB_CURRENT} install failed + autoport_log ${JOB_CURRENT} install failed $tmpfile continue } || { cat $tmpfile >> $logfile @@ -592,7 +601,7 @@ MESSAGE=`cgi_encodevar \"ported $JOB_CURRENT to $TARGET_ARCH and s JOB_MAYBEINSTALLED_NEW=(${JOB_MAYBEINSTALLED_NEW[*]} $JOB_CURRENT) JOB_COMPLETED=(${JOB_COMPLETED[*]} $JOB_CURRENT) echo -n "(OK)" - autoport_log ${JOB_CURRENT} install ok + autoport_log ${JOB_CURRENT} install ok $tmpfile } echo done @@ -639,7 +648,7 @@ MESSAGE=`cgi_encodevar \"ported $JOB_CURRENT to $TARGET_ARCH and s JOB_MAYBEINSTALLED_NEW=(${JOB_MAYBEINSTALLED_NEW[*]} $JOB_CURRENT) JOB_COMPLETED=(${JOB_COMPLETED[*]} $JOB_CURRENT) echo -n "(OK)" - autoport_log ${JOB_CURRENT} retryinstall ok + autoport_log ${JOB_CURRENT} retryinstall ok $tmpfile } echo done