autodist: create an operation state file for better parsing from cgi; return the number of faild operations as exit code

This commit is contained in:
Silvan Calarco 2012-10-18 00:52:41 +02:00
parent 0e1067c930
commit 12b3759777
2 changed files with 142 additions and 70 deletions

View File

@ -423,6 +423,21 @@ get_job_vector() {
JOB_VARNAMES=(${JOB[1]//,/ })
}
function autobuild_log() {
local pkg=$1
local operation=$2
local result=$3
local job=$4
local logfile=$5
[ "$pkg" -a "$operation" -a "$result" ] || return
if [ "$job" -a "$job" != "$pkg" ]; then
echo "$job/$pkg $operation $result $logfile" >> $LOGBASEDIR/autoupdate-current
else
echo "$pkg $operation $result $logfile" >> $LOGBASEDIR/autoupdate-current
fi
}
function launch_pkgs_loop() {
# This function iterates the passed operation for each package (column) in job
#
@ -618,6 +633,7 @@ function launch_pkgs_loop() {
echo "!! Warning: skipping ${pkg} package already up to date ($pkglinever >= ${passed_arguments/ *})."
echo "%% Adding ${pkg} job to skipped list."
echo "${pkg} $DATE_NOW 255" >> $SKIPPEDLISTDIR/auto.skip
[ "$do_autobuild" ] && autobuild_log $pkg $operation skipped $JOB_NAME
continue
}
fi
@ -630,6 +646,7 @@ function launch_pkgs_loop() {
echo "!! Warning: skipping ${pkg} package build because already present in delayed repository."
echo "%% Adding ${pkg} job to skipped list."
echo "${pkg} $DATE_NOW 254" >> $SKIPPEDLISTDIR/auto.skip
[ "$do_autobuild" ] && autobuild_log $pkg $operation skipped $JOB_NAME
continue
fi
fi
@ -719,7 +736,7 @@ function launch_pkgs_loop() {
if [ $ret -gt 0 ]; then
echo " FAILED (ret=$ret)"
echo " FAILED (ret=$ret)" >> $LOGFILE
[ "$quiet" ] || {
[ ! "$quiet" -a ! "$verbose" ] && {
echo "=> $command"
echo
echo " == begin of output log =="
@ -731,6 +748,7 @@ function launch_pkgs_loop() {
}
mv $LOGFILE_PKG $LOGDIR/failed/
echo "?= See $LOGDIR/failed/${LOGFILE_PKG_NAME}"
[ "$do_autobuild" ] && autobuild_log $pkg $operation failed $JOB_NAME $LOGDIR/failed/${LOGFILE_PKG_NAME}
case $operation in
autoupdate|update)
;;
@ -754,9 +772,12 @@ function launch_pkgs_loop() {
echo " OK" >> $LOGFILE
mv $LOGFILE_PKG $LOGDIR/ok/
echo "?= See $LOGDIR/ok/${LOGFILE_PKG_NAME}"
[ "$do_autobuild" ] && autobuild_log $pkg $operation ok $JOB_NAME $LOGDIR/ok/${LOGFILE_PKG_NAME}
case $operation in
prepare)
[ -e $spec_dir/.$pkg.spec.autodist.prebuild ] && \
rm -f $spec_dir/.$pkg.spec.autodist.prebuild
cp $spec_dir/$pkg.spec $spec_dir/.$pkg.spec.autodist.preupdate
;;
rebuild)
@ -1017,8 +1038,12 @@ if [ "$do_autobuild" = "1" ]; then
done
echo
rm -f $pkgtmpfile $srctmpfile
[ -e $LOGBASEDIR/autoupdate-current ] && mv $LOGBASEDIR/autoupdate-current $LOGBASEDIR/autoupdate-last
fi
OPERATION_ERRORS=0
echo "%% ${#JOBNAME[*]} job(s) scheduled"
echo "%% Starting main jobs loop"
for JOB_NUM in `seq 1 ${#JOBNAME[*]}`; do
@ -1057,9 +1082,10 @@ for JOB_NUM in `seq 1 ${#JOBNAME[*]}`; do
case $SEVERITY in
0) ;;
1) echo "%! Error during package prepare; skipping $JOB_NAME job."
OPERATION_ERRORS=`expr $OPERATION_ERRORS + 1`
continue ;;
*) echo "!! Error during package prepare; aborting."
exit 1 ;;
exit 255 ;;
esac
}
if [ ! "$do_update" -a "$do_autobuild" = "1" ]; then
@ -1069,9 +1095,10 @@ for JOB_NUM in `seq 1 ${#JOBNAME[*]}`; do
case $SEVERITY in
0) ;;
1) echo "%! Error during package update for rebuild; skipping $JOB_NAME job."
OPERATION_ERRORS=`expr $OPERATION_ERRORS + 1`
continue ;;
*) echo "!! Error during package update for rebuild; aborting."
exit 1 ;;
exit 255 ;;
esac
}
fi
@ -1105,9 +1132,10 @@ for JOB_NUM in `seq 1 ${#JOBNAME[*]}`; do
case $SEVERITY in
0) ;;
1) echo "%! Error during package update; skipping $JOB_NAME job."
OPERATION_ERRORS=`expr $OPERATION_ERRORS + 1`
continue ;;
*) echo "!! Error during package update; aborting."
exit 1 ;;
exit 255 ;;
esac
else
echo "%% Rebuilding package(s)"
@ -1117,9 +1145,10 @@ for JOB_NUM in `seq 1 ${#JOBNAME[*]}`; do
case $SEVERITY in
0) ;;
1) echo "%! Error during package update for rebuild; skipping $JOB_NAME job."
OPERATION_ERRORS=`expr $OPERATION_ERRORS + 1`
continue ;;
*) echo "!! Error during package update for rebuild; aborting."
exit 1 ;;
exit 255 ;;
esac
}
fi
@ -1141,9 +1170,10 @@ for JOB_NUM in `seq 1 ${#JOBNAME[*]}`; do
case $SEVERITY in
0) ;;
1) echo "%! Error during update; skipping $JOB_NAME job."
OPERATION_ERRORS=`expr $OPERATION_ERRORS + 1`
continue ;;
*) echo "!! Error during update; aborting."
exit 1 ;;
exit 255 ;;
esac
else
[ "${AUTOSPEC_ARGS/--changelog}" = "${AUTOSPEC_ARGS}" ] &&
@ -1152,9 +1182,10 @@ for JOB_NUM in `seq 1 ${#JOBNAME[*]}`; do
case $SEVERITY in
0) ;;
1) echo "%! Error during package update for rebuild; skipping $JOB_NAME job."
OPERATION_ERRORS=`expr $OPERATION_ERRORS + 1`
continue ;;
*) echo "!! Error during package update for rebuild; aborting."
exit 1 ;;
exit 255 ;;
esac
}
fi
@ -1171,9 +1202,10 @@ for JOB_NUM in `seq 1 ${#JOBNAME[*]}`; do
case $SEVERITY in
0) ;;
1) echo "%! Error during build; skipping $JOB_NAME job."
OPERATION_ERRORS=`expr $OPERATION_ERRORS + 1`
continue ;;
*) echo "!! Error during build; aborting."
exit 1 ;;
exit 255 ;;
esac
}
fi
@ -1188,9 +1220,10 @@ for JOB_NUM in `seq 1 ${#JOBNAME[*]}`; do
case $SEVERITY in
0) ;;
1) echo "%! Error during installation; skipping $JOB_NAME job."
OPERATION_ERRORS=`expr $OPERATION_ERRORS + 1`
continue ;;
*) echo "!! Error during installation; aborting."
exit 1 ;;
exit 255 ;;
esac
}
fi
@ -1239,4 +1272,6 @@ for JOB_NUM in `seq 1 ${#JOBNAME[*]}`; do
}
done
[ "$do_autobuild" = "1" ] && rm -f $PIDFILE
echo "%! $OPERATION_ERRORS error(s)"
echo "%% All jobs done @ `LANG=C date`"
exit $OPERATION_ERRORS

View File

@ -1,5 +1,6 @@
#!/bin/bash
. /etc/sysconfig/autoport
. /usr/share/autodist/webbuild-functions
NUM=`echo "$QUERY_STRING" | sed -n 's/^.*NUM=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"`
HOST=`echo "$QUERY_STRING" | sed -n 's/^.*HOST=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"`
@ -86,14 +87,18 @@ for i in `seq 0 ${#AUTOPORT_ARCH[*]}`; do
if [ "${AUTOPORT_UPDATE[$i]}" ]; then
LOGFILE="/var/autodist/log/autodist.log"
LASTLOGFILE="/var/autodist/log/autodist-last.log"
STATEFILE="/var/autodist/log/autoupdate-current"
LASTSTATEFILE="/var/autodist/log/autoupdate-last"
elif [ "${AUTOPORT_CHROOT[$i]}" ]; then
LOGFILE="/var/autodist/log/autoport-chroot-$r-${AUTOPORT_CHROOT[$i]}.log"
LASTLOGFILE="/var/autodist/log/autoport-chroot-$r-${AUTOPORT_CHROOT[$i]}-last.log"
AUTODISTLOGFILE="/var/autoport/${AUTOPORT_CHROOT[$i]}/home/${AUTOPORT_CHROOT_USER[$i]}/.autoport/${AUTOPORT_ARCH[$i]}/$r-current.log"
AUTODISTSTATEFILE="/var/autoport/${AUTOPORT_CHROOT[$i]}/var/autodist/autoport-$r-current"
elif [ "${AUTOPORT_NATIVE[$i]}" ]; then
LOGFILE="/var/autodist/log/autoport-native-$r-${AUTOPORT_ARCH[$i]}.log"
LASTLOGFILE="/var/autodist/log/autoport-native-$r-${AUTOPORT_ARCH[$i]}-last.log"
AUTODISTLOGFILE="/var/autodist/.autoport/${AUTOPORT_ARCH[$i]}/$r-current.log"
AUTODISTSTATEFILE="/var/autodist/log/autoport-$r-current"
elif [ "${AUTOPORT_CROSS[$i]}" ]; then
LOGFILE="/var/autodist/log/autoport-cross-$r-${AUTOPORT_CROSS[$i]}.log"
LASTLOGFILE="/var/autodist/log/autoport-cross-$r-${AUTOPORT_CROSS[$i]}-last.log"
@ -117,14 +122,24 @@ for i in `seq 0 ${#AUTOPORT_ARCH[*]}`; do
echo -n ":</h2>"
cat << _EOF
<div class=downloadbox align=center>
<div align=left style="background-color: white; height: 550px; width: 850px; overflow: auto; font-size: 8pt; border:1px solid #2b6600; margin: 4px; padding:4; background-color: #e0f2d0;">
<div align=left style="height: 550px; width: 850px;overflow:auto;font-size:8pt;border:1px solid #2b6600;margin:4px;padding:2px;color:lightgray;background-color:black;">
_EOF
echo -n "<pre><code>"
if [ "$SHOWLOG" = "current" ]; then
RELOADTIME=600000
cat $LOGFILE | sed "s|<|\&lt;|g"
RELOADTIME=3600000
cat $LOGFILE | parse_build_output
# sed "s|<|\&lt;|g;s|[[:cntrl:]]\[[0-9;]*m||g;
# s|^\([\+#] .*\)|<font style=\"color:gray\">\1</font>|;
# s|\(.*error[[:space:]]*:.*\)|<font style=\"background-color:red;color:white\">\1</font>|i;
# s|^\(== =.*\)|<font style=\"background-color:green;color:white\">\1</font>|;
# s|^\([%?=][!=>] .*\)|<font style=\"color:white\">\1</font>|;"
elif [ "$SHOWLOG" = "last" ]; then
cat $LASTLOGFILE | sed "s|<|\&lt;|g"
cat $LASTLOGFILE | parse_build_output
# sed "s|<|\&lt;|g;s|[[:cntrl:]]\[[0-9;]*m||g;
# s|^\([\+#] .*\)|<font style=\"color:gray\">\1</font>|;
# s|\(.*error[[:space:]]*:.*\)|<font style=\"background-color:red;color:white\">\1</font>|i;
# s|^\(== =.*\)|<font style=\"background-color:green;color:white\">\1</font>|;
# s|^\([%?=][!=>] .*\)|<font style=\"color:white\">\1</font>|;"
else
# Security check
[ "${SHOWLOG/\/\.}" != "${SHOWLOG}" ] && continue
@ -136,7 +151,7 @@ _EOF
if [ "${SHOWLOG/\/ok\/}" != "${SHOWLOG}" -o "${SHOWLOG/\failed\/}" != "${SHOWLOG}" ]; then
SHOWLOGPKG=${SHOWLOG/*\/}
[ "${SHOWLOGPKG}" ] || continue
echo "<table><tr><td colspan=3>Last ${SHOWLOGPKG} logs:</td>"
echo "<table style=\"color:black\"><tr><td colspan=3>Last ${SHOWLOGPKG} logs:</td>"
for t in prepare update build install send; do
echo -n "<tr><td>$t:</td><td>"
[ -e ${BUILDLOGDIR}${t}/ok/${SHOWLOGPKG} ] && echo -n "<td><a href=\"?NUM=$i$HOST_ADD&SHOWLOG=${t}/ok/${SHOWLOGPKG}\">OK</a></td>" || echo "<td></td>"
@ -145,9 +160,16 @@ _EOF
done
echo "</table>"
fi
[ -e ${BUILDLOGDIR}${SHOWLOG} ] && cat ${BUILDLOGDIR}${SHOWLOG} | sed "s|<|\&lt;|g"
[ -e ${BUILDLOGDIR}${SHOWLOG} ] && \
cat ${BUILDLOGDIR}${SHOWLOG} | parse_build_output
# sed "s|<|\&lt;|g;s|[[:cntrl:]]\[[0-9;]*m||g;
# s|^\([\+#] .*\)|<font style=\"color:gray\">\1</font>|;
# s|\(.*error:.*\)|<font style=\"background-color:red;color:white\">\1</font>|i;
# s|^\(== =.*\)|<font style=\"background-color:green;color:white\">\1</font>|;
# s|^\([%?=][!=>] .*\)|<font style=\"color:white\">\1</font>|;"
fi
echo "</code></pre></div></div>"
echo -n "</pre></code>"
echo -n "</div></div>"
break
fi
[ "${AUTOPORT_UPDATE[$i]}" ] && break
@ -180,55 +202,54 @@ _EOF
_EOF
echo "<pre>"
if [ "${AUTOPORT_UPDATE[$i]}" ]; then
unset STARTTIME ENDTIME JOBON BUILDNOW BUILDLOG SCHEDULED
BUILDNOWIDX=0
STARTTIME=`head $LOGFILE | grep "%% Autodist started" | sed "s|.* @ ||"`
AUTODIST_PID=`head $LOGFILE | grep "%% Autodist started" | sed "s|.*PID \([0-9]*\) .*|\1|"`
SCHEDULED=`head $LOGFILE | grep "%% autoupdate jobs:" | sed "s|.*autoupdate jobs: ||"`
ENDTIME=`tail $LOGFILE | grep "%% All jobs done @" | sed "s|.*@ ||"`
while read line; do
set -- $line
if [[ "${line:0:1}" = $'\r' ]]; then
PREF=`echo $1 | tr -d "\r"`
for c in `seq 0 ${#BUILDNOW[*]}`; do
if [ "${BUILDNOW[c]}" = "$1" ]; then
BUILDNOWIDX=$c
break
else
PREF=$1
BUILDNOWIDX=$c
fi
case $PREF in
"%%" ) if [ "$2 $3" == "Autodist started" ]; then
STARTTIME=`echo $line | sed "s|.* @ ||"`
AUTODIST_PID=`echo $line | sed "s|.*PID \([0-9]*\) .*|\1|"`
elif [ "$2 $3 $4" == "All jobs done" ]; then
ENDTIME=`echo $line | sed "s|.* @ ||"`
elif [ "$2 $3" == "autoupdate jobs:" ]; then
SCHEDULED=`echo $line | sed "s|.*autoupdate jobs: ||"`
elif [ "$2" == "Doing" ]; then
BUILDNOW[$BUILDNOWIDX]="$3"
BUILDLOG[$BUILDNOWIDX]=
done
BUILDNOW[$BUILDNOWIDX]="$1"
BUILDLOG[$BUILDNOWIDX]="${BUILDLOG[$BUILDNOWIDX]} `echo $4 | sed "s|.*autodist/log/||"`"
# BUILDLOG[$BUILDNOWIDX]="${BUILDLOG[$BUILDNOWIDX]} $2/$3/$1.i586"
BUILDNOWSTATUS[$BUILDNOWIDX]=
BUILDNOWIDX=`expr $BUILDNOWIDX + 1`
# elif [ "$2" == "Starting" ]; then
# case $3 in
# "PREPARE"|"UPDATE"|"BUILD"|"SEND") BUILDNOW[$BUILDNOWIDX-1]="${BUILDNOW[$BUILDNOWIDX-1]} $3" ;;
# esac
#elif [ "$2" != "Starting" -a "$2" != "Updating" ]; then
# [ "$JOBON" ] || echo $line
fi ;;
"?=") if [ "$2" == "See" ]; then
BUILDLOG[$BUILDNOWIDX-1]="${BUILDLOG[$BUILDNOWIDX-1]} `echo $line | sed 's|.*autodist/log/||'`"
fi ;;
"=>") JOBON=1 ;;
"%!"|"!!") BUILDNOWSTATUS[$BUILDNOWIDX-1]="$line"; JOBON= ;;
"==") if [ "${2:0:2}" == "==" ]; then
JOBON=
# else
# if [ "$2" != "${BUILDNOW[$BUILDNOWIDX-1]}" ]; then
# BUILDNOW[$BUILDNOWIDX]="$2"
# BUILDNOWIDX=`expr $BUILDNOWIDX + 1`
# fi
fi ;;
"Hunk"|"") ;;
*) [ "$JOBON" ] || {
line=`echo $line | sed "s|.*\r||"`
echo $line
} ;;
esac
done < $LOGFILE
done < $STATEFILE
elif [ -e $AUTODISTSTATEFILE ]; then
BUILDNOWIDX=0
STARTTIME=`head $LOGFILE | grep "^+ .*start at " | sed "s|.*start at ||"`
# AUTODIST_PID=`head $LOGFILE | grep "%% Autodist started" | sed "s|.*PID \([0-9]*\) .*|\1|"`
SCHEDULED=`head -n 40 $LOGFILE | grep -m 1 "^> .*Scheduled jobs: " | sed "s|.*Scheduled jobs: ||"`
ENDTIME=`tail $LOGFILE | grep "^- .*end at " | sed "s|.*end at ||"`
while read line; do
set -- $line
for c in `seq 0 ${#BUILDNOW[*]}`; do
if [ "${BUILDNOW[c]}" = "$1" ]; then
BUILDNOWIDX=$c
break
else
BUILDNOWIDX=$c
fi
done
BUILDNOW[$BUILDNOWIDX]="$1"
if [ "$2" = "port" ]; then
CURRLOGFILE=build/$3/$1.${AUTOPORT_ARCH[$i]}
else
CURRLOGFILE=$2/$3/$1.${AUTOPORT_ARCH[$i]}
fi
BUILDLOG[$BUILDNOWIDX]="${BUILDLOG[$BUILDNOWIDX]} $CURRLOGFILE"
# BUILDLOG[$BUILDNOWIDX]="${BUILDLOG[$BUILDNOWIDX]} $2/$3/$1.i586"
BUILDNOWSTATUS[$BUILDNOWIDX]=
BUILDNOWIDX=`expr $BUILDNOWIDX + 1`
done < $AUTODISTSTATEFILE
else
unset STARTTIME ENDTIME BUILDNOW BUILDLOG AUTODISTON LASTLINE SCHEDULED
while read line; do
@ -269,15 +290,25 @@ _EOF
*) [ "$JOBON" ] || echo $line ;;
esac
done < $AUTODISTLOGFILE
for p in `dirname $LOGFILE`/current.*; do
CURRENT_FILE=`basename $p`
AUTODIST_PID=${p/*.}
[ -e /proc/$AUTODIST_PID ] && break
AUTODIST_PID=
done
LASTLINE=$line
fi
echo "<b>Start:</b> $STARTTIME"
echo "<b>Scheduled jobs:</b> $SCHEDULED"
LASTJOBNAME=
for b in `seq 0 ${#BUILDNOW[*]}`; do
# set -- ${BUILDNOW[$b]}
JOBNAME=${BUILDNOW[$b]}
[ "$JOBNAME" ] || continue
if [ "$JOBNAME" != "$LASTJOBNAME" ]; then
[ "$LASTJOBNAME" -a "$JOBNAME" ] && echo ""
echo -n "<a href=\"/distribution/webbuild.php?ENVIRONMENT=$i&PACKAGE=$JOBNAME\" target=\"webbuild\"><b>$JOBNAME</b></a>: "
fi
rowcnt=0
if [ "${BUILDLOG[$b]}" ]; then
for l in ${BUILDLOG[$b]}; do
@ -285,12 +316,14 @@ _EOF
[ "${SUBJOBNAME:0:1}" = "_" ] && SUBJOBNAME=$l
[ "$SUBJOBNAME" = "$l" ] && SUBJOBNAME=`echo $l | sed "s|.*/\(.*\)\..*|\1|"`
SUBJOBNAME=${SUBJOBNAME/__*}
OPERATIONNAME=`echo $l | sed "s|\(.*\)/.*/.*|\1|"`
OPERATIONSTATUS=`echo $l | sed "s|.*/\(.*\)/.*|\1|"`
if [ $rowcnt -ge 4 ]; then
echo
rowcnt=0
fi
[ "${BUILDOPERATION[$b]}" ] && \
OPERATIONNAME=${BUILDOPERATION[$b]} || OPERATIONNAME=`echo $l | sed "s|\(.*\)/.*/.*|\1|"`
[ "${BUILDOPERATIONSTATUS[$b]}" ] && \
OPERATIONSTATUS=${BUILDOPERATIONSTATUS[$b]} || OPERATIONSTATUS=`echo $l | sed "s|.*/\(.*\)/.*|\1|"`
# if [ $rowcnt -ge 4 ]; then
# echo
# rowcnt=0
# fi
if [ "$OPERATIONNAME" = "send" -a "$OPERATIONSTATUS" = "ok" ]; then
echo -n "<a href=\"?SHOWLOG=$l&NUM=$i$HOST_ADD\">$OPERATIONNAME:$OPERATIONSTATUS</a>"
elif [ "$OPERATIONNAME" = "install" -a "$OPERATIONSTATUS" = "ok" ]; then
@ -305,13 +338,17 @@ _EOF
rowcnt=`expr $rowcnt + 1`
done
fi
[ "${BUILDNOWSTATUS[$b]}" ] && echo "<i>(${BUILDNOWSTATUS[$b]})</i>" || echo
[ "${BUILDNOWSTATUS[$b]}" ] && echo -n "<i>(${BUILDNOWSTATUS[$b]})</i>"
LASTJOBNAME=$JOBNAME
done
[ "$LASTJOBNAME" ] && echo
if [ "$LASTLINE" ]; then
set -- $LASTLINE
[ "$1" = "^" ] && echo "<font color=red><b>${2}</b> ${3} ${4} ${5} ${6} ${7}</font>"
fi
[ "$ENDTIME" ] && echo "<b>End:</b> $ENDTIME" || echo "<b>End: <a href=\"?SHOWLOG=current.$AUTODIST_PID&NUM=$i$HOST_ADD\">in progress</a></b>"
[ "$ENDTIME" ] && \
echo "<b>End:</b> $ENDTIME" || \
echo "<b>End: <a href=\"?SHOWLOG=current.$AUTODIST_PID&NUM=$i$HOST_ADD\">in progress</a></b>"
echo
# grep -v "^=" $LOGFILE | tail -n "$LINES" | sed "s|.*\r||"
# echo