openmamba/autodist
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

autodist/autoport: improve logging support for multiple packages targets; fix some checks with delayed repositories

Browse Source
This commit is contained in:
Silvan Calarco 2013-05-04 15:40:10 +02:00
parent 8576dc1d8c
commit d7f48ae5af
3 changed files with 74 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
autodist
View File

@ -56,6 +56,25 @@ ${me[0]} ${me[1]}
}
function fetch_repository_list() {
local REPOLIST=$1
if [ -r ${LOCAL_REPS_BASE_DIR}/$REPOLIST ]; then
echo ${LOCAL_REPS_BASE_DIR}/$REPOLIST
return 0
else
if [ ! -r $USERCONFDIR/$REPOLIST -o "`find $USERCONFDIR/$REPOLIST -mmin +60 2>/dev/null`" ]; then
mkdir -p `dirname $USERCONFDIR/$REPOLIST`
curl -s $REPS_BASE_URL/$REPOLIST -o $USERCONFDIR/$REPOLIST || {
echo "Error: unable to fetch $REPS_BASE_DIR/$REPOLIST"
exit 1
}
fi
echo $USERCONFDIR/$REPOLIST
return 0
fi
}
[ $# -gt 0 ] || { usage ; exit 1; }
DATE_NOW=`LANG=C date +%s`
@ -201,36 +220,6 @@ AUTOUPDATEDIR=${LOCAL_REPS_BASE_DIR}/$AUTODIST_REPOSITORY/autoupdate/
mkdir -p $AUTOUPDATEDIR
}
SOURCESDIR=$AUTOUPDATEDIR/sources/
SUCCESSLISTDIR=$AUTOUPDATEDIR
SKIPPEDLISTDIR=$AUTOUPDATEDIR
SRCPKGLIST=${LOCAL_REPS_BASE_DIR}/$AUTODIST_REPOSITORY/srcpkglist
[ -r $SRCPKGLIST ] || {
SRCPKGLIST=$USERCONFDIR/$AUTODIST_REPOSITORY/srcpkglist
if [ ! -r $SRCPKGLIST -o "`find $SRCPKGLIST -mmin +60`" ]; then
mkdir -p $USERCONFDIR/$AUTODIST_REPOSITORY
curl -s $REPS_BASE_URL/$AUTODIST_REPOSITORY/srcpkglist -o $USERCONFDIR/$AUTODIST_REPOSITORY/srcpkglist || {
echo "Error: unable to fetch $REPS_BASE_DIR/$AUTODIST_REPOSITORY/srcpkglist"
exit 1
}
fi
}
[ "$AUTODIST_DELAYED_REPOSITORY" ] && {
SRCPKGLIST_DELAYED=${LOCAL_REPS_BASE_DIR}/$AUTODIST_DELAYED_REPOSITORY/srcpkglist
[ -r $SRCPKGLIST_DELAYED ] || {
SRCPKGLIST_DELAYED=$USERCONFDIR/$AUTODIST_DELAYED_REPOSITORY/srcpkglist
if [ ! -r $SRCPKGLIST_DELAYED -o "`find $SRCPKGLIST -mmin +60`" ]; then
mkdir -p $USERCONFDIR/$AUTODIST_DELAYED_REPOSITORY
curl -s $REPS_BASE_URL/$AUTODIST_DELAYED_REPOSITORY/srcpkglist -o $USERCONFDIR/$AUTODIST_DELAYED_REPOSITORY/srcpkglist || {
echo "Error: unable to fetch $REPS_BASE_DIR/$AUTODIST_DELAYED_REPOSITORY/srcpkglist"
exit 1
}
fi
}
}
# perform arch names conversions
case $BUILDARCH in
i386|i486|i686) BUILDARCH=i586 ;;
@ -240,6 +229,16 @@ case $TARGETARCH in
i386|i486|i686) TARGETARCH=i586 ;;
esac
SOURCESDIR=$AUTOUPDATEDIR/sources/
SUCCESSLISTDIR=$AUTOUPDATEDIR
SKIPPEDLISTDIR=$AUTOUPDATEDIR
SRCPKGLIST=`fetch_repository_list $AUTODIST_REPOSITORY/srcpkglist`
[ "$AUTODIST_DELAYED_REPOSITORY" ] && SRCPKGLIST_DELAYED=`fetch_repository_list $AUTODIST_DELAYED_REPOSITORY/srcpkglist`
BUILDSLIST=`fetch_repository_list distromatic/$AUTODIST_REPOSITORY/builds-$TARGETARCH`
[ "$AUTODIST_DELAYED_REPOSITORY" ] && BUILDSLIST_DELAYED=`fetch_repository_list distromatic/$AUTODIST_DELAYED_REPOSITORY/builds-$TARGETARCH`
[ -e "$DISTDB" ] || {
echo "ERROR: missing distdb file $DISTDB; aborting."
exit 1
@ -588,7 +587,9 @@ function launch_pkgs_loop() {
[ -e "$SRCPKGLIST" ] && PKGLINE=`grep "^${pkg} " $SRCPKGLIST` || PKGLINE=
[ "$PKGLINE" -a -e "$BUILDSLIST" ] && BUILDSLINE=`grep "^${pkg}:" $BUILDSLIST` || BUILDSLINE=
[ -e "$SRCPKGLIST_DELAYED" ] && PKGLINE_DELAYED=`grep "^${pkg} " $SRCPKGLIST_DELAYED` || PKGLINE_DELAYED=
[ "$PKGLINE_DELAYED" -a -e "$BUILDSLIST_DELAYED" ] && BUILDSLINE_DELAYED=`grep "^${pkg} " $SRCPKGLIST_DELAYED` || BUILDSLINE_DELAYED=
if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then
set -- $PKGLINE
@ -647,7 +648,7 @@ function launch_pkgs_loop() {
command_opts="-a3:4" ;;
build)
# skip package in job if it is in the delayed repository
if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then
if [ "$PKGLINE_DELAYED" -a "$BUILDSLINE_DELAYED" -a "$do_autobuild" ]; then
set -- $PKGLINE_DELAYED
if [ "$SPEC_VERSION-$SPEC_RELEASE" = "$2-$6" ]; then
echo "!! Warning: skipping ${pkg} package build because already present in delayed repository."
@ -660,7 +661,7 @@ function launch_pkgs_loop() {
command_opts="-a5:6" ;;
buildinstall)
# skip package in job if up to date
if [ "$PKGLINE" ]; then
if [ "$PKGLINE" -a "$BUILDSLINE" ]; then
set -- $PKGLINE
pkglinever=$2
pkglinerel=$6
@ -675,7 +676,7 @@ function launch_pkgs_loop() {
}
fi
# skip package in job if it is in the delayed repository
if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then
if [ "$PKGLINE_DELAYED" -a "$BUILDSLINE_DELAYED" -a "$do_autobuild" ]; then
set -- $PKGLINE_DELAYED
if [ "$SPEC_VERSION-$SPEC_RELEASE" = "$2-$6" ]; then
echo "!! Warning: skipping ${pkg} package build because already present in delayed repository."
@ -685,7 +686,7 @@ function launch_pkgs_loop() {
command_opts="-a5,6,10,11 --force-install" ;;
send)
# skip package in job if up to date
if [ "$PKGLINE" -a "$do_autobuild" ]; then
if [ "$PKGLINE" -a "$BUILDSLINE" -a "$do_autobuild" ]; then
set -- $PKGLINE
pkglinever=$2
pkglinerel=$6
@ -700,7 +701,7 @@ function launch_pkgs_loop() {
}
fi
# skip package in job if it is in the delayed repository
if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then
if [ "$PKGLINE_DELAYED" -a "$BUILDSLINE_DELAYED" -a "$do_autobuild" ]; then
set -- $PKGLINE_DELAYED
if [ "$SPEC_VERSION-$SPEC_RELEASE" = "$2-$6" ]; then
echo "!! Warning: skipping ${pkg} package send because already present in delayed repository."

20
autodist-cgi
View File

@ -153,11 +153,7 @@ _EOF
# Security check
[ "${SHOWLOG/\/\.}" != "${SHOWLOG}" ] && continue
if [ "${AUTOPORT_CHROOT[$i]}" ]; then
if [ "${AUTOPORT_CHROOT_USER[$i]}" = "autodist" ]; then
BUILDLOGDIR="/var/autoport/${AUTOPORT_CHROOT[$i]}/var/autodist/log/"
else
BUILDLOGDIR="/var/autoport/${AUTOPORT_CHROOT[$i]}/home/${AUTOPORT_CHROOT_USER[$i]}/.autodist/log/"
fi
BUILDLOGDIR="$LOGDIR"
elif [ "${AUTOPORT_NATIVE[$i]}" -o "${AUTOPORT_UPDATE[$i]}" ]; then
BUILDLOGDIR="/var/autodist/log/"
fi
@ -175,6 +171,8 @@ _EOF
fi
if [ -r ${BUILDLOGDIR}${SHOWLOG} ]; then
LOGFILESIZE=`stat -c %s ${BUILDLOGDIR}${SHOWLOG}`
LOGDATE=`stat -c %y ${BUILDLOGDIR}${SHOWLOG}`
echo -n "<b>${SHOWLOG} ($LOGDATE; $LOGFILESIZE):</b>"
if [ $LOGFILESIZE -lt 131072 ]; then
cat ${BUILDLOGDIR}${SHOWLOG} | parse_build_output
else
@ -257,12 +255,16 @@ _EOF
fi
done
BUILDNOW[$BUILDNOWIDX]="$1"
if [ "$2" = "port" ]; then
CURRLOGFILE=build/$3/$1.${AUTOPORT_ARCH[$i]}
if [ "$4" ]; then
BUILDLOG[$BUILDNOWIDX]="${BUILDLOG[$BUILDNOWIDX]} `echo $4 | sed "s|.*/log/||"`"
else
CURRLOGFILE=$2/$3/$1.${AUTOPORT_ARCH[$i]}
if [ "$2" = "port" ]; then
CURRLOGFILE=build/$3/$1.${AUTOPORT_ARCH[$i]}
else
CURRLOGFILE=$2/$3/$1.${AUTOPORT_ARCH[$i]}
fi
BUILDLOG[$BUILDNOWIDX]="${BUILDLOG[$BUILDNOWIDX]} $CURRLOGFILE"
fi
BUILDLOG[$BUILDNOWIDX]="${BUILDLOG[$BUILDNOWIDX]} $CURRLOGFILE"
# BUILDLOG[$BUILDNOWIDX]="${BUILDLOG[$BUILDNOWIDX]} $2/$3/$1.i586"
BUILDNOWSTATUS[$BUILDNOWIDX]=
BUILDNOWIDX=`expr $BUILDNOWIDX + 1`

45
autoport
View File

@ -48,10 +48,14 @@ function autoport_log() {
local pkg=$1
local operation=$2
local result=$3
local logfile=$4
local loggrep=`grep "?= See" $4 | sed "s|?= See ||"`
local logfiles=""
for f in $loggrep; do
logfiles="$logfiles $f"
done
[ "$pkg" -a "$operation" -a "$result" -a "$BATCH_MODE" ] || return
echo "$pkg $operation $result $logfile" >> $DATADIR/autoport-$PORT_REPOSITORY-current
echo "$pkg $operation $result $logfiles" >> $DATADIR/autoport-$PORT_REPOSITORY-current
}
# for webbuild message
@ -442,7 +446,8 @@ for TARGET_ARCH in ${TARGET_ARCHS}; do
echo -n "prepare"
[ "$VERBOSE_MODE" ] && echo "
%% COMMAND: LANG=C LC_ALL=C autodist -p ${JOB_CURRENT} --server $PORT_REPOSITORY --repository $PORT_REPOSITORY --severity 2 -- $STAGEOPTS"
LANG=C LC_ALL=C autodist -p ${JOB_CURRENT} --server $PORT_REPOSITORY --repository $PORT_REPOSITORY --severity 2 -- $STAGEOPTS &>>$logfile
LANG=C LC_ALL=C autodist -p ${JOB_CURRENT} --server $PORT_REPOSITORY --repository $PORT_REPOSITORY --severity 2 -- $STAGEOPTS &>$tmpfile
cat $tmpfile >> $logfile
[ $? -gt 0 ] && {
RES=`find_source_by_provide $PORT_REPOSITORY $JOB_CURRENT`
for j in ${RES}; do
@ -455,37 +460,39 @@ for TARGET_ARCH in ${TARGET_ARCHS}; do
echo "(FAILED) "
fi
JOB_CANTPREPARE=(${JOB_CANTPREPARE[*]} $JOB_CURRENT)
autoport_log ${JOB_CURRENT} prepare failed
autoport_log ${JOB_CURRENT} prepare failed $tmpfile
continue
} || {
echo $JOB_CURRENT >> $DATAARCHDIR/preparedjobs
echo -n "(OK) "
autoport_log ${JOB_CURRENT} prepare ok
autoport_log ${JOB_CURRENT} prepare ok $tmpfile
}
if [ "$REBUILD_MODE" = "1" ]; then
echo -n "update"
[ "$VERBOSE_MODE" ] && echo "
%% COMMAND: LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"$REBUILD_CHANGELOG\""
LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"$REBUILD_CHANGELOG\" &>>$logfile
LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"$REBUILD_CHANGELOG\" &>$tmpfile
cat $tmpfile >> $logfile
[ $? -gt 0 ] && {
echo "(FAILED) "
autoport_log ${JOB_CURRENT} update failed
autoport_log ${JOB_CURRENT} update failed $tmpfile
continue
} || {
autoport_log ${JOB_CURRENT} update ok
autoport_log ${JOB_CURRENT} update ok $tmpfile
echo -n "(OK) "
}
elif [ "$PORT_REPOSITORY" != "$DEST_REPOSITORY" ]; then
echo -n "update"
[ "$VERBOSE_MODE" ] && echo "
%% COMMAND: LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"automatic port from $PORT_REPOSITORY\""
LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"automatic port from $PORT_REPOSITORY\" &>>$logfile
LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"automatic port from $PORT_REPOSITORY\" &>$tmpfile
cat $tmpfile >> $logfile
[ $? -gt 0 ] && {
echo "(FAILED) "
autoport_log ${JOB_CURRENT} update failed
autoport_log ${JOB_CURRENT} update failed $tmpfile
continue
} || {
autoport_log ${JOB_CURRENT} update ok
autoport_log ${JOB_CURRENT} update ok $tmpfile
echo -n "(OK) "
}
elif [ "$REBUILD_MILESTONE" -a "$pkg_milestone" != "$REBUILD_MILESTONE" ]; then
@ -511,12 +518,12 @@ for TARGET_ARCH in ${TARGET_ARCHS}; do
echo "(FAILED)"
JOB_LOG=`grep "^?= See " $tmpfile | awk '{ print $3 }'`
[ "$JOB_LOG" ] || JOB_LOG=$tmpfile
autoport_log ${JOB_CURRENT} update failed
autoport_log ${JOB_CURRENT} update failed $tmpfile
continue
} || {
cat $tmpfile >> $logfile
echo -n "(OK) "
autoport_log ${JOB_CURRENT} update ok
autoport_log ${JOB_CURRENT} update ok $tmpfile
}
fi
}
@ -532,6 +539,8 @@ for TARGET_ARCH in ${TARGET_ARCHS}; do
fi
[ "$PORT_REPOSITORY" != "$DEST_REPOSITORY" ] && pkg_repository="$DEST_REPOSITORY"
[ ! "$DONT_PREPARE_THIS" -a "$PORT_REPOSITORY" = "$DEST_REPOSITORY" -a ! "$REBUILD_MILESTONE" ] && NOSRPM_OPTS="--nosrpm" || NOSRPM_OPTS="--force"
[ "$VERBOSE_MODE" ] && echo "
%% COMMAND: LANG=C LC_ALL=C autodist -b -s ${JOB_CURRENT} --arch $TARGET_ARCH --server $pkg_repository --repository $PORT_REPOSITORY --severity 2 $FORCE_FLAG -- $STAGEOPTS $NOSRPM_OPTS"
LANG=C LC_ALL=C autodist -b -s ${JOB_CURRENT} --arch $TARGET_ARCH --server $pkg_repository --repository $PORT_REPOSITORY --severity 2 $FORCE_FLAG -- $STAGEOPTS $NOSRPM_OPTS &>$tmpfile
[ $? -gt 0 ] && {
if [ "$VERBOSE_MODE" ]; then
@ -548,14 +557,14 @@ for TARGET_ARCH in ${TARGET_ARCHS}; do
JOB_LOG=`grep "^?= See " $tmpfile | awk '{ print $3 }'`
[ "$JOB_LOG" ] || JOB_LOG=$tmpfile
[ ! "$REBUILD_MILESTONE" ] && find_requirements $JOB_LOG
autoport_log ${JOB_CURRENT} port failed
autoport_log ${JOB_CURRENT} port failed $tmpfile
continue
} || {
cat $tmpfile >> $logfile
JOB_SENT=(${JOB_SENT[*]} $JOB_CURRENT)
echo $JOB_CURRENT >> $DATAARCHDIR/sentjobs
echo -n "(OK) "
autoport_log ${JOB_CURRENT} port ok
autoport_log ${JOB_CURRENT} port ok $tmpfile
if [ "$WEBBUILD_URL" -a "$WEBBUILD_USER" ]; then
curl -s "$WEBBUILD_URL?REQUEST=message&USER=$WEBBUILD_USER&SECRET=$WEBBUILD_SECRET&USER_EMAIL=$WEBBUILD_EMAIL&\
MESSAGE=`cgi_encodevar \"ported <b>$JOB_CURRENT</b> to <b>$TARGET_ARCH</b> and sent it to <b>$pkg_repository</b>\"`" >/dev/null
@ -584,7 +593,7 @@ MESSAGE=`cgi_encodevar \"ported <b>$JOB_CURRENT</b> to <b>$TARGET_ARCH</b> and s
JOB_LOG=`grep "^?= See " $tmpfile | awk '{ print $3 }'`
[ "$JOB_LOG" ] || JOB_LOG=$tmpfile
[ ! "$REBUILD_MILESTONE" ] && find_requirements $JOB_LOG
autoport_log ${JOB_CURRENT} install failed
autoport_log ${JOB_CURRENT} install failed $tmpfile
continue
} || {
cat $tmpfile >> $logfile
@ -592,7 +601,7 @@ MESSAGE=`cgi_encodevar \"ported <b>$JOB_CURRENT</b> to <b>$TARGET_ARCH</b> and s
JOB_MAYBEINSTALLED_NEW=(${JOB_MAYBEINSTALLED_NEW[*]} $JOB_CURRENT)
JOB_COMPLETED=(${JOB_COMPLETED[*]} $JOB_CURRENT)
echo -n "(OK)"
autoport_log ${JOB_CURRENT} install ok
autoport_log ${JOB_CURRENT} install ok $tmpfile
}
echo
done
@ -639,7 +648,7 @@ MESSAGE=`cgi_encodevar \"ported <b>$JOB_CURRENT</b> to <b>$TARGET_ARCH</b> and s
JOB_MAYBEINSTALLED_NEW=(${JOB_MAYBEINSTALLED_NEW[*]} $JOB_CURRENT)
JOB_COMPLETED=(${JOB_COMPLETED[*]} $JOB_CURRENT)
echo -n "(OK)"
autoport_log ${JOB_CURRENT} retryinstall ok
autoport_log ${JOB_CURRENT} retryinstall ok $tmpfile
}
echo
done