From 179121211e38d984d6424aac869cb7f2a810bfb7 Mon Sep 17 00:00:00 2001 From: Silvan Calarco Date: Sat, 9 Jun 2012 16:35:51 +0200 Subject: [PATCH] autoport-cgi: more basic enhancements autodist,autoport: better loggin location and internal formatting for parsing --- autodist | 19 +- autodist-cgi | 193 ++++++++++++++---- autoport | 4 +- .../65-openmamba-autobuild-autoport | 4 +- 4 files changed, 170 insertions(+), 50 deletions(-) diff --git a/autodist b/autodist index b132eab..84b403f 100755 --- a/autodist +++ b/autodist @@ -557,10 +557,10 @@ function launch_pkgs_loop() { if [ "$REPNAME" ]; then command_opts="$command_opts --server ${REPNAME}" else - echo "Warning: undefined repository $4 in srcpkglist, check AUTODIST_REPOSITORIES in configuration file" + echo "!! Warning: undefined repository $4 in srcpkglist, check AUTODIST_REPOSITORIES in configuration file" fi else - echo "Warning: package $pkg is missing in srcpkglist file" + echo "!! Warning: package $pkg is missing in srcpkglist file" fi ;; rebuild) command_opts="-a4 --rebuild" ;; @@ -573,7 +573,7 @@ function launch_pkgs_loop() { # warning: assuming version is passed first version_find_bigger "${passed_arguments/ *}" "$pkglinever" [ $? -eq 1 ] || { - echo "Warning: skipping ${pkg} package already up to date ($pkglinever >= ${passed_arguments/ *})." + echo "!! Warning: skipping ${pkg} package already up to date ($pkglinever >= ${passed_arguments/ *})." continue } fi @@ -583,7 +583,7 @@ function launch_pkgs_loop() { if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then set -- $PKGLINE_DELAYED if [ "$SPEC_VERSION-$SPEC_RELEASE" = "$2-$6" ]; then - echo "Warning: skipping ${pkg} package build because already present in delayed repository." + echo "!! Warning: skipping ${pkg} package build because already present in delayed repository." continue fi fi @@ -599,7 +599,7 @@ function launch_pkgs_loop() { [ $? -eq 0 ] && { version_find_bigger "$SPEC_RELEASE" "$pkglinerel" if [ $? -ne 1 ]; then - echo "Warning: skipping ${pkg} package already up to date ($SPEC_VERSION-$SPEC_RELEASE >= $pkglinever-$pkglinerel)." + echo "!! Warning: skipping ${pkg} package already up to date ($SPEC_VERSION-$SPEC_RELEASE >= $pkglinever-$pkglinerel)." continue fi } @@ -608,7 +608,7 @@ function launch_pkgs_loop() { if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then set -- $PKGLINE_DELAYED if [ "$SPEC_VERSION-$SPEC_RELEASE" = "$2-$6" ]; then - echo "Warning: skipping ${pkg} package build because already present in delayed repository." + echo "!! Warning: skipping ${pkg} package build because already present in delayed repository." continue fi fi @@ -624,7 +624,7 @@ function launch_pkgs_loop() { [ $? -eq 0 ] && { version_find_bigger "$SPEC_RELEASE" "$pkglinerel" if [ $? -ne 1 ]; then - echo "Warning: skipping ${pkg} package already up to date ($SPEC_VERSION-$SPEC_RELEASE >= $pkglinever-$pkglinerel)." + echo "!! Warning: skipping ${pkg} package already up to date ($SPEC_VERSION-$SPEC_RELEASE >= $pkglinever-$pkglinerel)." continue fi } @@ -633,7 +633,7 @@ function launch_pkgs_loop() { if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then set -- $PKGLINE_DELAYED if [ "$SPEC_VERSION-$SPEC_RELEASE" = "$2-$6" ]; then - echo "Warning: skipping ${pkg} package send because already present in delayed repository." + echo "!! Warning: skipping ${pkg} package send because already present in delayed repository." continue fi fi @@ -940,8 +940,7 @@ for JOB_NUM in `seq 1 ${#JOBNAME[*]}`; do get_job_vector $JOB_NAME JOB_NAME=${JOB_NAME/*\/} - [ "$quiet" ] || echo " -== =======================$JOB_NAME=============================" + [ "$quiet" ] || echo "== =======================$JOB_NAME=============================" echo -n "%% Doing $JOB_NAME (VER=$JOB_VER" for k in ${JOB_VARNAMES[*]}; do echo -n " %${k}" diff --git a/autodist-cgi b/autodist-cgi index 1e341fc..b986088 100755 --- a/autodist-cgi +++ b/autodist-cgi @@ -3,15 +3,44 @@ NUM=`echo "$QUERY_STRING" | sed -n 's/^.*NUM=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"` SECONDARY=`echo "$QUERY_STRING" | sed -n 's/^.*SECONDARY=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"` +AS_SECONDARY=`echo "$QUERY_STRING" | sed -n 's/^.*AS_SECONDARY=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"` SHOWLOG=`echo "$QUERY_STRING" | sed -n 's/^.*SHOWLOG=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"` -SHOWLASTLOG=`echo "$QUERY_STRING" | sed -n 's/^.*SHOWLASTLOG=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"` +SHOWINDEX=`echo "$QUERY_STRING" | sed -n 's/^.*SHOWINDEX=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"` LINES=`echo "$QUERY_STRING" | sed -n 's/^.*LINES=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"` [ "$LINES" ] || LINES=5 -[ "$SECONDARY" ] && SECONDARY_ADD="&SECONDARY=1" +[ "$AS_SECONDARY" ] && SECONDARY_ADD="&SECONDARY=1" echo -e "Content-type: text/html; charset=UTF-8\n\n" +[ ! "$AS_SECONDARY" ] && echo "Show: ALL " + +[ "$SHOWINDEX" -o ! "$AS_SECONDARY" ] && \ +for i in `seq 0 ${#AUTOPORT_ARCH[*]}`; do + [ "${AUTOPORT_ARCH[$i]}" ] || continue + [ "${AUTOPORT_DISABLE[$i]}" -a "${AUTOPORT_DISABLE[$i]}" != "0" ] && continue + + if [ "${AUTOPORT_UPDATE[$i]}" ]; then + echo "${AUTOPORT_REPOSITORIES[$i]}(${AUTOPORT_ARCH[$i]}) " + elif [ "${AUTOPORT_CHROOT[$i]}" ]; then + echo "${AUTOPORT_REPOSITORIES[$i]}(${AUTOPORT_ARCH[$i]}) " + elif [ "${AUTOPORT_NATIVE[$i]}" ]; then + echo "${AUTOPORT_REPOSITORIES[$i]}(${AUTOPORT_ARCH[$i]}) " + elif [ "${AUTOPORT_CROSS[$i]}" ]; then + echo "${AUTOPORT_REPOSITORIES[$i]}(${AUTOPORT_CROSS[$i]};${AUTOPORT_ARCH[$i]}) " + fi +done + +if [ "$SHOWINDEX" ]; then + exit 0 +else + curl "$AUTOPORT_CGI_SECONDARY?AS_SECONDARY=1&SHOWINDEX=1" +fi + +[ ! "$AS_SECONDARY" ] && echo "
" + +[ ! "$SHOWLOG" -a "$SECONDARY" = "$AS_SECONDARY" ] && echo "

Host: `hostname -s` (`uname -m`, kernel `uname -r`)

" + for i in `seq 0 ${#AUTOPORT_ARCH[*]}`; do [ "${AUTOPORT_ARCH[$i]}" ] || continue [ "${AUTOPORT_DISABLE[$i]}" -a "${AUTOPORT_DISABLE[$i]}" != "0" ] && continue @@ -22,9 +51,11 @@ for i in `seq 0 ${#AUTOPORT_ARCH[*]}`; do elif [ "${AUTOPORT_CHROOT[$i]}" ]; then LOGFILE="/var/autodist/log/autoport-chroot-${AUTOPORT_CHROOT[$i]}.log" LASTLOGFILE="/var/autodist/log/autoport-chroot-${AUTOPORT_CHROOT[$i]}-last.log" + AUTODISTLOGFILE="/var/autoport/${AUTOPORT_CHROOT[$i]}/home/${AUTOPORT_CHROOT_USER[$i]}/.autoport/${AUTOPORT_ARCH[$i]}/${AUTOPORT_REPOSITORIES[$i]}.log" elif [ "${AUTOPORT_NATIVE[$i]}" ]; then - LOGFILE="/var/autodist/log/autoport-native-${AUTOPORT_ARCH[$i]}.log" - LASTLOGFILE="/var/autodist/log/autoport-native-${AUTOPORT_ARCH[$i]}-last.log" + LOGFILE="/var/autodist/log/autoport-native-${AUTOPORT_REPOSITORIES[$i]}-${AUTOPORT_ARCH[$i]}.log" + LASTLOGFILE="/var/autodist/log/autoport-native-${AUTOPORT_REPOSITORIES[$i]}-${AUTOPORT_ARCH[$i]}-last.log" + AUTODISTLOGFILE="/var/autodist/.autoport/${AUTOPORT_ARCH[$i]}/${AUTOPORT_REPOSITORIES[$i]}.log" elif [ "${AUTOPORT_CROSS[$i]}" ]; then LOGFILE="/var/autodist/log/autoport-cross-${AUTOPORT_CROSS[$i]}.log" LASTLOGFILE="/var/autodist/log/autoport-cross-${AUTOPORT_CROSS[$i]}-last.log" @@ -33,54 +64,144 @@ for i in `seq 0 ${#AUTOPORT_ARCH[*]}`; do fi if [ "$SHOWLOG" -a "$NUM" = "$i" ]; then - if [ ! "$SECONDARY" ]; then - echo "
"
-         cat $LOGFILE
-         echo "
" + if [ "$SECONDARY" = "$AS_SECONDARY" ]; then + echo "
"
+         if [ "$SHOWLOG" = "current" ]; then
+            cat $LOGFILE | sed "s|<|\<|g"
+         elif [ "$SHOWLOG" = "last" ]; then
+            cat $LASTLOGFILE | sed "s|<|\<|g"
+         else
+            if [ "${AUTOPORT_CHROOT[$i]}" ]; then
+               BUILDLOGFILE="/var/autoport/${AUTOPORT_CHROOT[$i]}/home/${AUTOPORT_CHROOT_USER[$i]}/.autodist/log/$SHOWLOG"
+           elif [ "${AUTOPORT_NATIVE[$i]}" -o "${AUTOPORT_UPDATE[$i]}" ]; then
+               BUILDLOGFILE="/var/autodist/log/$SHOWLOG"
+            fi
+            [ -e $BUILDLOGFILE ] && cat $BUILDLOGFILE | sed "s|<|\<|g"
+         fi
+         echo "
" else - curl "$AUTOPORT_CGI_SECONDARY?SHOWLOG=1&NUM=$i" + curl "$AUTOPORT_CGI_SECONDARY?AS_SECONDARY=1&SHOWLOG=$SHOWLOG&NUM=$i&LOG=$LOG" fi exit 0 - elif [ "$SHOWLASTLOG" -a "$NUM" = "$i" ]; then - if [ ! "$SECONDARY" ]; then - echo "
"
-         cat $LASTLOGFILE
-         echo "
" - else - curl "$AUTOPORT_CGI_SECONDARY?SHOWLASTLOG=1&NUM=$i" - fi - exit 0 - elif [ ! "$SHOWLOG" -a ! "$SHOWLASTLOG" ]; then + elif [ ! "$SHOWLOG" ]; then [[ "$NUM" && "$NUM" != "$i" ]] && continue - + [[ "$SECONDARY" = "$AS_SECONDARY" ]] || continue if [ "${AUTOPORT_UPDATE[$i]}" ]; then - echo "* autodist update (host:`hostname -s`,arch:${AUTOPORT_ARCH[$i]},channels:${AUTOPORT_REPOSITORIES[$i]})" - echo " [ last ]" + echo "* ${AUTOPORT_REPOSITORIES[$i]}(${AUTOPORT_ARCH[$i]}) (type:autodist update)" + echo " [ log last ]" fi if [ "${AUTOPORT_CHROOT[$i]}" ]; then - echo "* chroot autoport (host:`hostname -s`,arch:${AUTOPORT_ARCH[$i]},channels:${AUTOPORT_REPOSITORIES[$i]},chroot:${AUTOPORT_CHROOT[$i]})" - echo " [ last ]" + echo "* ${AUTOPORT_REPOSITORIES[$i]}(${AUTOPORT_ARCH[$i]}) (type:chroot autoport chroot:${AUTOPORT_CHROOT[$i]})" + echo " [ log last ]" fi if [ "${AUTOPORT_NATIVE[$i]}" ]; then - echo "* native autoport (host:`hostname -s`,arch:${AUTOPORT_ARCH[$i]},channels:${AUTOPORT_REPOSITORIES[$i]})" - echo " [ last ]" + echo "* ${AUTOPORT_REPOSITORIES[$i]}(${AUTOPORT_ARCH[$i]}) (type:native autoport)" + echo " [ log last ]" fi if [ "${AUTOPORT_CROSS[$i]}" ]; then - echo "* cross-platform autoport (host:`hostname -s`,arch:${AUTOPORT_CROSS[$i]})" - echo " [ last ]" + echo "* cross-platform autoport (arch:${AUTOPORT_CROSS[$i]})" + echo " [ last ]" fi - echo "
"
-      grep -v "^=" $LOGFILE | tail -n "$LINES" | sed "s|.*\r||"
-      echo
+      echo "
"
+      if [ "${AUTOPORT_UPDATE[$i]}" ]; then
+         unset STARTTIME ENDTIME JOBON BUILDNOW BUILDLOG SCHEDULED
+         BUILDNOWIDX=0
+         while read line; do
+            set -- $line
+            if [[ "${line:0:1}" = $'\r' ]]; then
+               PREF=`echo $1 | tr -d "\r"`
+            else
+               PREF=$1
+            fi
+            case $PREF in
+               "%%" ) if [ "$2 $3" == "Autodist started" ]; then
+                         STARTTIME=`echo $line | sed "s|.* @ ||"`
+                      elif [ "$2 $3 $4" == "All jobs done" ]; then
+                         ENDTIME=`echo $line | sed "s|.* @ ||"`
+                      elif [ "$2 $3" == "autoupdate jobs:" ]; then
+                         SCHEDULED=`echo $line | sed "s|.*autoupdate jobs: ||"`
+                      elif [ "$2" == "Doing" ]; then
+                         BUILDNOW[$BUILDNOWIDX]="$3"
+                         BUILDNOWIDX=`expr $BUILDNOWIDX + 1`
+                      #elif [ "$2" != "Starting" -a "$2" != "Updating" ]; then
+                      #   [ "$JOBON" ] || echo $line
+                      fi ;;
+               "?=") if [ "$2" == "See" ]; then
+                        BUILDLOG[$BUILDNOWIDX-1]=`echo $line | sed "s|.*autodist/log/||"`
+                     fi ;;
+               "=>") JOBON=1 ;;
+               "%!"|"!!") BUILDNOW[$BUILDNOWIDX-1]="${BUILDNOW[$BUILDNOWIDX-1]} $line"; JOBON= ;;
+               "==") [ "${2:0:2}" == "==" ] && JOBON= ;;
+               "Hunk"|"") ;;
+               *) [ "$JOBON" ] || {
+                     line=`echo $line | sed "s|.*\r||"`
+                     echo $line
+                  } ;;
+            esac
+         done < $LOGFILE
+      else
+         unset STARTTIME ENDTIME BUILDNOW BUILDLOG AUTODISTON LASTLINE SCHEDULED
+         while read line; do
+            line=`echo $line | sed "s|.*\r||"`
+            set -- $line
+            case $1 in
+               "%%" ) AUTODISTON=1 ;;
+               "^"|"="*|""|"!"|"|"|"#"|"->") ;; #comment
+               "+") [ "$STARTTIME" ] || STARTTIME=`echo $line | sed "s|.*start at ||"` ;;
+               "-") ENDTIME=`echo $line | sed "s|.*end at ||"` ;;
+               [A-Za-z0-9]*) ;;
+               ">") [ "$SCHEDULED" ] || SCHEDULED=`echo $line | sed "s|.*Scheduled jobs: ||"` ;;
+               *) [ ! "$AUTODISTON" ] && echo $line ;;
+            esac
+         done < $LOGFILE
+         BUILDNOWIDX=0
+         while read line; do
+            line=`echo $line | sed "s|.*\r||"`
+            set -- $line
+            case $1 in
+               "%%" ) if [ "$2" == "Doing" ]; then
+                         if [ "$3" != "${BUILDNOW[$BUILDNOWIDX-1]}" ]; then
+                            BUILDNOW[$BUILDNOWIDX]="$3"
+                            BUILDNOWIDX=`expr $BUILDNOWIDX + 1`
+                         fi
+                         JOBON=1
+                      fi ;;
+               "?=") if [ "$2" == "See" ]; then
+                        BUILDLOG[$BUILDNOWIDX-1]=`echo $line | sed "s|.*autodist/log/||"`
+                     fi ;;
+               "=>") JOBON=1 ;;
+               "%!"|"!!") BUILDNOW[$BUILDNOWIDX-1]="${BUILDNOW[$BUILDNOWIDX-1]} $line"; JOBON= ;;
+               "==") [ "${2:0:2}" == "==" ] && JOBON= ;;
+               "Hunk"|"") ;;
+               *) [ "$JOBON" ] || echo $line ;;
+            esac
+         done < $AUTODISTLOGFILE
+         LASTLINE=$line
+      fi
+         echo "Start: $STARTTIME"
+         echo "Scheduled jobs: $SCHEDULED"
+         for b in `seq 1 ${#BUILDNOW[*]}`; do
+            set -- ${BUILDNOW[$b-1]}
+            echo "$1: $2 $3 $4 $5 $6 "
+         done
+         if [ "$LASTLINE" ]; then
+            set -- $LASTLINE
+            [ "$1" = "^" ] && echo "${2} ${3} ${4} ${5} ${6} ${7}"
+         fi
+         [ "$ENDTIME" ] && echo "End: $ENDTIME" || echo "End: in progress"
+      echo 
+#      grep -v "^=" $LOGFILE | tail -n "$LINES" | sed "s|.*\r||"
+#      echo
       echo "
"
    fi
 done
-
-[ "$NUM" ] && NUM=`expr $NUM - $i`
-
-if [ "$AUTOPORT_CGI_SECONDARY" -a ! "$SHOWLOG" -a ! "$SHOWLASTLOG" ]; then
-   curl "$AUTOPORT_CGI_SECONDARY?SECONDARY=1&LINES=$LINES&NUM=$NUM"
+if [ "$AUTOPORT_CGI_SECONDARY" -a ! "$SHOWLOG" ]; then
+   if [ "$NUM" -a "$SECONDARY" ]; then
+      curl "$AUTOPORT_CGI_SECONDARY?AS_SECONDARY=1&LINES=$LINES&NUM=$NUM&SECONDARY=$SECONDARY"
+   elif [ ! "$NUM" ]; then
+      curl "$AUTOPORT_CGI_SECONDARY?AS_SECONDARY=1&LINES=$LINES"
+   fi
 fi
diff --git a/autoport b/autoport
index e8edfc8..85df1f8 100755
--- a/autoport
+++ b/autoport
@@ -245,8 +245,8 @@ for TARGET_ARCH in ${TARGET_ARCHS}; do
    [ -d $DATADIR/$BASE_REPOSITORY ] || mkdir -p $DATADIR/$BASE_REPOSITORY
    [ -d $DATADIR/$PORT_REPOSITORY ] || mkdir -p $DATADIR/$PORT_REPOSITORY
 
-   logfile=$DATAARCHDIR/log
-   [ -e $logfile ] && mv -f $logfile $DATAARCHDIR/loglast
+   logfile=$DATAARCHDIR/$PORT_REPOSITORY.log
+   [ -e $logfile ] && mv -f $logfile $DATAARCHDIR/$PORT_REPOSITORY-last.log
    > $logfile
 
    BUILD_PLATFORM=`rpm --target $TARGET_ARCH --eval %{_build}`
diff --git a/etc/cron.hourly/65-openmamba-autobuild-autoport b/etc/cron.hourly/65-openmamba-autobuild-autoport
index 192125a..ea4dab1 100755
--- a/etc/cron.hourly/65-openmamba-autobuild-autoport
+++ b/etc/cron.hourly/65-openmamba-autobuild-autoport
@@ -45,8 +45,8 @@ for i in `seq 0 ${#AUTOPORT_ARCH[*]}`; do
       LOGFILE="/var/autodist/log/autoport-chroot-${AUTOPORT_CHROOT[$i]}.log"
       LASTLOGFILE="/var/autodist/log/autoport-chroot-${AUTOPORT_CHROOT[$i]}-last.log"
    elif [ "${AUTOPORT_NATIVE[$i]}" ]; then
-      LOGFILE="/var/autodist/log/autoport-native-${AUTOPORT_ARCH[$i]}.log"
-      LASTLOGFILE="/var/autodist/log/autoport-native-${AUTOPORT_ARCH[$i]}-last.log"
+      LOGFILE="/var/autodist/log/autoport-native-${AUTOPORT_REPOSITORIES[$i]}-${AUTOPORT_ARCH[$i]}.log"
+      LASTLOGFILE="/var/autodist/log/autoport-native-${AUTOPORT_REPOSITORIES[$i]}-${AUTOPORT_ARCH[$i]}-last.log"
    elif [ "${AUTOPORT_CROSS[$i]}" ]; then
       LOGFILE="/var/autodist/log/autoport-cross-${AUTOPORT_CROSS[$i]}.log"
       LASTLOGFILE="/var/autodist/log/autoport-cross-${AUTOPORT_CROSS[$i]}-last.log"