autodist/autodist

#!/bin/bash
#
# Autodist -- batch build tool for RPM based distributions
# Copyright (C) 2006-2012 by Silvan Calarco <silvan.calarco@mambasoft.it>
#
# Released under the terms of the GNU GPL release 3 license
#
VERSION=0.9.3
me=(${0##*/} $VERSION "Sat Aug 20 2010")
exec 3>`readlink /proc/self/fd/0`

function usage() {
   echo "\
${me[0]} ${me[1]}
"$"Copyright (C) 2006-2012 Silvan Calarco <silvan.calarco@mambasoft.it>""
"$"Released under the terms of the GNU GPL v3 license"
   echo "
"$"Batch build tool for bulding a whole distribution and much more.""

"$"Usage"":
   $me [operations ...] [options ...] [job ...] [-- autospec_args]

"$"Operations"":
   -a,--autobuild     "$"Start batch build operations (implies -p -u -b -s --severity 1)""
   -p,--prepare       "$"Prepare packages (download and prepare for update)""
   -u,--update        "$"Update packages""
   -b,--build         "$"Build packages""
   -i,--install       "$"Install packages on local system""
   -s,--send          "$"Send packages to repository""

"$"Options"":
   -c,--cleanlogs     "$"Clean all logs before starting""
   -d,--debug         "$"Enable print of debugging information""
   -h,--help          "$"Print this help, then exit""
   -t,--test          "$"Just simulate the execution of programs""
   -q,--quiet         "$"Be quiet (don't display tail of error logs)""
   -r,--rebuild       "$"Rebuild packages""
   -v,--verbose       "$"Be verbose""
   --force            "$"Pass force parameter to autospec""
   --repository       "$"Pass given repository as base for operations""
   --server num       "$"Pass given server number to autospec for send operation""
   --severity num     "$"Error handling level:
                            0: don't stop (default)
                            1: skip current job on error
                            2: abort on error""
   --arch             "$"Build packages for specified target architecture""

   job                "$"Job specification in the form pkg#version, job#version, job/pkg#version, examples:
                            glibc#2.11.1
                            kernel-extra
                            kernel-packages/lirc""
   autospec_args      "$"Send specified arguments to autospec""
"

}

[ $# -gt 0 ] || { usage ; exit 1; }

DATE_NOW=`LANG=C date +%s`
BUILDTIME=`date -Iseconds`
SYSCONFDIR=/etc/autodist
LOCALSTATEDIR=/var/autodist
[ "$USER" = "autodist" ] && USERCONFDIR=$LOCALSTATEDIR || USERCONFDIR=$HOME/.autodist
LOGBASEDIR=$USERCONFDIR/log
RPMTMPPATH=/var/tmp/autodist
SCRIPTDIR=$SYSCONFDIR/scripts

DISTDB=$SYSCONFDIR/distdb
DISTDBDIR=$SYSCONFDIR/distdb.d/
BLACKLIST=$SYSCONFDIR/blacklist
LEGACYLIST=$SYSCONFDIR/legacy
BUILDARCH=`rpm --eval %{_build_cpu}`
TARGETARCH=$BUILDARCH
CONFIGFILE=$SYSCONFDIR/config
SCRIPT_UPDATE_SPECFILE=$SCRIPTDIR/update-specfile
AUTOSPEC_CMD="/usr/bin/autospec"
PIDFILE="/var/run/autodist/autodist.pid"

# Configuration defaults normally overriden in configuration file
AUTOBUILD_MAXNUM=50
AUTOBUILD_DATEFROM=20070101
GNOME_VER=2.24.0
AUTODIST_REPOSITORY=
SEND_SERVER=

[ -r $CONFIGFILE ] && . $CONFIGFILE


eval `autospec -q --eval="srpms_dir"`
eval `autospec -q --eval="rpms_dir"`
eval `autospec -q --eval="spec_dir"`
eval `autospec -q --eval="source_dir"`
eval `autospec -q --eval="tmppath_dir"`

# init environment especially to have a full PATH when executed by crond with
# system init PATH
. /etc/profile
. /etc/profile.d/*.sh
export PATH

# error handling severity: 0=don't stop; 1=skip; 2=stop
SEVERITY=1

for ((i=1; i<=$#; i++)); do
   case ${!i} in
      -a|--autobuild) do_autobuild=1;
                      do_prepare=1;
                      do_update=1;
                      do_build=1;
                      do_send=1; 
                      SEVERITY=1
		      [ -e $PIDFILE ] && {
	                 PIDCHECK=`cat $PIDFILE`
		         [ "$PIDCHECK" -a -e /proc/$PIDCHECK -a ! "$DISABLE_PID_CHECK" ] && {
		            echo "Error: autodist -a already running with PID=$PIDCHECK.Aborting."
		            exit 1
		         }
		      }
		      
		      ;;
      -c|--cleanlogs) do_cleanlogs=1 ;;
      -h|--help) usage 
                 exit 0
                 ;;
      -p|--prepare) do_prepare=1 ;;
      -u|--update) do_update=1 ;;
      -r|--rebuild) rebuild_packages=1 ;;
      -b|--build) do_build=1 ;;
      -d|--debug) debug=1 ;;
      -i|--install) do_install=1 ;;
      -s|--send) do_send=1 ;;
      -t|--test) simulate=1 ;; 
      -q|--quiet) quiet=1 ;;
      -v|--verbose) verbose=1 ;;
      --force) SEND_FORCE="--force" 
               SEND_FORCE_BUILD="--force-build" 
               ;;
      --server) let i+=1
                [ "${!i}" ] || {
                    echo "Error: server parameter not given, aborting."
                    exit 1
                }
                SEND_SERVER="${!i}"
                ;;
      --repository) let i+=1
                [ "${!i}" ] || {
                    echo "Error: repository parameter not given, aborting."
                    exit 1
                }
                AUTODIST_REPOSITORY="${!i}"
                ;;
      --severity) let i+=1
                [ "${!i}" ] || {
                    echo "Error: severity parameter not given, aborting."
                    exit 1
                }
                SEVERITY=${!i}
		;;
      --arch) let i+=1
                [ "${!i}" ] || {
                    echo "Error: arch parameter not given, aborting."
                    exit 1
                }
                TARGETARCH=${!i}
	        ;;
      -*) if [ "${!i}" = "--" ]; then
             AUTOSPEC_ARGS_MODE=1
          elif [ "$AUTOSPEC_ARGS_MODE" ]; then
             AUTOSPEC_ARGS="$AUTOSPEC_ARGS ${!i}"
          else
             echo "ERROR: invalid option \`${!i}'; aborting."
             exit 1
          fi
          ;;
      *) if [ "$AUTOSPEC_ARGS_MODE" != "1" ]; then
            JOBNAME[${#JOBNAME[@]}]="${!i/\#*}"
	       [ "${!i/*\#}" != "${!i}" ] && JOBVER[${#JOBNAME[@]}]="+${!i/*\#}"
	 else
	    AUTOSPEC_ARGS="$AUTOSPEC_ARGS ${!i}"
	 fi
         ;;
   esac
done

[ "$AUTODIST_REPOSITORY" ] || AUTODIST_REPOSITORY=devel-autodist
[ "$SEND_SERVER" ] || SEND_SERVER=$AUTODIST_REPOSITORY

SEND_SERVER_CMD="--server $SEND_SERVER"
AUTOUPDATEDIR=${LOCAL_REPS_BASE_DIR}/$AUTODIST_REPOSITORY/autoupdate/
[ -r $AUTOUPDATEDIR ] || {
   AUTOUPDATEDIR=$USERCONFDIR/$AUTODIST_REPOSITORY/autoupdate
   mkdir -p $AUTOUPDATEDIR
}

SOURCESDIR=$AUTOUPDATEDIR/sources/
SUCCESSLISTDIR=$AUTOUPDATEDIR
SKIPPEDLISTDIR=$AUTOUPDATEDIR

SRCPKGLIST=${LOCAL_REPS_BASE_DIR}/$AUTODIST_REPOSITORY/srcpkglist
[ -r $SRCPKGLIST ] || {
   SRCPKGLIST=$USERCONFDIR/$AUTODIST_REPOSITORY/srcpkglist
   if [ ! -r $SRCPKGLIST -o "`find $SRCPKGLIST -mmin +60`" ]; then
      mkdir -p $USERCONFDIR/$AUTODIST_REPOSITORY
      curl -s $REPS_BASE_URL/$AUTODIST_REPOSITORY/srcpkglist -o $USERCONFDIR/$AUTODIST_REPOSITORY/srcpkglist || {
         echo "Error: unable to fetch $REPS_BASE_DIR/$AUTODIST_REPOSITORY/srcpkglist"
         exit 1
      }
   fi
}

[ "$AUTODIST_DELAYED_REPOSITORY" ] && {
   SRCPKGLIST_DELAYED=${LOCAL_REPS_BASE_DIR}/$AUTODIST_DELAYED_REPOSITORY/srcpkglist
   [ -r $SRCPKGLIST_DELAYED ] || {
      SRCPKGLIST_DELAYED=$USERCONFDIR/$AUTODIST_DELAYED_REPOSITORY/srcpkglist
      if [ ! -r $SRCPKGLIST_DELAYED -o "`find $SRCPKGLIST -mmin +60`" ]; then
         mkdir -p $USERCONFDIR/$AUTODIST_DELAYED_REPOSITORY
         curl -s $REPS_BASE_URL/$AUTODIST_DELAYED_REPOSITORY/srcpkglist -o $USERCONFDIR/$AUTODIST_DELAYED_REPOSITORY/srcpkglist || {
            echo "Error: unable to fetch $REPS_BASE_DIR/$AUTODIST_DELAYED_REPOSITORY/srcpkglist"
            exit 1
         }
      fi
   }
}

# perform arch names conversions
case $BUILDARCH in
   i386|i486|i686) BUILDARCH=i586 ;;
esac

case $TARGETARCH in
   i386|i486|i686) TARGETARCH=i586 ;;
esac

[ -e "$DISTDB" ] || {
   echo "ERROR: missing distdb file $DISTDB; aborting."
   exit 1
}

[ -e $LOGBASEDIR ] || {
   mkdir -p $LOGBASEDIR/prepare/{ok,failed}
   mkdir -p $LOGBASEDIR/update/{ok,failed}
   mkdir -p $LOGBASEDIR/build/{ok,failed}
   mkdir -p $LOGBASEDIR/install/{ok,failed}
   mkdir -p $LOGBASEDIR/send/{ok,failed}
}

#[ "$PACKAGE" ] || {
#   echo "ERROR: missing target; aborting."
#   usage
#   exit 1
#}

. $DISTDB
[ -e $DISTDB.$TARGETARCH ] && . $DISTDB.$TARGETARCH
[ -d $DISTDBDIR ] && \
   for f in `ls $DISTDBDIR/*.db`; do
      . $f
   done

function tempfile() {
   local tmpfile
   tmpfile=`mktemp -q -t $me.XXXXXXXX` || {
      echo "ERROR: cannot create temporary files." >&2
      exit 1
   }
   trap "rm -f $tmpfile" 0 1 2 5 15
   
   > $tmpfile || {
      echo "ERROR: cannot write to temporary files."  >&2
      exit 1
   }
   echo $tmpfile
}

function version_compare()
{
   local A B
   A=$1
   B=$2

   if [ ${A/[a-zA-Z_]*} -gt ${B/[a-zA-Z_]*} ]; then
      return 1
   elif [ ${A/[a-zA-Z_]*} -lt ${B/[a-zA-Z_]*} ]; then
      return 2
   else
      if [[ "$A" > "$B" ]]; then
         return 1
      elif [[ "$A" < "$B" ]]; then
         return 2
      fi
   fi
   return 0
   
}

function version_find_bigger()
{
   local VER1 VER2 FPOS CUTVER1 CUTVER2
   
   VER1=$1
   VER2=$2
   FPOS=1
   while true; do
      CUTVER1=`echo $VER1. | cut -d. -f $FPOS`
      CUTVER2=`echo $VER2. | cut -d. -f $FPOS`
      if [ "$CUTVER1" -a ! "$CUTVER2" ]; then
         return 1
      elif [ "$CUTVER2" -a ! "$CUTVER1" ]; then
         return 2
      elif [ ! "$CUTVER1" -a "$CUTVER2" ]; then
         return 0
      else
         version_compare $CUTVER1 $CUTVER2
         case $? in
            1) return 1 ;;
	    2) return 2 ;;
         esac
      fi
      FPOS=`expr $FPOS + 1`
   done
   return 0
}

#for target in ${TARGET[*]}; do
#   echo ${target[*]}
#done
#exit 1

function tail_file() {
   local elapsedtime elapsedstring elapsedcount phase lastprinttime newtime begintime lines c
   c=0
   lines=0
   begintime=`LANG=C date +%s`
   lastlinetime=$begintime
   lastprinttime=0

   while read line; do
      echo "$line" >> $1
      let lines+=1
      newtime=`LANG=C date +%s`
      let difftime=$newtime-$lastprinttime

      case $verbose in
	    1) echo "$line"
	       ;;
	    *) 
               if [ $difftime -gt 1 ]; then
                  let c+=1
                  lastprinttime=$newtime
                  let elapsedtime=$newtime-$begintime
                  [ $elapsedtime -gt 0 ] && let speed=$lines/$elapsedtime

                  case $c in
                     1) phase="|" ;;
                     2) phase="/" ;;
                     3) phase="-" ;;
                     4) phase="\\"
                        let c=0 ;;
                  esac
                  elapsedstring="`expr $elapsedtime % 60`s"
                  elapsedcount=`expr $elapsedtime / 60`
                  [ $elapsedcount -gt 0 ] && elapsedstring="`expr $elapsedcount % 60`m $elapsedstring"
	          elapsedcount=`expr $elapsedcount / 60`
                  [ $elapsedcount -gt 0 ] && elapsedstring="${elapsedcount}h $elapsedstring"
                  echo -n -e "\r$2 $phase ($lines lines, $elapsedstring, $speed l/s)"
	       fi
	       ;;
      esac
   #echo -n -e "\rTailing...$phase (sleeped $diffdate seconds, $lines output lines) " 
   #savedate=newdate
   done
}

get_job_vector() {
   # note: if JOB_NAME contains a "-" translate it to "_"
   local JTARGET=${1/\/*}
   local JPKG=
   [ "$JTARGET" != "$1" ] && JPKG="${1/*\/}"
   local JNAME=`echo $JTARGET | tr - _`

   # resolve JOB_NAME from distdb
   local jobtmpfile=`tempfile`
   # hack to get an array variable named as $j assigned to the JOB array
   echo "echo \${$JNAME[*]}" > $jobtmpfile

   JOB=(`. $jobtmpfile`)
   rm -f $jobtmpfile

   if [ ${#JOB[*]} -eq 0 ]; then
       # create a default job with given JOB_NAME
       JOB=($JTARGET "" "")
   elif [ ${#JOB[*]} -eq 1 ]; then
       # no variables defined, add an empty job
       JOB=(${JOB[*]} "" "")
   elif [ "${JPKG}" ]; then
       TMPJOB=($JPKG)
       for i in `seq 4 ${#JOB[*]}`; do
          TMPJOB=(${TMPJOB[*]} "${JOB[$i]}")
       done
       JOB=${TMPJOB[*]}
   fi
   JOB_PKGS=(${JOB[0]//,/ })
   JOB_VARNAMES=(${JOB[1]//,/ })
}

function launch_pkgs_loop() {
# This function iterates the passed operation for each package (column) in job
#
# $1: operation (prepare, update, build, send, install)
# $2: options
# $3: job  variables (e.g. i386,std)

   local JOB_VALUES JOB_ARGS JOB_IDX DEFINES_STRING i a PKGLINE
   local cross_target_cpu autodist_crossonly operation

   # evaluate job variables
   JOB_VALUES=(${3//,/ })
   JOB_IDX=$4

   [ "$debug" = "1" ] && JOB_ARGS="--debug " || JOB_ARGS=""
   [ $JOB_IDX ] || JOB_IDX=0
   operation="$1"
   passed_arguments="$2"
   ret=0
   DEFINES_STRING=""

   if [ "$TARGETARCH" != "$BUILDARCH" ]; then
      TARGET_PLATFORM=`rpm --target ${TARGETARCH} --eval %{_target_platform}`
      JOB_ARGS="$JOB_ARGS-A \"${TARGETARCH}\" --root /usr/${TARGET_PLATFORM} "
      DEFINES_STRING="--define _host_cpu=\"${TARGETARCH}\",_target_cpu=\"${TARGETARCH}\""
   fi

   cross_target_cpu=""
   autodist_crossonly=""
   patch_operation=$operation

   for i in `seq 1 ${#JOB_VALUES[*]}`; do
      [ "${JOB_VALUES[$i-1]}" = "-" ] || \
      case ${JOB_VARNAMES[$i-1]} in
         %build_and_install)
	    [ "$operation" = "build" ] && operation=buildinstall
	    break
	    ;;
         %*)
            echo "!! Warning: skipping unknown internal variable: ${JOB_VARNAMES[$i-1]}"
	    break
	    ;;
         autodist_crossonly)
	    autodist_crossonly=1
	    ;;
         cross_target_cpu)
	    cross_target_cpu="${JOB_VALUES[$i-1]}"
	    # don't specify cross_target_cpu for non-cross-target builds
	    if [ "$cross_target_cpu" = "$TARGETARCH" ]; then
	       # set cross_target_cpu as don't use
	       JOB_VALUES[$i-1]="-"
	    else
               [ "$DEFINES_STRING" ] && 
                   DEFINES_STRING="${DEFINES_STRING}," ||
                   DEFINES_STRING="--define "
               DEFINES_STRING="${DEFINES_STRING}${JOB_VARNAMES[$i-1]}=$cross_target_cpu"
	    fi
            ;;
         target)
	       # specific target build was specified, don't build packages containing explicit different target
               [ "$BUILDARCH" != "$TARGETARCH" -a "${JOB_VALUES[$i-1]}" != "$TARGETARCH" ] && return 0;
               JOB_ARGS="$JOB_ARGS-A \"${JOB_VALUES[$i-1]}\" "
            ;;
         *)
            if [ "${JOB_VALUES[$i-1]}" ]; then
                [ "$DEFINES_STRING" ] && 
                    DEFINES_STRING="${DEFINES_STRING}," ||
                    DEFINES_STRING="--define "
                DEFINES_STRING="${DEFINES_STRING}${JOB_VARNAMES[$i-1]}=${JOB_VALUES[$i-1]}"
	    fi
	    ;;	
         esac
   done
   # don't build cross package for the same arch as target arch
   [[ ( "$autodist_crossonly" = "1" ) && ( "$cross_target_cpu" = "$TARGETARCH" ) ]] && return 0;

   JOB_ARGS="$JOB_ARGS$DEFINES_STRING"

   [ "$patch_operation" = "buildinstall" ] && patch_operation=build
   [ "$patch_operation" = "autoupdate" ] && patch_operation=update

   # launch loop for each package
   for pkg in ${JOB_PKGS[*]}; do

      case $operation in
         autoupdate|update|build|buildinstall)
            [ -e $spec_dir/$pkg.spec ] || {
               case $SEVERITY in
                  0) ;;
                  1) echo "!! Error: spec filename must be the same as package name or invalid job name; skipping $pkg package."
                     continue ;;
                  *) echo "!! Error: spec filename must be the same as package name or invalid job name; aborting."
                     exit 1 ;;
               esac
            }

            # apply patches
	    SPEC_DATE=`stat $spec_dir/$pkg.spec --printf=%y`
            if [ $JOB_IDX -eq 0 -a "$do_autobuild" ]; then
	       # preserve specfile modification date
               if [ -e $AUTOUPDATEDIR/spec-patches-$patch_operation/$pkg.spec.patch ]; then
                  echo -n "== "
	          patch -p1 $spec_dir/$pkg.spec $AUTOUPDATEDIR/spec-patches-$patch_operation/$pkg.spec.patch || {
                     case $SEVERITY in
                        0) ;;
                        1) echo "%! Error: could not apply patch to specfile; skipping $pkg package."
		           ret=1
                           continue ;;
                        *) echo "!! Error: could not apply patch to specfile; aborting."
                           exit 1 ;;
                     esac
   	          }
		  [ "$operation" = "update" -o "$operation" = "autoupdate" ] && continue
	       fi
	    fi
            $SCRIPT_UPDATE_SPECFILE $spec_dir/$pkg.spec
	    ;;
      esac

      [ -e $spec_dir/$pkg.spec ] && {
            SPEC_EPOCH=`grep "Epoch:" $spec_dir/$pkg.spec | sed "s|Epoch:[[:space:]]*||"`
            SPEC_VERSION=`grep "Version:" $spec_dir/$pkg.spec | sed "s|Version:[[:space:]]*||"`
            SPEC_RELEASE=`grep "Release:" $spec_dir/$pkg.spec | sed "s|Release:[[:space:]]*||"`
      }

      unset LOGFILE_SUFFIX
      for a in ${JOB_VALUES[*]}; do
         tr_job=`echo $a | tr / _`
         LOGFILE_SUFFIX=${LOGFILE_SUFFIX}__$tr_job
      done
      LOGFILE_SUFFIX=$LOGFILE_SUFFIX.$TARGETARCH
      [ "$JOB_NAME" = "${pkg}" ] &&
         LOGFILE_PKG_NAME=${pkg}${LOGFILE_SUFFIX} ||
         LOGFILE_PKG_NAME=${JOB_NAME}_${pkg}${LOGFILE_SUFFIX}
      LOGFILE_PKG=${LOGDIR}/${LOGFILE_PKG_NAME}

      # clean previous logs for this package 
      [ -e ${LOGDIR}/ok/${LOGFILE_PKG_NAME}${LOGFILE_SUFFIX} ] &&
         rm -f ${LOGDIR}/ok/${LOGFILE_PKG_NAME}${LOGFILE_SUFFIX}
      [ -e ${LOGDIR}/failed/${LOGFILE_PKG_NAME}${LOGFILE_SUFFIX} ] &&
         rm -f ${LOGDIR}/failed/${LOGFILE_PKG_NAME}-${LOGFILE_SUFFIX}

      [ -e "$SRCPKGLIST" ] && PKGLINE=`grep "^${pkg} " $SRCPKGLIST` || PKGLINE=
      [ -e "$SRCPKGLIST_DELAYED" ] && PKGLINE_DELAYED=`grep "^${pkg} " $SRCPKGLIST_DELAYED` || PKGLINE_DELAYED=

      case $operation in
         prepare)
            command_opts="-a1"
            if [ "$PKGLINE" ]; then
               set -- $PKGLINE
               [[ $4 =~ "^[0-9]+$" ]] && REPNAME=${AUTODIST_REPOSITORIES[$4]} || REPNAME="$4"
               if [ "$REPNAME" ]; then
                  command_opts="$command_opts --server ${REPNAME}"
               else
                  echo "!! Warning: undefined repository $4 in srcpkglist, check AUTODIST_REPOSITORIES in configuration file"
               fi
            else
               echo "!! Warning: package $pkg is missing in srcpkglist file"
            fi
            ;;
         rebuild) command_opts="-a4 --rebuild" ;;
         autoupdate) command_opts="-a2:4" ;;
         update)
            # skip package in job if up to date
            if [ "$PKGLINE" ]; then
               set -- $PKGLINE
               pkglinever=$2
               # warning: assuming version is passed first
               version_find_bigger "${passed_arguments/ *}" "$pkglinever"
               [ $? -eq 1 ] || {
                  echo "!! Warning: skipping ${pkg} package already up to date ($pkglinever >= ${passed_arguments/ *})."
                  continue
               }
            fi
            command_opts="-a3:4" ;;
         build)
            # skip package in job if it is in the delayed repository
            if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then
               set -- $PKGLINE_DELAYED
               if [ "$SPEC_VERSION-$SPEC_RELEASE" = "$2-$6" ]; then
                  echo "!! Warning: skipping ${pkg} package build because already present in delayed repository."
                  continue
               fi
            fi
            command_opts="-a5:6" ;;
         buildinstall)
            # skip package in job if up to date
            if [ "$PKGLINE" ]; then
               set -- $PKGLINE
               pkglinever=$2
               pkglinerel=$6
               # warning: assuming version is passed first
               version_find_bigger "$SPEC_VERSION" "$pkglinever"
               [ $? -eq 0 ] && {
                  version_find_bigger "$SPEC_RELEASE" "$pkglinerel"
                  if [ $? -ne 1 ]; then
                     echo "!! Warning: skipping ${pkg} package already up to date ($SPEC_VERSION-$SPEC_RELEASE >= $pkglinever-$pkglinerel)."
                     continue
                  fi
               }
            fi
            # skip package in job if it is in the delayed repository
            if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then
               set -- $PKGLINE_DELAYED
               if [ "$SPEC_VERSION-$SPEC_RELEASE" = "$2-$6" ]; then
                  echo "!! Warning: skipping ${pkg} package build because already present in delayed repository."
                  continue
               fi
            fi
            command_opts="-a5,6,10,11 --force-install" ;;
         send)
            # skip package in job if up to date
            if [ "$PKGLINE" -a "$do_autobuild" ]; then
               set -- $PKGLINE
               pkglinever=$2
               pkglinerel=$6
               # warning: assuming version is passed first
               version_find_bigger "$SPEC_VERSION" "$pkglinever"
               [ $? -eq 0 ] && {
                  version_find_bigger "$SPEC_RELEASE" "$pkglinerel"
                  if [ $? -ne 1 ]; then
                     echo "!! Warning: skipping ${pkg} package already up to date ($SPEC_VERSION-$SPEC_RELEASE >= $pkglinever-$pkglinerel)."
                     continue
                 fi
               }
            fi
            # skip package in job if it is in the delayed repository
            if [ "$PKGLINE_DELAYED" -a "$do_autobuild" ]; then
               set -- $PKGLINE_DELAYED
               if [ "$SPEC_VERSION-$SPEC_RELEASE" = "$2-$6" ]; then
                  echo "!! Warning: skipping ${pkg} package send because already present in delayed repository."
                  continue
               fi
            fi
            command_opts="-a10" ;;
         install) command_opts="-a11" ;;
         *)
            echo "!! Internal error: unknown operation $operation; aborting."
            exit 1
            ;;
      esac

      command="$AUTOSPEC_CMD -u $pkg $command_opts $passed_arguments $JOB_ARGS" 

      tmpfile=`tempfile`
      tmpfile1=`tempfile`
      echo -n "== "`log_date`" $JOB_NAME with args: $JOB_ARGS..." >> $LOGFILE
      echo "
=> $command" >> $LOGFILE_PKG
      [ "$verbose" ] && echo "== "`log_date`" $JOB_NAME with args: $JOB_ARGS...
=> $command"

      if [ "$simulate" ]; then
         echo "Command: $command"
      else
#         eval "$command 2>&1 && echo '0'>$tmpfile1 || echo '1'>$tmpfile1" | tail_file $tmpfile "== $pkg (${JOB_VALUES[*]})..."
         (export LANG=C; $command 2>&1; echo $? >$tmpfile1) | tail_file $tmpfile "== $pkg (${JOB_VALUES[*]})..."
	 if [ -e $tmpfile1 ]; then
	    ret=`cat $tmpfile1`
            rm -f $tmpfile1
	 else
	    ret=0
	 fi

         cat $tmpfile >> $LOGFILE_PKG

         if [ $ret -gt 0 ]; then
            echo " FAILED (ret=$ret)"
	    echo " FAILED (ret=$ret)" >> $LOGFILE
            [ "$quiet" ] || {
	       echo "=> $command"
	       echo
	       echo "                         == begin of output log =="
	       echo
	       tail -n 40 $tmpfile
	       echo
	       echo "                          == end of output log =="
	       echo
	    }
	    mv $LOGFILE_PKG $LOGDIR/failed/
            echo "?= See $LOGDIR/failed/${LOGFILE_PKG_NAME}"
	    case $operation in
	       autoupdate|update)
	          ;;
	       build|buildinstall)
	          # copy new patches to sourcesdir for sharing
	          [ "$do_autobuild" ] && {
	             PATCHES=`diff $spec_dir/$pkg.spec $spec_dir/.$pkg.spec.autodist.preupdate | grep "> Patch" | sed "s|.*[[:space:]]\([^[:space:]]*\)|\1|"`
		     for p in $PATCHES; do
		         echo "== Copying patch $p to public dir"
		        [ -e $source_dir/$p ] && cp $source_dir/$p $SOURCESDIR/
		     done
		  }
	          ;;
	    esac
	    [ $SEVERITY -gt 0 ] && {
               rm -f $tmpfile
	       return $ret
	    }
         else
            echo
	    echo " OK" >> $LOGFILE
   	    mv $LOGFILE_PKG $LOGDIR/ok/

            case $operation in
               prepare)
	          cp $spec_dir/$pkg.spec $spec_dir/.$pkg.spec.autodist.preupdate
	          ;;
	       rebuild)
                  cp $spec_dir/$pkg.spec $spec_dir/.$pkg.spec.autodist.prebuild
	          ;;
               update|autoupdate)
                  cp $spec_dir/$pkg.spec $spec_dir/.$pkg.spec.autodist.prebuild
                  SPEC_EPOCH=`grep "Epoch:" $spec_dir/$pkg.spec | sed "s|Epoch:[[:space:]]*||"`
                  SPEC_VERSION=`grep "Version:" $spec_dir/$pkg.spec | sed "s|Version:[[:space:]]*||"`
                  SPEC_RELEASE=`grep "Release:" $spec_dir/$pkg.spec | sed "s|Release:[[:space:]]*||"`
                  if [ "$PKGLINE" ]; then
                     set -- $PKGLINE
                     echo "== Specfile updated from version $2-$6 to $SPEC_VERSION-$SPEC_RELEASE"
                     if [ "`echo $SPEC_VERSION | grep -i alpha`" -o \
                          "`echo $SPEC_VERSION | grep -i beta`" -o \
                          "`echo $SPEC_VERSION | grep -i rc`" -o \
                          "`echo $SPEC_VERSION | grep -i pre`" -o \
                          "`echo $SPEC_VERSION | grep -i pl`" ]; then
                        if [ ! "`echo $2 | grep -i alpha`" -a \
                             ! "`echo $2 | grep -i beta`" -a \
                             ! "`echo $2 | grep -i rc`" -a \
                             ! "`echo $2 | grep -i pre`" -a \
                             ! "`echo $2 | grep -i pl`" ]; then
                             echo "!! Looks like an update to an unstable release; skipping."
                             [ $SEVERITY -gt 0 ] && {
                                rm -f $tmpfile
                                return 3
                             }
                        fi
                     fi
                  fi
                  ;;
               buildinstall|send)
                  if [ $JOB_IDX -eq $JOB_LAST -a "$do_autobuild" ]; then
                     # archive patches
                     [ -e $AUTOUPDATEDIR/spec-patches-update/$pkg.spec.patch ] && \
	                mv $AUTOUPDATEDIR/spec-patches-update/$pkg.spec.patch \
                           $AUTOUPDATEDIR/spec-patches-update/old/
                     [ -e $AUTOUPDATEDIR/spec-patches-build/$pkg.spec.patch ] && \
	                mv $AUTOUPDATEDIR/spec-patches-build/$pkg.spec.patch \
                           $AUTOUPDATEDIR/spec-patches-build/old/
	             PATCHES=`diff $spec_dir/$pkg.spec $spec_dir/.$pkg.spec.autodist.preupdate | grep "> Patch" | sed "s|.*[[:space:]]\([^[:space:]]*\)|\1|"`
		     for p in $PATCHES; do
		        [ -e $SOURCESDIR/$p ] && rm -f $SOURCESDIR/$p
		     done
		  fi
		  ;;
	    esac
	 fi
         rm -f $tmpfile
      fi
   done
   return $ret
}

function launch_job_loop() {
   # Iterates launch_pkgs_loop for each of the lines of current job
   # (defined from JOB_FIRST to JOB_LAST)
   # This is needed for build, send and install operations
   for j in `seq $JOB_FIRST $JOB_LAST`; do
      launch_pkgs_loop "$1" "$2" "${JOBS[$j]}" $j
      [ $? -gt 0 ] && return 1;
   done
   return 0
}

function log_date() {
   echo -n `LANG=C date +%Y%m%d@%H%M%S`
}

for a in $LOGBASEDIR/{,prepare/{,ok,failed},update/{,ok,failed},build/{,ok,failed},install/{,ok,failed},send/{,ok,failed}}; do

   [ ! -w $a ] && {
      echo "!! Error: directory $a is missing or read-only; aborting."
      exit 1
   }

   [ "$do_cleanlogs" ] && {
      find $a -type f -exec rm -f {} \;
   }
   
done

echo "%% Autodist started with PID $$ @ `LANG=C date`"

# autobuild: add jobs ordered by oldest builds
if [ "$do_autobuild" = "1" ]; then
   trap "rm -f $PIDFILE; exit 1" SIGQUIT SIGINT SIGKILL SIGTERM
   echo $$ > $PIDFILE
   [ -e "$SRCPKGLIST" ] || {
      echo "ERROR: missing srcpkglist file $SRCPKGLIST; aborting."
      exit 1
   }
   srctmpfile=`tempfile`
   cat $AUTOUPDATEDIR/*.in > $srctmpfile
   cat $SRCPKGLIST | sort --key=3 >> $srctmpfile

   pkgtmpfile=`tempfile`
   [ "$pkgtmpfile" ] || {
      echo "ERROR: cannot create temporary files; aborting."
      exit 1
   }

   ## when autobuilding always rebuild packages
   #SEND_FORCE_BUILD="--force-build"
   AUTOBUILD_NUMADDED=0
   BUILDDATE=0
   PKGCOUNT=`wc -l $srctmpfile | awk '{ print $1 }'`
   SKIPPEDCOUNT=0
   BLACKLISTCOUNT=0
   SUCCESSLISTCOUNT=0

   > $pkgtmpfile

#   cat $srctmpfile | sort --key=3 | \
   while read line; do
       set -- $line
       PKGNAME=${1}
       VERSION=${2}
       BUILDDATE=${3}
       SOURCEREP=${4}
       RELEASE=${5}
       EPOCH=${6}

       # skip duplicates
       grep "^$PKGNAME " $pkgtmpfile >/dev/null && continue

       # check blacklist
       awk '{ print $1 }' $BLACKLIST | grep "^$PKGNAME$" >/dev/null && {
          BLACKLISTCOUNT=`expr $BLACKLISTCOUNT + 1`
          #echo "?= Job $PKGNAME is in the blacklist; skipping"
          continue
       }

       if [ "${VERSION:0:1}" == "+" -a "$VERSION" != "+0" ]; then
            TRVER=`echo ${VERSION:1} | tr - .`
	    get_job_vector $PKGNAME
	    NEED_REBUILD=0
	    if [ ${#JOB_PKGS[*]} -gt 1 ]; then
	       # jobs are not fully rebuilt
	       for j in ${JOB_PKGS[*]}; do
	          REPVER=`grep "^$j " $SRCPKGLIST | gawk '{ print $2 }'`
	          version_find_bigger "$TRVER" "$REPVER"
	          [ $? -eq 1 ] && NEED_REBUILD=1
	       done
	    else
	       # single packages are rebuilt if scheduled
	       NEED_REBUILD=1
	    fi
	    if [ "$NEED_REBUILD" = "0" ]; then
	          echo "?= Job $PKGNAME is already up to date; skipping"
	          continue
	    fi
       elif [ "$VERSION" != "+0" ]; then

	     # check successlist
	     SUCCESSLINE=`awk '{ print $1" "$2 }' $SUCCESSLISTDIR/*.success | grep -m1 "^$PKGNAME "`
	     if [ "$SUCCESSLINE" ]; then
	        set -- $SUCCESSLINE
		SUCCESSTIME=$2
		[ "$SUCCESSTIME" ] || SUCCESSTIME=0
		SUCCESSDAYS=`expr \( $DATE_NOW - $SUCCESSTIME \) / 86400`
		if [ "$SUCCESSDAYS" -le "$AUTOBUILD_SUCCESS_DAYS" ]; then
		   SUCCESSLISTCOUNT=`expr $SUCCESSLISTCOUNT + 1`
                   #echo "?= Job $PKGNAME has been in the successlist for $SUCCESSDAYS days; skipping"
                   continue
		else
		   sed -i "/^$PKGNAME /d" $SUCCESSLISTDIR/*.success
		fi
	     fi
	     
             # check skippedlist
             SKIPPEDLINE=`awk '{ print $1" "$2 }' $SKIPPEDLISTDIR/*.skip | grep -m1 "^$PKGNAME "`
	     if [ "$SKIPPEDLINE" ]; then
	        set -- $SKIPPEDLINE
		SKIPPEDTIME=$2
		SKIPPEDDAYS=`expr \( $DATE_NOW - $SKIPPEDTIME \) / 86400`
		if [ "$SKIPPEDDAYS" -le "$AUTOBUILD_SKIP_DAYS" ]; then
		   SKIPPEDCOUNT=`expr $SKIPPEDCOUNT + 1`
#                   echo "?= Package $PKGNAME has been in the skippedlist for $SKIPPEDDAYS days; skipping"
                   continue
		else
		   sed -i "/^$PKGNAME /d" $SKIPPEDLISTDIR/*.skip
		fi
             fi
	     
             # check legacylist
             awk '{ print $1 }' $LEGACYLIST | grep "^$PKGNAME$" >/dev/null && {
                echo "?= Job $PKGNAME is in the legacy list; skipping automatic update"
		echo "$PKGNAME $DATE_NOW 100" >> $SKIPPEDLISTDIR/auto.skip
                continue
	     }

       fi

       if [ $AUTOBUILD_NUMADDED -ge $AUTOBUILD_MAXNUM ]; then
          break
       elif [ $BUILDDATE -ge $AUTOBUILD_DATEFROM -o $BUILDDATE -eq 0 ]; then
          AUTOBUILD_NUMADDED=`expr $AUTOBUILD_NUMADDED + 1`
          echo "$PKGNAME $VERSION" >> $pkgtmpfile
       fi
   done < $srctmpfile

   echo "%% Total pkgs: $PKGCOUNT, skipped list: $SKIPPEDCOUNT, blacklist: $BLACKLISTCOUNT, success list: $SUCCESSLISTCOUNT"
   while read line; do
         set -- $line
         PKGNAME=${1}
         VERSION=${2}
         JOBVER[${#JOBNAME[@]}]=$VERSION
         JOBNAME[${#JOBNAME[@]}]=$PKGNAME
   done < $pkgtmpfile

   echo -n "%% autoupdate jobs: "
   for i in `seq 1 ${#JOBNAME[@]}`; do
      echo -n "${JOBNAME[$i-1]}(${JOBVER[$i-1]}) "
   done
   echo
   rm -f $pkgtmpfile $srctmpfile
fi

echo "%% ${#JOBNAME[*]} job(s) scheduled"
echo "%% Starting main jobs loop"
for JOB_NUM in `seq 1 ${#JOBNAME[*]}`; do
   JOB_NAME=${JOBNAME[$JOB_NUM-1]}
   JOB_VER=${JOBVER[$JOB_NUM-1]}
   JOB_VER=`echo $JOB_VER | tr - .`

   # resolve JOB_NAME from distdb
   get_job_vector $JOB_NAME
   JOB_NAME=${JOB_NAME/*\/}

   [ "$quiet" ] || echo "== =======================$JOB_NAME============================="
   echo -n "%% Doing $JOB_NAME (VER=$JOB_VER"
   for k in ${JOB_VARNAMES[*]}; do
      echo -n " %${k}"
   done
   echo ") job"

   DATE_NOW=`LANG=C date +%s`
   JOBS=()
   for k in `seq 3 ${#JOB[*]}`; do
      JOBS=(${JOBS[*]} "${JOB[$k-1]}")
   done

   JOB_NUMBER=${#JOBS[*]}
   JOB_FIRST=0
   JOB_LAST=`expr $JOB_NUMBER - 1`

   if [ "$do_prepare" ]; then
      LOGDIR=$LOGBASEDIR/prepare
      LOGFILE=$LOGDIR/last
      > $LOGFILE
      echo "%% Starting PREPARE operation @ `LANG=C date`"
      echo "%% "`log_date`" Starting PREPARE operation" > $LOGFILE
      launch_pkgs_loop prepare "$AUTOSPEC_ARGS --force" || {
            case $SEVERITY in
                 0) ;;
                 1) echo "%! Error during package prepare; skipping $JOB_NAME job."
                    continue ;;
                 *) echo "!! Error during package prepare; aborting."
                    exit 1 ;;
            esac
      }
      if [ ! "$do_update" -a "$do_autobuild" = "1" ]; then
         [ "${AUTOSPEC_ARGS/--changelog}" = "${AUTOSPEC_ARGS}" ] &&
            AUTOSPEC_ARGS="$AUTOSPEC_ARGS --changelog \"automatic rebuild by autodist\""
         launch_pkgs_loop rebuild "$AUTOSPEC_ARGS" || {
            case $SEVERITY in
                 0) ;;
                 1) echo "%! Error during package update for rebuild; skipping $JOB_NAME job."
                    continue ;;
                 *) echo "!! Error during package update for rebuild; aborting."
                    exit 1 ;;
            esac
         }
      fi
   fi

   if [ "$do_update" ]; then
      LOGDIR=$LOGBASEDIR/update
      LOGFILE=$LOGDIR/last
      echo "%% Starting UPDATE operation @ `LANG=C date`"
      echo "%% "`log_date`" Starting UPDATE operation" > $LOGFILE

      if [ "${JOB_VER:0:1}" != "+" -o "${JOB_VER}" = "+0" ]; then
#	    --force-update \
         # check legacylist
         awk '{ print $1 }' $LEGACYLIST | grep "^$JOB_NAME$" >/dev/null && {
            echo "?= Job $JOB_NAME is in the legacy list; forcing rebuild"
	    ret=1
	 } || {
	    [ "${AUTOSPEC_ARGS/--changelog}" = "${AUTOSPEC_ARGS}" ] &&
               AUTOSPEC_ARGS="$AUTOSPEC_ARGS --changelog \"automatic update by autodist\""
            launch_pkgs_loop autoupdate "$AUTOSPEC_ARGS $SEND_FORCE \
	    --define gnomever=$GNOME_VER"
	    ret=$?
	 }
	 if [ $ret != 0 ]; then
	    if [ ! "$rebuild_packages" -a "${JOB_VER}" != "+0" ]; then
	       [ $ret -ge 2 -a "$do_autobuild" ] && {
                  echo "%% Adding $JOB_NAME job to skipped list." 
	          echo "$JOB_NAME $DATE_NOW $?" >> $SKIPPEDLISTDIR/auto.skip
	       }
               case $SEVERITY in
                  0) ;;
                  1) echo "%! Error during package update; skipping $JOB_NAME job." 
                     continue ;;
                  *) echo "!! Error during package update; aborting."
                     exit 1 ;;
               esac
	    else
               echo "%% Rebuilding package(s)"
               [ "${AUTOSPEC_ARGS/--changelog}" = "${AUTOSPEC_ARGS}" ] &&
                  AUTOSPEC_ARGS="$AUTOSPEC_ARGS --changelog \"automatic rebuild by autodist\""
               launch_pkgs_loop rebuild "$AUTOSPEC_ARGS" || {
                  case $SEVERITY in
                     0) ;;
                     1) echo "%! Error during package update for rebuild; skipping $JOB_NAME job."
                        continue ;;
                     *) echo "!! Error during package update for rebuild; aborting."
                        exit 1 ;;
                  esac
	       }
	    fi
         fi
      else
         ## skip package in job if up to date
         #for j in ${JOB_PKGS[*]}; do
      	#    REPVER=`grep "^$j " $SRCPKGLIST | gawk '{ print $2 }'`
	#    version_find_bigger "$TRVER" "$REPVER"
	#    [ $? -eq 1 ] && NEED_REBUILD=1
	# done
         echo "%% Updating to version ${JOB_VER:1}"
         # WARNING: JOB_VER must be passed to launch_pkgs_loop as the first string in the second parameter
         [ "${AUTOSPEC_ARGS/--changelog}" = "${AUTOSPEC_ARGS}" ] &&
            AUTOSPEC_ARGS="$AUTOSPEC_ARGS --changelog \"automatic version update by autodist\""
         launch_pkgs_loop update "${JOB_VER:1} $AUTOSPEC_ARGS $SEND_FORCE \
	    --define gnomever=$GNOME_VER" || {
	    if [ ! "$rebuild_packages" ]; then
               case $SEVERITY in
                  0) ;;
                  1) echo "%! Error during update; skipping $JOB_NAME job." 
                     continue ;;
                  *) echo "!! Error during update; aborting."
                     exit 1 ;;
               esac
	    else
	       [ "${AUTOSPEC_ARGS/--changelog}" = "${AUTOSPEC_ARGS}" ] &&
                  AUTOSPEC_ARGS="$AUTOSPEC_ARGS --changelog \"automatic rebuild by autodist\""
               launch_pkgs_loop rebuild "$AUTOSPEC_ARGS" || {
                  case $SEVERITY in
                     0) ;;
                     1) echo "%! Error during package update for rebuild; skipping $JOB_NAME job."
                        continue ;;
                     *) echo "!! Error during package update for rebuild; aborting."
                        exit 1 ;;
                  esac
	       }
	    fi
            }
      fi
   fi

   if [ "$do_build" ]; then
      LOGDIR=$LOGBASEDIR/build
      LOGFILE=$LOGDIR/last
      echo "%% Starting BUILD operation @ `LANG=C date`"
      echo "%% "`log_date`" Starting BUILD operation" > $LOGFILE
      launch_job_loop build "$AUTOSPEC_ARGS $SEND_FORCE_BUILD $SEND_SERVER_CMD" || {
            case $SEVERITY in
                 0) ;;
                 1) echo "%! Error during build; skipping $JOB_NAME job."
                    continue ;;
                 *) echo "!! Error during build; aborting."
                    exit 1 ;;
            esac
      }
   fi

   if [ "$do_install" ]; then
      LOGDIR=$LOGBASEDIR/install
      LOGFILE=$LOGDIR/last
      > $LOGFILE
      echo "%% Starting INSTALL operation @ `LANG=C date`"
      echo "%% "`log_date`" Starting INSTALL operation" > $LOGFILE
      launch_job_loop install "$AUTOSPEC_ARGS" || {
            case $SEVERITY in
                 0) ;;
                 1) echo "%! Error during installation; skipping $JOB_NAME job." 
                    continue ;;
                 *) echo "!! Error during installation; aborting."
                    exit 1 ;;
            esac
      }
   fi

   if [ "$do_send" ]; then
      LOGDIR=$LOGBASEDIR/send
      LOGFILE=$LOGDIR/last
      > $LOGFILE
      echo "%% Starting SEND operation @ `LANG=C date`"
      echo "%% "`log_date`" Starting SEND operation" > $LOGFILE
      launch_job_loop send "$AUTOSPEC_ARGS --nosrpm $SEND_SERVER_CMD" || {
            case $SEVERITY in
                 0) ;;
                 1) echo "%! Error sending packages; skipping $JOB_NAME job." 
                    continue ;;
                 *) echo "!! Error sending packages; aborting."
                    exit 1 ;;
            esac
      }
      echo "== sending source rpms"
      launch_pkgs_loop send "$AUTOSPEC_ARGS --norpm $SEND_SERVER_CMD"
      if [ $? -gt 0 ]; then
            case $SEVERITY in
                 0) ;;
                 1) echo "%! Error sending sources; skipping $JOB_NAME job." 
                    continue ;;
                 *) echo "!! Error sending sources; aborting."
                    exit 1 ;;
            esac
      else
         # archive patches
         if [ "$do_autobuild" ]; then
            [ -e $AUTOUPDATEDIR/spec-patches-update/$JOB_NAME.spec.patch ] && \
	       mv $AUTOUPDATEDIR/spec-patches-update/$JOB_NAME.spec.patch \
                  $AUTOUPDATEDIR/spec-patches-update/old/${JOB_NAME}.spec.${DATE_NOW}.patch
            [ -e $AUTOUPDATEDIR/spec-patches-build/$JOB_NAME.spec.patch ] && \
	       mv $AUTOUPDATEDIR/spec-patches-build/${JOB_NAME}.spec.${DATE_NOW}.patch \
                  $AUTOUPDATEDIR/spec-patches-build/old/${JOB_NAME}.spec.${DATE_NOW}.patch
         fi
      fi
   fi
   # if we are here the job was completed; remove manual entry, if any, and add to successlist
   [ "$do_autobuild" ] && {
      sed -i "/^$JOB_NAME .*/d" $AUTOUPDATEDIR/*.in
      echo "$JOB_NAME $DATE_NOW" >> $SUCCESSLISTDIR/auto.success
   }
done
[ "$do_autobuild" = "1" ] && rm -f $PIDFILE
echo "%% All jobs done @ `LANG=C date`"