autodist/autoport

#!/bin/bash
# autoport -- batch cross-build tool based on autodist
# Copyright (C) 2011-2012 by Silvan Calarco <silvan.calarco@mambasoft.it>
#
# Released under the terms of the GNU GPL release 3 license
#
VERSION=0.9.4
BASE_ARCH=i586
BASE_REPOSITORY=devel
PORT_REPOSITORY=devel

. /etc/autodist/config

me=(${0##*/} $VERSION "Sat Mar 20 2011")

function usage() {
   echo "\
${me[0]} ${me[1]}
"$"Copyright (C) 2011-2012 Silvan Calarco <silvan.calarco@mambasoft.it>""
"$"Released under the terms of the GNU GPL v3 license"
   echo "
"$"Batch port and cross-build tool based on autodist.""

"$"Usage"":
   $me [-a][-f][-x arch1[,arch2],..][-r repository][-d release_repository]{--fix|job ...}

   -a                 "$"Automatic mode (use cache)
   -b                 "$"Batch port all packages in port repository to sync with base repository
   -f                 "$"Force operations
   -np                "$"Don't prepare jobs passed to command line unless specfile is not found
   -npa               "$"Don't prepare any jobs unless specfile is not found
   -nr                "$"Don't recurse build
   --fix              "$"Fix .la files for cross linking and exit
   -r                 "$"Work on given repository (default: $PORT_REPOSITORY)
   -d                 "$"Release packages to given repository (default: $PORT_REPOSITORY)
   -x                 "$"Operate in cross build mode
   -h                 "$"Show this help and exit
   -v                 "$"More verbose output

   arch               "$"Target architecture
   job                "$"Job specification for autodist""
"

}

# get_pkg_buildinfo - uses distromatic generated build file for
#                     getting information on the repository
# 
# $1: distromatic files base url for repository
# $2: architecture
# $3: pkg name
function get_pkg_buildinfo() {

   local pkg i a local_arch local_distromatic

   [ $1 ] && local_distromatic=$1 || exit 1
   [ $2 ] && local_arch=$2 || exit 1
   [ $3 ] && pkg=$3

   if [ ! "$pkg" ]; then
      [ -e $local_distromatic/builds-${local_arch}.sh ] && . $local_distromatic/builds-${local_arch}.sh
      return 0
   fi

   pkg_archs=();
   for a in ${archs[*]}; do
      [ "$a" = "${local_arch}" ] && continue
      pkg_header=();
      DISTROMATIC_BUILD_FILE=$local_distromatic/builds-$a.sh
      [ -e $DISTROMATIC_BUILD_FILE ] && . $DISTROMATIC_BUILD_FILE
      [ ${pkg_header[0]} ] && pkg_archs=(${pkg_archs[*]} $a)
   done

   pkg_header=();
   pkg_builds=();
   pkg_obsoletes=();
   pkg_list=();

   DISTROMATIC_BUILD_FILE=$local_distromatic/builds-${local_arch}.sh
   [ -e $DISTROMATIC_BUILD_FILE ] && . $DISTROMATIC_BUILD_FILE
   [ ${pkg_header[0]} ] && pkg_archs=(${pkg_archs[*]} ${local_arch})

#    {
#      echo "Error: missing distromatic generated file $DISTROMATIC_BUILD_FILE; aborting."
#      return;
#   }
#  . $DISTROMATIC_BUILD_FILE

   for i in ${pkg_list[*]}; do
      if [ "$i" == "${pkg_header[0]}" ]; then
         pkg_name=${pkg_header[0]};
         # Note: pkg_arch reported in builds file is just last arch source was
         # built for, so we use repository arch instead
         pkg_arch=${pkg_header[1]};
         [ "$pkg_arch" = "noarch" ] || pkg_arch=${local_arch}
         pkg_version=${pkg_header[2]};
         pkg_release=${pkg_header[3]};
         pkg_group=${pkg_header[4]};
         pkg_license=${pkg_header[5]};
         pkg_size=${pkg_header[6]};
         pkg_buildtime=${pkg_header[7]};
         pkg_altrep=${pkg_header[8]};
         pkg_repository=${pkg_header[9]};
         return 0
      fi
   done
   unset pkg_name pkg_arch pkg_version pkg_release \
         pkg_group pkg_license pkg_size pkg_buildtime pkg_altrep pkg_repository

   return 1
}

function write_last_file() {
   [ "$AUTO_MODE" == "1" ] && LASTFILE=$DATADIR/last || LASTFILE=$DATADIR/lastmanual
   cat > $LASTFILE << _EOF
JOB_CANTPREPARE=(${JOB_CANTPREPARE[*]})
JOB_MISSINGREQ=(${JOB_MISSINGREQ[*]})
JOB_SENT=(${JOB_SENT[*]})
JOB_CANTINSTAL=(${JOB_CANTINSTALL[*]})
JOB_COMPLETED=(${JOB_COMPLETED[*]})
JOB_FAILED=(${JOB_FAILED[*]})
_EOF
}

function find_source_by_provide() {
   local pkg=`echo $2 | sed "s|(|\\\\\[|g;s|)|\\\\\]|g;s|:|_|g"`
   grep "^$pkg:" $DATADIR/$1/sources-$BASE_ARCH | \
      awk '{ $1=""; print $0 }' | \
      sed "s|[[:space:]]_| |g"
}

function find_requirements() {
   [ "$1" ] || return 1
   local tf=`mktemp` || {
      echo "Error: cannot create tempfile"
      return 1
   }
   local tf1=`mktemp`
   local OUTFILE=$1

   if [ "`grep " is needed by " $OUTFILE`" ]; then
      grep " is needed by " $OUTFILE | awk '{ print $1 }' | sort -u > $tf1
   elif [ "`grep " matches no packages" $OUTFILE`" ]; then
      grep " matches no packages" $OUTFILE | awk '{ print $2 }' | tr -d \' | sort -u > $tf1
   elif [ "`grep " no package provides " $OUTFILE`" ]; then
      grep " no package provides " $OUTFILE | sed "s|.* no package provides ||" | sort -u > $tf1
   fi

   [ `stat -c %s  $tf1` -gt 0 ] && {
      echo -n " | missing requirements: "
      > $tf
      cat $tf1 | while read line; do
         REQ=`find_source_by_provide $PORT_REPOSITORY $line`
         [ "$REQ" ] && \
            echo "${REQ}" >> $tf || \
            echo -n "$line(unresolved) "
      done
      > $tf1
      cat $tf | sort -u 2>/dev/null | while read line; do
         if [ "`echo " ${JOB[*]} ${JOB_MISSINGREQ[*]} ${JOB_CANTINSTALL[*]} " | grep " $line "`" ]; then
            echo -n "($line) "
         else
            echo -n "$line "
            echo -n "$line " >> $tf1
         fi
      done
      JOB_MISSINGREQ=(${JOB_MISSINGREQ[*]} `cat $tf1`)
      echo
   }
   rm -f $tf $tf1
   return 0
}

[ $# -gt 0 ] || { usage ; exit 1; }

BUILD_ARCH=`uname -i`

case $BUILD_ARCH in
   i386|i486|i686) BUILD_ARCH=i586 ;;
   arm*) BUILD_ARCH=arm ;;
esac

TARGET_ARCHS=$BUILD_ARCH
JOB_CMDLINE=()
CROSS_MODE=

for ((i=1; i<=$#; i++)); do
   case ${!i} in
      -a) AUTO_MODE=1 ;;
      -b) BATCH_MODE=1 ;;
      -f) FORCE_MODE=1 ;;
      -v) VERBOSE_MODE=1 ;;
      -npa) DONT_PREPARE_ANY=1
            DONT_PREPARE=1 ;;
      -np) DONT_PREPARE=1 ;;
      -nr) DONT_RECURSE=1 ;;
      --fix) FIX_MODE=1 ;;
      -r) shift
          PORT_REPOSITORY="${!i}"
          ;;
      -d) shift
          DEST_REPOSITORY="${!i}"
          ;;
      -x) shift
          TARGET_ARCHS="`echo ${!i} | tr , ' '`"
          CROSS_MODE=1
          ;;
      -h|--help)
          usage
          exit 0
          ;;
      *) JOB_CMDLINE=(${JOB_CMDLINE[*]} ${!i})
         ;;
   esac
done

[ "$DEST_REPOSITORY" ] || DEST_REPOSITORY=$PORT_REPOSITORY

for TARGET_ARCH in ${TARGET_ARCHS}; do

   [ "$BASE_ARCH" = "$TARGET_ARCH" -a "$PORT_REPOSITRY" = "$DEST_REPOSITORY" ] && {
      echo "Error: base and target arch ($BASE_ARCH) are the same as well as port and release repositories ($PORT_REPOSITORY); skipping."
      continue
   }

   archs=($BASE_ARCH $TARGET_ARCH)
   JOBS_SKIPPED=()
   JOBS_CANTPREPARE=()
   JOBS_CANTINSTALL=()
   JOB_MISSINGREQ=()
   JOB_COMPLETED=()
   JOB_FAILED=()

   DATADIR=$HOME/.autoport
   DATAARCHDIR=$DATADIR/$TARGET_ARCH
   DISTROMATIC_BASE_URL=${REPS_BASE_URL}/distromatic/
   BASE_REPOSITORY_DISTROMATIC_URL=${DISTROMATIC_BASE_URL}$BASE_REPOSITORY
   PORT_REPOSITORY_DISTROMATIC_URL=${DISTROMATIC_BASE_URL}$PORT_REPOSITORY

   [ "$AUTO_MODE" -a -e $DATAARCHDIR/last ] && . $DATAARCHDIR/last
   JOB=(${JOB_CMDLINE[*]} ${JOB_MISSINGREQ[*]} ${JOB_FAILED[*]})
   [ "$FIX_MODE" -o "$BATCH_MODE" -o "${JOB[*]}" ] || { usage ; exit 1; }

   [ -d $DATAARCHDIR ] || mkdir -p $DATAARCHDIR
   [ -d $DATADIR/$BASE_REPOSITORY ] || mkdir -p $DATADIR/$BASE_REPOSITORY
   [ -d $DATADIR/$PORT_REPOSITORY ] || mkdir -p $DATADIR/$PORT_REPOSITORY

   logdate=`date +"%F@%T"`
   logfile=$DATAARCHDIR/$logdate.log
#   [ -e $logfile ] && mv -f $logfile $DATAARCHDIR/$PORT_REPOSITORY-last.log
   > $logfile
   ln -sf $logdate.log $DATAARCHDIR/current.log

   BUILD_PLATFORM=`rpm --target $TARGET_ARCH --eval %{_build}`
   BUILD_CPU=`echo $BUILD_PLATFORM | cut -d- -f1`
   TARGET_PLATFORM=`rpm --target $TARGET_ARCH --eval %{_target_platform}`
   TARGET_CPU=`echo $TARGET_PLATFORM | cut -d- -f1`
   SPECDIR=`rpm --eval %{_specdir}`

   echo "Build platform is $BUILD_PLATFORM"
   echo "Target platform is $TARGET_PLATFORM"
   echo "Port repository is $PORT_REPOSITORY"
   echo "Release repository is $DEST_REPOSITORY"

   curl -s $PORT_REPOSITORY_DISTROMATIC_URL/sources-$BASE_ARCH -o $DATADIR/$PORT_REPOSITORY/sources-$BASE_ARCH ||
      echo "Warning: unable to fetch $PORT_REPOSITORY_DISTROMATIC_URL/sources-$BASE_ARCH"
   curl -s $PORT_REPOSITORY_DISTROMATIC_URL/sources-$TARGET_CPU -o $DATADIR/$PORT_REPOSITORY/sources-$TARGET_CPU ||
      echo "Warning: unable to fetch $PORT_REPOSITORY_DISTROMATIC_URL/sources-$TARGET_CPU"
   curl -s $PORT_REPOSITORY_DISTROMATIC_URL/builds-$BASE_ARCH.sh -o $DATADIR/$PORT_REPOSITORY/builds-$BASE_ARCH.sh ||
      echo "Error: unable to fetch $PORT_REPOSITORY_DISTROMATIC_URL/builds-$BASE_ARCH.sh"
   curl -s $PORT_REPOSITORY_DISTROMATIC_URL/builds-$TARGET_ARCH.sh -o $DATADIR/$PORT_REPOSITORY/builds-$TARGET_ARCH.sh ||
      echo "Error: unable to fetch $PORT_REPOSITORY_DISTROMATIC_URL/builds-$TARGET_ARCH.sh"

   if [ "$BATCH_MODE" ]; then
      [ "${JOB[*]}" ] && {
         echo "Error: no jobs allowed in batch mode; aborting."
         exit 1
      }
      curl -s $BASE_REPOSITORY_DISTROMATIC_URL/builds-$BASE_ARCH.sh -o $DATADIR/$BASE_REPOSITORY/builds-$BASE_ARCH.sh ||
         echo "Error: unable to fetch $BASE_REPOSITORY_DISTROMATIC_URL/builds-$BASE_ARCH.sh"
      curl -s $BASE_REPOSITORY_DISTROMATIC_URL/builds-$TARGET_ARCH.sh -o $DATADIR/$BASE_REPOSITORY/builds-$TARGET_ARCH.sh ||
         echo "Error: unable to fetch $BASE_REPOSITORY_DISTROMATIC_URL/builds-$TARGET_ARCH.sh"
      get_pkg_buildinfo $DATADIR/$PORT_REPOSITORY $BASE_ARCH
      port_pkg_list=${pkg_list[*]}

      for p in ${port_pkg_list}; do
         get_pkg_buildinfo $DATADIR/$PORT_REPOSITORY $TARGET_ARCH $p
         target_pkg_name=$pkg_name
         target_pkg_ver=$pkg_version
         target_pkg_rel=$pkg_release
         [ "$pkg_name" ] || get_pkg_buildinfo $DATADIR/$BASE_REPOSITORY $TARGET_ARCH $p
         if [ "$pkg_name" ]; then
            base_pkg_ver=$pkg_version
            base_pkg_rel=$pkg_release

            get_pkg_buildinfo $DATADIR/$PORT_REPOSITORY $BASE_ARCH $p
            [ "$pkg_name" ] || {
               echo "Error: internal error, should never be here; aborting."
               exit 1
            }
            port_pkg_ver=$pkg_version
            port_pkg_rel=$pkg_release

            if [ "$target_pkg_name" ]; then
               pkg_milestone=`echo $target_pkg_rel | sed "s|[0-9.]*\(.*\)|\1|"`
               if [ "$target_pkg_ver" != "$port_pkg_ver" ]; then
                  echo "Package $pkg_name#${port_pkg_ver}-${port_pkg_rel}($TARGET_ARCH) differs from $pkg_name#${target_pkg_ver}-${target_pkg_rel}($BASE_ARCH); port needed."
                  JOB=(${JOB[*]} $p)
               elif [ "$target_pkg_rel" != "$port_pkg_rel" ]; then
                  echo "Package $pkg_name#${port_pkg_ver}-${port_pkg_rel}($TARGET_ARCH) differs from $pkg_name#${target_pkg_ver}-${target_pkg_rel}($BASE_ARCH); port needed."
                  JOB=(${JOB[*]} $p)
               elif [ "$REBUILD_MILESTONE" -a "$pkg_milestone" != "$REBUILD_MILESTONE" ]; then
                  echo "Package $pkg_name($TARGET_ARCH) was not built for '$REBUILD_MILESTONE' milestone; rebuild needed."
                  JOB=(${JOB[*]} $p)
               else
                  [ "$VERBOSE_MODE" ] && echo "Package $pkg_name#${port_pkg_ver}-${port_pkg_rel}($TARGET_ARCH) is up to date."
               fi
            else
               echo "$p#${port_pkg_ver}-${port_pkg_rel}($BASE_ARCH) is in port repository but $p#${base_pkg_ver}-${base_pkg_rel}($TARGET_ARCH) is in base repository; port needed."
               JOB=(${JOB[*]} $p)
            fi
         fi
      done
   fi

   step=0
   tmpfile=`mktemp`
   tmpfile1=`mktemp`

   [ -e $DATAARCHDIR/sentjobs ] || > $DATAARCHDIR/sentjobs
   find $DATAARCHDIR/sentjobs -cmin 120 && {
      > $DATAARCHDIR/sentjobs
   }

   [ -e $DATAARCHDIR/preparedjobs ] || > $DATAARCHDIR/preparedjobs
   find $DATAARCHDIR/preparedjobs -ctime 1 && {
      > $DATAARCHDIR/preparedjobs
   }

   sudo autoport-fix-environment $TARGET_ARCH

   [ "$FIX_MODE" ] && exit 0

   [ "$FORCE_MODE" ] && FORCE_FLAG="--force"

  while [ "${JOB[*]}" ]; do
      step=`expr $step + 1`
      [ "$BATCH_MODE" -a $step -gt $AUTOPORT_MAX_STEPS ] && {
         echo "Maximum steps count reached for batch mode; exiting. See $logfile for details."
         break
      }
      echo "=========="
      echo "# Step $step"
      echo "> Scheduled jobs: ${JOB[*]}"
      echo
      JOB_MISSINGREQ_OLD=(${JOB_MISSINGREQ[*]})
      JOB_MISSINGREQ=()
      JOB_MAYBEINSTALLED_NEW=()
      JOB_FAILED_OLD=(${JOB_FAILED[*]})
      JOB_FAILED=()
      JOB_CANTINSTALL_OLD=(${JOB_CANTINSTALL[*]})
      for JOB_NUM in `seq 1 ${#JOB[*]}`; do
         JOB_CURRENT=${JOB[$JOB_NUM-1]}
         if [ "`grep "^${JOB_CURRENT}$" /etc/autodist/unstage 2>/dev/null`" -o \
              "`grep "^${JOB_CURRENT}$" /etc/autodist/unstage.${TARGET_ARCH} 2>/dev/null`" ]; then
            echo -n "^ $JOB_CURRENT: "
            STAGEOPTS=
         else
            echo -n "^ $JOB_CURRENT(staging): "
            STAGEOPTS="$AUTOPORT_STAGEVARS"
         fi
         if [ "$CROSS_MODE" -a "${JOB_CURRENT:0:5}" == "perl-" ]; then
            echo "< (SKIPPED) [unable to cross-build perl modules]"
            JOB_SKIPPED=(${JOB_SKIPPED[*]} $JOB_CURRENT)
            continue
         fi
         [ "`grep ^$JOB_CURRENT\$ $DATAARCHDIR/sentjobs`" -a ! "$FORCE_MODE" ] && {
            echo "< (SKIPPED) [already sent]"
            JOB_SKIPPED=(${JOB_SKIPPED[*]} $JOB_CURRENT)
            continue
         }
         if [ "${JOB_MAYBEINSTALLED_NEW[*]}" = "" -a \
              "${JOB_MISSINGREQ_OLD[*]}" != "" -a \
              "`echo "( ${JOB_FAILED_OLD[*]} )"| grep " $JOB_CURRENT "`" ]; then
            echo "< (FAILED) [nothing new since previous attempt]"
            JOB_FAILED=(${JOB_FAILED[*]} $JOB_CURRENT)
            continue
         fi
         pkg_release=
         pkg_version=
         get_pkg_buildinfo $DATADIR/$PORT_REPOSITORY $TARGET_ARCH ${JOB_CURRENT} || \
         get_pkg_buildinfo $DATADIR/$PORT_REPOSITORY $BASE_ARCH ${JOB_CURRENT} || \
         get_pkg_buildinfo $DATADIR/$BASE_REPOSITORY $TARGET_ARCH ${JOB_CURRENT} || \
         get_pkg_buildinfo $DATADIR/$BASE_REPOSITORY $BASE_ARCH ${JOB_CURRENT}
         pkg_milestone=`echo $pkg_release | sed "s|[0-9.]*\(.*\)|\1|"`
         pkg_numrelease=`echo $pkg_release | sed "s|\([0-9.]*\).*|\1|"`
         DONT_PREPARE_THIS=
         for p in ${JOB_CMDLINE[*]}; do
            [ "$p" = "${JOB_CURRENT}" -o "$DONT_PREPARE_ANY" ] || continue
            [ -e ${SPECDIR}/${JOB_CURRENT}.spec ] && DONT_PREPARE_THIS="$DONT_PREPARE"
         done
         [ ! "$DONT_PREPARE_THIS" -a ! "`grep ^$JOB_CURRENT\$ $DATAARCHDIR/preparedjobs`" ] && {
            echo -n "prepare"
            [ "$VERBOSE_MODE" ] && echo "
%% COMMAND: LANG=C LC_ALL=C autodist -p ${JOB_CURRENT} --server $PORT_REPOSITORY --repository $PORT_REPOSITORY --severity 2 -- $STAGEOPTS"
            LANG=C LC_ALL=C autodist -p ${JOB_CURRENT} --server $PORT_REPOSITORY --repository $PORT_REPOSITORY --severity 2 -- $STAGEOPTS &>>$logfile
            [ $? -gt 0 ] && {
               RES=`find_source_by_provide $PORT_REPOSITORY $JOB_CURRENT`
               for j in ${RES}; do
                  [ "$j" != "$JOB_CURRENT" ] && RESOLVED_JOB="$RESOLVED_JOB $j"
               done
               if [ "$RESOLVED_JOB" ]; then
                  echo "(FAILED) [but scheduling $RESOLVED_JOB]"
                  JOB_FAILED=(${JOB_FAILED[*]} $RESOLVED_JOB)
               else
                  echo "(FAILED) "
               fi
               JOB_CANTPREPARE=(${JOB_CANTPREPARE[*]} $JOB_CURRENT)
               continue
            } || {
               echo $JOB_CURRENT >> $DATAARCHDIR/preparedjobs
               echo -n "(OK) "
            }
            if [ "$PORT_REPOSITORY" != "$DEST_REPOSITORY" ]; then
               echo -n "update"
               [ "$VERBOSE_MODE" ] && echo "
%% COMMAND: LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"automatic port from $PORT_REPOSITORY\""
               LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"automatic port from $PORT_REPOSITORY\" &>>$logfile
               [ $? -gt 0 ] && {
                  echo "(FAILED) "
                  continue
               } || {
                  echo -n "(OK) "
               }
            elif [ "$REBUILD_MILESTONE" -a "$pkg_milestone" != "$REBUILD_MILESTONE" ]; then
               echo -n "update"
               [ "$pkg_repository" -a "$pkg_version" -a "$pkg_release" ] || {
                  echo "(FAILED) [can't get repository information for this package]"
                  continue
               }
               # FIXME: should be implemented in autodist
               LANG=C LC_ALL=C autospec -u ${JOB_CURRENT} -a4 ${pkg_version} ${pkg_numrelease}${REBUILD_MILESTONE} \
                  --changelog "$REBUILD_MILESTONE milestone rebuild" &>$tmpfile
               [ $? -gt 0 ] && {
                  if [ "$VERBOSE_MODE" ]; then
                     echo
                     echo "= Failed output:"
                     echo "================"
                     cat $tmpfile
                     echo "================"
                     echo
                  fi
                  cat $tmpfile >> $logfile
                  JOB_FAILED=(${JOB_FAILED[*]} $JOB_CURRENT)
                  echo "(FAILED)"
                  JOB_LOG=`grep "^?= See " $tmpfile | awk '{ print $3 }'`
                  [ "$JOB_LOG" ] || JOB_LOG=$tmpfile
                  continue
               } || {
                  cat $tmpfile >> $logfile
                  echo -n "(OK) "
               }
            fi
         }
         echo -n "port"
         if [ ! "$pkg_repository" -a "$DONT_PREPARE_THIS" != "1" ]; then
            echo "(FAILED) [can't get repository information for this package]"
            continue
         fi
         if [ "$DONT_PREPARE_THIS" = "1" -a "$pkg_repository" != "$PORT_REPOSITORY" ]; then
            # if both -np and -r were given send to PORT_REPOSITORY instead of pkg_repository
            pkg_repository=$PORT_REPOSITORY
         fi
         [ "$PORT_REPOSITORY" != "$DEST_REPOSITORY" ] && pkg_repository="$DEST_REPOSITORY"
         [ ! "$DONT_PREPARE_THIS" -a "$PORT_REPOSITORY" = "$DEST_REPOSITORY" -a ! "$REBUILD_MILESTONE" ] && NOSRPM_OPTS="--nosrpm" || NOSRPM_OPTS="--force"
         LANG=C LC_ALL=C autodist -b -s ${JOB_CURRENT} --arch $TARGET_ARCH --server $pkg_repository --repository $PORT_REPOSITORY --severity 2 $FORCE_FLAG -- $STAGEOPTS $NOSRPM_OPTS &>$tmpfile
         [ $? -gt 0 ] && {
            if [ "$VERBOSE_MODE" ]; then
               echo
               echo "= Failed output:"
               echo "================"
               cat $tmpfile
               echo "================"
               echo
            fi
            cat $tmpfile >> $logfile
            JOB_FAILED=(${JOB_FAILED[*]} $JOB_CURRENT)
            echo "(FAILED)"
            JOB_LOG=`grep "^?= See " $tmpfile | awk '{ print $3 }'`
            [ "$JOB_LOG" ] || JOB_LOG=$tmpfile
            [ ! "$REBUILD_MILESTONE" ] && find_requirements $JOB_LOG
            continue
         } || {
            cat $tmpfile >> $logfile
            JOB_SENT=(${JOB_SENT[*]} $JOB_CURRENT)
            echo $JOB_CURRENT >> $DATAARCHDIR/sentjobs
            echo -n "(OK) "
         }
         echo -n "install"
         LANG=C LC_ALL=C autodist -i ${JOB_CURRENT} --arch $TARGET_ARCH --force --severity 2 -- $STAGEOPTS &>$tmpfile
         # --server $DEST_REPOSITORY --repository $PORT_REPOSITORY
         [ $? -gt 0 ] && {
            if [ "$VERBOSE_MODE" ]; then
               echo
               echo "= Failed output:"
               echo "================"
               cat $tmpfile
               echo "================"
               echo
            fi
            cat $tmpfile >> $logfile
            JOB_CANTINSTALL=(${JOB_CANTINSTALL[*]} $JOB_CURRENT)
            echo $JOB_CURRENT >> $DATAARCHDIR/checkinstalljobs
            echo "(FAILED)"
            # sometimes script fails but package is installed; run fix_la_files anyway
            sudo autoport-fix-environment $TARGET_ARCH

            JOB_MAYBEINSTALLED_NEW=(${JOB_MAYBEINSTALLED_NEW[*]} $JOB_CURRENT)
            JOB_LOG=`grep "^?= See " $tmpfile | awk '{ print $3 }'`
            [ "$JOB_LOG" ] || JOB_LOG=$tmpfile
            [ ! "$REBUILD_MILESTONE" ] && find_requirements $JOB_LOG
            continue
         } || {
            cat $tmpfile >> $logfile
            sudo autoport-fix-environment $TARGET_ARCH
            JOB_MAYBEINSTALLED_NEW=(${JOB_MAYBEINSTALLED_NEW[*]} $JOB_CURRENT)
            JOB_COMPLETED=(${JOB_COMPLETED[*]} $JOB_CURRENT)
            echo -n "(OK)"
         }
         echo
      done
      if [ "${JOB[*]}" == "${JOB_FAILED[*]}" -a "${JOB_MISSINGREQ[*]}" == "" ]; then
         # some new build, retry cantinstall jobs
         JOB_CANTINSTALL=()
         for JOB_NUM in `seq 1 ${#JOB_CANTINSTALL_OLD[*]}`; do
            JOB_CURRENT=${JOB_CANTINSTALL_OLD[$JOB_NUM-1]}
            if [ "`grep "^${JOB_CURRENT}$" /etc/autodist/unstage 2>/dev/null`" -o \
                 "`grep "^${JOB_CURRENT}$" /etc/autodist/unstage.${TARGET_ARCH} 2>/dev/null`" ]; then
               echo -n "$JOB_CURRENT: "
               STAGEOPTS=
            else
               echo -n "$JOB_CURRENT(staging): "
               STAGEOPTS="$AUTOPORT_STAGEVARS"
            fi
            echo -n "retryinstall"
            LANG=C LC_ALL=C autodist -i ${JOB_CURRENT} --arch $TARGET_ARCH --force --severity 2 -- $STAGEOPTS &>>$tmpfile
            # --server $DEST_REPOSITORY --repository $PORT_REPOSITORY
            [ $? -gt 0 ] && {
               if [ "$VERBOSE_MODE" ]; then
                  echo
                  echo "= Failed output:"
                  echo "================"
                  cat $tmpfile
                  echo "================"
                  echo
               fi
               cat $tmpfile >> $logfile
               JOB_CANTINSTALL=(${JOB_CANTINSTALL[*]} $JOB_CURRENT)
               echo $JOB_CURRENT >> $DATAARCHDIR/checkinstalljobs
               echo "(FAILED)"
               # sometimes script fails but package is installed; run fix_la_files anyway
               sudo autoport-fix-environment $TARGET_ARCH

               JOB_MAYBEINSTALLED_NEW=(${JOB_MAYBEINSTALLED_NEW[*]} $JOB_CURRENT)
               JOB_LOG=`grep "^?= See " $tmpfile | awk '{ print $3 }'`
               [ "$JOB_LOG" ] || JOB_LOG=$tmpfile
               find_requirements $JOB_LOG
               continue
            } || {
               cat $tmpfile >> $logfile
               sudo autoport-fix-environment $TARGET_ARCH
               JOB_MAYBEINSTALLED_NEW=(${JOB_MAYBEINSTALLED_NEW[*]} $JOB_CURRENT)
               JOB_COMPLETED=(${JOB_COMPLETED[*]} $JOB_CURRENT)
               echo -n "(OK)"
            }
            echo
         done
      fi
      echo "| Skipped jobs: ${JOB_SKIPPED[*]}"
      echo "| Unpreparable jobs: ${JOB_CANTPREPARE[*]}"
      echo "| Missing requirements: ${JOB_MISSINGREQ[*]}"
      echo "| Completed jobs: ${JOB_COMPLETED[*]}"
      echo "| Installation error for: ${JOB_CANTINSTALL[*]}"
      echo "| Failed jobs: ${JOB_FAILED[*]}"
      write_last_file

      [ "${JOB[*]}" == "${JOB_FAILED[*]}" -a \
        "${JOB_MISSINGREQ[*]}" == "" -a \
        "${JOB_CANTINSTALL_OLD[*]}" == "${JOB_CANTINSTALL[*]}"  ] && {
         echo "! Loop detected. Finishing. See $logfile for details."
         rm -f $tmpfile $tmpfile1
         exit 1
      }
      [ "$DONT_RECURSE" ] && break
      JOB=(${JOB_MISSINGREQ[*]} ${JOB_FAILED[*]})
   done
   rm -f $tmpfile $tmpfile1
done
echo "* All jobs completed. See $logfile for details."
exit 0