autodist/autoport

483 lines
18 KiB
Bash
Executable File

#!/bin/bash
# autoport -- batch cross-build tool based on autodist
# Copyright (C) 2011 by Silvan Calarco <silvan.calarco@mambasoft.it>
#
# Released under the terms of the GNU GPL release 3 license
#
VERSION=0.8.5
BASE_ARCH=i586
BASE_REPOSITORY=devel
PORT_REPOSITORY=devel
. /etc/autodist/config
me=(${0##*/} $VERSION "Sat Mar 20 2011")
function usage() {
echo "\
${me[0]} ${me[1]}
"$"Copyright (C) 2010 Silvan Calarco <silvan.calarco@mambasoft.it>""
"$"Released under the terms of the GNU GPL v3 license"
echo "
"$"Batch port and cross-build tool based on autodist.""
"$"Usage"":
$me [-a][-f][-x arch1[,arch2],..][-r repository][-d release_repository]{--fix|job ...}
-a "$"Automatic mode (use cache)
-b "$"Batch port all packages in port repository to sync with base repository
-f "$"Force operations
-np "$"Don't prepare jobs passed to command line unless specfile is not found
-npa "$"Don't prepare any jobs unless specfile is not found
-nr "$"Don't recurse build
--fix "$"Fix .la files for cross linking and exit
-r "$"Work on given repository (default: $PORT_REPOSITORY)
-d "$"Release packages to given repository (default: $PORT_REPOSITORY)
-x "$"Operate in cross build mode
-h "$"Show this help and exit
-v "$"More verbose output
arch "$"Target architecture
job "$"Job specification for autodist""
"
}
# get_pkg_buildinfo - uses distromatic generated build file for
# getting information on the repository
#
# $1: distromatic files base url for repository
# $2: architecture
# $3: pkg name
function get_pkg_buildinfo() {
local pkg i a local_arch local_distromatic
[ $1 ] && local_distromatic=$1 || exit 1
[ $2 ] && local_arch=$2 || exit 1
[ $3 ] && pkg=$3
if [ ! "$pkg" ]; then
[ -e $local_distromatic/builds-${local_arch}.sh ] && . $local_distromatic/builds-${local_arch}.sh
return 0
fi
pkg_archs=();
for a in ${archs[*]}; do
pkg_header=();
DISTROMATIC_BUILD_FILE=$local_distromatic/builds-$a.sh
[ -e $DISTROMATIC_BUILD_FILE ] && . $DISTROMATIC_BUILD_FILE
[ ${pkg_header[0]} ] && pkg_archs=(${pkg_archs[*]} $a)
done
pkg_header=();
pkg_builds=();
pkg_obsoletes=();
pkg_list=();
DISTROMATIC_BUILD_FILE=$local_distromatic/builds-${local_arch}.sh
[ -e $DISTROMATIC_BUILD_FILE ] && . $DISTROMATIC_BUILD_FILE
# {
# echo "Error: missing distromatic generated file $DISTROMATIC_BUILD_FILE; aborting."
# return;
# }
# . $DISTROMATIC_BUILD_FILE
for i in ${pkg_list[*]}; do
if [ "$i" == "${pkg_header[0]}" ]; then
pkg_name=${pkg_header[0]};
# Note: pkg_arch reported in builds file is just last arch source was
# built for, so we use repository arch instead
pkg_arch=${pkg_header[1]};
[ "$pkg_arch" = "noarch" ] || pkg_arch=${local_arch}
pkg_version=${pkg_header[2]};
pkg_release=${pkg_header[3]};
pkg_group=${pkg_header[4]};
pkg_license=${pkg_header[5]};
pkg_size=${pkg_header[6]};
pkg_buildtime=${pkg_header[7]};
pkg_altrep=${pkg_header[8]};
pkg_repository=${pkg_header[9]};
return 0
fi
done
unset pkg_name pkg_arch pkg_version pkg_release \
pkg_group pkg_license pkg_size pkg_buildtime pkg_altrep pkg_repository
return 1
}
function write_last_file() {
[ "$AUTO_MODE" == "1" ] && LASTFILE=$DATADIR/last || LASTFILE=$DATADIR/lastmanual
cat > $LASTFILE << _EOF
JOB_CANTPREPARE=(${JOB_CANTPREPARE[*]})
JOB_MISSINGREQ=(${JOB_MISSINGREQ[*]})
JOB_SENT=(${JOB_SENT[*]})
JOB_CANTINSTAL=(${JOB_CANTINSTALL[*]})
JOB_COMPLETED=(${JOB_COMPLETED[*]})
JOB_FAILED=(${JOB_FAILED[*]})
_EOF
}
function find_requirements() {
[ "$1" ] || return 1
local tf=`mktemp` || {
echo "Error: cannot create tempfile"
return 1
}
local tf1=`mktemp`
local OUTFILE=$1
if [ "`grep " is needed by " $OUTFILE`" ]; then
grep " is needed by " $OUTFILE | awk '{ print $1 }' | sort -u > $tf1
elif [ "`grep " matches no packages" $OUTFILE`" ]; then
grep " matches no packages" $OUTFILE | awk '{ print $2 }' | tr -d \' | sort -u > $tf1
elif [ "`grep " no package provides " $OUTFILE`" ]; then
grep " no package provides " $OUTFILE | sed "s|.* no package provides ||" | sort -u > $tf1
fi
[ `stat -c %s $tf1` -gt 0 ] && {
echo -n " | missing requirements: "
> $tf
cat $tf1 | while read line; do
REQ=`grep "^$line:" $DATADIR/$PORT_REPOSITORY/sources-$BASE_ARCH | awk '{ print $2 }'`
[ "$REQ" ] && \
echo "${REQ/_}" >> $tf || \
echo -n "$line(unresolved) "
done
> $tf1
cat $tf | sort -u 2>/dev/null | while read line; do
if [ "`echo " ${JOB[*]} ${JOB_MISSINGREQ[*]} ${JOB_CANTINSTALL[*]} " | grep " $line "`" ]; then
echo -n "($line) "
else
echo -n "$line "
echo -n "$line " >> $tf1
fi
done
JOB_MISSINGREQ=(${JOB_MISSINGREQ[*]} `cat $tf1`)
echo
}
rm -f $tf $tf1
return 0
}
[ $# -gt 0 ] || { usage ; exit 1; }
BUILD_ARCH=`uname -i`
case $BUILD_ARCH in
i386|i486|i686) BUILD_ARCH=i586 ;;
arm*) BUILD_ARCH=arm ;;
esac
TARGET_ARCHS=$BUILD_ARCH
JOB_CMDLINE=()
CROSS_MODE=
for ((i=1; i<=$#; i++)); do
case ${!i} in
-a) AUTO_MODE=1 ;;
-b) BATCH_MODE=1 ;;
-f) FORCE_MODE=1 ;;
-v) VERBOSE_MODE=1 ;;
-npa) DONT_PREPARE_ANY=1
DONT_PREPARE=1 ;;
-np) DONT_PREPARE=1 ;;
-nr) DONT_RECURSE=1 ;;
--fix) FIX_MODE=1 ;;
-r) shift
PORT_REPOSITORY="${!i}"
;;
-d) shift
DEST_REPOSITORY="${!i}"
;;
-x) shift
TARGET_ARCHS="`echo ${!i} | tr , ' '`"
CROSS_MODE=1
;;
-h|--help)
usage
exit 0
;;
*) JOB_CMDLINE=(${JOB_CMDLINE[*]} ${!i})
;;
esac
done
[ "$DEST_REPOSITORY" ] || DEST_REPOSITORY=$PORT_REPOSITORY
for TARGET_ARCH in ${TARGET_ARCHS}; do
[ "$BASE_ARCH" = "$TARGET_ARCH" -a "$PORT_REPOSITRY" = "$DEST_REPOSITORY" ] && {
echo "Error: base and target arch ($BASE_ARCH) are the same as well as port and release repositories ($PORT_REPOSITORY); skipping."
continue
}
archs=($BASE_ARCH $TARGET_ARCH)
JOBS_SKIPPED=()
JOBS_CANTPREPARE=()
JOBS_CANTINSTALL=()
JOB_MISSINGREQ=()
JOB_COMPLETED=()
JOB_FAILED=()
DATADIR=$HOME/.autoport
DATAARCHDIR=$DATADIR/$TARGET_ARCH
DISTROMATIC_BASE_URL=${REPS_BASE_URL}/distromatic/
BASE_REPOSITORY_DISTROMATIC_URL=${DISTROMATIC_BASE_URL}$BASE_REPOSITORY
PORT_REPOSITORY_DISTROMATIC_URL=${DISTROMATIC_BASE_URL}$PORT_REPOSITORY
[ "$AUTO_MODE" -a -e $DATAARCHDIR/last ] && . $DATAARCHDIR/last
JOB=(${JOB_CMDLINE[*]} ${JOB_MISSINGREQ[*]} ${JOB_FAILED[*]})
[ "$FIX_MODE" -o "$BATCH_MODE" -o "${JOB[*]}" ] || { usage ; exit 1; }
[ -d $DATAARCHDIR ] || mkdir -p $DATAARCHDIR
[ -d $DATADIR/$BASE_REPOSITORY ] || mkdir -p $DATADIR/$BASE_REPOSITORY
[ -d $DATADIR/$PORT_REPOSITORY ] || mkdir -p $DATADIR/$PORT_REPOSITORY
logfile=$DATAARCHDIR/log
[ -e $logfile ] && mv -f $logfile $DATAARCHDIR/loglast
> $logfile
BUILD_PLATFORM=`rpm --target $TARGET_ARCH --eval %{_build}`
BUILD_CPU=`echo $BUILD_PLATFORM | cut -d- -f1`
TARGET_PLATFORM=`rpm --target $TARGET_ARCH --eval %{_target_platform}`
TARGET_CPU=`echo $TARGET_PLATFORM | cut -d- -f1`
SPECDIR=`rpm --eval %{_specdir}`
echo "Build platform is $BUILD_PLATFORM"
echo "Target platform is $TARGET_PLATFORM"
echo "Port repository is $PORT_REPOSITORY"
echo "Release repository is $DEST_REPOSITORY"
curl -s $PORT_REPOSITORY_DISTROMATIC_URL/sources-$BASE_ARCH -o $DATADIR/$PORT_REPOSITORY/sources-$BASE_ARCH ||
echo "Warning: unable to fetch $PORT_REPOSITORY_DISTROMATIC_URL/sources-$BASE_ARCH"
curl -s $PORT_REPOSITORY_DISTROMATIC_URL/sources-$TARGET_CPU -o $DATADIR/$PORT_REPOSITORY/sources-$TARGET_CPU ||
echo "Warning: unable to fetch $PORT_REPOSITORY_DISTROMATIC_URL/sources-$TARGET_CPU"
curl -s $PORT_REPOSITORY_DISTROMATIC_URL/builds-$BASE_ARCH.sh -o $DATADIR/$PORT_REPOSITORY/builds-$BASE_ARCH.sh ||
echo "Error: unable to fetch $PORT_REPOSITORY_DISTROMATIC_URL/builds-$BASE_ARCH.sh"
curl -s $PORT_REPOSITORY_DISTROMATIC_URL/builds-$TARGET_ARCH.sh -o $DATADIR/$PORT_REPOSITORY/builds-$TARGET_ARCH.sh ||
echo "Error: unable to fetch $PORT_REPOSITORY_DISTROMATIC_URL/builds-$TARGET_ARCH.sh"
if [ "$BATCH_MODE" ]; then
[ "${JOB[*]}" ] && {
echo "Error: no jobs allowed in batch mode; aborting."
exit 1
}
curl -s $BASE_REPOSITORY_DISTROMATIC_URL/builds-$BASE_ARCH.sh -o $DATADIR/$BASE_REPOSITORY/builds-$BASE_ARCH.sh ||
echo "Error: unable to fetch $BASE_REPOSITORY_DISTROMATIC_URL/builds-$BASE_ARCH.sh"
curl -s $BASE_REPOSITORY_DISTROMATIC_URL/builds-$TARGET_ARCH.sh -o $DATADIR/$BASE_REPOSITORY/builds-$TARGET_ARCH.sh ||
echo "Error: unable to fetch $BASE_REPOSITORY_DISTROMATIC_URL/builds-$TARGET_ARCH.sh"
get_pkg_buildinfo $DATADIR/$PORT_REPOSITORY $BASE_ARCH
port_pkg_list=${pkg_list[*]}
for p in ${port_pkg_list}; do
get_pkg_buildinfo $DATADIR/$PORT_REPOSITORY $TARGET_ARCH $p || get_pkg_buildinfo $DATADIR/$BASE_REPOSITORY $TARGET_ARCH $p
if [ "$pkg_name" ]; then
base_pkg_ver=$pkg_version
get_pkg_buildinfo $DATADIR/$PORT_REPOSITORY $BASE_ARCH $p
[ "$pkg_name" ] || {
echo "Error: internal error, should never be here; aborting."
exit 1
}
port_pkg_ver=$pkg_version
get_pkg_buildinfo $DATADIR/$PORT_REPOSITORY $TARGET_ARCH $p
if [ "$pkg_name" ]; then
if [ "$pkg_version" != "$port_pkg_ver" ]; then
echo "Package $pkg_name#${port_pkg_ver}($TARGET_ARCH) differs from $pkg_name#${pkg_version}($BASE_ARCH); port needed."
JOB=(${JOB[*]} $p)
else
echo "Package $pkg_name#${port_pkg_ver}($TARGET_ARCH) is up to date."
fi
else
echo "$p#${port_pkg_ver}($BASE_ARCH) is in port repository but $p#${base_pkg_ver}($TARGET_ARCH) is in base repository; port needed."
JOB=(${JOB[*]} $p)
fi
fi
done
fi
step=0
tmpfile=`mktemp`
tmpfile1=`mktemp`
[ -e $DATAARCHDIR/sentjobs ] || > $DATAARCHDIR/sentjobs
find $DATAARCHDIR/sentjobs -cmin 120 && {
> $DATAARCHDIR/sentjobs
}
[ -e $DATAARCHDIR/preparedjobs ] || > $DATAARCHDIR/preparedjobs
find $DATAARCHDIR/preparedjobs -ctime 1 && {
> $DATAARCHDIR/preparedjobs
}
sudo autoport-fix-environment $TARGET_ARCH
[ "$FIX_MODE" ] && exit 0
[ "$FORCE_MODE" ] && FORCE_FLAG="--force"
while [ "${JOB[*]}" ]; do
step=`expr $step + 1`
[ "$BATCH_MODE" -a $step -gt $AUTOPORT_MAX_STEPS ] && {
echo "Maximum steps count reached for batch mode; exiting. See $logfile for details."
break
}
echo "=========="
echo "Step $step"
echo "Scheduled jobs: ${JOB[*]}"
echo
JOB_MISSINGREQ_OLD=(${JOB_MISSINGREQ[*]})
JOB_MISSINGREQ=()
JOB_MAYBEINSTALLED_NEW=()
JOB_FAILED_OLD=(${JOB_FAILED[*]})
JOB_FAILED=()
for JOB_NUM in `seq 1 ${#JOB[*]}`; do
JOB_CURRENT=${JOB[$JOB_NUM-1]}
if [ "`grep "^${JOB_CURRENT}$" /etc/autodist/unstage 2>/dev/null`" -o \
"`grep "^${JOB_CURRENT}$" /etc/autodist/unstage.${TARGET_ARCH} 2>/dev/null`" ]; then
echo -n "$JOB_CURRENT: "
STAGEOPTS=
else
echo -n "$JOB_CURRENT(staging): "
STAGEOPTS="$AUTOPORT_STAGEVARS"
fi
if [ "$CROSS_MODE" -a "${JOB_CURRENT:0:5}" == "perl-" ]; then
echo " (SKIPPED) [unable to cross-build perl modules]"
JOB_SKIPPED=(${JOB_SKIPPED[*]} $JOB_CURRENT)
continue
fi
[ "`grep ^$JOB_CURRENT\$ $DATAARCHDIR/sentjobs`" -a ! "$FORCE_MODE" ] && {
echo " (SKIPPED) [already sent]"
JOB_SKIPPED=(${JOB_SKIPPED[*]} $JOB_CURRENT)
continue
}
if [ "${JOB_MAYBEINSTALLED_NEW[*]}" = "" -a \
"${JOB_MISSINGREQ_OLD[*]}" != "" -a \
"`echo "( ${JOB_FAILED_OLD[*]} )"| grep " $JOB_CURRENT "`" ]; then
echo " (FAILED) [nothing new since previous attempt]"
JOB_FAILED=(${JOB_FAILED[*]} $JOB_CURRENT)
continue
fi
DONT_PREPARE_THIS=
for p in ${JOB_CMDLINE[*]}; do
[ "$p" = "${JOB_CURRENT}" -o "$DONT_PREPARE_ANY" ] || continue
[ -e ${SPECDIR}/${JOB_CURRENT}.spec ] && DONT_PREPARE_THIS="$DONT_PREPARE"
done
[ ! "$DONT_PREPARE_THIS" -a ! "`grep ^$JOB_CURRENT\$ $DATAARCHDIR/preparedjobs`" ] && {
echo -n "prepare"
[ "$VERBOSE_MODE" ] && echo "
%% COMMAND: LANG=C LC_ALL=C autodist -p ${JOB_CURRENT} --server $PORT_REPOSITORY --repository $PORT_REPOSITORY --severity 2 -- $STAGEOPTS"
LANG=C LC_ALL=C autodist -p ${JOB_CURRENT} --server $PORT_REPOSITORY --repository $PORT_REPOSITORY --severity 2 -- $STAGEOPTS &>>$logfile
[ $? -gt 0 ] && {
JOB_CANTPREPARE=(${JOB_CANTPREPARE[*]} $JOB_CURRENT)
echo "(FAILED) "
continue
} || {
echo $JOB_CURRENT >> $DATAARCHDIR/preparedjobs
echo -n "(OK) "
}
if [ "$PORT_REPOSITORY" != "$DEST_REPOSITORY" ]; then
echo -n "update"
[ "$VERBOSE_MODE" ] && echo "
%% COMMAND: LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"automatic port from $PORT_REPOSITORY\""
LANG=C LC_ALL=C autodist -u -r ${JOB_CURRENT} --severity 2 -- $STAGEOPTS --changelog \"automatic port from $PORT_REPOSITORY\" &>>$logfile
[ $? -gt 0 ] && {
echo "(FAILED) "
continue
} || {
echo -n "(OK) "
}
fi
}
echo -n "port"
get_pkg_buildinfo $DATADIR/$PORT_REPOSITORY $TARGET_ARCH ${JOB_CURRENT} || \
get_pkg_buildinfo $DATADIR/$PORT_REPOSITORY $BASE_ARCH ${JOB_CURRENT} || \
get_pkg_buildinfo $DATADIR/$BASE_REPOSITORY $TARGET_ARCH ${JOB_CURRENT} || \
get_pkg_buildinfo $DATADIR/$BASE_REPOSITORY $BASE_ARCH ${JOB_CURRENT}
[ "$pkg_repository" ] || {
echo "(FAILED) [can't get repository information for this package]"
continue
}
[ "$PORT_REPOSITORY" != "$DEST_REPOSITORY" ] && pkg_repository="$DEST_REPOSITORY"
[ ! "$DONT_PREPARE_THIS" -a "$PORT_REPOSITORY" = "$DEST_REPOSITORY" ] && NOSRPM_OPTS="--nosrpm" || NOSRPM_OPTS="--force"
LANG=C LC_ALL=C autodist -b -s ${JOB_CURRENT} --arch $TARGET_ARCH --server $pkg_repository --repository $PORT_REPOSITORY --severity 2 $FORCE_FLAG -- $STAGEOPTS $NOSRPM_OPTS &>$tmpfile
[ $? -gt 0 ] && {
if [ "$VERBOSE_MODE" ]; then
echo
echo "Failed output:"
echo "=============="
cat $tmpfile
echo "=============="
echo
fi
cat $tmpfile >> $logfile
JOB_FAILED=(${JOB_FAILED[*]} $JOB_CURRENT)
echo "(FAILED)"
JOB_LOG=`grep "^?= See " $tmpfile | awk '{ print $3 }'`
[ "$JOB_LOG" ] || JOB_LOG=$tmpfile
find_requirements $JOB_LOG
continue
} || {
cat $tmpfile >> $logfile
JOB_SENT=(${JOB_SENT[*]} $JOB_CURRENT)
echo $JOB_CURRENT >> $DATAARCHDIR/sentjobs
echo -n "(OK) "
}
echo -n "install"
LANG=C LC_ALL=C autodist -i ${JOB_CURRENT} --arch $TARGET_ARCH --force --severity 2 -- $STAGEOPTS &>>$tmpfile
# --server $DEST_REPOSITORY --repository $PORT_REPOSITORY
[ $? -gt 0 ] && {
if [ "$VERBOSE_MODE" ]; then
echo
echo "Failed output:"
echo "=============="
cat $tmpfile
echo "=============="
echo
fi
cat $tmpfile >> $logfile
JOB_CANTINSTALL=(${JOB_CANTINSTALL[*]} $JOB_CURRENT)
echo $JOB_CURRENT >> $DATAARCHDIR/checkinstalljobs
echo "(FAILED)"
# sometimes script fails but package is installed; run fix_la_files anyway
sudo autoport-fix-environment $TARGET_ARCH
JOB_MAYBEINSTALLED_NEW=(${JOB_MAYBEINSTALLED_NEW[*]} $JOB_CURRENT)
JOB_LOG=`grep "^?= See " $tmpfile | awk '{ print $3 }'`
[ "$JOB_LOG" ] || JOB_LOG=$tmpfile
find_requirements $JOB_LOG
continue
} || {
cat $tmpfile >> $logfile
sudo autoport-fix-environment $TARGET_ARCH
JOB_MAYBEINSTALLED_NEW=(${JOB_MAYBEINSTALLED_NEW[*]} $JOB_CURRENT)
JOB_COMPLETED=(${JOB_COMPLETED[*]} $JOB_CURRENT)
echo -n "(OK)"
}
echo
done
echo "Skipped jobs: ${JOB_SKIPPED[*]}"
echo "Unpreparable jobs: ${JOB_CANTPREPARE[*]}"
echo "Missing requirements: ${JOB_MISSINGREQ[*]}"
echo "Completed jobs: ${JOB_COMPLETED[*]}"
echo "Installation error for: ${JOB_CANTINSTALL[*]}"
echo "Failed jobs: ${JOB_FAILED[*]}"
write_last_file
[ "${JOB[*]}" == "${JOB_FAILED[*]}" -a "${JOB_MISSINGREQ[*]}" == "" ] && {
echo "Loop detected. Finishing. See $logfile for details."
rm -f $tmpfile $tmpfile1
exit 1
}
[ "$DONT_RECURSE" ] && break
JOB=(${JOB_MISSINGREQ[*]} ${JOB_FAILED[*]})
done
rm -f $tmpfile $tmpfile1
done
echo "All jobs completed. See $logfile for details."
exit 0