#!/bin/bash # # openmamba inter-repository import script from remote ftp to local # Copyright (c) 2007-2009 by Silvan Calarco # . /etc/sysconfig/openmamba-central me=${0##*/} function usage() { echo "openmamba inter-repository import script" echo "Copyright (c) 2007-2008 by Silvan Calarco" echo echo "Usage:" echo "$me list" echo "$me archive REPOSITORY PKGS ..." echo "$me query REPOSITORY PKGS ..." echo "$me verify REPOSITORY [PKGS ...]" echo "$me inspect REPOSITORY {PKGS ...} [-d REPOSITORY]" echo "$me setwarning REPOSITORY {PKG ...} -t \"TEXT\"" echo "$me diff REPOSITORY [PKGS ...] [-d REPOSITORY]" echo "$me import REPOSITORY [PKGS ...] [-d REPOSITORY] [-s]" echo "$me release REPOSITORY [PKGS ...] [-d REPOSITORY] [-s]" echo "$me distromatic REPOSITORY" echo echo " -d use given repository as destination (default: devel)" echo " -s simulate operations to see if it would work" echo " -t warning text" } # get_pkg_buildinfo - uses distromatic generated build file for # getting information on the repository # # $1: repositoy name # $2: pkg name # $3: architecture function get_pkg_buildinfo() { local pkg i [ $1 ] && rep=$1 || exit 1 [ $2 ] && pkg=$2 [ $3 ] && buildarch="$3" || buildarch="i586" DISTROMATIC_BUILD_FILE=$local_distromatic/$rep/builds-$buildarch [ -e $DISTROMATIC_BUILD_FILE ] || { echo "Error: missing distromatic generated file $DISTROMATIC_BUILD_FILE; aborting." exit 1; } pkg_header=(); pkg_builds=(); pkg_obsoletes=(); . $DISTROMATIC_BUILD_FILE for i in ${pkg_list[*]}; do if [ "$i" == "${pkg_header[0]}" ]; then pkg_name=${pkg_header[0]}; # Note: pkg_arch reported in builds file is just last arch source was # built for, so we use repository arch instead pkg_arch=${pkg_header[1]}; [ "$pkg_arch" = "noarch" ] || pkg_arch=$buildarch pkg_version=${pkg_header[2]}; pkg_release=${pkg_header[3]}; pkg_group=${pkg_header[4]}; pkg_license=${pkg_header[5]}; pkg_size=${pkg_header[6]}; pkg_buildtime=${pkg_header[7]}; pkg_altrep=${pkg_header[8]}; return 0 fi done unset pkg_name pkg_arch pkg_version pkg_release \ pkg_group pkg_license pkg_size pkg_buildtime pkg_altrep return 0 } function import_file() { [ $1 ] || exit 1 local f import_mode f=$1 import_mode=$2 if [ "$import_mode" = "backup" ]; then curl_delete_add="-Q '-DELE $f'" else curl_delete_add="" fi [ "$simulate" != "1" ] && if [ "$ORIG_MODE" = "remote" ]; then if [ $ORIG_URL_LOCAL_ARCH -a ! -f $ORIG_URL_LOCAL_ARCH/$f ]; then echo "Warning: package missing in local mirror; setting copy from remote repository." fi if [ "$DEST_MODE" = "local" ]; then # remote -> local if [ $ORIG_URL_LOCAL_ARCH -a -f $ORIG_URL_LOCAL_ARCH/$f ]; then echo -n "(L) " # if file exists in a local mirror use it by preference cp $ORIG_URL_LOCAL_ARCH/$f $DEST_URL_ARCH/ || { echo "Error: cannot move file $ORIG_URL_LOCAL_ARCH/$f to $DEST_URL_ARCH/$f; aborting." exit 1 } eval curl -s -u$ftpuser:$ftppass $ORIG_URL_ARCH $curl_delete_add >/dev/null && { rm -f $ORIG_URL_LOCAL_ARCH/$f } || { echo "Warning: cannot delete remote file $ORIG_URL_ARCH/$f; you'll have to delete it." echo curl -s -u$ftpuser:$ftppass $ORIG_URL_ARCH $curl_delete_add } else echo -n "(R) " curl -s -u$ftpuser:$ftppass \ --get $ORIG_URL_ARCH/$f \ -o $DEST_URL_ARCH/$f $curl_delete_add || { echo "Error: cannot get file $ORIG_URL_ARCH/$f; aborting." echo curl -s -u$ftpuser:$ftppass \ --get $ORIG_URL_ARCH/$f \ -o $DEST_URL_ARCH/$f $curl_delete_add exit 1 } fi else # remote -> remote echo "Error: remote to remote file import is not implemented yet; aborting." exit 1 fi else if [ "$DEST_MODE" = "local" ]; then # local -> local cp $ORIG_URL_ARCH/$f $DEST_URL_ARCH/ || { echo "Error: cannot copy file $ORIG_URL_ARCH/$f to $DEST_URL_ARCH/$f; aborting." exit 1 } if [ "$import_mode" = "backup" ]; then rm -f $ORIG_URL_ARCH/$f || { echo "Error: cannot remove file $ORIG_URL_ARCH/$f; aborting." exit 1 } fi else # local -> remote echo "Error: local to remote file import is not implemented yet; aborting." exit 1 fi fi } function backup_local_file() { [ $1 ] || return [ -e $LOCAL_BACKUP ] || mkdir -p $LOCAL_BACKUP movefiles=$1 #`find $DEST_URL_ARCH -maxdepth 1 -regex ".*/${pkgname}-[^-]*-[^-]*"` for m in $movefiles; do echo "backing up $m" if [ "$simulate" != "1" ]; then mv $m $LOCAL_BACKUP/ || { echo "Error: can't move $m to $LOCAL_BACKUP; aborting." exit 1 } fi done } function backup_package() { local rep reg i [ "$1" ] || return [ "$2" ] && rep=$2 || rep=$destrepository [ "$3" ] && reg=$3 || reg=$DESTREGFILE archive_pkg=$1 get_pkg_buildinfo $rep $archive_pkg if [ "$pkg_name" ]; then PKG_FILENAME="$pkg_name-$pkg_version-$pkg_release.src.rpm" [ -f $DEST_URL_LOCAL/SRPMS.base/$PKG_FILENAME ] || { echo "Error: package $PKG_FILENAME does not exist in local repository; aborting." exit 1 } backup_local_file $DEST_URL_LOCAL/SRPMS.base/$PKG_FILENAME if [ "$simulate" != "1" -a "$DEST_MODE" = "remote" ]; then curl -s -u$ftpuser:$ftppass $DEST_URL/SRPMS.base/ -Q "-DELE $PKG_FILENAME" >/dev/null || { echo "Warning: cannot delete remote file $DEST_URL/SRPMS.base/$PKG_FILENAME; you'll have to delete it." } fi for i in ${pkg_builds[*]}; do PKG_FILENAME="$i-$pkg_version-$pkg_release.$pkg_arch.rpm" [ -f $DEST_URL_LOCAL/RPMS.i586/$PKG_FILENAME ] || { echo "Error: package $PKG_FILENAME does not exist in local repository; aborting." exit 1 } backup_local_file $DEST_URL_LOCAL/RPMS.i586/$PKG_FILENAME if [ "$simulate" != "1" -a "$DEST_MODE" = "remote" ]; then curl -s -u$ftpuser:$ftppass $DEST_URL/RPMS.i586/ -Q "-DELE $PKG_FILENAME" >/dev/null || { echo "Warning: cannot delete remote file $DEST_URL/RPMS.i586/$PKG_FILENAME; you'll have to delete it." } fi done # write register [ "$simulate" != "1" ] && { echo "`date +%Y%m%d%H%M` Package $pkg_name ($pkg_version-$pkg_release) archived" >> $reg # | \ # tee -a $ORIGREGFILE $DESTREGFILE >/dev/null # echo "`date +%Y%m%d%H%M` \"\" \"package $pkg_name ($pkg_version-$pkg_release) archived from $rep\"" >> $reg } else echo "Warning: package $archive_pkg does not exists in $rep; skipping." fi } function import_package() { [ $1 ] || exit 1 local import_pkg import_mode import_pkg=$1 import_mode=$2 # check release in dest repository get_pkg_buildinfo $destrepository $import_pkg [ "$pkg_version" ] && { destpkgname="$pkg_name" destpkgversion="$pkg_version-$pkg_release" destpkgarch="$pkg_arch" } || destpkgversion="none" get_pkg_buildinfo $origrepository $import_pkg [ "$pkg_version" ] && { origpkgname="$pkg_name" origpkgversion="$pkg_version-$pkg_release" origpkgarch="$pkg_arch" } || origpkgversion="none" [ $origpkgname ] || { echo "Error: package $import_pkg does not exist; aborting." exit 1 } DEST_URL_ARCH=$DEST_URL/SRPMS.base/ ORIG_URL_ARCH=$ORIG_URL/SRPMS.base/ ORIG_URL_LOCAL_ARCH=$ORIG_URL_LOCAL/SRPMS.base/ PKG_FILENAME="$origpkgname-$origpkgversion.src.rpm" [ "$ORIG_MODE" = "remote" ] && ORIG_FILELIST=`curl -s -l -u$ftpuser:$ftppass --url $ORIG_URL_ARCH/` || ORIG_FILELIST=`ls $ORIG_URL_ARCH` check_existence=0; for i in $ORIG_FILELIST; do [ "$i" = "$PKG_FILENAME" ] && check_existence=1; done if [ $check_existence = 1 ]; then [ "$destpkgversion" = "$origpkgversion" ] && { echo "Warning: same version of $origpkgname exists in destination" } echo -n "Import $PKG_FILENAME ($origpkgversion -> $destpkgversion) [y/N]?" read ans [ "$ans" = "y" -o "$ans" = "Y" ] && { echo -n "Importing $PKG_FILENAME " import_file $PKG_FILENAME $import_mode for i in ${pkg_builds[*]}; do PKG_FILENAME="$i-$origpkgversion.$origpkgarch.rpm" DEST_URL_ARCH=$DEST_URL/RPMS.i586/ ORIG_URL_ARCH=$ORIG_URL/RPMS.i586/ ORIG_URL_LOCAL_ARCH=$ORIG_URL_LOCAL/RPMS.i586/ echo -n "$PKG_FILENAME " import_file $PKG_FILENAME $import_mode done echo # write register [ "$simulate" != "1" ] && { echo "`date +%Y%m%d%H%M` Package $import_pkg ($origpkgversion -> $destpkgversion) imported from $origrepository to $destrepository" | \ tee -a $ORIGREGFILE $DESTREGFILE >/dev/null } if [ "$import_mode" = "backup" ]; then # backup stuff [ "$destpkgversion" != "none" -a \ "$destpkgversion" != "$origpkgversion" ] && { backup_package $import_pkg $destrepository $DESTREGFILE } # remove distromatic extra files associated with this package [ -e $local_distromatic/$rep/warnings/$import_pkg.in ] && { rm -f $local_distromatic/$rep/warnings/$import_pkg.in || echo "Warning: cannot remove file $local_distromatic/$rep/warnings/$import_pkg.in" } fi #for i in ${pkg_obsoletes}; do # PKG_FILENAME="$i-$pkg_version-$pkg_release.$namearch.rpm" # DEST_URL_ARCH=$DEST_URL/RPMS.i586/$PKG_FILENAME # [ -e $DEST_URL_ARCH ] && echo "Warning: obsoleted package $i exists" # backup_package $i $destrepository $DESTREGFILE # #echo rm $DEST_URL_ARCH #done } # ans = y else # check_existence != 1 echo "Warning: $import_pkg reported by distromatic does no longer exist" fi } function extract_diffinfo() { PKG=$1 REP=$2 TMP=$3 local i get_pkg_buildinfo $REP $PKG if [ "$pkg_name" ]; then PKG_FILENAME="$local_ftp/$REP/SRPMS.base/$pkg_name-$pkg_version-$pkg_release.src.rpm" [ -e "$PKG_FILENAME" ] || { echo "Error: package $PKG_FILENAME missing in $origrepository; skipping" return 1 } rpm -qp $PKG_FILENAME --requires > $TMP/buildrequires autospec -q -x $PKG_FILENAME -F \*.spec --destdir $TMP >/dev/null || { echo "Error: could not extract specfile from $PKG_FILENAME; skipping package" return 1 } [ -e "$TMP_SPEC_DIR/$pkg_name.spec" ] || { SPEC_FOUND="`ls $TMP_SPEC_DIR/*.spec`" mv $SPEC_FOUND $TMP_SPEC_DIR/$pkg_name.spec echo "Warning: specfile name should be $pkg_name.spec instead of ${SPEC_FOUND/*\//} in $REP repository" } > $TMP/requires > $TMP/provides for i in ${pkg_builds[*]}; do PKG_FILENAME="$local_ftp/$REP/RPMS.i586/$i-$pkg_version-$pkg_release.$pkg_arch.rpm" [ -e "$PKG_FILENAME" ] || { echo "Error: package $PKG_FILENAME missing in $origrepository; skipping" return 1 } rpm -qp $PKG_FILENAME --requires >> $TMP/requires rpm -qp $PKG_FILENAME --provides >> $TMP/provides rpm -qlp $PKG_FILENAME >> $TMP/files done else #echo "Warning: can't find package $PKG in $REP repository" return 1 fi return 0 } [ $1 ] || { usage; exit 1; } origrepository= destrepository=devel packages= command= simulate=0 while [ "$1" ]; do case $1 in -d) destrepository=$2; shift ;; -s) simulate=1 ;; -t) shift warningtext="$@" break ;; *) if [ "$command" ]; then case "$command" in "import"|"release"|"query"|"verify"|"archive"|"diff"|"inspect"|"setwarning"|"distromatic") [ "$origrepository" ] && packages="$packages $1" || origrepository=$1 ;; *) usage echo "Error: invalid option $1; aborting." exit 1 ;; esac else case "$1" in "import"|"release"|"query"|"verify"|"archive"|"list"|"diff"|"inspect"|"setwarning"|"distromatic") command=$1 ;; *) usage echo "Errror: $1 is not a valid command; aborting." exit 1 ;; esac fi ;; esac shift done [ "$command" = "" ] && { usage; exit 1; } [ "$command" = "list" ] && echo "Local repositories:" for a in ${local_reps[*]}; do [ "$a" = "$destrepository" ] && DEST_MODE=local; [ "$a" = "$origrepository" ] && ORIG_MODE=local; [ "$command" = "list" ] && echo "$a" done [ "$do_list" ] && echo "Remote repositories:" for a in ${remote_reps[*]}; do [ "$a" = "$destrepository" ] && DEST_MODE=remote; [ "$a" = "$origrepository" ] && ORIG_MODE=remote; [ "$command" = "list" ] && echo "$a" done [ "$command" = "list" ] && exit 0; [ "$origrepository" ] || { usage; exit 1; } [ "$DEST_MODE" ] || { echo "Error: $destrepository is not a valid repository; aborting."; exit 1; } [ "$ORIG_MODE" ] || { echo "Error: $origrepository is not a valid repository; aborting."; exit 1; } [ "$DEST_MODE" = "remote" ] && { echo "Waring: destination is a remote repository; this is an EXPERIMENTAL feature."; } [ "$command" = "query" ] && { [ "$packages" ] || { usage; exit 1; } for i in $packages; do get_pkg_buildinfo $origrepository $i if [ ! "$pkg_name" ]; then echo "$i: package not found in $origrepository repository" else echo "Name: $pkg_name" echo "BuildArch: $pkg_arch" echo "Version: $pkg_version" echo "Release: $pkg_release" echo "Group: $pkg_group" echo "License: $pkg_license" echo "Size: $pkg_size" echo "Builds: ${pkg_builds[*]}" echo "Obsoletes: ${pkg_obsoletes[*]}" echo fi done exit 0; } [ "$command" = "verify" ] && { [ "$packages" ] || { get_pkg_buildinfo $origrepository packages=${pkg_list[*]} } for i in $packages; do get_pkg_buildinfo $origrepository $i if [ ! "$pkg_name" ]; then echo "$i: package not found in $origrepository repository" else PKG_FILENAME="$i-$pkg_version-$pkg_release.src.rpm" rpm2cpio $local_ftp/${origrepository}/SRPMS.base/$PKG_FILENAME &>/dev/null || { echo "Warning: source package $PKG_FILENAME is empty or corrupted." } for l in ${pkg_builds[*]}; do PKG_FILENAME="$l-$pkg_version-$pkg_release.$pkg_arch.rpm" rpm2cpio $local_ftp${origrepository}/RPMS.i586/$PKG_FILENAME &>/dev/null || { echo "Warning: package $PKG_FILENAME is empty or corrupted." } done fi done exit 0; } [ "$command" = "diff" ] && { [ "$packages" ] || { get_pkg_buildinfo $origrepository packages=${pkg_list[*]} } TMP_SPEC_DIR=`mktemp -d` for i in $packages; do echo echo "*******************************************************************" echo "$i package check:" echo "*******************************************************************" extract_diffinfo $i $origrepository $TMP_SPEC_DIR || continue [ -e $TMP_SPEC_DIR/$i.spec ] || { echo "Error: could not extract specfile for $i in $origrepository repository; skipping" continue } mv $TMP_SPEC_DIR/$i.spec $TMP_SPEC_DIR/$i.spec.origrep mv $TMP_SPEC_DIR/files $TMP_SPEC_DIR/files.origrep sort -u $TMP_SPEC_DIR/buildrequires > $TMP_SPEC_DIR/buildrequires.origrep sort -u $TMP_SPEC_DIR/requires > $TMP_SPEC_DIR/requires.origrep sort -u $TMP_SPEC_DIR/provides > $TMP_SPEC_DIR/provides.origrep extract_diffinfo $i $destrepository $TMP_SPEC_DIR || { echo "Looks like a new package; inspecting data:" echo "" echo "SPECFILE:" echo "=========" cat $TMP_SPEC_DIR/$i.spec.origrep echo "" echo "REQUIRES:" echo "=========" cat $TMP_SPEC_DIR/requires.origrep echo "" echo "PROVIDES:" echo "=========" cat $TMP_SPEC_DIR/provides.origrep echo "" echo "FILES:" echo "======" cat $TMP_SPEC_DIR/files.origrep continue } [ -e $TMP_SPEC_DIR/$i.spec ] || { echo "Error: could not extract specfile for $i in $destrepository repository; skipping" } mv $TMP_SPEC_DIR/$i.spec $TMP_SPEC_DIR/$i.spec.destrep mv $TMP_SPEC_DIR/files $TMP_SPEC_DIR/files.destrep sort -u $TMP_SPEC_DIR/buildrequires > $TMP_SPEC_DIR/buildrequires.destrep sort -u $TMP_SPEC_DIR/requires > $TMP_SPEC_DIR/requires.destrep sort -u $TMP_SPEC_DIR/provides > $TMP_SPEC_DIR/provides.destrep echo "Showing differences between package version in $origrepository and $destrepository:" echo "" echo "SPECFILE:" echo "=========" diff -u $TMP_SPEC_DIR/$i.spec.destrep $TMP_SPEC_DIR/$i.spec.origrep echo "" echo "BUILDREQUIRES:" echo "==============" diff -u $TMP_SPEC_DIR/buildrequires.destrep $TMP_SPEC_DIR/buildrequires.origrep echo "" echo "REQUIRES:" echo "=========" diff -u $TMP_SPEC_DIR/requires.destrep $TMP_SPEC_DIR/requires.origrep echo "" echo "PROVIDES:" echo "=========" diff -u $TMP_SPEC_DIR/provides.destrep $TMP_SPEC_DIR/provides.origrep echo "" echo "FILES:" echo "======" diff -u $TMP_SPEC_DIR/files.destrep $TMP_SPEC_DIR/files.origrep echo done rm -rf $TMP_SPEC_DIR exit 0; } [ "$command" = "setwarning" ] && { [ "$packages" ] || { usage; exit 1; } TMP_SPEC_DIR=`mktemp -d` for i in $packages; do extract_diffinfo $i $origrepository $TMP_SPEC_DIR [ -e $TMP_SPEC_DIR/$i.spec ] || { echo "Error: could not extract specfile for $i in $origrepository repository; aborting." exit 1 } echo "$warningtext" > $local_ftp/distromatic/$origrepository/warnings/$i.in done [ "$TMP_SPEC_DIR" != "/" ] && rm -rf $TMP_SPEC_DIR exit 0 } [ "$command" = "inspect" ] && { [ "$packages" ] || { usage; exit 1; } TMP_SPEC_DIR=`mktemp -d` for i in $packages; do extract_diffinfo $i $origrepository $TMP_SPEC_DIR [ -e $TMP_SPEC_DIR/$i.spec ] || { echo "Error: could not extract specfile for $i in $origrepository repository; aborting." exit 1 } echo "$i: details of package in $origrepository repository" echo "" echo "SPECFILE:" echo "=========" cat $TMP_SPEC_DIR/$i.spec echo "" echo "REQUIRES:" echo "=========" cat $TMP_SPEC_DIR/requires echo "" echo "PROVIDES:" echo "=========" cat $TMP_SPEC_DIR/provides done [ "$TMP_SPEC_DIR" != "/" ] && rm -rf $TMP_SPEC_DIR exit 0; } [ "$command" = "distromatic" ] && { SRCPKGLIST="$local_ftp/$origrepository/srcpkglist" [ -r $SRCPKGLIST ] || { echo "Error: srcpkglist file missing for $origrepository repository; aborting." exit 1 } [ -d $local_ftp/$origrepository/specs ] || mkdir $local_ftp/$origrepository/specs [ -d $local_ftp/$origrepository/patches ] || mkdir $local_ftp/$origrepository/patches while read line; do set -- $line [ -e $local_ftp/$origrepository/SRPMS.base/$1-$2-$6.src.rpm ] && { [ $local_ftp/$origrepository/SRPMS.base/$1-$2-$6.src.rpm -nt \ $local_ftp/$origrepository/specs/$1.spec ] && { # echo $local_ftp/$origrepository/SRPMS.base/$1-$2-$6.src.rpm autospec -x $local_ftp/$origrepository/SRPMS.base/$1-$2-$6.src.rpm -F '*.spec' \ --destdir $local_ftp/$origrepository/specs/ -q >/dev/null touch $local_ftp/$origrepository/specs/$1.spec grep -i "^Patch[0-9]*:" $local_ftp/$origrepository/specs/$1.spec &>/dev/null && { autospec -x $local_ftp/$origrepository/SRPMS.base/$1-$2-$6.src.rpm -F '*.patch' \ --destdir $local_ftp/$origrepository/patches/ -q >/dev/null } } } done < $SRCPKGLIST exit 0 } [ "$simulate" = "1" ] && echo "Simulation mode enabled." # # import and other active commands # DEST_URL=$local_ftp/$destrepository DEST_URL_LOCAL=$local_ftp/$destrepository LOCAL_BACKUP=$DEST_URL/old # operation files are always in the local copy of the repository ORIGREGFILE=$local_ftp/$origrepository/operations.log.html DESTREGFILE=$local_ftp/$destrepository/operations.log.html # ORIG_URL_LOCAL is set if a local copy of the repository exists # and will be preferred for file transfer optimizations if [ "$ORIG_MODE" = "remote" ]; then ORIG_URL=$dest_ftp/pub/openmamba/$origrepository else ORIG_URL=$local_ftp/$origrepository fi ORIG_URL_LOCAL=$local_ftp/$origrepository [ "$command" = "archive" ] && { DEST_URL=$ORIG_URL DEST_URL_LOCAL=$ORIG_URL_LOCAL DEST_MODE=$ORIG_MODE LOCAL_BACKUP=$DEST_URL_LOCAL/old/archived [ "$packages" ] || { usage; exit 1; } for i in $packages; do backup_package $i $origrepository $ORIGREGFILE done exit 0 } [ "$command" = "import" -o "$command" = "release" ] && { LOCAL_BACKUP=$DEST_URL/old/import-$origrepository [ "$origrepository" = "$destrepository" ] && { echo "Error: source and destination repository cannot be the same; aborting."; exit 1; } if [ "$command" = "import" ]; then echo "Importing $1: $origrepository ($ORIG_MODE) => $destrepository ($DEST_MODE)" backup_mode=backup else echo "Releasing $1: $origrepository ($ORIG_MODE) => $destrepository ($DEST_MODE)" backup_mode=release fi if [ ! "$packages" ]; then get_pkg_buildinfo $origrepository for i in ${pkg_list[*]}; do import_package $i $backup_mode done else for i in ${packages[*]}; do import_package $i $backup_mode done fi exit 0 } usage echo "Error: $command is not a valid command; aborting." exit 1