diff --git a/autodist-upstream-updates b/autodist-upstream-updates index 74f7942..731c54d 100755 --- a/autodist-upstream-updates +++ b/autodist-upstream-updates @@ -43,6 +43,7 @@ function usage() echo echo " -h: generate distromatic HTML output" echo " -m: report missing packages only" + echo " -s: skip fetching and parsing upstream updates" echo " -u: output not up-to-date packages only" echo " -q: produces quite output" echo " -r repository: specify the distromatic base (default: devel)" @@ -78,13 +79,13 @@ function version_compare() A=$1 B=$2 - if [[ ${1} =~ "^[0-9]+$" && ${2} =~ "^[0-9]+$" ]]; then - if [ ${A/[a-zA-Z_]*} -gt ${B/[a-zA-Z_]*} ]; then + if [[ ${1} =~ ^[0-9]+$ && ${2} =~ ^[0-9]+$ ]]; then + if [ $((10#${A/[a-zA-Z_]*})) -gt $((10#${B/[a-zA-Z_]*})) ]; then return 1 - elif [ ${A/[a-zA-Z_]*} -lt ${B/[a-zA-Z_]*} ]; then + elif [ $((10#${A/[a-zA-Z_]*})) -lt $((10#${B/[a-zA-Z_]*})) ]; then return 2 fi - else + else if [[ "$A" > "$B" ]]; then return 1 elif [[ "$A" < "$B" ]]; then @@ -132,6 +133,7 @@ while [ "$1" ]; do else missing_only=1 fi ;; + -s) skip_parsing=1 ;; -u) needupdate_only=1 ;; -r) [ "$2" ] || { echo "Error: option -r requires repository name as parameter" @@ -166,116 +168,132 @@ UPDATES_DB=$CACHE_DIR/upstream-updates ALIASES_DB=$CACHE_DIR/aliases MANUALVER_DB=$CACHE_DIR/manualver -tmpfile=`mktemp -q -t autodist-upstream-updates.XXXXXXXX` -buildstmp=`mktemp -q -t autodist-upstream-updates.XXXXXXXX` -tail -n+2 $BUILDS_FILE > $buildstmp -# parse Arch Linux package list -[ "$quiet" ] || echo -n "Parsing Arch Linux packages list..." >&2 -#for page in `seq 1 45`; do -for rep in core community extra; do -# SOURCEURL="https://www.archlinux.org/packages/?page=$page&sort=-last_update&q=&arch=i686&maintainer=&flagged=" - SOURCEURL="http://lug.mtu.edu/archlinux/$rep/os/i686/" - curl -s "$SOURCEURL" | \ - grep ".pkg." | grep -v ".sig\"" | \ - while read line; do - line=`echo $line | sed "s|.*href=\"\([^\"]*\)\">.*|\1|"` - pkg=`echo $line | sed "s|\(.*\)-[^-]*-[^-]*-[^-]*|\1|"` - ver=`echo $line | sed "s|.*-\([^-]*\)-[^-]*-[^-]*|\1|"` - alias=`grep "^$pkg " $ALIASES_DB` - [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB` - [ "$alias" ] && pkgalias=${alias/* /} || pkgalias=$pkg - line=`grep -i "^$pkgalias:" $buildstmp || grep -i "^lib$pkgalias:" $buildstmp || grep -i " $pkgalias[^-_A-Za-z0-9]" $buildstmp` - if [ "$line" ]; then - [ "$pkg" != "$pkgalias" ] && alias=$pkgalias || alias= - [ "$pkg" -a "$ver" ] && { - echo "$pkg $ver $SOURCEURL ${alias}" >> $tmpfile - } - fi - done -done -rm -f $buildstmp - -# parse X.org stable packages list -[ "$quiet" ] || echo "Parsing X.org release ftp directory..." >&2 -SOURCEURL="ftp://ftp.x.org/pub/$XORG_RELEASE/src/everything/" -curl -s $SOURCEURL -l | sed "s|\.tar\..*||" | sort -u | -while read line; do - if [ "$line" ]; then - ver=`echo $line | sed "s|.*-||"` - pkg="${line/-$ver}" - alias=`grep "^$pkg " $ALIASES_DB` - if [ ! "$alias" -a "${pkg:0:5}" == "xf86-" ]; then - alias="$pkg xorg-drv-${pkg/xf86-}" - else - [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB` - fi - [ "$pkg" -a "$ver" ] && echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile - fi -done - -# parse Gnome stable packages list -[ "$quiet" ] || echo "Parsing GNOME stable versions file..." >&2 -for f in versions-stable versions-stable-extras; do - SOURCEURL="http://people.gnome.org/~vuntz/tmp/versions/$f" - curl -s $SOURCEURL | grep -v "^#" | - while read line; do - if [ "$line" ]; then - IFS=":" - set -- $line - pkg="$2" - ver="$3" +parse_arch_linux() { + # parse Arch Linux package list + [ "$quiet" ] || echo -n "Parsing Arch Linux packages list..." >&2 + #for page in `seq 1 45`; do + for rep in core community extra; do + # SOURCEURL="https://www.archlinux.org/packages/?page=$page&sort=-last_update&q=&arch=i686&maintainer=&flagged=" + SOURCEURL="http://lug.mtu.edu/archlinux/$rep/os/i686/" + curl -L -s "$SOURCEURL" | \ + grep ".pkg." | grep -v ".sig\"" | \ + while read line; do + line=`echo $line | sed "s|.*href=\"\([^\"]*\)\">.*|\1|"` + pkg=`echo $line | sed "s|\(.*\)-[^-]*-[^-]*-[^-]*|\1|"` + ver=`echo $line | sed "s|.*-\([^-]*\)-[^-]*-[^-]*|\1|"` alias=`grep "^$pkg " $ALIASES_DB` [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB` + [ "$alias" ] && pkgalias=${alias/* /} || pkgalias=$pkg + line=`grep -i "^$pkgalias:" $buildstmp || grep -i "^lib$pkgalias:" $buildstmp || grep -i " $pkgalias[^-_A-Za-z0-9]" $buildstmp` + if [ "$line" ]; then + [ "$pkg" != "$pkgalias" ] && alias=$pkgalias || alias= + [ "$pkg" -a "$ver" ] && { + echo "$pkg $ver $SOURCEURL ${alias}" >> $tmpfile + } + fi + done + done + rm -f $buildstmp +} + +parse_xorg() { + # parse X.org stable packages list + [ "$quiet" ] || echo "Parsing X.org release ftp directory..." >&2 + SOURCEURL="ftp://ftp.x.org/pub/$XORG_RELEASE/src/everything/" + curl -L -s $SOURCEURL -l | sed "s|\.tar\..*||" | sort -u | + while read line; do + if [ "$line" ]; then + ver=`echo $line | sed "s|.*-||"` + pkg="${line/-$ver}" + alias=`grep "^$pkg " $ALIASES_DB` + if [ ! "$alias" -a "${pkg:0:5}" == "xf86-" ]; then + alias="$pkg xorg-drv-${pkg/xf86-}" + else + [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB` + fi [ "$pkg" -a "$ver" ] && echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile fi done -done +} -# parse distrowatch.com packages list -[ "$quiet" ] || echo "Parsing Distrowatch packages list..." >&2 -SOURCEURL="http://distrowatch.com/packages.php" -lynx -width 300 -dump $SOURCEURL | -while read line; do - [ "`echo $line | grep "Package Version Note"`" ] && start_print=1 - [ "`echo $line | grep "____________________"`" ] && unset start_print - [ "$start_print" ] && { - set -- $line - pkg="${1/\[*\]/}" - ver="${2/\[*\]/}" - alias=`grep "^$pkg " $ALIASES_DB` - [ "$pkg" != "chromium" ] && \ - echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile - } -done - -cat $tmpfile | sort -uf > $UPDATES_DB.tmp -rm -f $tmpfile - -> $UPDATES_DB -unset lastpkg -while read pkg ver upsource alias; do - # skip updates to unstable versions - unset found_beta - for b in alpha beta rc "~"; do - [ "${ver/$b}" != "${ver}" ] && found_beta=1 +parse_gnome() { + # parse Gnome stable packages list + [ "$quiet" ] || echo "Parsing GNOME stable versions file..." >&2 + for f in versions-stable versions-stable-extras; do + SOURCEURL="https://people.gnome.org/~vuntz/tmp/versions/$f" + curl -s -L $SOURCEURL | grep -v "^#" | + while read line; do + if [ "$line" ]; then + IFS=":" + set -- $line + pkg="$2" + ver="$3" + alias=`grep "^$pkg " $ALIASES_DB` + [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB` + [ "$pkg" -a "$ver" ] && echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile + fi + done done - [ "$found_beta" ] && continue - if [ "$pkg" = "$lastpkg" ]; then +} + +parse_distromatic() { + # parse distrowatch.com packages list + [ "$quiet" ] || echo "Parsing Distrowatch packages list..." >&2 + SOURCEURL="http://distrowatch.com/packages.php" + lynx -width 300 -dump $SOURCEURL | + while read line; do + [ "`echo $line | grep "Package Version Note"`" ] && start_print=1 + [ "`echo $line | grep "____________________"`" ] && unset start_print + [ "$start_print" ] && { + set -- $line + pkg="${1/\[*\]/}" + ver="${2/\[*\]/}" + alias=`grep "^$pkg " $ALIASES_DB` + [ "$pkg" != "chromium" ] && \ + echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile + } + done +} + +if [ ! "$skip_parsing" ]; then + tmpfile=`mktemp -q -t autodist-upstream-updates.XXXXXXXX` + buildstmp=`mktemp -q -t autodist-upstream-updates.XXXXXXXX` + tail -n+2 $BUILDS_FILE > $buildstmp + + parse_arch_linux + parse_xorg + parse_gnome + parse_distromatic + + cat $tmpfile | sort -uf > $UPDATES_DB.tmp + rm -f $tmpfile + + > $UPDATES_DB + unset lastpkg + while read pkg ver upsource alias; do + # skip updates to unstable versions + unset found_beta + for b in alpha beta rc "~"; do + [ "${ver/$b}" != "${ver}" ] && found_beta=1 + done + [ "$found_beta" ] && continue + if [ "$pkg" = "$lastpkg" ]; then # echo "Warning: duplicate found: $pkg lastver: $lastver ver: $ver" >&2 - version_find_bigger $lastver $ver - vercmp=$? - if [ $vercmp -eq 2 ]; then - sed -i "/^$lastpkg $lastver /d" $UPDATES_DB + version_find_bigger $lastver $ver + vercmp=$? + if [ $vercmp -eq 2 ]; then + sed -i "/^$lastpkg $lastver /d" $UPDATES_DB + echo "$pkg $ver $upsource $alias" >> $UPDATES_DB + fi + else echo "$pkg $ver $upsource $alias" >> $UPDATES_DB fi - else - echo "$pkg $ver $upsource $alias" >> $UPDATES_DB - fi - lastpkg=$pkg - lastver=$ver -done < $UPDATES_DB.tmp -rm -f $UPDATES_DB.tmp + lastpkg=$pkg + lastver=$ver + done < $UPDATES_DB.tmp + rm -f $UPDATES_DB.tmp +fi > $UPDATES_DB.missing > $BUILDLIST_FILE @@ -311,6 +329,7 @@ while read pkg ver upsource alias; do pkgrep=$4 version_find_bigger $pkgver ${ver/-/.} vercmp=$? +#echo "$pkgname - version_find_bigger $pkgver ${ver/-/.} result=$vercmp" [ "${vercmp}" != "2" -a "$needupdate_only" ] && continue [ "$found_manual" ] && pkgname=$3 [ "$found_alias" -o "$found_manual" ] && nameadd="$pkg" || unset nameadd