From 6cf3e9f99fcba1e04bf2bee4f380e0b421c26fac Mon Sep 17 00:00:00 2001 From: Silvan Calarco Date: Sat, 1 Dec 2012 17:18:36 +0100 Subject: [PATCH] Added script autodust-upstream-updates with daily cron script --- Makefile | 2 + autodist-upstream-updates | 327 ++++++++++++++++++++ etc/cron.daily/40-autodist-upstream-updates | 16 + 3 files changed, 345 insertions(+) create mode 100755 autodist-upstream-updates create mode 100755 etc/cron.daily/40-autodist-upstream-updates diff --git a/Makefile b/Makefile index 6925471..738bd29 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,7 @@ install-programs: @$(INSTALL_SCRIPT) autodist-tool $(DESTDIR)$(bindir)/autodist-tool @$(INSTALL_SCRIPT) autoport $(DESTDIR)$(bindir)/autoport @$(INSTALL_SCRIPT) autoport-chroot $(DESTDIR)$(sbindir)/autoport-chroot + @$(INSTALL_SCRIPT) autodist-upstream-updates $(DESTDIR)$(sbindir)/autodist-upstream-updates @$(INSTALL_SCRIPT) autoport-fix-environment $(DESTDIR)$(bindir)/autoport-fix-environment @$(INSTALL_SCRIPT) etc/autodist/scripts/* $(DESTDIR)$(configdir)/scripts/ @$(INSTALL_SCRIPT) autodist-cgi $(DESTDIR)$(cgidir)/autodist @@ -101,6 +102,7 @@ install-data: @$(INSTALL_SCRIPT) etc/cron.hourly/65-autoport-native $(DESTDIR)$(sysconfdir)/cron.hourly/ @$(INSTALL_SCRIPT) etc/cron.hourly/66-autoport-chroot $(DESTDIR)$(sysconfdir)/cron.hourly/ @$(INSTALL_SCRIPT) etc/cron.daily/40-autodist-cleanold $(DESTDIR)$(sysconfdir)/cron.daily/ + @$(INSTALL_SCRIPT) etc/cron.daily/40-autodist-upstream-updates $(DESTDIR)$(sysconfdir)/cron.daily/ @$(INSTALL_DATA) autospec-conf $(DESTDIR)$(pck_statedir)/.autospec @touch $(DESTDIR)$(pck_statedir)/template/autoupdate/auto.success @touch $(DESTDIR)$(pck_statedir)/template/autoupdate/auto.skip diff --git a/autodist-upstream-updates b/autodist-upstream-updates new file mode 100755 index 0000000..10294e1 --- /dev/null +++ b/autodist-upstream-updates @@ -0,0 +1,327 @@ +#!/bin/bash +# +# autodist upstream updates - find upstream packages updates from different internet resources +# Copyright (c) 2004-2012 by Silvan Calarco +# + +#[ -r /etc/sysconfig/openmamba-central ] || { +# echo "Error: this program must be run as root; aborting." >&2 +# exit 1 +#} +. /etc/autodist/config +DISTROMATIC_PREFIX=/distribution/distromatic.html? +DISTROMATIC_REPOSITORY=devel +DISTDB=/etc/autodist/distdb +DISTDBDIR=/etc/autodist/distdb.d +XORG_RELEASE=current + +[ -r $DISTDB ] && { + . $DISTDB + [ -d $DISTDBDIR ] && \ + for f in `ls $DISTDBDIR/*.db`; do + . $f + done +} + +[ -r $PKGLIST_FILE ] || { + echo "Error: file $PKGLIST_FILE cannot be read; aborting." >&2 + exit 1 +} + +[ -r $ALIASES_DB ] || { + echo "Error: file $ALIASES_DB cannot be read; aborting." >&2 + exit 1 +} + +function usage() +{ + echo "openmamba-upstream-updates - finds upstream packages updates from different internet resources" + echo + echo "Usage:" + echo "openmamba-upstream-updates [-h|-m][-r repository][-o output_repository]" + echo + echo " -h: generate distromatic HTML output" + echo " -m: show missing packages only" + echo " -q: produces quite output" + echo " -r repository: specify the repository (default: devel)" + echo " -o repository: specify the repository for output data (default: same as base repository)" + echo +} + +get_job_vector() { + local JNAME=$1 + + # resolve JOB_NAME from distdb + # note: if JOB_NAME contains a "-" it can't be a distdb JOB, so skip it + if [ "${JNAME/-/}" = "${JNAME}" ]; then + local jobtmpfile=`tempfile` + # hack to get an array variable named as $j assigned to the JOB array + echo "echo \${$JNAME[*]}" > $jobtmpfile + JOB=(`. $jobtmpfile`) + rm -f $jobtmpfile + else + JOB=() + fi + + if [ ${#JOB[*]} -eq 0 ]; then + # create a default job with given JOB_NAME + JOB=($JNAME "" "") + elif [ ${#JOB[*]} -eq 1 ]; then + # no variables defined, add an empty job + JOB=(${JOB[*]} "" "") + fi + JOB_PKGS=(${JOB[0]//,/ }) + JOB_VARNAMES=(${JOB[1]//,/ }) +} + +function version_compare() +{ + local A B + A=$1 + B=$2 + + if [[ ${1} =~ "^[0-9]+$" && ${2} =~ "^[0-9]+$" ]]; then + if [ ${A/[a-zA-Z_]*} -gt ${B/[a-zA-Z_]*} ]; then + return 1 + elif [ ${A/[a-zA-Z_]*} -lt ${B/[a-zA-Z_]*} ]; then + return 2 + fi + else + if [[ "$A" > "$B" ]]; then + return 1 + elif [[ "$A" < "$B" ]]; then + return 2 + fi + fi + return 0 +} + +function version_find_bigger() +{ + local VER1 VER2 FPOS CUTVER1 CUTVER2 + + VER1=$1 + VER2=$2 + FPOS=1 + while true; do + CUTVER1=`echo $VER1. | cut -d. -f $FPOS` + CUTVER2=`echo $VER2. | cut -d. -f $FPOS` + if [ "$CUTVER1" -a ! "$CUTVER2" ]; then + return 1 + elif [ "$CUTVER2" -a ! "$CUTVER1" ]; then + return 2 + elif [ ! "$CUTVER1" -a "$CUTVER2" ]; then + return 0 + else + version_compare $CUTVER1 $CUTVER2 + case $? in + 1) return 1 ;; + 2) return 2 ;; + esac + fi + FPOS=`expr $FPOS + 1` + done + return 0 +} + +while [ "$1" ]; do + case $1 in + -h) distromatic_html=1 ;; + -m) if [ "$distromatic_html" ]; then + echo "Error: options -h and -m cannot be used together." + usage + exit 1 + else + missing_only=1 + fi ;; + -r) [ "$2" ] || { + echo "Error: option -r requires repository name as parameter" + usage + exit 1 + } + DISTROMATIC_REPOSITORY=$2 + shift + ;; + -o) [ "$2" ] || { + echo "Error: option -o requires repository name as parameter" + usage + exit 1 + } + OUTPUT_REPOSITORY=$2 + shift + ;; + -q) quiet=1 ;; + *) echo "Error: invalid option $1."; usage; exit 1 ;; + esac + shift +done + +[ "$OUTPUT_REPOSITORY" ] || OUTPUT_REPOSITORY=$DISTROMATIC_REPOSITORY + +CACHE_DIR=${LOCAL_REPS_BASE_DIR}/$OUTPUT_REPOSITORY/autoupdate/ +PKGLIST_FILE=${LOCAL_REPS_BASE_DIR}/$DISTROMATIC_REPOSITORY/srcpkglist +BUILDS_FILE=${LOCAL_REPS_BASE_DIR}/distromatic/$DISTROMATIC_REPOSITORY/builds-i586 +BUILDLIST_FILE=$CACHE_DIR/upstream-updates.in +CONFIG_DIR=$CACHE_DIR +UPDATES_DB=$CACHE_DIR/upstream-updates +ALIASES_DB=$CACHE_DIR/aliases +MANUALVER_DB=$CACHE_DIR/manualver + +tmpfile=`mktemp -q -t autodist-upstream-updates.XXXXXXXX` +buildstmp=`mktemp -q -t autodist-upstream-updates.XXXXXXXX` +tail -n+2 $BUILDS_FILE > $buildstmp + +# parse Arch Linux package list +[ "$quiet" ] || echo "Parsing Arch Linux packages list..." >&2 +curl -s "https://www.archlinux.org/packages/?sort=-last_update&arch=i686&q=&maintainer=&last_update=&flagged=&limit=all" | \ +grep "View package details" | \ +while read line; do + line=`echo $line | html2text -nobs` + set -- $line + pkg=$3 + ver=${4/-*} + ver=${ver/*:} + alias=`grep "^$pkg" $ALIASES_DB` + [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB` + [ "$alias" ] && pkg=$alias + line=`grep -i " $pkg[^-_A-Za-z0-9]" $buildstmp || grep -i " lib$pkg[^-_A-Za-z0-9]" $buildstmp` + if [ "$line" ]; then + pkg=${line/:*} + [ "$pkg" -a "$ver" ] && echo "$pkg $ver ${alias/* /}" >> $tmpfile + fi +done +rm -f $buildstmp + +# parse X.org stable packages list +[ "$quiet" ] || echo "Parsing X.org release ftp directory..." >&2 +curl -s ftp://ftp.x.org/pub/$XORG_RELEASE/src/everything/ -l | sed "s|\.tar\..*||" | sort -u | +while read line; do + if [ "$line" ]; then + ver=`echo $line | sed "s|.*-||"` + pkg="${line/-$ver}" + alias=`grep "^$pkg " $ALIASES_DB` + if [ ! "$alias" -a "${pkg:0:5}" == "xf86-" ]; then + alias="$pkg xorg-drv-${pkg/xf86-}" + else + [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB` + fi + [ "$pkg" -a "$ver" ] && echo "$pkg $ver ${alias/* /}" >> $tmpfile + fi +done + +# parse Gnome stable packages list +[ "$quiet" ] || echo "Parsing GNOME stable versions file..." >&2 +for f in versions-stable versions-stable-extras; do + curl -s http://people.gnome.org/~vuntz/tmp/versions/$f | grep -v "^#" | + while read line; do + if [ "$line" ]; then + IFS=":" + set -- $line + pkg="$2" + ver="$3" + alias=`grep "^$pkg " $ALIASES_DB` + [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB` + [ "$pkg" -a "$ver" ] && echo "$pkg $ver ${alias/* /}" >> $tmpfile + fi + done +done + +# parse distrowatch.com packages list +[ "$quiet" ] || echo "Parsing Distrowatch packages list..." >&2 +lynx -width 300 -dump http://distrowatch.com/packages.php | +while read line; do + [ "`echo $line | grep "Package Version Note"`" ] && start_print=1 + [ "`echo $line | grep "____________________"`" ] && unset start_print + [ "$start_print" ] && { + set -- $line + pkg="${1/\[*\]/}" + ver="${2/\[*\]/}" + alias=`grep "^$pkg " $ALIASES_DB` + [ "$pkg" != "chromium" ] && \ + echo "$pkg $ver ${alias/* /}" >> $tmpfile + } +done + +cat $tmpfile | sort -uf > $UPDATES_DB.tmp +rm -f $tmpfile + +> $UPDATES_DB +unset lastpkg +while read pkg ver alias; do + if [ "$pkg" = "$lastpkg" ]; then +# echo "Warning: duplicate found: $pkg lastver: $lastver ver: $ver" >&2 + version_find_bigger $lastver $ver + vercmp=$? + if [ $vercmp -eq 2 ]; then + sed -i "/^$lastpkg $lastver /d" $UPDATES_DB + echo "$pkg $ver $alias" >> $UPDATES_DB + fi + else + echo "$pkg $ver $alias" >> $UPDATES_DB + fi + lastpkg=$pkg + lastver=$ver +done < $UPDATES_DB.tmp +rm -f $UPDATES_DB.tmp + +> $UPDATES_DB.missing +> $BUILDLIST_FILE +while read pkg ver alias; do + unset pkgline + unset found_manual + unset found_alias + pkgline=`grep "^$pkg " $MANUALVER_DB` && found_manual=1 + if [ ! "$found_manual" ]; then + if [ "$alias" ]; then + get_job_vector $alias + lastjob=${#JOB_PKGS[*]} + pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE` && found_alias=1 + else + get_job_vector $pkg + lastjob=${#JOB_PKGS[*]} + pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE` + [ "$pkgline" ] || { + get_job_vector lib${pkg} + lastjob=${#JOB_PKGS[*]} + pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE` && { + found_alias=1 + alias=lib${pkg} + } + } + fi + fi + if [ "$pkgline" ]; then + set -- $pkgline + pkgname=$1 + pkgver=$2 + version_find_bigger $pkgver ${ver/-/.} + vercmp=$? + [ "$found_manual" ] && pkgname=$3 + if [ ! "$missing_only" ]; then + [ "$found_alias" -o "$found_manual" ] && nameadd="$pkg" || unset nameadd + unset veradd + [ ${vercmp} = 2 ] && veradd="$ver" + [ ${vercmp} = 1 ] && veradd="$ver" +# [ "${ver/-/}" != "$pkgver" ] && veradd="$ver" || unset veradd + [ "$veradd" -o "$nameadd" ] && { + [ "$veradd" -a "$nameadd" ] && \ + verappend="($nameadd;$veradd)" || + verappend="(${nameadd}${veradd})" + } || unset verappend + if [ "$distromatic_html" = "1" ]; then + echo "${JOB_PKGS[0]} $pkgver $verappend
" + else + echo "$pkg $pkgver ($ver)" + fi + [ $vercmp = 2 ] && { + if [ "$found_alias" ]; then + echo "$alias +$ver 0" >> $BUILDLIST_FILE + else + echo "$pkg +$ver 0" >> $BUILDLIST_FILE + fi + } + fi + else + echo "$pkg ($ver)" >> $UPDATES_DB.missing + fi +done < $UPDATES_DB diff --git a/etc/cron.daily/40-autodist-upstream-updates b/etc/cron.daily/40-autodist-upstream-updates new file mode 100755 index 0000000..34fc416 --- /dev/null +++ b/etc/cron.daily/40-autodist-upstream-updates @@ -0,0 +1,16 @@ +#!/bin/bash +# +# autodist-upstream-updates daily cron script +# (c) 2008-2012 by Silvan Calarco +# +. /etc/autodist/config +REPOSITORY=devel +/usr/sbin/autodist-upstream-updates -h -r $REPOSITORY > $LOCAL_REPS_BASE_DIR/distromatic/$REPOSITORY/_popular.html + +REPOSITORY=milestone1-1.1 +/usr/sbin/autodist-upstream-updates -h -r $REPOSITORY > $LOCAL_REPS_BASE_DIR/distromatic/$REPOSITORY/_popular.html + +REPOSITORY=milestone2 +/usr/sbin/autodist-upstream-updates -h -r $REPOSITORY > $LOCAL_REPS_BASE_DIR/distromatic/$REPOSITORY/_popular.html + +exit 0