autodist/autodist-upstream-updates

#!/bin/bash
#
# autodist upstream updates - find upstream packages updates from different internet resources
# Copyright (c) 2004-2013 by Silvan Calarco <silvan.calarco@mambasoft.it>
#

#[ -r /etc/sysconfig/openmamba-central ] || { 
#   echo "Error: this program must be run as root; aborting." >&2
#   exit 1
#}
. /etc/autodist/config
DISTROMATIC_PREFIX=/distribution/distromatic.html?
DISTROMATIC_REPOSITORY=devel
DISTDB=/etc/autodist/distdb
DISTDBDIR=/etc/autodist/distdb.d
XORG_RELEASE=current

[ -r $DISTDB ] && {
   . $DISTDB
   [ -d $DISTDBDIR ] && \
      for f in `ls $DISTDBDIR/*.db`; do
         . $f
      done
}

[ -r $PKGLIST_FILE ] || {
   echo "Error: file $PKGLIST_FILE cannot be read; aborting." >&2
   exit 1
}

[ -r $ALIASES_DB ] || {
   echo "Error: file $ALIASES_DB cannot be read; aborting." >&2
   exit 1
}

function usage() 
{
   echo "openmamba-upstream-updates - finds upstream packages updates from different internet resources"
   echo
   echo "Usage:"
   echo "openmamba-upstream-updates [-h|-m][-r repository][-o output_repository]"
   echo
   echo " -h: generate distromatic HTML output"
   echo " -m: report missing packages only"
   echo " -u: output not up-to-date packages only"
   echo " -q: produces quite output"
   echo " -r repository: specify the distromatic base (default: devel)"
   echo " -o repository: specify the repository for output data (default: same as base repository)"
   echo
}

get_job_vector() {
   local JNAME=$1

   # resolve JOB_NAME from distdb
   # note: if JOB_NAME contains a "-" it can't be a distdb JOB, so skip it
   if [ "${JNAME/-/}" = "${JNAME}" ]; then
      local jobtmpfile=`tempfile`
      # hack to get an array variable named as $j assigned to the JOB array
      echo "echo \${$JNAME[*]}" > $jobtmpfile
      JOB=(`. $jobtmpfile`)
      rm -f $jobtmpfile
   else
      JOB=()
   fi

   if [ ${#JOB[*]} -eq 0 ]; then
       # create a default job with given JOB_NAME
       JOB=($JNAME "" "")
   elif [ ${#JOB[*]} -eq 1 ]; then
       # no variables defined, add an empty job
       JOB=(${JOB[*]} "" "")
   fi
   JOB_PKGS=(${JOB[0]//,/ })
   JOB_VARNAMES=(${JOB[1]//,/ })
}

function version_compare()
{
   local A B
   A=$1
   B=$2

   if [[ ${1} =~ "^[0-9]+$" && ${2} =~ "^[0-9]+$" ]]; then
      if [ ${A/[a-zA-Z_]*} -gt ${B/[a-zA-Z_]*} ]; then
         return 1
      elif [ ${A/[a-zA-Z_]*} -lt ${B/[a-zA-Z_]*} ]; then
         return 2
      fi
   else
      if [[ "$A" > "$B" ]]; then
         return 1
      elif [[ "$A" < "$B" ]]; then
         return 2
      fi
   fi
   return 0
}

function version_find_bigger()
{
   local VER1 VER2 FPOS CUTVER1 CUTVER2
   
   VER1=$1
   VER2=$2
   FPOS=1
   while true; do
      CUTVER1=`echo $VER1. | cut -d. -f $FPOS`
      CUTVER2=`echo $VER2. | cut -d. -f $FPOS`
      if [ "$CUTVER1" -a ! "$CUTVER2" ]; then
         return 1
      elif [ "$CUTVER2" -a ! "$CUTVER1" ]; then
         return 2
      elif [ ! "$CUTVER1" -a "$CUTVER2" ]; then
         return 0
      else
         version_compare $CUTVER1 $CUTVER2
         case $? in
            1) return 1 ;;
	    2) return 2 ;;
         esac
      fi
      FPOS=`expr $FPOS + 1`
   done
   return 0
}

while [ "$1" ]; do
   case $1 in
      -h) distromatic_html=1 ;;
      -m) if [ "$distromatic_html" ]; then
             echo "Error: options -h and -m cannot be used together."
	     usage
	     exit 1
	  else
             missing_only=1
	  fi ;;
      -u) needupdate_only=1 ;;
      -r) [ "$2" ] || {
             echo "Error: option -r requires repository name as parameter"
	     usage
	     exit 1
          }
	  DISTROMATIC_REPOSITORY=$2
	  shift
	  ;;
      -o) [ "$2" ] || {
             echo "Error: option -o requires repository name as parameter"
             usage
             exit 1
          }
          OUTPUT_REPOSITORY=$2
          shift
          ;;
      -q) quiet=1 ;;
      *) echo "Error: invalid option $1."; usage; exit 1 ;;  
   esac
   shift
done

[ "$OUTPUT_REPOSITORY" ] || OUTPUT_REPOSITORY=$DISTROMATIC_REPOSITORY

CACHE_DIR=${LOCAL_REPS_BASE_DIR}/$OUTPUT_REPOSITORY/autoupdate/
PKGLIST_FILE=${LOCAL_REPS_BASE_DIR}/$DISTROMATIC_REPOSITORY/srcpkglist
BUILDS_FILE=${LOCAL_REPS_BASE_DIR}/distromatic/$DISTROMATIC_REPOSITORY/builds-i586
BUILDLIST_FILE=$CACHE_DIR/upstream-updates.in
CONFIG_DIR=$CACHE_DIR
UPDATES_DB=$CACHE_DIR/upstream-updates
ALIASES_DB=$CACHE_DIR/aliases
MANUALVER_DB=$CACHE_DIR/manualver

tmpfile=`mktemp -q -t autodist-upstream-updates.XXXXXXXX`
buildstmp=`mktemp -q -t autodist-upstream-updates.XXXXXXXX`
tail -n+2 $BUILDS_FILE > $buildstmp

# parse Arch Linux package list
[ "$quiet" ] || echo -n "Parsing Arch Linux packages list..." >&2
#for page in `seq 1 45`; do
for rep in core community extra; do
#   SOURCEURL="https://www.archlinux.org/packages/?page=$page&sort=-last_update&q=&arch=i686&maintainer=&flagged="
   SOURCEURL="http://lug.mtu.edu/archlinux/$rep/os/i686/"
   curl -s "$SOURCEURL" | \
   grep ".pkg." | grep -v ".sig\"" | \
   while read line; do
       line=`echo $line | sed "s|.*href=\"\([^\"]*\)\">.*|\1|"`
       pkg=`echo $line | sed "s|\(.*\)-[^-]*-[^-]*-[^-]*|\1|"`
       ver=`echo $line | sed "s|.*-\([^-]*\)-[^-]*-[^-]*|\1|"`
       alias=`grep "^$pkg" $ALIASES_DB`
       [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
       [ "$alias" ] && pkg=$alias
       line=`grep -i "^$pkg:" $buildstmp || grep -i "^lib$pkg:" $buildstmp || grep -i " $pkg[^-_A-Za-z0-9]" $buildstmp`
       if [ "$line" ]; then
          pkg=${line/:*}
          [ "$pkg" -a "$ver" ] && { 
             echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
          }
       fi
   done
done
rm -f $buildstmp

# parse X.org stable packages list
[ "$quiet" ] || echo "Parsing X.org release ftp directory..." >&2
SOURCEURL="ftp://ftp.x.org/pub/$XORG_RELEASE/src/everything/"
curl -s $SOURCEURL -l | sed "s|\.tar\..*||" | sort -u |
while read line; do
   if [ "$line" ]; then
      ver=`echo $line | sed "s|.*-||"`
      pkg="${line/-$ver}"
      alias=`grep "^$pkg " $ALIASES_DB`
      if [ ! "$alias" -a "${pkg:0:5}" == "xf86-" ]; then
         alias="$pkg xorg-drv-${pkg/xf86-}"
      else
         [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
      fi
      [ "$pkg" -a "$ver" ] && echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
   fi
done

# parse Gnome stable packages list
[ "$quiet" ] || echo "Parsing GNOME stable versions file..." >&2
for f in versions-stable versions-stable-extras; do
   SOURCEURL="http://people.gnome.org/~vuntz/tmp/versions/$f"
   curl -s $SOURCEURL | grep -v "^#" |
   while read line; do
      if [ "$line" ]; then
         IFS=":"
         set -- $line
         pkg="$2"
         ver="$3"
         alias=`grep "^$pkg " $ALIASES_DB`
         [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
         [ "$pkg" -a "$ver" ] && echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
      fi
   done
done

# parse distrowatch.com packages list
[ "$quiet" ] || echo "Parsing Distrowatch packages list..." >&2
SOURCEURL="http://distrowatch.com/packages.php"
lynx -width 300 -dump $SOURCEURL |
while read line; do
   [ "`echo $line | grep "Package Version Note"`" ] && start_print=1
   [ "`echo $line | grep "____________________"`" ] && unset start_print
   [ "$start_print" ] && {
      set -- $line
      pkg="${1/\[*\]/}" 
      ver="${2/\[*\]/}"
      alias=`grep "^$pkg " $ALIASES_DB`
      [ "$pkg" != "chromium" ] && \
         echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
   }
done

cat $tmpfile | sort -uf > $UPDATES_DB.tmp
rm -f $tmpfile

> $UPDATES_DB
unset lastpkg
while read pkg ver upsource alias; do
   # skip updates to unstable versions
   unset found_beta
   for b in alpha beta rc "~"; do
      [ "${ver/$b}" != "${ver}" ] && found_beta=1
   done
   [ "$found_beta" ] && continue
   if [ "$pkg" = "$lastpkg" ]; then
#      echo "Warning: duplicate found: $pkg lastver: $lastver ver: $ver" >&2
      version_find_bigger $lastver $ver
      vercmp=$?
      if [ $vercmp -eq 2 ]; then
         sed -i "/^$lastpkg $lastver /d" $UPDATES_DB
         echo "$pkg $ver $upsource $alias" >> $UPDATES_DB
      fi
   else
      echo "$pkg $ver $upsource $alias" >> $UPDATES_DB
   fi
   lastpkg=$pkg
   lastver=$ver
done < $UPDATES_DB.tmp
rm -f $UPDATES_DB.tmp

> $UPDATES_DB.missing
> $BUILDLIST_FILE
while read pkg ver upsource alias; do
   unset pkgline
   unset found_manual
   unset found_alias
   pkgline=`grep "^$pkg " $MANUALVER_DB` && found_manual=1
   if [ ! "$found_manual" ]; then
      if [ "$alias" ]; then
         get_job_vector $alias
         lastjob=${#JOB_PKGS[*]}
         pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE` && found_alias=1
      else
         get_job_vector $pkg
         lastjob=${#JOB_PKGS[*]}
         pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE`
         [ "$pkgline" ] || {
            get_job_vector lib${pkg}
            lastjob=${#JOB_PKGS[*]}
            pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE` && {
               found_alias=1
               alias=lib${pkg}
            }
         }
      fi
   fi
   if [ "$pkgline" -a ! "$missing_only" ]; then
      set -- $pkgline
      pkgname=$1
      pkgver=$2
      pkgrep=$4
      version_find_bigger $pkgver ${ver/-/.}
      vercmp=$?
      [ "${vercmp}" != "2" -a "$needupdate_only" ] && continue
      [ "$found_manual" ] && pkgname=$3
      [ "$found_alias" -o "$found_manual" ] && nameadd="$pkg" || unset nameadd
      unset veradd
      [ ${vercmp} = 2 ] && veradd="<a href=\"$upsource\"><font color=red>$ver</font></a>"
      [ ${vercmp} = 1 ] && veradd="$ver"
      [ "$veradd" -o "$nameadd" ] && {
         [ "$veradd" -a "$nameadd" ] && \
            verappend="($nameadd;$veradd)" || 
            verappend="(${nameadd}${veradd})"
      } || unset verappend
      if [ "$distromatic_html" = "1" ]; then
         echo "<a href=\"${DISTROMATIC_PREFIX}tag=${pkgrep}&pkg=${JOB_PKGS[0]}.source\">${JOB_PKGS[0]} $pkgver</a> $verappend<br>"
      else
         echo "$pkg $pkgver ($ver)"
      fi
      [ $vercmp = 2 ] && {
         if [ "$found_alias" ]; then
            echo "$alias +$ver 0 $upsource" >> $BUILDLIST_FILE
         else
            echo "$pkg +$ver 0 $upsource" >> $BUILDLIST_FILE
         fi
      }
   elif [ ! "$pkgline" ]; then
      echo "$pkg ($ver) $upsource" >> $UPDATES_DB.missing
   fi
done < $UPDATES_DB