autodist/autodist-upstream-updates

#!/bin/bash
#
# autodist upstream updates - find upstream packages updates from different internet resources
# Copyright (c) 2004-2016 by Silvan Calarco <silvan.calarco@mambasoft.it>
#

#[ -r /etc/sysconfig/openmamba-central ] || { 
#   echo "Error: this program must be run as root; aborting." >&2
#   exit 1
#}
. /etc/autodist/config
DISTROMATIC_PREFIX=/distribution/distromatic.html?
DISTROMATIC_REPOSITORY=devel
DISTDB=/etc/autodist/distdb
BLACKLISTDB=/etc/autodist/blacklist
DISTDBDIR=/etc/autodist/distdb.d
XORG_RELEASE=current

[ -r $DISTDB ] && {
   . $DISTDB
   [ -d $DISTDBDIR ] && \
      for f in `ls $DISTDBDIR/*.db`; do
         . $f
      done
}

function usage() 
{
   echo "openmamba-upstream-updates - finds upstream packages updates from different internet resources"
   echo
   echo "Usage:"
   echo "openmamba-upstream-updates [-h|-m][-r repository][-o output_repository]"
   echo
   echo " -h: generate distromatic HTML output"
   echo " -m: report missing packages only"
   echo " -s: skip fetching and parsing upstream updates"
   echo " -u: output not up-to-date packages only"
   echo " -q: produces quite output"
   echo " -r repository: specify the distromatic base (default: devel)"
   echo " -o repository: specify the repository for output data (default: same as base repository)"
   echo
}

get_job_vector() {
   local JNAME=$1
   local JNAME_ESCAPED=`echo $JNAME | tr - _ | tr . _ | tr @ _`

   # resolve JOB_NAME from distdb
   local jobtmpfile=`tempfile`
   # hack to get an array variable named as $j assigned to the JOB array
   echo "echo \${$JNAME_ESCAPED[*]}" > $jobtmpfile
   JOB=(`. $jobtmpfile`)
   rm -f $jobtmpfile

   if [ ${#JOB[*]} -eq 0 ]; then
       # create a default job with given JOB_NAME
       JOB=($JNAME "" "")
   elif [ ${#JOB[*]} -eq 1 ]; then
       # no variables defined, add an empty job
       JOB=(${JOB[*]} "" "")
   fi
   JOB_PKGS=(${JOB[0]//,/ })
   JOB_VARNAMES=(${JOB[1]//,/ })
}

function version_compare()
{
   local A B
   A=$1
   B=$2

   if [[ ${1} =~ ^[0-9]+$ && ${2} =~ ^[0-9]+$ ]]; then
      if [ $((10#${A/[a-zA-Z_]*})) -gt $((10#${B/[a-zA-Z_]*})) ]; then
         return 1
      elif [ $((10#${A/[a-zA-Z_]*})) -lt $((10#${B/[a-zA-Z_]*})) ]; then
         return 2
      fi
  else
      if [[ ${1} =~ ^[0-9]+$ || "$A" > "$B" ]]; then
         return 1
      elif [[ "$A" < "$B" ]]; then
         return 2
      fi
   fi
   return 0
}

function version_find_bigger()
{
   local VER1 VER2 FPOS CUTVER1 CUTVER2

   VER1=$1
   VER2=$2
   FPOS=1
   while true; do
      CUTVER1=`echo $VER1. | cut -d. -f $FPOS`
      CUTVER2=`echo $VER2. | cut -d. -f $FPOS`
      if [ "$CUTVER1" -a ! "$CUTVER2" ]; then
         return 1
      elif [ "$CUTVER2" -a ! "$CUTVER1" ]; then
         return 2
      elif [ ! "$CUTVER1" -a "$CUTVER2" ]; then
         return 0
      else
         version_compare $CUTVER1 $CUTVER2
         case $? in
            1) return 1 ;;
	    2) return 2 ;;
         esac
      fi
      FPOS=`expr $FPOS + 1`
   done
   return 0
}

while [ "$1" ]; do
   case $1 in
      -h) distromatic_html=1 ;;
      -m) if [ "$distromatic_html" ]; then
             echo "Error: options -h and -m cannot be used together."
	     usage
	     exit 1
	  else
             missing_only=1
	  fi ;;
      -s) skip_parsing=1 ;;
      -u) needupdate_only=1 ;;
      -r) [ "$2" ] || {
             echo "Error: option -r requires repository name as parameter"
	     usage
	     exit 1
          }
	  DISTROMATIC_REPOSITORY=$2
	  shift
	  ;;
      -o) [ "$2" ] || {
             echo "Error: option -o requires repository name as parameter"
             usage
             exit 1
          }
          OUTPUT_REPOSITORY=$2
          shift
          ;;
      -q) quiet=1 ;;
      *) echo "Error: invalid option $1."; usage; exit 1 ;;  
   esac
   shift
done

[ "$OUTPUT_REPOSITORY" ] || OUTPUT_REPOSITORY=$DISTROMATIC_REPOSITORY

CACHE_DIR=${LOCAL_REPS_BASE_DIR}/$OUTPUT_REPOSITORY/autoupdate/
PKGLIST_FILE=${LOCAL_REPS_BASE_DIR}/$DISTROMATIC_REPOSITORY/srcpkglist
BUILDS_FILE=${LOCAL_REPS_BASE_DIR}/distromatic/$DISTROMATIC_REPOSITORY/builds-i586
BUILDLIST_FILE=$CACHE_DIR/upstream-updates.in
CONFIG_DIR=$CACHE_DIR
UPDATES_DB=$CACHE_DIR/upstream-updates
ALIASES_DB=$CACHE_DIR/aliases
MANUALVER_DB=$CACHE_DIR/manualver
BLACKLIST_DB_PREFIX=$CACHE_DIR/blacklist.

[ -s $PKGLIST_FILE ] || {
   echo "Error: file $PKGLIST_FILE is missing or empty; aborting." >&2
   exit 1
}

[ -r $ALIASES_DB ] || {
   echo "Error: file $ALIASES_DB cannot be read; aborting." >&2
   exit 1
}

parse_arch_linux() {
   # parse Arch Linux package list
   [ "$quiet" ] || echo -n "Parsing Arch Linux packages list..." >&2
   #for page in `seq 1 45`; do
   for rep in core community extra; do
   #   SOURCEURL="https://www.archlinux.org/packages/?page=$page&sort=-last_update&q=&arch=i686&maintainer=&flagged="
      SOURCEURL="http://lug.mtu.edu/archlinux/$rep/os/x86_64/"
      curl -L -s "$SOURCEURL" | \
      grep ".pkg." | grep -v ".sig\"" | \
      while read line; do
         line=`echo $line | sed "s|.*href=\"\([^\"]*\)\">.*|\1|"`
         pkg=`echo $line | sed "s|\(.*\)-[^-]*-[^-]*-[^-]*|\1|"`
         ver=`echo $line | sed "s|.*-\([^-]*\)-[^-]*-[^-]*|\1|;s|.*%3a||i;s|%2b.*||i"`
         alias=`grep "^$pkg " $ALIASES_DB`
         # Skip blackisted
         if [ -e ${BLACKLIST_DB_PREFIX}arch ]; then
            grep -q "^$pkg$" ${BLACKLIST_DB_PREFIX}arch && continue
         fi
         [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
         [ "$alias" ] && pkgalias=${alias/* /} || pkgalias=$pkg
         line=`grep -i "^$pkgalias:" $buildstmp || grep -i "^lib$pkgalias:" $buildstmp || grep -i " $pkgalias[^-_A-Za-z0-9]" $buildstmp`
         if [ "$line" ]; then
            [ "$pkg" != "$pkgalias" ] && alias=$pkgalias || alias=
            [ "$pkg" -a "$ver" ] && { 
               echo "$pkg $ver $SOURCEURL ${alias}" >> $tmpfile
            }
         fi
      done
   done
   rm -f $buildstmp
}

parse_xorg() {
   # parse X.org stable packages list
   [ "$quiet" ] || echo "Parsing X.org release ftp directory..." >&2
   SOURCEURL="ftp://ftp.x.org/pub/$XORG_RELEASE/src/everything/"
   curl -L -s $SOURCEURL -l | sed "s|\.tar\..*||" | sort -u |
   while read line; do
      if [ "$line" ]; then
         ver=`echo $line | sed "s|.*-||"`
         pkg="${line/-$ver}"
         if [ -e ${BLACKLIST_DB_PREFIX}xorg ]; then
            grep -q "^$pkg$" ${BLACKLIST_DB_PREFIX}xorg && continue
         fi
         alias=`grep "^$pkg " $ALIASES_DB`
         if [ ! "$alias" -a "${pkg:0:5}" == "xf86-" ]; then
            alias="$pkg xorg-drv-${pkg/xf86-}"
         else
            [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
         fi
         [ "$pkg" -a "$ver" ] && echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
      fi
   done
}

parse_gnome() {
   # parse Gnome stable packages list
   [ "$quiet" ] || echo "Parsing GNOME stable versions file..." >&2
   for f in versions-stable versions-stable-extras; do
      SOURCEURL="https://people.gnome.org/~vuntz/tmp/versions/$f"
      curl -s -L $SOURCEURL | grep -v "^#" |
      while read line; do
         if [ "$line" ]; then
            IFS=":"
            set -- $line
            pkg="$2"
            if [ -e ${BLACKLIST_DB_PREFIX}gnome ]; then
               grep -q "^$pkg$" ${BLACKLIST_DB_PREFIX}gnome && continue
            fi
            ver="$3"
            alias=`grep "^$pkg " $ALIASES_DB`
            [ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
            if [ "$pkg" -a "$ver" ]; then
               ver1=$(echo $ver | cut -d. -f1)
               ver2=$(echo $ver | cut -d. -f2)
               if [ "$ver2" -a $(($ver2 % 2)) -ne 0 ]; then
                  # skip unstable versions with odd numbers
                  continue
               else
                  echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
               fi
            fi
         fi
      done
   done
}

parse_distromatic() {
   # parse distrowatch.com packages list
   [ "$quiet" ] || echo "Parsing Distrowatch packages list..." >&2
   SOURCEURL="https://distrowatch.com/packages.php"
   lynx -width 300 -dump $SOURCEURL |
   while read line; do
      [ "`echo $line | grep "Package Version Note"`" ] && start_print=1
      [ "`echo $line | grep "____________________"`" ] && unset start_print
      [ "$start_print" ] && {
         set -- $line
         pkg="${1/\[*\]/}" 
         ver="${2/\[*\]/}"
         alias=`grep "^$pkg " $ALIASES_DB`
         [ "$pkg" != "chromium" ] && \
            echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
      }
   done
}

if [ ! "$skip_parsing" ]; then
   #tmpfile=`mktemp -q -t autodist-upstream-updates.XXXXXXXX`
   buildstmp=`mktemp -q -t autodist-upstream-updates.XXXXXXXX`
   tail -n+2 $BUILDS_FILE > $buildstmp

   #parse_arch_linux
   #parse_xorg
   #parse_gnome
   #parse_distromatic
   /usr/share/autodist/repology-openmamba.py > $UPDATES_DB.tmp

   #cat $tmpfile | sort -uf > $UPDATES_DB.tmp
   #rm -f $tmpfile

   > $UPDATES_DB
   unset lastpkg
   while read pkg ver upsource alias; do
      # skip updates to unstable versions
      unset found_beta
      for b in alpha beta rc "~"; do
         [ "${ver/$b}" != "${ver}" ] && found_beta=1
      done
      [ "$found_beta" ] && continue
      if [ "$pkg" = "$lastpkg" ]; then
#      echo "Warning: duplicate found: $pkg lastver: $lastver ver: $ver" >&2
         version_find_bigger $lastver $ver
         vercmp=$?
         if [ $vercmp -eq 2 ]; then
            sed -i "/^$lastpkg $lastver /d" $UPDATES_DB
            echo "$pkg $ver $upsource $alias" >> $UPDATES_DB
         fi
      else
         echo "$pkg $ver $upsource $alias" >> $UPDATES_DB
      fi
      lastpkg=$pkg
      lastver=$ver
   done < $UPDATES_DB.tmp
   rm -f $UPDATES_DB.tmp
fi

> $UPDATES_DB.missing
> $BUILDLIST_FILE
while read pkg ver upsource alias; do
   grep "^$pkg$" $BLACKLISTDB >/dev/null && continue
   unset pkgline
   unset found_manual
   unset found_alias
   pkgline=`grep -i "^$pkg " $MANUALVER_DB` && found_manual=1
   if [ ! "$found_manual" ]; then
      if [ "$alias" ]; then
         get_job_vector $alias
         lastjob=${#JOB_PKGS[*]}
         pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE` && found_alias=1
      else
         get_job_vector $pkg
         lastjob=${#JOB_PKGS[*]}
         pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE`
         [ "$pkgline" ] || {
            get_job_vector lib${pkg}
            lastjob=${#JOB_PKGS[*]}
            pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE` && {
               found_alias=1
               alias=lib${pkg}
            }
         }
      fi
   fi
   if [ "$pkgline" -a ! "$missing_only" ]; then
      set -- $pkgline
      pkgname=$1
      pkgver=$2
      pkgrep=$4
      version_find_bigger $pkgver ${ver/-/.}
      vercmp=$?
#echo "$pkgname - version_find_bigger $pkgver ${ver/-/.} result=$vercmp"
      [ "${vercmp}" != "2" -a "$needupdate_only" ] && continue
      [ "$found_manual" ] && pkgname=$3
      [ "$found_alias" -o "$found_manual" ] && nameadd="$pkg" || unset nameadd
      unset veradd
      [ ${vercmp} = 2 ] && veradd="<a href=\"$upsource\"><font color=red>$ver</font></a>"
      [ ${vercmp} = 1 ] && veradd="$ver"
      [ "$veradd" -o "$nameadd" ] && {
         [ "$veradd" -a "$nameadd" ] && \
            verappend="($nameadd;$veradd)" || 
            verappend="(${nameadd}${veradd})"
      } || unset verappend
      if [ "$distromatic_html" = "1" ]; then
         echo "<a href=\"${DISTROMATIC_PREFIX}tag=${pkgrep}&pkg=${JOB_PKGS[0]}.source\">${JOB_PKGS[0]} $pkgver</a> $verappend<br>"
      else
         echo "$pkg $pkgver ($ver)"
      fi
      [ $vercmp = 2 ] && {
         if [ "$found_alias" ]; then
            echo "$alias +$ver 0 $upsource" >> $BUILDLIST_FILE
         else
            echo "$pkg +$ver 0 $upsource" >> $BUILDLIST_FILE
         fi
      }
   elif [ ! "$pkgline" ]; then
      echo "$pkg ($ver) $upsource" >> $UPDATES_DB.missing
   fi
done < $UPDATES_DB