380 lines
12 KiB
Bash
Executable File
380 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# autodist upstream updates - find upstream packages updates from different internet resources
|
|
# Copyright (c) 2004-2016 by Silvan Calarco <silvan.calarco@mambasoft.it>
|
|
#
|
|
|
|
#[ -r /etc/sysconfig/openmamba-central ] || {
|
|
# echo "Error: this program must be run as root; aborting." >&2
|
|
# exit 1
|
|
#}
|
|
. /etc/autodist/config
|
|
DISTROMATIC_PREFIX=/distribution/distromatic.html?
|
|
DISTROMATIC_REPOSITORY=devel
|
|
DISTDB=/etc/autodist/distdb
|
|
BLACKLISTDB=/etc/autodist/blacklist
|
|
DISTDBDIR=/etc/autodist/distdb.d
|
|
XORG_RELEASE=current
|
|
|
|
[ -r $DISTDB ] && {
|
|
. $DISTDB
|
|
[ -d $DISTDBDIR ] && \
|
|
for f in `ls $DISTDBDIR/*.db`; do
|
|
. $f
|
|
done
|
|
}
|
|
|
|
function usage()
|
|
{
|
|
echo "openmamba-upstream-updates - finds upstream packages updates from different internet resources"
|
|
echo
|
|
echo "Usage:"
|
|
echo "openmamba-upstream-updates [-h|-m][-r repository][-o output_repository]"
|
|
echo
|
|
echo " -h: generate distromatic HTML output"
|
|
echo " -m: report missing packages only"
|
|
echo " -s: skip fetching and parsing upstream updates"
|
|
echo " -u: output not up-to-date packages only"
|
|
echo " -q: produces quite output"
|
|
echo " -r repository: specify the distromatic base (default: devel)"
|
|
echo " -o repository: specify the repository for output data (default: same as base repository)"
|
|
echo
|
|
}
|
|
|
|
get_job_vector() {
|
|
local JNAME=$1
|
|
local JNAME_ESCAPED=`echo $JNAME | tr - _ | tr . _ | tr @ _`
|
|
|
|
# resolve JOB_NAME from distdb
|
|
local jobtmpfile=`tempfile`
|
|
# hack to get an array variable named as $j assigned to the JOB array
|
|
echo "echo \${$JNAME_ESCAPED[*]}" > $jobtmpfile
|
|
JOB=(`. $jobtmpfile`)
|
|
rm -f $jobtmpfile
|
|
|
|
if [ ${#JOB[*]} -eq 0 ]; then
|
|
# create a default job with given JOB_NAME
|
|
JOB=($JNAME "" "")
|
|
elif [ ${#JOB[*]} -eq 1 ]; then
|
|
# no variables defined, add an empty job
|
|
JOB=(${JOB[*]} "" "")
|
|
fi
|
|
JOB_PKGS=(${JOB[0]//,/ })
|
|
JOB_VARNAMES=(${JOB[1]//,/ })
|
|
}
|
|
|
|
function version_compare()
|
|
{
|
|
local A B
|
|
A=$1
|
|
B=$2
|
|
|
|
if [[ ${1} =~ ^[0-9]+$ && ${2} =~ ^[0-9]+$ ]]; then
|
|
if [ $((10#${A/[a-zA-Z_]*})) -gt $((10#${B/[a-zA-Z_]*})) ]; then
|
|
return 1
|
|
elif [ $((10#${A/[a-zA-Z_]*})) -lt $((10#${B/[a-zA-Z_]*})) ]; then
|
|
return 2
|
|
fi
|
|
else
|
|
if [[ ${1} =~ ^[0-9]+$ || "$A" > "$B" ]]; then
|
|
return 1
|
|
elif [[ "$A" < "$B" ]]; then
|
|
return 2
|
|
fi
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
function version_find_bigger()
|
|
{
|
|
local VER1 VER2 FPOS CUTVER1 CUTVER2
|
|
|
|
VER1=$1
|
|
VER2=$2
|
|
FPOS=1
|
|
while true; do
|
|
CUTVER1=`echo $VER1. | cut -d. -f $FPOS`
|
|
CUTVER2=`echo $VER2. | cut -d. -f $FPOS`
|
|
if [ "$CUTVER1" -a ! "$CUTVER2" ]; then
|
|
return 1
|
|
elif [ "$CUTVER2" -a ! "$CUTVER1" ]; then
|
|
return 2
|
|
elif [ ! "$CUTVER1" -a "$CUTVER2" ]; then
|
|
return 0
|
|
else
|
|
version_compare $CUTVER1 $CUTVER2
|
|
case $? in
|
|
1) return 1 ;;
|
|
2) return 2 ;;
|
|
esac
|
|
fi
|
|
FPOS=`expr $FPOS + 1`
|
|
done
|
|
return 0
|
|
}
|
|
|
|
while [ "$1" ]; do
|
|
case $1 in
|
|
-h) distromatic_html=1 ;;
|
|
-m) if [ "$distromatic_html" ]; then
|
|
echo "Error: options -h and -m cannot be used together."
|
|
usage
|
|
exit 1
|
|
else
|
|
missing_only=1
|
|
fi ;;
|
|
-s) skip_parsing=1 ;;
|
|
-u) needupdate_only=1 ;;
|
|
-r) [ "$2" ] || {
|
|
echo "Error: option -r requires repository name as parameter"
|
|
usage
|
|
exit 1
|
|
}
|
|
DISTROMATIC_REPOSITORY=$2
|
|
shift
|
|
;;
|
|
-o) [ "$2" ] || {
|
|
echo "Error: option -o requires repository name as parameter"
|
|
usage
|
|
exit 1
|
|
}
|
|
OUTPUT_REPOSITORY=$2
|
|
shift
|
|
;;
|
|
-q) quiet=1 ;;
|
|
*) echo "Error: invalid option $1."; usage; exit 1 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
[ "$OUTPUT_REPOSITORY" ] || OUTPUT_REPOSITORY=$DISTROMATIC_REPOSITORY
|
|
|
|
CACHE_DIR=${LOCAL_REPS_BASE_DIR}/$OUTPUT_REPOSITORY/autoupdate/
|
|
PKGLIST_FILE=${LOCAL_REPS_BASE_DIR}/$DISTROMATIC_REPOSITORY/srcpkglist
|
|
BUILDS_FILE=${LOCAL_REPS_BASE_DIR}/distromatic/$DISTROMATIC_REPOSITORY/builds-i586
|
|
BUILDLIST_FILE=$CACHE_DIR/upstream-updates.in
|
|
CONFIG_DIR=$CACHE_DIR
|
|
UPDATES_DB=$CACHE_DIR/upstream-updates
|
|
ALIASES_DB=$CACHE_DIR/aliases
|
|
MANUALVER_DB=$CACHE_DIR/manualver
|
|
BLACKLIST_DB_PREFIX=$CACHE_DIR/blacklist.
|
|
|
|
[ -s $PKGLIST_FILE ] || {
|
|
echo "Error: file $PKGLIST_FILE is missing or empty; aborting." >&2
|
|
exit 1
|
|
}
|
|
|
|
[ -r $ALIASES_DB ] || {
|
|
echo "Error: file $ALIASES_DB cannot be read; aborting." >&2
|
|
exit 1
|
|
}
|
|
|
|
parse_arch_linux() {
|
|
# parse Arch Linux package list
|
|
[ "$quiet" ] || echo -n "Parsing Arch Linux packages list..." >&2
|
|
#for page in `seq 1 45`; do
|
|
for rep in core community extra; do
|
|
# SOURCEURL="https://www.archlinux.org/packages/?page=$page&sort=-last_update&q=&arch=i686&maintainer=&flagged="
|
|
SOURCEURL="http://lug.mtu.edu/archlinux/$rep/os/x86_64/"
|
|
curl -L -s "$SOURCEURL" | \
|
|
grep ".pkg." | grep -v ".sig\"" | \
|
|
while read line; do
|
|
line=`echo $line | sed "s|.*href=\"\([^\"]*\)\">.*|\1|"`
|
|
pkg=`echo $line | sed "s|\(.*\)-[^-]*-[^-]*-[^-]*|\1|"`
|
|
ver=`echo $line | sed "s|.*-\([^-]*\)-[^-]*-[^-]*|\1|;s|.*%3a||i;s|%2b.*||i"`
|
|
alias=`grep "^$pkg " $ALIASES_DB`
|
|
# Skip blackisted
|
|
if [ -e ${BLACKLIST_DB_PREFIX}arch ]; then
|
|
grep -q "^$pkg$" ${BLACKLIST_DB_PREFIX}arch && continue
|
|
fi
|
|
[ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
|
|
[ "$alias" ] && pkgalias=${alias/* /} || pkgalias=$pkg
|
|
line=`grep -i "^$pkgalias:" $buildstmp || grep -i "^lib$pkgalias:" $buildstmp || grep -i " $pkgalias[^-_A-Za-z0-9]" $buildstmp`
|
|
if [ "$line" ]; then
|
|
[ "$pkg" != "$pkgalias" ] && alias=$pkgalias || alias=
|
|
[ "$pkg" -a "$ver" ] && {
|
|
echo "$pkg $ver $SOURCEURL ${alias}" >> $tmpfile
|
|
}
|
|
fi
|
|
done
|
|
done
|
|
rm -f $buildstmp
|
|
}
|
|
|
|
parse_xorg() {
|
|
# parse X.org stable packages list
|
|
[ "$quiet" ] || echo "Parsing X.org release ftp directory..." >&2
|
|
SOURCEURL="ftp://ftp.x.org/pub/$XORG_RELEASE/src/everything/"
|
|
curl -L -s $SOURCEURL -l | sed "s|\.tar\..*||" | sort -u |
|
|
while read line; do
|
|
if [ "$line" ]; then
|
|
ver=`echo $line | sed "s|.*-||"`
|
|
pkg="${line/-$ver}"
|
|
if [ -e ${BLACKLIST_DB_PREFIX}xorg ]; then
|
|
grep -q "^$pkg$" ${BLACKLIST_DB_PREFIX}xorg && continue
|
|
fi
|
|
alias=`grep "^$pkg " $ALIASES_DB`
|
|
if [ ! "$alias" -a "${pkg:0:5}" == "xf86-" ]; then
|
|
alias="$pkg xorg-drv-${pkg/xf86-}"
|
|
else
|
|
[ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
|
|
fi
|
|
[ "$pkg" -a "$ver" ] && echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
|
|
fi
|
|
done
|
|
}
|
|
|
|
parse_gnome() {
|
|
# parse Gnome stable packages list
|
|
[ "$quiet" ] || echo "Parsing GNOME stable versions file..." >&2
|
|
for f in versions-stable versions-stable-extras; do
|
|
SOURCEURL="https://people.gnome.org/~vuntz/tmp/versions/$f"
|
|
curl -s -L $SOURCEURL | grep -v "^#" |
|
|
while read line; do
|
|
if [ "$line" ]; then
|
|
IFS=":"
|
|
set -- $line
|
|
pkg="$2"
|
|
if [ -e ${BLACKLIST_DB_PREFIX}gnome ]; then
|
|
grep -q "^$pkg$" ${BLACKLIST_DB_PREFIX}gnome && continue
|
|
fi
|
|
ver="$3"
|
|
alias=`grep "^$pkg " $ALIASES_DB`
|
|
[ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
|
|
if [ "$pkg" -a "$ver" ]; then
|
|
ver1=$(echo $ver | cut -d. -f1)
|
|
ver2=$(echo $ver | cut -d. -f2)
|
|
if [ "$ver2" -a $(($ver2 % 2)) -ne 0 ]; then
|
|
# skip unstable versions with odd numbers
|
|
continue
|
|
else
|
|
echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
done
|
|
}
|
|
|
|
parse_distromatic() {
|
|
# parse distrowatch.com packages list
|
|
[ "$quiet" ] || echo "Parsing Distrowatch packages list..." >&2
|
|
SOURCEURL="https://distrowatch.com/packages.php"
|
|
lynx -width 300 -dump $SOURCEURL |
|
|
while read line; do
|
|
[ "`echo $line | grep "Package Version Note"`" ] && start_print=1
|
|
[ "`echo $line | grep "____________________"`" ] && unset start_print
|
|
[ "$start_print" ] && {
|
|
set -- $line
|
|
pkg="${1/\[*\]/}"
|
|
ver="${2/\[*\]/}"
|
|
alias=`grep "^$pkg " $ALIASES_DB`
|
|
[ "$pkg" != "chromium" ] && \
|
|
echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
|
|
}
|
|
done
|
|
}
|
|
|
|
if [ ! "$skip_parsing" ]; then
|
|
#tmpfile=`mktemp -q -t autodist-upstream-updates.XXXXXXXX`
|
|
buildstmp=`mktemp -q -t autodist-upstream-updates.XXXXXXXX`
|
|
tail -n+2 $BUILDS_FILE > $buildstmp
|
|
|
|
#parse_arch_linux
|
|
#parse_xorg
|
|
#parse_gnome
|
|
#parse_distromatic
|
|
/usr/share/autodist/repology-openmamba.py > $UPDATES_DB.tmp
|
|
|
|
#cat $tmpfile | sort -uf > $UPDATES_DB.tmp
|
|
#rm -f $tmpfile
|
|
|
|
> $UPDATES_DB
|
|
unset lastpkg
|
|
while read pkg ver upsource alias; do
|
|
# skip updates to unstable versions
|
|
unset found_beta
|
|
for b in alpha beta rc "~"; do
|
|
[ "${ver/$b}" != "${ver}" ] && found_beta=1
|
|
done
|
|
[ "$found_beta" ] && continue
|
|
if [ "$pkg" = "$lastpkg" ]; then
|
|
# echo "Warning: duplicate found: $pkg lastver: $lastver ver: $ver" >&2
|
|
version_find_bigger $lastver $ver
|
|
vercmp=$?
|
|
if [ $vercmp -eq 2 ]; then
|
|
sed -i "/^$lastpkg $lastver /d" $UPDATES_DB
|
|
echo "$pkg $ver $upsource $alias" >> $UPDATES_DB
|
|
fi
|
|
else
|
|
echo "$pkg $ver $upsource $alias" >> $UPDATES_DB
|
|
fi
|
|
lastpkg=$pkg
|
|
lastver=$ver
|
|
done < $UPDATES_DB.tmp
|
|
rm -f $UPDATES_DB.tmp
|
|
fi
|
|
|
|
> $UPDATES_DB.missing
|
|
> $BUILDLIST_FILE
|
|
while read pkg ver upsource alias; do
|
|
grep "^$pkg$" $BLACKLISTDB >/dev/null && continue
|
|
unset pkgline
|
|
unset found_manual
|
|
unset found_alias
|
|
pkgline=`grep -i "^$pkg " $MANUALVER_DB` && found_manual=1
|
|
if [ ! "$found_manual" ]; then
|
|
if [ "$alias" ]; then
|
|
get_job_vector $alias
|
|
lastjob=${#JOB_PKGS[*]}
|
|
pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE` && found_alias=1
|
|
else
|
|
get_job_vector $pkg
|
|
lastjob=${#JOB_PKGS[*]}
|
|
pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE`
|
|
[ "$pkgline" ] || {
|
|
get_job_vector lib${pkg}
|
|
lastjob=${#JOB_PKGS[*]}
|
|
pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE` && {
|
|
found_alias=1
|
|
alias=lib${pkg}
|
|
}
|
|
}
|
|
fi
|
|
fi
|
|
if [ "$pkgline" -a ! "$missing_only" ]; then
|
|
set -- $pkgline
|
|
pkgname=$1
|
|
pkgver=$2
|
|
pkgrep=$4
|
|
version_find_bigger $pkgver ${ver/-/.}
|
|
vercmp=$?
|
|
#echo "$pkgname - version_find_bigger $pkgver ${ver/-/.} result=$vercmp"
|
|
[ "${vercmp}" != "2" -a "$needupdate_only" ] && continue
|
|
[ "$found_manual" ] && pkgname=$3
|
|
[ "$found_alias" -o "$found_manual" ] && nameadd="$pkg" || unset nameadd
|
|
unset veradd
|
|
[ ${vercmp} = 2 ] && veradd="<a href=\"$upsource\"><font color=red>$ver</font></a>"
|
|
[ ${vercmp} = 1 ] && veradd="$ver"
|
|
[ "$veradd" -o "$nameadd" ] && {
|
|
[ "$veradd" -a "$nameadd" ] && \
|
|
verappend="($nameadd;$veradd)" ||
|
|
verappend="(${nameadd}${veradd})"
|
|
} || unset verappend
|
|
if [ "$distromatic_html" = "1" ]; then
|
|
echo "<a href=\"${DISTROMATIC_PREFIX}tag=${pkgrep}&pkg=${JOB_PKGS[0]}.source\">${JOB_PKGS[0]} $pkgver</a> $verappend<br>"
|
|
else
|
|
echo "$pkg $pkgver ($ver)"
|
|
fi
|
|
[ $vercmp = 2 ] && {
|
|
if [ "$found_alias" ]; then
|
|
echo "$alias +$ver 0 $upsource" >> $BUILDLIST_FILE
|
|
else
|
|
echo "$pkg +$ver 0 $upsource" >> $BUILDLIST_FILE
|
|
fi
|
|
}
|
|
elif [ ! "$pkgline" ]; then
|
|
echo "$pkg ($ver) $upsource" >> $UPDATES_DB.missing
|
|
fi
|
|
done < $UPDATES_DB
|