autodist-upstream-updates: fixes and some code rework
This commit is contained in:
parent
094174a2b6
commit
dd9fb71c1b
@ -43,6 +43,7 @@ function usage()
|
|||||||
echo
|
echo
|
||||||
echo " -h: generate distromatic HTML output"
|
echo " -h: generate distromatic HTML output"
|
||||||
echo " -m: report missing packages only"
|
echo " -m: report missing packages only"
|
||||||
|
echo " -s: skip fetching and parsing upstream updates"
|
||||||
echo " -u: output not up-to-date packages only"
|
echo " -u: output not up-to-date packages only"
|
||||||
echo " -q: produces quite output"
|
echo " -q: produces quite output"
|
||||||
echo " -r repository: specify the distromatic base (default: devel)"
|
echo " -r repository: specify the distromatic base (default: devel)"
|
||||||
@ -78,13 +79,13 @@ function version_compare()
|
|||||||
A=$1
|
A=$1
|
||||||
B=$2
|
B=$2
|
||||||
|
|
||||||
if [[ ${1} =~ "^[0-9]+$" && ${2} =~ "^[0-9]+$" ]]; then
|
if [[ ${1} =~ ^[0-9]+$ && ${2} =~ ^[0-9]+$ ]]; then
|
||||||
if [ ${A/[a-zA-Z_]*} -gt ${B/[a-zA-Z_]*} ]; then
|
if [ $((10#${A/[a-zA-Z_]*})) -gt $((10#${B/[a-zA-Z_]*})) ]; then
|
||||||
return 1
|
return 1
|
||||||
elif [ ${A/[a-zA-Z_]*} -lt ${B/[a-zA-Z_]*} ]; then
|
elif [ $((10#${A/[a-zA-Z_]*})) -lt $((10#${B/[a-zA-Z_]*})) ]; then
|
||||||
return 2
|
return 2
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
if [[ "$A" > "$B" ]]; then
|
if [[ "$A" > "$B" ]]; then
|
||||||
return 1
|
return 1
|
||||||
elif [[ "$A" < "$B" ]]; then
|
elif [[ "$A" < "$B" ]]; then
|
||||||
@ -132,6 +133,7 @@ while [ "$1" ]; do
|
|||||||
else
|
else
|
||||||
missing_only=1
|
missing_only=1
|
||||||
fi ;;
|
fi ;;
|
||||||
|
-s) skip_parsing=1 ;;
|
||||||
-u) needupdate_only=1 ;;
|
-u) needupdate_only=1 ;;
|
||||||
-r) [ "$2" ] || {
|
-r) [ "$2" ] || {
|
||||||
echo "Error: option -r requires repository name as parameter"
|
echo "Error: option -r requires repository name as parameter"
|
||||||
@ -166,116 +168,132 @@ UPDATES_DB=$CACHE_DIR/upstream-updates
|
|||||||
ALIASES_DB=$CACHE_DIR/aliases
|
ALIASES_DB=$CACHE_DIR/aliases
|
||||||
MANUALVER_DB=$CACHE_DIR/manualver
|
MANUALVER_DB=$CACHE_DIR/manualver
|
||||||
|
|
||||||
tmpfile=`mktemp -q -t autodist-upstream-updates.XXXXXXXX`
|
|
||||||
buildstmp=`mktemp -q -t autodist-upstream-updates.XXXXXXXX`
|
|
||||||
tail -n+2 $BUILDS_FILE > $buildstmp
|
|
||||||
|
|
||||||
# parse Arch Linux package list
|
parse_arch_linux() {
|
||||||
[ "$quiet" ] || echo -n "Parsing Arch Linux packages list..." >&2
|
# parse Arch Linux package list
|
||||||
#for page in `seq 1 45`; do
|
[ "$quiet" ] || echo -n "Parsing Arch Linux packages list..." >&2
|
||||||
for rep in core community extra; do
|
#for page in `seq 1 45`; do
|
||||||
# SOURCEURL="https://www.archlinux.org/packages/?page=$page&sort=-last_update&q=&arch=i686&maintainer=&flagged="
|
for rep in core community extra; do
|
||||||
SOURCEURL="http://lug.mtu.edu/archlinux/$rep/os/i686/"
|
# SOURCEURL="https://www.archlinux.org/packages/?page=$page&sort=-last_update&q=&arch=i686&maintainer=&flagged="
|
||||||
curl -s "$SOURCEURL" | \
|
SOURCEURL="http://lug.mtu.edu/archlinux/$rep/os/i686/"
|
||||||
grep ".pkg." | grep -v ".sig\"" | \
|
curl -L -s "$SOURCEURL" | \
|
||||||
while read line; do
|
grep ".pkg." | grep -v ".sig\"" | \
|
||||||
line=`echo $line | sed "s|.*href=\"\([^\"]*\)\">.*|\1|"`
|
while read line; do
|
||||||
pkg=`echo $line | sed "s|\(.*\)-[^-]*-[^-]*-[^-]*|\1|"`
|
line=`echo $line | sed "s|.*href=\"\([^\"]*\)\">.*|\1|"`
|
||||||
ver=`echo $line | sed "s|.*-\([^-]*\)-[^-]*-[^-]*|\1|"`
|
pkg=`echo $line | sed "s|\(.*\)-[^-]*-[^-]*-[^-]*|\1|"`
|
||||||
alias=`grep "^$pkg " $ALIASES_DB`
|
ver=`echo $line | sed "s|.*-\([^-]*\)-[^-]*-[^-]*|\1|"`
|
||||||
[ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
|
|
||||||
[ "$alias" ] && pkgalias=${alias/* /} || pkgalias=$pkg
|
|
||||||
line=`grep -i "^$pkgalias:" $buildstmp || grep -i "^lib$pkgalias:" $buildstmp || grep -i " $pkgalias[^-_A-Za-z0-9]" $buildstmp`
|
|
||||||
if [ "$line" ]; then
|
|
||||||
[ "$pkg" != "$pkgalias" ] && alias=$pkgalias || alias=
|
|
||||||
[ "$pkg" -a "$ver" ] && {
|
|
||||||
echo "$pkg $ver $SOURCEURL ${alias}" >> $tmpfile
|
|
||||||
}
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
done
|
|
||||||
rm -f $buildstmp
|
|
||||||
|
|
||||||
# parse X.org stable packages list
|
|
||||||
[ "$quiet" ] || echo "Parsing X.org release ftp directory..." >&2
|
|
||||||
SOURCEURL="ftp://ftp.x.org/pub/$XORG_RELEASE/src/everything/"
|
|
||||||
curl -s $SOURCEURL -l | sed "s|\.tar\..*||" | sort -u |
|
|
||||||
while read line; do
|
|
||||||
if [ "$line" ]; then
|
|
||||||
ver=`echo $line | sed "s|.*-||"`
|
|
||||||
pkg="${line/-$ver}"
|
|
||||||
alias=`grep "^$pkg " $ALIASES_DB`
|
|
||||||
if [ ! "$alias" -a "${pkg:0:5}" == "xf86-" ]; then
|
|
||||||
alias="$pkg xorg-drv-${pkg/xf86-}"
|
|
||||||
else
|
|
||||||
[ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
|
|
||||||
fi
|
|
||||||
[ "$pkg" -a "$ver" ] && echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# parse Gnome stable packages list
|
|
||||||
[ "$quiet" ] || echo "Parsing GNOME stable versions file..." >&2
|
|
||||||
for f in versions-stable versions-stable-extras; do
|
|
||||||
SOURCEURL="http://people.gnome.org/~vuntz/tmp/versions/$f"
|
|
||||||
curl -s $SOURCEURL | grep -v "^#" |
|
|
||||||
while read line; do
|
|
||||||
if [ "$line" ]; then
|
|
||||||
IFS=":"
|
|
||||||
set -- $line
|
|
||||||
pkg="$2"
|
|
||||||
ver="$3"
|
|
||||||
alias=`grep "^$pkg " $ALIASES_DB`
|
alias=`grep "^$pkg " $ALIASES_DB`
|
||||||
[ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
|
[ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
|
||||||
|
[ "$alias" ] && pkgalias=${alias/* /} || pkgalias=$pkg
|
||||||
|
line=`grep -i "^$pkgalias:" $buildstmp || grep -i "^lib$pkgalias:" $buildstmp || grep -i " $pkgalias[^-_A-Za-z0-9]" $buildstmp`
|
||||||
|
if [ "$line" ]; then
|
||||||
|
[ "$pkg" != "$pkgalias" ] && alias=$pkgalias || alias=
|
||||||
|
[ "$pkg" -a "$ver" ] && {
|
||||||
|
echo "$pkg $ver $SOURCEURL ${alias}" >> $tmpfile
|
||||||
|
}
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
done
|
||||||
|
rm -f $buildstmp
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_xorg() {
|
||||||
|
# parse X.org stable packages list
|
||||||
|
[ "$quiet" ] || echo "Parsing X.org release ftp directory..." >&2
|
||||||
|
SOURCEURL="ftp://ftp.x.org/pub/$XORG_RELEASE/src/everything/"
|
||||||
|
curl -L -s $SOURCEURL -l | sed "s|\.tar\..*||" | sort -u |
|
||||||
|
while read line; do
|
||||||
|
if [ "$line" ]; then
|
||||||
|
ver=`echo $line | sed "s|.*-||"`
|
||||||
|
pkg="${line/-$ver}"
|
||||||
|
alias=`grep "^$pkg " $ALIASES_DB`
|
||||||
|
if [ ! "$alias" -a "${pkg:0:5}" == "xf86-" ]; then
|
||||||
|
alias="$pkg xorg-drv-${pkg/xf86-}"
|
||||||
|
else
|
||||||
|
[ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
|
||||||
|
fi
|
||||||
[ "$pkg" -a "$ver" ] && echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
|
[ "$pkg" -a "$ver" ] && echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
done
|
}
|
||||||
|
|
||||||
# parse distrowatch.com packages list
|
parse_gnome() {
|
||||||
[ "$quiet" ] || echo "Parsing Distrowatch packages list..." >&2
|
# parse Gnome stable packages list
|
||||||
SOURCEURL="http://distrowatch.com/packages.php"
|
[ "$quiet" ] || echo "Parsing GNOME stable versions file..." >&2
|
||||||
lynx -width 300 -dump $SOURCEURL |
|
for f in versions-stable versions-stable-extras; do
|
||||||
while read line; do
|
SOURCEURL="https://people.gnome.org/~vuntz/tmp/versions/$f"
|
||||||
[ "`echo $line | grep "Package Version Note"`" ] && start_print=1
|
curl -s -L $SOURCEURL | grep -v "^#" |
|
||||||
[ "`echo $line | grep "____________________"`" ] && unset start_print
|
while read line; do
|
||||||
[ "$start_print" ] && {
|
if [ "$line" ]; then
|
||||||
set -- $line
|
IFS=":"
|
||||||
pkg="${1/\[*\]/}"
|
set -- $line
|
||||||
ver="${2/\[*\]/}"
|
pkg="$2"
|
||||||
alias=`grep "^$pkg " $ALIASES_DB`
|
ver="$3"
|
||||||
[ "$pkg" != "chromium" ] && \
|
alias=`grep "^$pkg " $ALIASES_DB`
|
||||||
echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
|
[ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
|
||||||
}
|
[ "$pkg" -a "$ver" ] && echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
|
||||||
done
|
fi
|
||||||
|
done
|
||||||
cat $tmpfile | sort -uf > $UPDATES_DB.tmp
|
|
||||||
rm -f $tmpfile
|
|
||||||
|
|
||||||
> $UPDATES_DB
|
|
||||||
unset lastpkg
|
|
||||||
while read pkg ver upsource alias; do
|
|
||||||
# skip updates to unstable versions
|
|
||||||
unset found_beta
|
|
||||||
for b in alpha beta rc "~"; do
|
|
||||||
[ "${ver/$b}" != "${ver}" ] && found_beta=1
|
|
||||||
done
|
done
|
||||||
[ "$found_beta" ] && continue
|
}
|
||||||
if [ "$pkg" = "$lastpkg" ]; then
|
|
||||||
|
parse_distromatic() {
|
||||||
|
# parse distrowatch.com packages list
|
||||||
|
[ "$quiet" ] || echo "Parsing Distrowatch packages list..." >&2
|
||||||
|
SOURCEURL="http://distrowatch.com/packages.php"
|
||||||
|
lynx -width 300 -dump $SOURCEURL |
|
||||||
|
while read line; do
|
||||||
|
[ "`echo $line | grep "Package Version Note"`" ] && start_print=1
|
||||||
|
[ "`echo $line | grep "____________________"`" ] && unset start_print
|
||||||
|
[ "$start_print" ] && {
|
||||||
|
set -- $line
|
||||||
|
pkg="${1/\[*\]/}"
|
||||||
|
ver="${2/\[*\]/}"
|
||||||
|
alias=`grep "^$pkg " $ALIASES_DB`
|
||||||
|
[ "$pkg" != "chromium" ] && \
|
||||||
|
echo "$pkg $ver $SOURCEURL ${alias/* /}" >> $tmpfile
|
||||||
|
}
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ ! "$skip_parsing" ]; then
|
||||||
|
tmpfile=`mktemp -q -t autodist-upstream-updates.XXXXXXXX`
|
||||||
|
buildstmp=`mktemp -q -t autodist-upstream-updates.XXXXXXXX`
|
||||||
|
tail -n+2 $BUILDS_FILE > $buildstmp
|
||||||
|
|
||||||
|
parse_arch_linux
|
||||||
|
parse_xorg
|
||||||
|
parse_gnome
|
||||||
|
parse_distromatic
|
||||||
|
|
||||||
|
cat $tmpfile | sort -uf > $UPDATES_DB.tmp
|
||||||
|
rm -f $tmpfile
|
||||||
|
|
||||||
|
> $UPDATES_DB
|
||||||
|
unset lastpkg
|
||||||
|
while read pkg ver upsource alias; do
|
||||||
|
# skip updates to unstable versions
|
||||||
|
unset found_beta
|
||||||
|
for b in alpha beta rc "~"; do
|
||||||
|
[ "${ver/$b}" != "${ver}" ] && found_beta=1
|
||||||
|
done
|
||||||
|
[ "$found_beta" ] && continue
|
||||||
|
if [ "$pkg" = "$lastpkg" ]; then
|
||||||
# echo "Warning: duplicate found: $pkg lastver: $lastver ver: $ver" >&2
|
# echo "Warning: duplicate found: $pkg lastver: $lastver ver: $ver" >&2
|
||||||
version_find_bigger $lastver $ver
|
version_find_bigger $lastver $ver
|
||||||
vercmp=$?
|
vercmp=$?
|
||||||
if [ $vercmp -eq 2 ]; then
|
if [ $vercmp -eq 2 ]; then
|
||||||
sed -i "/^$lastpkg $lastver /d" $UPDATES_DB
|
sed -i "/^$lastpkg $lastver /d" $UPDATES_DB
|
||||||
|
echo "$pkg $ver $upsource $alias" >> $UPDATES_DB
|
||||||
|
fi
|
||||||
|
else
|
||||||
echo "$pkg $ver $upsource $alias" >> $UPDATES_DB
|
echo "$pkg $ver $upsource $alias" >> $UPDATES_DB
|
||||||
fi
|
fi
|
||||||
else
|
lastpkg=$pkg
|
||||||
echo "$pkg $ver $upsource $alias" >> $UPDATES_DB
|
lastver=$ver
|
||||||
fi
|
done < $UPDATES_DB.tmp
|
||||||
lastpkg=$pkg
|
rm -f $UPDATES_DB.tmp
|
||||||
lastver=$ver
|
fi
|
||||||
done < $UPDATES_DB.tmp
|
|
||||||
rm -f $UPDATES_DB.tmp
|
|
||||||
|
|
||||||
> $UPDATES_DB.missing
|
> $UPDATES_DB.missing
|
||||||
> $BUILDLIST_FILE
|
> $BUILDLIST_FILE
|
||||||
@ -311,6 +329,7 @@ while read pkg ver upsource alias; do
|
|||||||
pkgrep=$4
|
pkgrep=$4
|
||||||
version_find_bigger $pkgver ${ver/-/.}
|
version_find_bigger $pkgver ${ver/-/.}
|
||||||
vercmp=$?
|
vercmp=$?
|
||||||
|
#echo "$pkgname - version_find_bigger $pkgver ${ver/-/.} result=$vercmp"
|
||||||
[ "${vercmp}" != "2" -a "$needupdate_only" ] && continue
|
[ "${vercmp}" != "2" -a "$needupdate_only" ] && continue
|
||||||
[ "$found_manual" ] && pkgname=$3
|
[ "$found_manual" ] && pkgname=$3
|
||||||
[ "$found_alias" -o "$found_manual" ] && nameadd="$pkg" || unset nameadd
|
[ "$found_alias" -o "$found_manual" ] && nameadd="$pkg" || unset nameadd
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user