Added script autodust-upstream-updates with daily cron script
This commit is contained in:
parent
878663f2f0
commit
6cf3e9f99f
2
Makefile
2
Makefile
@ -69,6 +69,7 @@ install-programs:
|
||||
@$(INSTALL_SCRIPT) autodist-tool $(DESTDIR)$(bindir)/autodist-tool
|
||||
@$(INSTALL_SCRIPT) autoport $(DESTDIR)$(bindir)/autoport
|
||||
@$(INSTALL_SCRIPT) autoport-chroot $(DESTDIR)$(sbindir)/autoport-chroot
|
||||
@$(INSTALL_SCRIPT) autodist-upstream-updates $(DESTDIR)$(sbindir)/autodist-upstream-updates
|
||||
@$(INSTALL_SCRIPT) autoport-fix-environment $(DESTDIR)$(bindir)/autoport-fix-environment
|
||||
@$(INSTALL_SCRIPT) etc/autodist/scripts/* $(DESTDIR)$(configdir)/scripts/
|
||||
@$(INSTALL_SCRIPT) autodist-cgi $(DESTDIR)$(cgidir)/autodist
|
||||
@ -101,6 +102,7 @@ install-data:
|
||||
@$(INSTALL_SCRIPT) etc/cron.hourly/65-autoport-native $(DESTDIR)$(sysconfdir)/cron.hourly/
|
||||
@$(INSTALL_SCRIPT) etc/cron.hourly/66-autoport-chroot $(DESTDIR)$(sysconfdir)/cron.hourly/
|
||||
@$(INSTALL_SCRIPT) etc/cron.daily/40-autodist-cleanold $(DESTDIR)$(sysconfdir)/cron.daily/
|
||||
@$(INSTALL_SCRIPT) etc/cron.daily/40-autodist-upstream-updates $(DESTDIR)$(sysconfdir)/cron.daily/
|
||||
@$(INSTALL_DATA) autospec-conf $(DESTDIR)$(pck_statedir)/.autospec
|
||||
@touch $(DESTDIR)$(pck_statedir)/template/autoupdate/auto.success
|
||||
@touch $(DESTDIR)$(pck_statedir)/template/autoupdate/auto.skip
|
||||
|
||||
327
autodist-upstream-updates
Executable file
327
autodist-upstream-updates
Executable file
@ -0,0 +1,327 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# autodist upstream updates - find upstream packages updates from different internet resources
|
||||
# Copyright (c) 2004-2012 by Silvan Calarco <silvan.calarco@mambasoft.it>
|
||||
#
|
||||
|
||||
#[ -r /etc/sysconfig/openmamba-central ] || {
|
||||
# echo "Error: this program must be run as root; aborting." >&2
|
||||
# exit 1
|
||||
#}
|
||||
. /etc/autodist/config
|
||||
DISTROMATIC_PREFIX=/distribution/distromatic.html?
|
||||
DISTROMATIC_REPOSITORY=devel
|
||||
DISTDB=/etc/autodist/distdb
|
||||
DISTDBDIR=/etc/autodist/distdb.d
|
||||
XORG_RELEASE=current
|
||||
|
||||
[ -r $DISTDB ] && {
|
||||
. $DISTDB
|
||||
[ -d $DISTDBDIR ] && \
|
||||
for f in `ls $DISTDBDIR/*.db`; do
|
||||
. $f
|
||||
done
|
||||
}
|
||||
|
||||
[ -r $PKGLIST_FILE ] || {
|
||||
echo "Error: file $PKGLIST_FILE cannot be read; aborting." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -r $ALIASES_DB ] || {
|
||||
echo "Error: file $ALIASES_DB cannot be read; aborting." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
function usage()
|
||||
{
|
||||
echo "openmamba-upstream-updates - finds upstream packages updates from different internet resources"
|
||||
echo
|
||||
echo "Usage:"
|
||||
echo "openmamba-upstream-updates [-h|-m][-r repository][-o output_repository]"
|
||||
echo
|
||||
echo " -h: generate distromatic HTML output"
|
||||
echo " -m: show missing packages only"
|
||||
echo " -q: produces quite output"
|
||||
echo " -r repository: specify the repository (default: devel)"
|
||||
echo " -o repository: specify the repository for output data (default: same as base repository)"
|
||||
echo
|
||||
}
|
||||
|
||||
get_job_vector() {
|
||||
local JNAME=$1
|
||||
|
||||
# resolve JOB_NAME from distdb
|
||||
# note: if JOB_NAME contains a "-" it can't be a distdb JOB, so skip it
|
||||
if [ "${JNAME/-/}" = "${JNAME}" ]; then
|
||||
local jobtmpfile=`tempfile`
|
||||
# hack to get an array variable named as $j assigned to the JOB array
|
||||
echo "echo \${$JNAME[*]}" > $jobtmpfile
|
||||
JOB=(`. $jobtmpfile`)
|
||||
rm -f $jobtmpfile
|
||||
else
|
||||
JOB=()
|
||||
fi
|
||||
|
||||
if [ ${#JOB[*]} -eq 0 ]; then
|
||||
# create a default job with given JOB_NAME
|
||||
JOB=($JNAME "" "")
|
||||
elif [ ${#JOB[*]} -eq 1 ]; then
|
||||
# no variables defined, add an empty job
|
||||
JOB=(${JOB[*]} "" "")
|
||||
fi
|
||||
JOB_PKGS=(${JOB[0]//,/ })
|
||||
JOB_VARNAMES=(${JOB[1]//,/ })
|
||||
}
|
||||
|
||||
function version_compare()
|
||||
{
|
||||
local A B
|
||||
A=$1
|
||||
B=$2
|
||||
|
||||
if [[ ${1} =~ "^[0-9]+$" && ${2} =~ "^[0-9]+$" ]]; then
|
||||
if [ ${A/[a-zA-Z_]*} -gt ${B/[a-zA-Z_]*} ]; then
|
||||
return 1
|
||||
elif [ ${A/[a-zA-Z_]*} -lt ${B/[a-zA-Z_]*} ]; then
|
||||
return 2
|
||||
fi
|
||||
else
|
||||
if [[ "$A" > "$B" ]]; then
|
||||
return 1
|
||||
elif [[ "$A" < "$B" ]]; then
|
||||
return 2
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
function version_find_bigger()
|
||||
{
|
||||
local VER1 VER2 FPOS CUTVER1 CUTVER2
|
||||
|
||||
VER1=$1
|
||||
VER2=$2
|
||||
FPOS=1
|
||||
while true; do
|
||||
CUTVER1=`echo $VER1. | cut -d. -f $FPOS`
|
||||
CUTVER2=`echo $VER2. | cut -d. -f $FPOS`
|
||||
if [ "$CUTVER1" -a ! "$CUTVER2" ]; then
|
||||
return 1
|
||||
elif [ "$CUTVER2" -a ! "$CUTVER1" ]; then
|
||||
return 2
|
||||
elif [ ! "$CUTVER1" -a "$CUTVER2" ]; then
|
||||
return 0
|
||||
else
|
||||
version_compare $CUTVER1 $CUTVER2
|
||||
case $? in
|
||||
1) return 1 ;;
|
||||
2) return 2 ;;
|
||||
esac
|
||||
fi
|
||||
FPOS=`expr $FPOS + 1`
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
while [ "$1" ]; do
|
||||
case $1 in
|
||||
-h) distromatic_html=1 ;;
|
||||
-m) if [ "$distromatic_html" ]; then
|
||||
echo "Error: options -h and -m cannot be used together."
|
||||
usage
|
||||
exit 1
|
||||
else
|
||||
missing_only=1
|
||||
fi ;;
|
||||
-r) [ "$2" ] || {
|
||||
echo "Error: option -r requires repository name as parameter"
|
||||
usage
|
||||
exit 1
|
||||
}
|
||||
DISTROMATIC_REPOSITORY=$2
|
||||
shift
|
||||
;;
|
||||
-o) [ "$2" ] || {
|
||||
echo "Error: option -o requires repository name as parameter"
|
||||
usage
|
||||
exit 1
|
||||
}
|
||||
OUTPUT_REPOSITORY=$2
|
||||
shift
|
||||
;;
|
||||
-q) quiet=1 ;;
|
||||
*) echo "Error: invalid option $1."; usage; exit 1 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
[ "$OUTPUT_REPOSITORY" ] || OUTPUT_REPOSITORY=$DISTROMATIC_REPOSITORY
|
||||
|
||||
CACHE_DIR=${LOCAL_REPS_BASE_DIR}/$OUTPUT_REPOSITORY/autoupdate/
|
||||
PKGLIST_FILE=${LOCAL_REPS_BASE_DIR}/$DISTROMATIC_REPOSITORY/srcpkglist
|
||||
BUILDS_FILE=${LOCAL_REPS_BASE_DIR}/distromatic/$DISTROMATIC_REPOSITORY/builds-i586
|
||||
BUILDLIST_FILE=$CACHE_DIR/upstream-updates.in
|
||||
CONFIG_DIR=$CACHE_DIR
|
||||
UPDATES_DB=$CACHE_DIR/upstream-updates
|
||||
ALIASES_DB=$CACHE_DIR/aliases
|
||||
MANUALVER_DB=$CACHE_DIR/manualver
|
||||
|
||||
tmpfile=`mktemp -q -t autodist-upstream-updates.XXXXXXXX`
|
||||
buildstmp=`mktemp -q -t autodist-upstream-updates.XXXXXXXX`
|
||||
tail -n+2 $BUILDS_FILE > $buildstmp
|
||||
|
||||
# parse Arch Linux package list
|
||||
[ "$quiet" ] || echo "Parsing Arch Linux packages list..." >&2
|
||||
curl -s "https://www.archlinux.org/packages/?sort=-last_update&arch=i686&q=&maintainer=&last_update=&flagged=&limit=all" | \
|
||||
grep "View package details" | \
|
||||
while read line; do
|
||||
line=`echo $line | html2text -nobs`
|
||||
set -- $line
|
||||
pkg=$3
|
||||
ver=${4/-*}
|
||||
ver=${ver/*:}
|
||||
alias=`grep "^$pkg" $ALIASES_DB`
|
||||
[ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
|
||||
[ "$alias" ] && pkg=$alias
|
||||
line=`grep -i " $pkg[^-_A-Za-z0-9]" $buildstmp || grep -i " lib$pkg[^-_A-Za-z0-9]" $buildstmp`
|
||||
if [ "$line" ]; then
|
||||
pkg=${line/:*}
|
||||
[ "$pkg" -a "$ver" ] && echo "$pkg $ver ${alias/* /}" >> $tmpfile
|
||||
fi
|
||||
done
|
||||
rm -f $buildstmp
|
||||
|
||||
# parse X.org stable packages list
|
||||
[ "$quiet" ] || echo "Parsing X.org release ftp directory..." >&2
|
||||
curl -s ftp://ftp.x.org/pub/$XORG_RELEASE/src/everything/ -l | sed "s|\.tar\..*||" | sort -u |
|
||||
while read line; do
|
||||
if [ "$line" ]; then
|
||||
ver=`echo $line | sed "s|.*-||"`
|
||||
pkg="${line/-$ver}"
|
||||
alias=`grep "^$pkg " $ALIASES_DB`
|
||||
if [ ! "$alias" -a "${pkg:0:5}" == "xf86-" ]; then
|
||||
alias="$pkg xorg-drv-${pkg/xf86-}"
|
||||
else
|
||||
[ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
|
||||
fi
|
||||
[ "$pkg" -a "$ver" ] && echo "$pkg $ver ${alias/* /}" >> $tmpfile
|
||||
fi
|
||||
done
|
||||
|
||||
# parse Gnome stable packages list
|
||||
[ "$quiet" ] || echo "Parsing GNOME stable versions file..." >&2
|
||||
for f in versions-stable versions-stable-extras; do
|
||||
curl -s http://people.gnome.org/~vuntz/tmp/versions/$f | grep -v "^#" |
|
||||
while read line; do
|
||||
if [ "$line" ]; then
|
||||
IFS=":"
|
||||
set -- $line
|
||||
pkg="$2"
|
||||
ver="$3"
|
||||
alias=`grep "^$pkg " $ALIASES_DB`
|
||||
[ "$alias" ] || alias=`grep "^lib$pkg " $ALIASES_DB`
|
||||
[ "$pkg" -a "$ver" ] && echo "$pkg $ver ${alias/* /}" >> $tmpfile
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# parse distrowatch.com packages list
|
||||
[ "$quiet" ] || echo "Parsing Distrowatch packages list..." >&2
|
||||
lynx -width 300 -dump http://distrowatch.com/packages.php |
|
||||
while read line; do
|
||||
[ "`echo $line | grep "Package Version Note"`" ] && start_print=1
|
||||
[ "`echo $line | grep "____________________"`" ] && unset start_print
|
||||
[ "$start_print" ] && {
|
||||
set -- $line
|
||||
pkg="${1/\[*\]/}"
|
||||
ver="${2/\[*\]/}"
|
||||
alias=`grep "^$pkg " $ALIASES_DB`
|
||||
[ "$pkg" != "chromium" ] && \
|
||||
echo "$pkg $ver ${alias/* /}" >> $tmpfile
|
||||
}
|
||||
done
|
||||
|
||||
cat $tmpfile | sort -uf > $UPDATES_DB.tmp
|
||||
rm -f $tmpfile
|
||||
|
||||
> $UPDATES_DB
|
||||
unset lastpkg
|
||||
while read pkg ver alias; do
|
||||
if [ "$pkg" = "$lastpkg" ]; then
|
||||
# echo "Warning: duplicate found: $pkg lastver: $lastver ver: $ver" >&2
|
||||
version_find_bigger $lastver $ver
|
||||
vercmp=$?
|
||||
if [ $vercmp -eq 2 ]; then
|
||||
sed -i "/^$lastpkg $lastver /d" $UPDATES_DB
|
||||
echo "$pkg $ver $alias" >> $UPDATES_DB
|
||||
fi
|
||||
else
|
||||
echo "$pkg $ver $alias" >> $UPDATES_DB
|
||||
fi
|
||||
lastpkg=$pkg
|
||||
lastver=$ver
|
||||
done < $UPDATES_DB.tmp
|
||||
rm -f $UPDATES_DB.tmp
|
||||
|
||||
> $UPDATES_DB.missing
|
||||
> $BUILDLIST_FILE
|
||||
while read pkg ver alias; do
|
||||
unset pkgline
|
||||
unset found_manual
|
||||
unset found_alias
|
||||
pkgline=`grep "^$pkg " $MANUALVER_DB` && found_manual=1
|
||||
if [ ! "$found_manual" ]; then
|
||||
if [ "$alias" ]; then
|
||||
get_job_vector $alias
|
||||
lastjob=${#JOB_PKGS[*]}
|
||||
pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE` && found_alias=1
|
||||
else
|
||||
get_job_vector $pkg
|
||||
lastjob=${#JOB_PKGS[*]}
|
||||
pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE`
|
||||
[ "$pkgline" ] || {
|
||||
get_job_vector lib${pkg}
|
||||
lastjob=${#JOB_PKGS[*]}
|
||||
pkgline=`grep "^${JOB_PKGS[$lastjob-1]} " $PKGLIST_FILE` && {
|
||||
found_alias=1
|
||||
alias=lib${pkg}
|
||||
}
|
||||
}
|
||||
fi
|
||||
fi
|
||||
if [ "$pkgline" ]; then
|
||||
set -- $pkgline
|
||||
pkgname=$1
|
||||
pkgver=$2
|
||||
version_find_bigger $pkgver ${ver/-/.}
|
||||
vercmp=$?
|
||||
[ "$found_manual" ] && pkgname=$3
|
||||
if [ ! "$missing_only" ]; then
|
||||
[ "$found_alias" -o "$found_manual" ] && nameadd="$pkg" || unset nameadd
|
||||
unset veradd
|
||||
[ ${vercmp} = 2 ] && veradd="<font color=red>$ver</font>"
|
||||
[ ${vercmp} = 1 ] && veradd="$ver"
|
||||
# [ "${ver/-/}" != "$pkgver" ] && veradd="<font color=red>$ver</font>" || unset veradd
|
||||
[ "$veradd" -o "$nameadd" ] && {
|
||||
[ "$veradd" -a "$nameadd" ] && \
|
||||
verappend="($nameadd;$veradd)" ||
|
||||
verappend="(${nameadd}${veradd})"
|
||||
} || unset verappend
|
||||
if [ "$distromatic_html" = "1" ]; then
|
||||
echo "<a href=\"${DISTROMATIC_PREFIX}tag=${DISTROMATIC_REPOSITORY}&pkg=${JOB_PKGS[0]}.source\">${JOB_PKGS[0]} $pkgver</a> $verappend<br>"
|
||||
else
|
||||
echo "$pkg $pkgver ($ver)"
|
||||
fi
|
||||
[ $vercmp = 2 ] && {
|
||||
if [ "$found_alias" ]; then
|
||||
echo "$alias +$ver 0" >> $BUILDLIST_FILE
|
||||
else
|
||||
echo "$pkg +$ver 0" >> $BUILDLIST_FILE
|
||||
fi
|
||||
}
|
||||
fi
|
||||
else
|
||||
echo "$pkg ($ver)" >> $UPDATES_DB.missing
|
||||
fi
|
||||
done < $UPDATES_DB
|
||||
16
etc/cron.daily/40-autodist-upstream-updates
Executable file
16
etc/cron.daily/40-autodist-upstream-updates
Executable file
@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# autodist-upstream-updates daily cron script
|
||||
# (c) 2008-2012 by Silvan Calarco <silvan.calarco@mambasoft.it>
|
||||
#
|
||||
. /etc/autodist/config
|
||||
REPOSITORY=devel
|
||||
/usr/sbin/autodist-upstream-updates -h -r $REPOSITORY > $LOCAL_REPS_BASE_DIR/distromatic/$REPOSITORY/_popular.html
|
||||
|
||||
REPOSITORY=milestone1-1.1
|
||||
/usr/sbin/autodist-upstream-updates -h -r $REPOSITORY > $LOCAL_REPS_BASE_DIR/distromatic/$REPOSITORY/_popular.html
|
||||
|
||||
REPOSITORY=milestone2
|
||||
/usr/sbin/autodist-upstream-updates -h -r $REPOSITORY > $LOCAL_REPS_BASE_DIR/distromatic/$REPOSITORY/_popular.html
|
||||
|
||||
exit 0
|
||||
Loading…
Reference in New Issue
Block a user