#!/bin/bash # libnetwork.lib -- @package@ library to get info from internet repositories # Copyright (C) 2008,2010,2012 Davide Madrisan [ -z "$BASH" ] || [ ${BASH_VERSION:0:1} -lt 3 ] && { echo $"this script requires bash version 3 or better" >&2 && exit 1; } me=(${0##*/} "@version@" "@date@") [ -r @libdir@/libmsgmng.lib ] || { echo "$me: "$"library not found"": @libdir@/libmsgmng.lib" 1>&2 exit 1; } . @libdir@/libmsgmng.lib if [[ -z "$LANG" && -r /etc/sysconfig/i18n ]]; then . /etc/sysconfig/i18n [ "$LANG" ] && export LANG fi TEXTDOMAIN="libnetwork"; export TEXTDOMAIN # check if all the needed tools are available # note: 'host' has been removed from this list because the package # bind-utils is not available in early stage platform development for tool in curl mktemp; do [ "$(type -p curl)" ] || notify.error $"utility not found"": \`curl'" done # function repository.is_reachable() # check if the server '$1' is reachable (test = DNS lookup query) # args: # $1 : server name function repository.is_reachable() { local ftpserver="${1/[a-z]*\:\/\//}" [ "$ftpserver" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (#1)" if [ "$(type -p host)" ]; then host -W3 $ftpserver >&/dev/null else notify.warning $"utility not found"": \`host'" fi return $? } # function repository.get_srpm_pckname_from_site() # get the list of SRPM packages by looking at the curl dump of the # ftp or html site # args: # --ftp | --html # $1 : package name or regexpr to look for # $2 : a (temporary) file containing the curl dump # return value: # the list of srpm files function repository.get_srpm_pckname_from_site() { local ARGS ARGS=`LC_ALL=C getopt \ -o fh --long ftp,html \ -n "$FUNCNAME" -- "$@"` [ $? = 0 ] || notify.error $"(bug)"" -- $FUNCNAME: "$"\`getopt' error" local is_html="0" is_ftp="0" eval set -- "$ARGS" while :; do case "$1" in -f|--ftp) is_ftp="1" ;; -h|--html) is_html="1" ;; --) shift; break ;; *) notify.error $"\ (bug)"" -- $FUNCNAME: "$"\`getopt' error: bad command \`$1'" ;; esac shift done [ "$is_ftp" != "$is_html" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"usage error (--ftp/--html)" local pck_name="$1" local infile="$2" [ "$pck_name" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (#1)" [ "$infile" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (#2)" notify.debug "$FUNCNAME: pck_name = \"$pck_name\"" case "$pck_name" in */*) notify.error $"\ (bug)"" -- $FUNCNAME: "$"'\\' characters detected in \$pck_name" ;; esac notify.debug "$FUNCNAME: infile = \"$infile\"" local regexpr [[ "$pck_name" =~ ^[a-zA-Z0-9-]+$ ]] && \ regexpr="^$pck_name-[^-]\+-[^-]\+$" || regexpr="$pck_name" notify.debug "$FUNCNAME: regexpr = \"$regexpr\"" if [ "$is_html" = "1" ]; then notify.debug "$FUNCNAME: parsing HTML dump..." local pckname_from_HTML=($(sed -n "\ # remove HTML tags --> get a better input /0 otherwise # example of output: # got_RPM_FILENAME = ( # i586:mbrowse-0.3.1-1qilnx.i586.rpm # i586:mbrowse-contrib-0.3.1-1qilnx.noarch.rpm # ppc:mbrowse-contrib-0.3.1-1qilnx.noarch.rpm ) function repository.get_RPMS_name() { ARGS=`LC_ALL=C getopt \ -o a:x:c:p:u: \ --long archlist:,exclude:,target-cpu:,proxy:,proxy-user:,user: \ -n "$FUNCNAME" -- "$@"` [ $? = 0 ] || notify.error $"(bug)"" -- $FUNCNAME: "$"\`getopt' error" eval set -- "$ARGS" local archlist excludeopt proxy proxy_user user target_cpu while :; do case $1 in -a|--archlist) archlist="$2"; shift notify.debug "$FUNCNAME: archlist = \"$archlist\"" ;; -x|--exclude) excludeopt="$2"; shift notify.debug "$FUNCNAME: excludeopt = \"$excludeopt\"" ;; -c|--target-cpu) target_cpu="$2"; shift notify.debug "$FUNCNAME: target_cpu = \"$target_cpu\"" ;; -p|--proxy) proxy="$2"; shift notify.debug "$FUNCNAME: proxy = \"$proxy\"" ;; -u|--proxy-user) proxy_user="$2"; shift notify.debug "$FUNCNAME: proxy_user = \"$proxy_user\"" ;; -l|--user) user="$2"; shift notify.debug "$FUNCNAME: user = \"$user\"" ;; --) shift; break ;; *) notify.error $"\ (bug)"" -- $FUNCNAME: "$"\`getopt' error" ;; esac shift done local scanurl="$1" [ "$scanurl" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (#1)" shift local pck_names="$@" notify.debug "$FUNCNAME: pck_names = \"${pck_names[@]}\"" [ "$pck_names" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (#2)" [ "$archlist" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"no list of architectures set"" (\$archlist)" local flist=`mktemp -q -t flist.XXXXXXXX` || notify.error $"can't create temporary files" local curr_arch errors pck_name curr_match let "errors = 0" local curr_arch_list case "$target_cpu" in "") curr_arch_list=( $archlist ) ;; "noarch") #curr_arch_list=( ${arch_noarch_upload[*]} ) # FIXME : if the noarch directory cannot be found a warning # message is displayed curr_arch_list=( $archlist noarch ) ;; *) # if the architecture is set by the user do ignore packages # built for other architectures (aka: please do not remove # packages built for other architectures!) curr_arch_list="$target_cpu" ;; esac notify.debug "\ $FUNCNAME: curr_arch_list = ( ${curr_arch_list[*]} )" # FIXME: 'curl_opts_netlink' is an external variable local curlopts="\ -L --silent --list-only $curl_opts_netlink -o $flist\ ${user:+ --user $user}\ ${proxy:+ --proxy $proxy}${proxy_user:+ --proxy-user $proxy_user}" local curlopts_debug="\ -L --silent --list-only $curl_opts_netlink -o $flist\ ${user:+ --user ***}\ ${proxy:+ --proxy ***}${proxy_user:+ --proxy-user ***}" got_RPM_FILENAME= for curr_arch in ${curr_arch_list[*]}; do currurl="$(repository.resolveURL "$scanurl" "$curr_arch")" #[ "$currurl" ] || notify.error $"(bug)"" -- $FUNCNAME: currurl = \"\"" # get the list of files from the repository notify.note $"getting list of files from"" \`${NOTE}$currurl/${NORM}'..." > $flist # NOTE: the trailing slash is necessary, do not remote it notify.debug "$FUNCNAME: curl $curlopts_debug --url $currurl/" curl $curlopts --url $currurl/ let "retval = $?" notify.debug "$FUNCNAME: curl return code: $retval" case "$retval" in 0) ;; 6) notify.warning $"couldn't resolve host" ;; 7) notify.warning $"failed to connect to host" ;; 56) notify.warning $"failure in receiving network data" ;; *) notify.warning $"curl error (exit code: $retval)" ;; esac if [ -s "$flist" ]; then sed -i 's/\x0D$//' $flist case $currurl in ftp://*|http://*) for pck_name in ${pck_names[@]}; do got_RPM_FILENAME=( ${got_RPM_FILENAME[@]} $(\ if [ -n "$excludeopt" ]; then sed -n "\ / $destdir/$file_name" pushd $destdir >/dev/null local retval notify.debug "curl $curl_options\ ${proxy:+ --proxy $proxy}${proxyuser:+ --proxy-user $proxy_user} $file_url" curl $curl_options \ ${proxy:+ --proxy $proxy} ${proxyuser:+ --proxy-user $proxy_user} \ "$file_url" retval=$? notify.debug "$FUNCNAME: curl return code: $retval" case "$retval" in 0) ;; 6) notify.warning $"couldn't resolve host" ;; 7) notify.warning $"failed to connect to host" ;; 56) notify.warning $"failure in receiving network data" ;; *) notify.warning $"curl error (exit code: $retval)" ;; esac [ -e $destdir/$file_name ] || if [ "$exitonerror" = 1 ]; then notify.error $"cannot download"": \`$file_name'" else popd >/dev/null notify.warning $"cannot download"": \`$file_name'" return 1 fi popd >/dev/null } # function curl.upload # upload a file using curl # function curl.upload() { local ARGS ARGS=`LC_ALL=C getopt \ -o f:l:o:p:u:x \ --long upload-file:,user:,options:,proxy:,proxy-user:,\ repository-alias:,exit-on-err,debug-unsecure \ -n "$FUNCNAME" -- "$@"` [ $? = 0 ] || notify.error $"(bug)"" -- $FUNCNAME: "$"\`getopt' error" local upfile user options proxy proxy_user local exitonerror=0 local debug_unsecure=0 eval set -- "$ARGS" while :; do case "$1" in -f|--upload-file) upfile="$2"; shift ;; -l|--user) user="$2"; shift #notify.debug "$FUNCNAME: user = \"$user\"" ;; -o|--options) options="$2"; shift #notify.debug "$FUNCNAME: options = \"$options\"" ;; -p|--proxy) proxy="$2"; shift #notify.debug "$FUNCNAME: proxy = \"$proxy\"" ;; -u|--proxy-user) proxy_user="$2"; shift #notify.debug "$FUNCNAME: proxy_user = \"$proxy_user\"" ;; -x|--exit-on-err) exitonerror="1" #notify.debug "$FUNCNAME: exitonerror = \"$exitonerror\"" ;; --debug-unsecure) debug_unsecure="1" ;; --) shift; break;; *) notify.error $"\ (bug)"" -- $FUNCNAME: "$"\`getopt' error: bad command \`$1'" ;; esac shift done #notify.debug "$FUNCNAME: debug_unsecure = \"$debug_unsecure\"" local destfile="$1" [ "$destfile" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (#1)" [ -e "$upfile" ] || notify.error $"package not found"": \`$upfile'" # add an extra newline to workaroung a bug in curl notify.note " * ${NOTE}$upfile${NORM} --> $destfile" # option: --write-out [%{url_effective}]\\n # the option '--progress-bar' is currently buggy in curl local curr_curl_cmd="$options --progress-bar --fail -o /dev/null \ ${proxy:+ --proxy $proxy}${proxyuser:+ --proxy-user $proxy_user} \ --upload-file $upfile $destfile" if [ "$debug_unsecure" = "1" ]; then notify.debug "curl --user $user $curr_curl_cmd" else notify.debug "curl --user ***:*** $curr_curl_cmd" fi curl --user $user $curr_curl_cmd [ $? -eq 0 ] || { [ "$exitonerror" = 1 ] && exit 1 || return 1; } } # function curl.ftp_command # execute an ftp command (creating a directory, renamig a file, # deleting) using curl # function curl.ftp_command() { local ARGS ARGS=`LC_ALL=C getopt \ -o l:o:p:u:a: \ --long user:,options:,proxy:,proxy-user:,debug-unsecure,\ action:,directory:,rename-from:,rename-to:,delete:,ftp-passive \ -n "$FUNCNAME" -- "$@"` [ $? = 0 ] || notify.error $"(bug)"" -- $FUNCNAME: "$"\`getopt' error" local user options proxy proxy_user ftp_passive local action directory fromfile tofile deletefile local exitonerror=0 eval set -- "$ARGS" while :; do case "$1" in -l|--user) user="$2"; shift #notify.debug "$FUNCNAME: user = \"$user\"" ;; -o|--options) options="$2"; shift #notify.debug "$FUNCNAME: options = \"$options\"" ;; -p|--proxy) proxy="$2"; shift #notify.debug "$FUNCNAME: proxy = \"$proxy\"" ;; -u|--proxy-user) proxy_user="$2"; shift #notify.debug "$FUNCNAME: proxy_user = \"$proxy_user\"" ;; --debug-unsecure) debug_unsecure="1" ;; -a|--action) action="$2"; shift ;; --directory) directory="$2"; shift ;; --rename-from) fromfile="$2"; shift ;; --rename-to) tofile="$2"; shift ;; --delete) deletefile="$2"; shift ;; --ftp-passive) ftp_passive="--ftp-pasv" ;; -x|--exit-on-err) exitonerror="1" ;; --) shift; break;; *) notify.error $"\ (bug)"" -- $FUNCNAME: "$"\`getopt' error: bad command \`$1'" ;; esac shift done local backup_ftp_server="$1" [ "$backup_ftp_server" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (#1)" [ "$ftp_passive" ] && options="$options $ftp_passive" # NOTE: see RFC959 for the sintax of the 'quote' commands # curl --ftp-pasv --user user:password \ # -Q "MKD /devel/old/prova.dir" \ # -Q "RNFR /devel/old/prova.tmp" \ # -Q "RNTO /devel/old/prova.dir/prova.tmp" ftp.linuxdistro.org case "$action" in mkdir) [ "$directory" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (--directory)" notify.debug "\ curl $options --user \"***:***\" --quote \"MKD $directory\" $1" curl $options --user "$user" --quote "MKD $directory" $1 &>/dev/null ;; rename-file) [ "$fromfile" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (--rename-from)" [ "$tofile" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (--rename-to)" notify.debug "curl $options --user \"***:***\"\ --quote \"RNFR $fromfile\" --quote \"RNTO $tofile\" $1" curl $options --user "$user" \ --quote "RNFR $fromfile" --quote "RNTO $tofile" $1 &>/dev/null ;; delete) [ "$deletefile" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (--delete)" # example: # curl -v --ftp-pasv --user "$user" \ # --quote "CWD SRPMS.base" \ # --quote "DELE filename-0.0.1-1distro.src.rpm" \ # ftp://ftpaccounts.openmamba.org # -or- # curl -v --ftp-pasv --user "$user" \ # --quote "DELE SRPMS.base/filename-0.0.1-1distro.src.rpm" \ # ftp://ftpaccounts.openmamba.org notify.debug "\ curl $options --user \"***:***\" --quote \"DELE $deletefile\" $1" curl $options --user "$user" --quote "DELE $deletefile" $1 &>/dev/null ;; *) notify.error $"\ (bug)"" -- $FUNCNAME: "$"unknown action"": ($action)" esac retval=$? case "$retval" in 0) ;; 6) notify.warning $"couldn't resolve host" ;; 7) notify.warning $"failed to connect to host" ;; *) notify.warning $"curl error (exit code: $retval)" ;; esac [[ "$retval" != "0" && "$exitonerror" = 1 ]] && exit 1 } # function git.create_tarball # clone a git repository and convert is into a tarball file # # args: # -d,--destdir : target directory # $@ : git repository # # return value: # 0 # # set: pck_version, spec_source, spec_source_comment, pck_tarball function git.create_tarball() { local ARGS ARGS=`LC_ALL=C getopt \ -o d:p:u: \ --long destdir:,proxy:,proxy-user: \ -n "$FUNCNAME" -- "$@"` [ $? = 0 ] || notify.error $"(bug)"" -- $FUNCNAME: "$"\`getopt' error" local destdir # FIXME: add support for proxy # HINT: git config --global http.proxy http://user:pwd@proxy.srv.com:port # see: http://bardofschool.blogspot.fr/2008/11/use-git-behind-proxy.html local proxy proxy_user eval set -- "$ARGS" while :; do case "$1" in -d|--destdir) destdir="$2"; shift notify.debug "$FUNCNAME: destdir = \"$destdir\"" ;; -p|--proxy) proxy="$2"; shift notify.debug "$FUNCNAME: proxy = \"$proxy\"" ;; -u|--proxy-user) proxy_user="$2"; shift notify.debug "$FUNCNAME: proxy_user = \"$proxy_user\"" ;; --) shift; break;; *) notify.error $"\ (bug)"" -- $FUNCNAME: "$"\`getopt' error: bad command \`$1'" ;; esac shift done local git_repository="$1" notify.debug "$FUNCNAME: git_repository = \"$git_repository\"" [ "$git_repository" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (#1)" [ "$destdir" ] || notify.error $"\ (bug)"" -- $FUNCNAME: "$"missing mandatory arg"" (--destdir)" which git &>/dev/null || notify.error $"utility not found"": \`git'" local tmpgitdir=$(mktemp -q -d -t tmpgit.XXXXXXXX) [ $? -eq 0 ] || notify.error $"can't create temporary files" notify.debug "$FUNCNAME: tmpgitdir = $tmpgitdir" pushd $tmpgitdir &>/dev/null # create a tarball by cloning the git repository notify.note \ "${NOTE}"$"cloning git repository"" <$git_repository>${NORM}""..." git clone --quiet "$git_repository" [ $? -eq 0 ] || { popd &>/dev/null rm -fr $tmpgitdir notify.error $"cannot clone git repository"; } pck_tarball="$(find -mindepth 1 -maxdepth 1 -type d -printf "%f")" # remove git files rm -fr $pck_tarball/.git rm -f $pck_tarball/.gitignore pck_version="$(date "+%Y%m%dgit")" mv $pck_tarball ${pck_tarball}-${pck_version} pck_tarball="${pck_tarball}-${pck_version}" # create a compressed tarball tar -cvf ${pck_tarball}.tar $pck_tarball/ >/dev/null && bzip2 ${pck_tarball}.tar && mv ${pck_tarball}.tar.bz2 $destdir/ [ $? -eq 0 ] || { popd &>/dev/null rm -fr $tmpgitdir notify.error $"\ an error occurred while creating"": ${pck_tarball}.tar.bz2"; } spec_source="${pck_tarball}.tar.bz2" spec_source_comment="## GITSOURCE $git_repository" pck_tarball="$destdir/${spec_source}" popd &>/dev/null rm -fr $tmpgitdir return 0 }