This is an automated email from the git hooks/post-receive script.

robert.savoye pushed a commit to branch md5sums in repository toolchain/abe.

commit 3d390f9e5151ccbbe2dcee2bc37b827b749c424f Author: Rob Savoye rob.savoye@linaro.org Date: Mon Aug 10 14:20:16 2015 -0600

Remove the need for an md5sums file. Remote URLS are now in sources.conf.

Change-Id: I12902a1f416afdc9c4aa2900eee106a812aeb1b5 --- abe.sh | 39 +------------------ config/sources.conf | 8 ++++ lib/checkout.sh | 25 ++++++------ lib/common.sh | 28 ++------------ lib/depend.sh | 6 --- lib/fetch.sh | 109 +++++++++++++++++----------------------------------- 6 files changed, 61 insertions(+), 154 deletions(-)

diff --git a/abe.sh b/abe.sh index 3a5d494..8a4fb2a 100755 --- a/abe.sh +++ b/abe.sh @@ -350,8 +350,8 @@ OPTIONS # Matches an identifier in sources.conf: glibc=glibc.git

- # Matches a tar snapshot in md5sums: - glibc=eglibc-linaro-2.17-2013.07 + # Matches a tar snapshot + glibc=eglibc-linaro-2.17-2013.07.tar.xz

# Direct URL: glibc=git://sourceware.org/git/glibc.git @@ -385,8 +385,6 @@ ABE GENERATED FILES AND DIRECTORIES snapshots/infrastructure Infrastructure (non-distributed) sources are stored here.

- snapshots/md5sums The snapshots file of maintained package tarballs. - AUTHOR Rob Savoye rob.savoye@linaro.org Ryan S. Arnold ryan.arnold@linaro.org @@ -653,33 +651,6 @@ check_optional_directive() return 0 }

-# This gets a list from a remote server of the available tarballs. We use HTTP -# instead of SSH so it's more accessible to those behind nasty firewalls. -# base - already checkout out source trees -# snapshots - tarballs of various source snapshots -# prebuilt - prebuilt executables -get_list() -{ - echo "Get version list for $1..." - - # http://abe.validation.linaro.org/snapshots - case $1 in - testcode|t*) - testcode="`grep testcode ${local_snapshots}/testcode/md5sums | cut -d ' ' -f [...] - echo "${testcode}" - ;; - snapshots|s*) - snapshots="`egrep -v ".asc|.diff|.txt|xdelta|base|infrastructure|testcode" [...] - echo "${snapshots}" - ;; - infrastructure|i*) - infrastructure="`grep infrastructure ${local_snapshots}/md5sums | cut -d ' ' [...] - echo "${infrastructure}" - ;; - esac - return 0 -} - # Get some info on the build system # $1 - If specified, it's the hostname of the remote system get_build_machine_info() @@ -1195,12 +1166,6 @@ if test x"${force}" = xyes -a x"$supdate" = xno; then echo " Using "--force" overrides "--disable update". Sources will [...] fi

-timeout_save=${wget_timeout} -wget_timeout=10 -# Get the md5sums file, which is used later to get the URL for remote files -fetch_md5sums -wget_timeout=${timeout_save} - if test ! -z "${do_makecheck}"; then # If we encounter 'all' in ${do_makecheck} anywhere we just overwrite # runtests with ${all_unit_tests} and ignore the rest. diff --git a/config/sources.conf b/config/sources.conf index 3adfaa7..8884cb2 100644 --- a/config/sources.conf +++ b/config/sources.conf @@ -3,6 +3,14 @@ # NOTE: use spaces instead of TABs #

+# These are infrastructure files, and commonly tarballs +gmp http://148.251.136.42/snapshots-ref/infrastructure/ +mpc http://148.251.136.42/snapshots-ref/infrastructure/ +mpfr http://148.251.136.42/snapshots-ref/infrastructure/ +linux http://148.251.136.42/snapshots-ref/infrastructure/ +installjammer http://148.251.136.42/snapshots-ref/infrastructure/ + +# These URLS are for repositories, ie... git dejagnu.git http://git.linaro.org/git/toolchain/dejagnu.git kernel http://git.linaro.org/git/kernel/linux-linaro-stable.git linux-linaro http://git.linaro.org/git/kernel/linux-linaro-stable.git diff --git a/lib/checkout.sh b/lib/checkout.sh index 489585c..49b6812 100644 --- a/lib/checkout.sh +++ b/lib/checkout.sh @@ -41,12 +41,6 @@ checkout_infrastructure() return 1 fi

- # This shouldn't happen, but it's nice for regression verification. - if test ! -e ${local_snapshots}/md5sums; then - error "Missing ${local_snapshots}/md5sums file needed for infrastructure libraries." - return 1 - fi - # We have to grep each dependency separately to preserve the order, as # some libraries depend on other libraries being bult first. Egrep # unfortunately sorts the files, which screws up the order. @@ -59,7 +53,7 @@ checkout_infrastructure() mpfr) version=${mpfr_version} ;; mpc) version=${mpc_version} ;; gmp) version=${gmp_version} ;; - dejagnu) version=${dejagnu_version} ;; + dejagnu) version=${dejagnu_version} ; continue ;; *) error "config/infrastructure.conf contains an unknown dependency: $i" return 1 @@ -81,9 +75,8 @@ checkout_infrastructure() return 1 fi

- # Hopefully we only download the exact match for each one. Depending # how vague the user is it might download multiple tarballs. - files="${files} `grep /${version} ${local_snapshots}/md5sums | cut -d ' ' -f3 | uniq`" + files="${files} ${version}" unset version done

@@ -108,11 +101,15 @@ checkout_infrastructure() return 1 fi else - fetch ${gitinfo} - if test $? -gt 0; then - error "Couldn't fetch tarball ${gitinfo}" - return 1 - fi + for i in xz bz2 gz; do + fetch ${gitinfo}.$i + if test $? -gt 0; then + warning "Couldn't fetch tarball ${gitinfo}.$i" + continue + fi + break + done + notice "Fetched tarball ${gitinfo}.$i" extract ${gitinfo} if test $? -gt 0; then error "Couldn't extract tarball ${gitinfo}" diff --git a/lib/common.sh b/lib/common.sh index 51dc1b6..faff993 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -407,35 +407,15 @@ find_snapshot() dir="" fi

- #rm -f ${local_snapshots}/md5sums - #fetch_http md5sums - #fetch_rsync ${remote_snapshots}/md5sums - - # Search for the snapshot in the md5sum file, and filter out anything we don't want. - snapshot="`grep $1 ${local_snapshots}/md5sums | egrep -v ".asc|.diff|.txt|x [...] + local name="`echo $1 | cut -d '-' -f 1`" + local snapshot="`grep ^${name} ${sources_conf}| tr -s ' ' | cut -d ' ' -f 2`" if test x"${snapshot}" != x; then - if test `echo "${snapshot}" | grep -c $1` -gt 1; then - warning "Too many results for $1!" - echo "${snapshot}" - return 1 - fi - echo "${snapshot}" + echo "$1.tar" return 0 - fi - -# snapshot="`grep $1 ${local_snapshots}/${dir}md5sums | cut -d ' ' -f 3`" - if test x"${snapshot}" = x; then + else warning "No results for $1!" return 1 fi - if test `echo "${snapshot}" | grep -c $1` -gt 1; then - warning "Too many results for $1!" - echo "${snapshot}" - return 1 - fi - - echo ${snapshot} - return 0 }

# Get the full path or URL to checkout or download sources of a toolchain diff --git a/lib/depend.sh b/lib/depend.sh index 7a33b7e..294d860 100755 --- a/lib/depend.sh +++ b/lib/depend.sh @@ -148,12 +148,6 @@ infrastructure() return 1 fi

- # This shouldn't happen, but it's nice for regression verification. - if test ! -e ${local_snapshots}/md5sums; then - error "Missing ${local_snapshots}/md5sums file needed for infrastructure libraries." - return 1 - fi - # We have to grep each dependency separately to preserve the order, as # some libraries depend on other libraries being bult first. Egrep # unfortunately sorts the files, which screws up the order. diff --git a/lib/fetch.sh b/lib/fetch.sh index 3559f1c..5cd7325 100755 --- a/lib/fetch.sh +++ b/lib/fetch.sh @@ -16,64 +16,29 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #

-# The fetch_md5sums() function is special because the md5sums file is used by -# ABE for knowing where to fetch other files, i.e. it's used by fetch(). This -# function should only be called once at the start of every ABE run. This -# function does not respect supdate=no. It is harmless to download new versions -# of md5sums. -fetch_md5sums() -{ - if test "${git_reference_dir:+set}" = "set" -a -e "${git_reference_dir}/md5sum [...] - # The user specified that they want to fetch from the reference dir. This - # will always fetch if the version in the reference dir is newer. - fetch_reference md5sums - else - # The fetch_http function will always attempt to fetch the remote file - # if the version on the server is newer than the local version. - fetch_http md5sums - fi - - # If the fetch_*() fails we might have a previous version of md5sums in - # ${local_snapshots}. Use that, otherwise we have no choice but to fail. - if test ! -s ${local_snapshots}/md5sums; then - return 1 - fi - return 0 -} - # Fetch a file from a remote machine. All decision logic should be in this # function, not in the fetch_<protocol> functions to avoid redundancy. fetch() { -# trace "$*" + trace "$*" + if test x"$1" = x; then error "No file name specified to fetch!" return 1 fi

- # Peel off 'infrastructure/' local file="`basename $1`" - - # The md5sums file should have been downloaded before fetch() was - # ever called. - if test ! -e "${local_snapshots}/md5sums"; then - error "${local_snapshots}/md5sums is missing." - return 1 - fi - - # We can grab the full file name by searching for it in the md5sums file. - # Match on the first hit. This might be prepended with the - # 'infrastructure/' directory name if it's an infrastructure file. - local getfile="$(grep ${file} -m 1 ${local_snapshots}/md5sums | cut -d ' ' -f 3)" + local name="`basename $1 | cut -d '-' -f 1`" + local getfile="`grep ^${name} ${sources_conf}| tr -s ' ' | cut -d ' ' -f 2`/${file}" if test x"${getfile}" = x; then - error "${file} not in md5sum!" + error "${file} not in ${sources_conf}!" return 1 fi

# Forcing trumps ${supdate} and always results in sources being updated. if test x"${force}" != xyes; then if test x"${supdate}" = xno; then - if test -e "${local_snapshots}/${getfile}"; then + if test -e "${getfile}"; then notice "${getfile} already exists and updating has been disabled." return 0 fi @@ -100,8 +65,13 @@ fetch() return 1 fi

- # Fetch only supports fetching files which have an entry in the md5sums file. - # An unlisted file should never get this far anyway. + # download from the file server or copy the file from the reference dir + fetch_${protocol} ${getfile}.asc + if test $? -gt 0; then + warning "No md5sum file for ${getfile}.asc" + return 1 + fi + dryrun "check_md5sum ${getfile}" if test $? -gt 0 -a x"${force}" != xyes; then error "md5sums don't match!" @@ -209,20 +179,10 @@ extract() # on the server is newer than the destination file. fetch_http() { -# trace "$*" + trace "$*"

local getfile=$1

- # This provides the infrastructure/ directory if ${getfile} contains it. - local dir="`dirname $1`/" - if test x"${dir}" = x"./"; then - local dir="" - else - if test ! -d ${local_snapshots}/${dir}; then - mkdir -p ${local_snapshots}/${dir} - fi - fi - # You MUST have " " around ${wget_bin} or test ! -x will # 'succeed' if ${wget_bin} is an empty string. if test ! -x "${wget_bin}"; then @@ -230,27 +190,35 @@ fetch_http() return 1 fi

+ # Extract the last directory + local dir="`dirname ${getfile}`" + local dir="`basename ${dir}`" + if test x"${dir}" = x"snapshots"; then + dir="" + fi + # Force will cause us to overwrite the version in local_snapshots unconditionally. local overwrite_or_timestamp= if test x${force} = xyes; then # This is the only way to explicitly overwrite the destination file # with wget. overwrite_or_timestamp="-O ${local_snapshots}/${getfile}" - notice "Downloading ${getfile} to ${local_snapshots} unconditionally." + notice "Downloading ${getfile} to ${local_snapshots}/${dir} unconditionally." else # We only every download if the version on the server is newer than # the local version. overwrite_or_timestamp="-N" - notice "Downloading ${getfile} to ${local_snapshots} if version on server [...] + notice "Downloading ${getfile} to ${local_snapshots}/${dir} if version on [...] fi

# NOTE: the timeout is short, and we only try twice to access the # remote host. This is to improve performance when offline, or # the remote host is offline. - dryrun "${wget_bin} ${wget_quiet:+-q} --timeout=${wget_timeout}${wget_progress [...] - if test x"${dryrun}" != xyes -a ! -s ${local_snapshots}/${getfile}; then - warning "downloaded file ${getfile} has zero data!" - return 1 + dryrun "${wget_bin} ${wget_quiet:+-q} --timeout=${wget_timeout}${wget_progress [...] + local name="`basename ${getfile}`" + if test x"${dryrun}" != xyes -a ! -s ${local_snapshots}/${dir}/${name}; then + warning "downloaded file ${getfile},$i has zero data!" + return 1 fi

return 0 @@ -295,25 +263,20 @@ fetch_reference() # the actual file's downloaded md5sum. check_md5sum() { -# trace "$*" - - # ${local_snapshots}/md5sums is a pre-requisite. - if test ! -e ${local_snapshots}/md5sums; then - error "${local_snapshots}/md5sums is missing." - return 1 - fi + trace "$*"

- local entry= - entry=$(grep "${1}" ${local_snapshots}/md5sums) - if test x"${entry}" = x; then - error "No md5sum entry for $1!" + local dir="`dirname $1`" + local dir="/`basename ${dir}`" + local entry="`basename $1.asc`" + if test ! -e "${local_snapshots}${dir}/${entry}"; then + error "No md5sum file for ${entry}!" return 1 fi

# Ask md5sum to verify the md5sum of the downloaded file against the hash in # the index. md5sum must be executed from the snapshots directory. - pushd ${local_snapshots} &>/dev/null - dryrun "echo "${entry}" | md5sum --status --check -" + pushd ${local_snapshots}${dir} &>/dev/null + dryrun "md5sum --status --check ${entry}" md5sum_ret=$? popd &>/dev/null