mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2024-11-27 02:10:55 +08:00
Code Issues Packages Projects Releases Wiki Activity
edcbc9966f
hdf5/bin/snapshot

835 lines
23 KiB
Bash
Executable File
Raw Blame History

#!/bin/sh
#
# Copyright by The HDF Group.
# Copyright by the Board of Trustees of the University of Illinois.
# All rights reserved.
#
# This file is part of HDF5. The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the COPYING file, which can be found at the root of the source code
# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
# If you do not have access to either file, you may request a copy from
# help@hdfgroup.org.
# This script should be run nightly from cron. It checks out the source
# from the source repository and compares it against the previous
# snapshot. If anything significant changed then a new snapshot is
# created, the minor version number is incremented, and the change is
# checked back into the source repository.
#
# function definitions
TIMESTAMP()
{
echo "=====" "$1": "`date`" "====="
}
EXIT_BANNER()
{
TIMESTAMP "Exit $PROGNAME with status=$?"
}
# Show current total disk usage.
DISKUSAGE()
{
du -ks | \
( read x y; echo "Disk Usage=$x KB" )
}
# function provided for testing software downloaded as tar files. A version of
# this function that properly extracts the downloaded files can be provided in
# the snapshots-${sw}-overrides file.
EXTRACT()
{
echo "Error: ${SWVERSION} is in source repository - does not need extraction."
}
# Standard procedure for checking out or updating source code from an hdfgroup
# git repository. Override the function for other repositories or procedures.
SOURCE_CHECKOUT()
{
if test -n $GIT_URL; then
if [ -n "$AUTOGEN" ]; then
echo "Creating fresh clone of $GIT_URL in $BASEDIR/current_src"
# Check out the current version from source repository.
(cd $BASEDIR; rm -rf current_src
if test -z $GIT_BRANCH; then
echo "Testing empty branch $GIT_BRANCH."
git clone $GIT_URL current_src
else
echo "Testing branch $GIT_BRANCH."
git clone $GIT_URL -b $GIT_BRANCH current_src
fi
) || exit 1
else
echo "Creating fresh clone of $GIT_URL in $BASEDIR/current"
# Check out the current version from source repository.
(cd $BASEDIR; rm -rf current
if test -n $GIT_BRANCH; then
git clone $GIT_URL -b $GIT_BRANCH current
else
git clone $GIT_URL current
fi ) || exit 1
fi
else
echo "Warning! Source directory ("current") is not checked out from git."
fi
}
# Standard procedure for running the configure command in a build (test)
# directory
RUNCONFIGURE()
{
if [ "${CURRENT}" != "${TESTDIR}" -a "$CPSRC" = "yes" ]; then
echo "Copying source files to ${TESTDIR}."
cp -pr ${CURRENT}/* ${TESTDIR}
cd ${TESTDIR}
./${CONFIGURE}
elif [ -n "${AUTOGEN}" ]; then
${CURRENTSRC}/${CONFIGURE}
else
${CURRENT}/${CONFIGURE}
fi
}
# Sometimes "make distclean" doesn't adequately remove files from the previous
# build. If a build (test) directory was used, its contents can be entirely
# deleted to provide a clean start. If the test is building in the source
# directory, the contents can't be deleted, so run "make distclean".
DISTCLEAN()
{
if [ "${srcdir}" = "yes" -a -n "${SRCDIRNAME}" -a -d ${BASEDIR}/TestDir/${SRCDIRNAME} ]; then
echo "Remove contents of $SRCDIRNAME.\n"
rm -rf ${BASEDIR}/TestDir/${SRCDIRNAME}/*
else
echo "$MAKE distclean"
(cd ${TESTDIR} && ${MAKE} distclean)
fi
}
# Several of the software packages tested do not support make check-install.
# Those that support it should have a version of this function in their
# override with the following lines:
# TIMESTAMP "check-install $1"
# ${MAKE} check-install $1
CHECKINSTALL()
{
echo "check-install is not supported for ${SWVERSION}"
}
# Function for hdf4 and hdf5 to override to check in changes after snapshot.
# Safety measure to avoid unintended checkins to other repositories.
COMMITSNAPSHOT()
{
echo "original hdf5 script committed code changes back into git."
}
DISPLAYUSAGE()
{
set -
cat <<EOF
Usage: $PROGNAME [all] [checkout] [ftp <URL> [diff] [test] [srcdir] [release] [help]
[clean] [distclean] [echo] [deploy <dir>] [deploydir <dir>]
[zlib <zlib_path>] [releasedir <dir>] [srcdirname <dir>] [check-vfd]
[check-vol]
[exec <command>] [module-load <module-list>] [op-configure <option>]
[--<option>]
all: Run all commands (checkout, test & release)
[Default is all]
checkout: Run source checkout
diff: Run diff on current and previous versions. Exit 0 if
no significant differences are found. Otherwise, non-zero.
deploy: deploy binary to directory <dir>
deploydir: use <dir> as the default directory for deployment
test: Run test
release: Run release
clean: Run make clean
distclean:Run make distclean
echo: Turn on echo mode (set -x)
setenv <name> <value>:
Set environment variable <name> to <value>.
setenvN <N> <name> <value> ...:
Set environment variable with <N> values.
E.g., setenvN 3 x a b c is same as setenv x="a b c".
srcdir: Use srcdir option (does not imply other commands)
"snapshot srcdir" is equivalent to "snapshot srcdir all"
"snapshot srcdir checkout" is equivalent to "snapshot checkout"
srcdirname <dir>:
Use <dir> as the srcdir testing directory if srcdir is choosen.
If <dir> starts with '-', it is append to the default name
E.g., "snapshot srcdir srcdirname -xx" uses hostname-xx
[Default is hostname]
help: Print this message
echo: Turn on shell echo
zlib <zlib_path>:
Use <zlib_path> as the ZLIB locations
[Default is $ZLIB_default]
releasedir <dir>:
Use <dir> as the release directory
[Default is $ReleaseDir_default]
check-vfd:
Run make check-vfd instead of just make check.
check-vol:
Run make check-vol instead of just make check.
exttest <testscript>;
Run testscript;
exec <command>:
Run <command>;
module-load <module-list>:
Load modules in comma-separated <module-list>;
op-configure <option>:
Pass <option> to the configure command
E.g., "snapshot op-configure --enable-parallel"
configures for parallel mode
--<option>:
Pass --<option> to the configure command
E.g., "snapshot --enable-parallel"
configures for parallel mode
EOF
exit $errcode
}
# MAIN
# SGI /bin/sh replaces $0 as function name if used in a function.
# Set the name here to avoid that ambiguity and better style too.
PROGNAME=$0
SNAPSHOTNAME=
HDFREPOS=
DOCVERSION=""
MODULELIST=""
if [ -f bin/snapshot_params ]; then
. bin/snapshot_params
echo "Added snapshot_params."
fi
if [ -z "$SWVER" -a -f bin/snapshot_version ]
then
. bin/snapshot_version
echo "Added snapshot_version."
fi
if [ -n ${HDFREPOS} -a -f bin/snapshot-${HDFREPOS}-overrides ]; then
. bin/snapshot-${HDFREPOS}-overrides
echo "Added snapshot-${HDFREPOS}-overrides."
fi
echo "====================================="
echo "$PROGNAME $*"
echo "====================================="
TIMESTAMP MAIN
uname -a
# setup exit banner message
trap EXIT_BANNER 0 1 2 9 15
# Dump environment variables before option parsing
echo ===Dumping environment variables before option parsing ===
printenv | sort
echo ===Done Dumping environment variables before option parsing ===
# snapshots release directory. Default relative to $BASEDIR.
ReleaseDir_default=release_dir
# Where is the zlib library?
# At NCSA, half of the machines have it in /usr/lib, the other half at
# /usr/ncsa/lib. Leave it unset.
ZLIB_default=
ZLIB=$ZLIB_default
# What compression methods to use? (md5 does checksum). Doc was apparently
# added as a compression method to create a separate tarfile containing the
# documentation files for v 1.8 and above.
if [ "${SWVERSION}" = "hdf5_1_6" ]; then
METHODS="gzip bzip2 md5"
else
METHODS="gzip bzip2 doc"
fi
# Use User's MAKE if set. Else use generic make.
MAKE=${MAKE:-make}
# Default check action.
CHECKVAL=check
#
# Command options
cmd="all"
test_opt=""
errcode=0
AUTOGEN=""
EXTTEST=""
EXEC_CMD_ARG=""
while [ $# -gt 0 ] ; do
case "$1" in
all)
cmd="all"
;;
checkout-autogen)
cmdcheckout="checkout"
AUTOGEN="autogen"
cmd=""
;;
checkout)
cmdcheckout="checkout"
cmd=""
;;
ftp)
echo "Setting ftp flags in snapshot script"
cmdcheckout="checkout"
cmdftp="ftp"
cmd=""
shift
if [ $# -lt 1 ]; then
echo "URL missing"
errcode=1
cmd="help"
break
fi
ftp_url="$1"
echo "ftp_url is $ftp_url"
;;
diff)
cmddiff="diff"
cmd=""
;;
deploy)
# deploy the built binary.
shift
if [ $# -lt 1 ]; then
echo "deploy <dir> missing"
errcode=1
cmd="help"
break
fi
cmddeploy="deploy"
DEPLOYDIRNAME="$1"
;;
deploydir)
# default directory for deployment.
shift
if [ $# -lt 1 ]; then
echo "deploydir <dir> missing"
errcode=1
cmd="help"
break
fi
deploydir="$1"
;;
test)
cmdtest="test"
cmd=""
;;
setenv)
# set environment variable
shift
eval $1="$2"
export $1
shift
;;
setenvN)
# set environment variable with $1 values
# e.g., setenvN 3 x a b c is same as setenv x="a b c".
# a kludge now--the extra single quotes are needed
# else eval complains.
shift
envN=$1
shift
envname=$1
envalue=
while test $envN -gt 0; do
shift
envalue="$envalue $1"
envN=`expr $envN - 1`
done
eval $envname="'$envalue'"
export $envname
;;
srcdir)
#use srcdir option for test
srcdir="yes"
;;
srcdirname)
shift
if [ $# -lt 1 ]; then
echo "srcdirname <dir> missing"
errcode=1
cmd="help"
break
fi
SRCDIRNAME="$1"
;;
release)
cmdrel="release"
cmd=""
;;
autogen-release)
cmdrel="autogen-release"
cmd=""
;;
clean | distclean)
cmdclean="$1"
cmd=""
;;
help)
cmd="help"
break
;;
echo)
set -x
break
;;
zlib)
shift
if [ $# -lt 1 ]; then
echo "ZLIB information missing"
errcode=1
cmd="help"
break
fi
ZLIB="$1"
;;
releasedir)
shift
if [ $# -lt 1 ]; then
echo "Release directory name missing"
errcode=1
cmd="help"
break
fi
ReleaseDir="$1"
;;
exttest)
shift
if [ $# -lt 1 ]; then
echo "exttest script name missing"
errcode=1
cmd="help"
break
fi
cmd=""
EXTTEST="$1"
;;
exec)
shift
if [ $# -lt 1 ]; then
echo "exec command name missing"
errcode=1
cmd="help"
break
fi
cmd=""
EXEC_CMD_ARG="$@"
# exit the parsing while loop since all arguments have been consummed.
break
;;
check-vfd)
CHECKVAL=check-vfd
;;
check-vol)
CHECKVAL=check-vol
;;
module-load)
shift
if [ $# -lt 1 ]; then
echo "missing module list to load"
errcode=1
cmd="help"
break
fi
MODULELIST="$1"
;;
--*)
OP_CONFIGURE="$OP_CONFIGURE $1"
;;
op-configure)
shift
if [ $# -lt 1 ]; then
echo "op-configure option missing"
errcode=1
cmd="help"
break
fi
OP_CONFIGURE="$OP_CONFIGURE $1"
;;
*)
echo "Unkown option $1"
errcode=1
cmd="help"
break
;;
esac
shift
done
if [ -n "$MODULELIST" ]; then
. ~/.bashrc
module use /opt/pkgs/modules/all
# load module command will take a space separated list of modules.
# If we have a comma separated list, convert ',' to ' '.
MODULELIST="$( echo -e "$MODULELIST" | tr ',' ' ' )"
module load $MODULELIST
fi
# Dump environment variables after option parsing
echo ===Dumping environment variables after option parsing ===
printenv | sort
echo ===Done Dumping environment variables after option parsing ===
if [ "$cmd" = help ]; then
DISPLAYUSAGE
fi
# Setup the proper configure option (--with-zlib) to use zlib library
# provide ZLIB is non-empty.
ZLIB=${ZLIB:+"--with-zlib="$ZLIB}
# Adding --prefix as a configure option will put the path to the deploy
# directory in the initial libhdf5*.la files
if [ -n "$DEPLOYDIRNAME" ]; then
OP_CONFIGURE="$OP_CONFIGURE --prefix=${deploydir}/${DEPLOYDIRNAME}"
fi
CONFIGURE="configure $OP_CONFIGURE"
# echo "Configure command is $CONFIGURE"
# Execute the requests
snapshot=yes
BASEDIR=${HOME}/snapshots-${SNAPSHOTNAME}
if [ ! -d ${BASEDIR} ]; then
echo "BASEDIR ($BASEDIR) does not exist"
exit 1
fi
CURRENT=${BASEDIR}/current
PREVIOUS=${BASEDIR}/previous
ReleaseDir=${ReleaseDir:=${BASEDIR}/${ReleaseDir_default}}
HOSTNAME=`hostname | cut -f1 -d.` # no domain part
# Try finding a version of diff that supports the -I option too.
DIFF=diff
for d in `echo $PATH | sed -e 's/:/ /g'` ; do
test -x $d/diff && $d/diff -I XYZ /dev/null /dev/null > /dev/null 2>&1 &&
DIFF=$d/diff && break
done
#=============================
# Run source checkout
#=============================
if [ "$cmd" = "all" -o -n "$cmdcheckout" ]; then
TIMESTAMP "checkout"
# ${BASEDIR}/bin is now updated from git by EveningMaint or DailyMaint
# to avoid updating the scripts in ${BASEDIR}/bin while they are running.
if [ -z "$AUTOGEN" ]; then
# If there is a Makefile in ${CURRENT}, the last test done in it
# has not been distclean'ed. They would interfere with other
# --srcdir build since make considers the files in ${CURRENT}
# take precedence over files in its own build-directory. Run
# a "make distclean" to clean them all out. This is not really
# part of the "checkout" functions but this is the most convenient
# spot to do the distclean. We will also continue the checkout process
# regardless of the return code of distclean.
( cd ${CURRENT}; test -f Makefile && ${MAKE} distclean)
fi
# echo "cmdftp is $cmdftp; ftp_url is $ftp_url"
if [ -n "$cmdftp" ]; then
echo "Get the NetCDF4 source from their ftp server."
echo "Command executed is: 2>&1 wget -N $ftp_url"
cd ${BASEDIR};
WGET_OUTPUT="`2>&1 wget -N $ftp_url`"
errcode=$?
if [[ $errcode -ne 0 ]]; then
exit $errcode
fi
if [ $? -ne 0 ];then
echo $0: "$WGET_OUTPUT" Exiting.
exit 1
fi
# echo "Wget output was $WGET_OUTPUT"
if echo "$WGET_OUTPUT" | fgrep 'not retrieving' &> /dev/null
then
echo "Snapshot unchanged"
else
echo "New snapshot downloaded"
EXTRACT
fi
else
SOURCE_CHECKOUT
fi
fi # Do source checkout
#=============================
# Run Test the HDF5 library
#=============================
if [ "$cmd" = "all" -o -n "$cmdtest" -o -n "$cmddiff" ]; then
TIMESTAMP "Run Tests"
# setup if srcdir is used.
if [ -z "$srcdir" ]; then
TESTDIR=${CURRENT}
else
#create TESTDIR if not exist yet
case "$SRCDIRNAME" in
"")
SRCDIRNAME=$HOSTNAME
;;
-*)
SRCDIRNAME="$HOSTNAME$SRCDIRNAME"
;;
esac
TESTDIR=${BASEDIR}/TestDir/${SRCDIRNAME}
test -d ${TESTDIR} || mkdir ${TESTDIR}
# set TESTDIR to use the direct path to the local test directory
# rather than the path through ${BASEDIR}.
cd ${TESTDIR}
TESTDIR=`pwd -P`
cd ${CURRENT}
fi
# Make sure current version exists and is clean
if [ -d ${TESTDIR} ]; then
DISTCLEAN
else
errcode=$?
snapshot=no
exit $errcode
fi
# Compare it with the previous version. Compare only files listed in
# the MANIFEST plus the MANIFEST itself.
if [ -d ${PREVIOUS} ]; then
if [ -z "${AUTOGEN}" ]; then
CURRENTSRC=${CURRENT}
else
CURRENTSRC=${BASEDIR}/current_src
fi
if (${DIFF} -c ${PREVIOUS}/MANIFEST ${CURRENTSRC}/MANIFEST); then
snapshot=no
for src in `grep '^\.' ${CURRENTSRC}/MANIFEST|expand|cut -f1 -d' '`; do
if ${DIFF} -I H5_VERS_RELEASE -I " released on " \
-I " currently under development" \
${PREVIOUS}/$src ${CURRENTSRC}/$src
then
: #continue
else
snapshot=yes
break
fi
done
fi
fi
# if diff is choosen, exit 0 if no significant differences are found.
# otherwise, exit 1. This includes cases of other failures.
if [ -n "$cmddiff" ]; then
if [ $snapshot = no ]; then
exit 0
else
exit 1
fi
fi
#=============================
# Execute command if defined
#=============================
#echo BEFORE EXEC command
#echo EXEC_CMD_ARG=${EXEC_CMD_ARG}
if [ -n "$EXEC_CMD_ARG" ]; then
TIMESTAMP ${EXEC_CMD_ARG}
TESTDIR=${BASEDIR}/TestDir/${SRCDIRNAME}
test -d ${TESTDIR} || mkdir ${TESTDIR}
if cd ${TESTDIR}; then
# clean up the directory before executing the command
# Do we need to clean first?
# rm -rf *
#
# If EXEC_CMD_ARG starts with a '/', it has an absolute path, else it is
# relative to the BASEDIR.
case "$EXEC_CMD_ARG" in
/*)
${EXEC_CMD_ARG}
;;
*)
${BASEDIR}/${EXEC_CMD_ARG}
;;
esac
errcode=$?
else
echo "${TESTDIR} not accessible"
errcode=1
fi
# exit snapshot since nothing else to do, for now.
exit $errcode
fi
# Build, run tests and install procedures
if [ "$snapshot" = "yes" ] && [ "$NOMAKE" != "yes" ]; then
FAIL_SECTION=""
if [ -f ${TESTDIR}/failsection ]; then
rm ${TESTDIR}/failsection
fi
if (cd ${TESTDIR} && \
TIMESTAMP "configure" && echo "configure" > ${TESTDIR}/failsection && \
RUNCONFIGURE && \
sleep 2 && \
TIMESTAMP "make" && echo "make" > ${TESTDIR}/failsection && \
${MAKE} && DISKUSAGE \
TIMESTAMP ${CHECKVAL} && echo "make check" > ${TESTDIR}/failsection && \
${MAKE} ${CHECKVAL} && DISKUSAGE \
TIMESTAMP "install" && echo "make install" > ${TESTDIR}/failsection && \
${MAKE} install && DISKUSAGE \
TIMESTAMP "check-install" && echo "make check-install" > ${TESTDIR}/failsection && \
CHECKINSTALL && DISKUSAGE \
TIMESTAMP "uninstall" && echo "make uninstall" > ${TESTDIR}/failsection && \
${MAKE} uninstall && DISKUSAGE); then
:
else
errcode=$?
FAIL_SECTION=`cat ${TESTDIR}/failsection`
echo "Failed running ${FAIL_SECTION}"
snapshot=no
exit $errcode
fi
elif [ $CPSRC ]; then
cp -pr ${CURRENT}/* ${TESTDIR}
else
cmdclean=""
fi
fi # Test the HDF5 library
# Run external test if configured
#=============================
#=============================
#if [ -d "$CURRENT" ]; then
if [ "$EXTTEST" != "" ]; then
TIMESTAMP ${EXTTEST}
TESTDIR=${BASEDIR}/TestDir/${SRCDIRNAME}
test -d ${TESTDIR} || mkdir ${TESTDIR}
cd ${TESTDIR}
sleep 1
TIMESTAMP $pwd
ls
${BASEDIR}/${EXTTEST}
errcode=$?
exit $errcode
fi
#=============================
# Run deployment if requested.
#=============================
if [ -n "$DEPLOYDIRNAME" ]; then
# The daily tests deploy to .../hdf5/... or .../hdf4/... except on cobalt where the
# deploy directory is in .../HDF5/... lc will take care of this. If hdf4 or hdf5
# either upper or lower case isn't in the path, RELEASE.txt won't be found unless
# it is in $CURRENT.
POS4=`perl -e "print index(lc(\"${deploydir}/${DEPLOYDIRNAME}\"), 'hdf4')"`
POS5=`perl -e "print index(lc(\"${deploydir}/${DEPLOYDIRNAME}\"), 'hdf5')"`
if [ "${POS4}" -ge "0" ]; then
RELEASE_TXT_LOC="release_notes"
elif [ "${POS5}" -ge "0" ]; then
RELEASE_TXT_LOC="release_docs"
else
RELEASE_TXT_LOC=""
fi
if [ "$snapshot" = "yes" ]; then
TIMESTAMP "deploy"
if (cd ${TESTDIR} &&
${BASEDIR}/bin/deploy ${deploydir}/${DEPLOYDIRNAME} && \
TIMESTAMP "clean" && \
${MAKE} clean && \
TIMESTAMP "check-install prefix=${deploydir}/${DEPLOYDIRNAME}" && \
CHECKINSTALL prefix=${deploydir}/${DEPLOYDIRNAME}); then
cd ${CURRENT}
cp ${RELEASE_TXT_LOC}/RELEASE.txt ${deploydir}/${DEPLOYDIRNAME}
cp COPYING ${deploydir}/${DEPLOYDIRNAME}
#: #continue
else
errcode=$?
exit $errcode
fi
fi
fi # Deploy
#=============================
# Run Release snapshot, update version, and commit to source repository
#=============================
if [ "$cmd" = "all" -o -n "$cmdrel" ]; then
if [ "$snapshot" = "yes" ]; then
TIMESTAMP "release"
DISTCLEAN
(
# Turn on exit on error in the sub-shell so that it does not
# commit source if errors encounter here.
set -e
if [ "$cmdrel" = "autogen-release" ]; then
cd ${BASEDIR}/current_src
else
cd ${CURRENT}
fi
if [ "$HDFREPOS" = "hdf4" ]; then
RELEASE_VERSION="`perl bin/h4vers -v`"
echo "Making snapshot release ($RELEASE_VERSION) to ${ReleaseDir}..."
bin/release -d $ReleaseDir $METHODS
perl bin/h4vers -i
elif [ "$HDFREPOS" = "hdf5" ]; then
RELEASE_VERSION="`perl bin/h5vers -v`"
echo "Making snapshot release ($RELEASE_VERSION) to ${ReleaseDir}..."
if [ "${DOCVERSION}" ]; then
bin/release -d $ReleaseDir --docver ${DOCVERSION} $METHODS
else
bin/release -d $ReleaseDir $METHODS
fi
perl bin/h5vers -i
else
echo "need real release steps. For now, only move current version to previous"
fi
COMMITSNAPSHOT
)
errcode=$?
fi
# Replace the previous version with the current version.
# Should check if the errcode of the release process but there
# are other failures after release was done (e.g. h5vers or git failures)
# that should allow the replacement to occure.
rm -rf ${PREVIOUS}
mv ${CURRENT} ${PREVIOUS}
fi #Release snapshot
#=============================
# Clean the test area. Default is no clean.
#=============================
if [ -n "$cmdclean" ] && [ "$NOMAKE" != "yes" ]; then
TIMESTAMP "clean"
# setup if srcdir is used.
if [ -z "$srcdir" ]; then
TESTDIR=${CURRENT}
else
case "$SRCDIRNAME" in
"")
SRCDIRNAME=$HOSTNAME
;;
-*)
SRCDIRNAME="$HOSTNAME$SRCDIRNAME"
;;
esac
TESTDIR=${BASEDIR}/TestDir/${SRCDIRNAME}
fi
# clean it
if (cd ${TESTDIR} && ${MAKE} $cmdclean ) then
:
else
errcode=$?
snapshot=no
exit $errcode
fi
fi # Clean the Test directory
exit $errcode
Reference in New Issue View Git Blame Copy Permalink