Index: /issm/trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python.sh (revision 25893) +++ /issm/trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python.sh (revision 25893) @@ -0,0 +1,229 @@ +#!/bin/bash + +################################################################################ +# Commits ISSM distributable package for macOS with Python API to repository +# for signing. This repository is polled by a project running on a JPL +# Cybersecurity Jenkins server and performs the actual signing and +# notarization. +# +# Options: +# -r/--resign Skip ISSM compilation and packaging. Use to retrigger +# signing/notarization if it fails but build and package +# are valid. +# -u/--unlock Remove lock file from signed package repository. Use if +# build is aborted to allow for subsequent fresh build. +# +# NOTE: +# - Assumes that the following constants are defined, +# +# COMPRESSED_PKG +# ISSM_BINARIES_REPO_PASS +# ISSM_BINARIES_REPO_USER +# SIGNED_REPO_COPY +# SIGNED_REPO_URL +# +# See also: +# - packagers/mac/complete-issm-mac-binaries-python.sh +# - packagers/mac/sign-issm-mac-binaries-python.sh +################################################################################ + +# Expand aliases within the context of this script +shopt -s expand_aliases + +# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes, +# +# Command line tool support for Subversion — including svn, git-svn, and +# related commands — is no longer provided by Xcode. (50266910) +# +# which results in, +# +# svn: error: The subversion command line tools are no longer provided by +# Xcode. +# +# when calling svn, even when subversion is installed via Homebrew and its path +# is available in PATH. +# +# NOTE: May be able to remove this after updating macOS. +# +alias svn='/usr/local/bin/svn' + +## Override certain other aliases +# +alias cp=$(which cp) +alias grep=$(which grep) + +## Constants +# +MAX_SIGNING_CHECK_ATTEMPTS=30 +NOTARIZATION_LOGFILE="notarization.log" +RETRIGGER_SIGNING_FILE="retrigger.txt" +SIGNING_CHECK_PERIOD=60 # in seconds +SIGNING_LOCK_FILE="signing.lock" +UNSIGNED_REPO_COPY="./unsigned" +UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned" + +## Functions +# +checkout_signed_repo_copy(){ + echo "Checking out copy of repository for signed packages" + + # NOTE: Get empty copy because we do not want to have to check out package + # from previous signing. + # + svn checkout \ + --trust-server-cert \ + --non-interactive \ + --depth empty \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + ${SIGNED_REPO_URL} \ + ${SIGNED_REPO_COPY} > /dev/null 2>&1 +} +checkout_unsigned_repo_copy(){ + echo "Checking out copy of repository for unsigned packages" + svn checkout \ + --trust-server-cert \ + --non-interactive \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + ${UNSIGNED_REPO_URL} \ + ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 +} +validate_signed_repo_copy(){ + # Validate copy of repository for signed binaries (e.g. + # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible'; + # initial checkout failed) + if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then + rm -rf ${SIGNED_REPO_COPY} + checkout_signed_repo_copy + fi +} + +## Parse options +# +if [ $# -gt 1 ]; then + echo "Can use only one option at a time" + exit 1 +fi + +retrigger_signing=0 +unlock=0 + +if [ $# -eq 1 ]; then + case $1 in + -r|--resign) retrigger_signing=1; ;; + -u|--unlock) unlock=1; ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac +fi + +validate_signed_repo_copy + +if [ ${unlock} -eq 1 ]; then + # Remove signing lock file from signed package repository so that a new + # build can run + echo "Removing lock file from repository for signed packages" + svn update \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 + svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 + svn commit \ + --trust-server-cert \ + --non-interactive \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} > /dev/null 2>&1 + svn cleanup ${SIGNED_REPO_COPY} > /dev/null 2>&1 + + echo "Remove -u/--unlock option from configuration and run again" + exit 1 +fi + +# If lock file exists, a signing build is still in process by JPL Cybersecurity +svn update \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 + +if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then + echo "Previous signing job still in process by JPL Cybersecurity. Please try again later." + exit 1 +fi + +# Commit lock file to repository for signed packages +echo "Committing lock file to repository for signed packages" +touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} +svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 +svn commit \ + --trust-server-cert \ + --non-interactive \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + --message "ADD: New lock file" ${SIGNED_REPO_COPY} > /dev/null 2>&1 + +# Check out copy of repository for unsigned packages +checkout_unsigned_repo_copy + +if [ ${retrigger_signing} -eq 0 ]; then + # Commit new compressed package to repository for unsigned binaries + echo "Committing package to repository for unsigned packages" + cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY} + svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1 + svn commit \ + --trust-server-cert \ + --non-interactive \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 +else + # NOTE: If notarize_only == 1, we commit a dummy file as we do not want to + # have to commit the entire compressed package again simply to + # retrigger the signing build on the remote JPL Cybersecurity Jenkins + # server. + # + echo "Attempting to sign existing package again" + echo $(date +'%Y-%m-%d-%H-%M-%S') > ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} # Write datetime stamp to file to ensure modification is made + svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1 + svn commit \ + --trust-server-cert \ + --non-interactive \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 +fi + +# Check status of signing +echo "Checking progress of signing..." +SIGNING_CHECK_ATTEMPT=0 +while [ ${SIGNING_CHECK_ATTEMPT} -lt ${MAX_SIGNING_CHECK_ATTEMPTS} ]; do + echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds" + sleep ${SIGNING_CHECK_PERIOD} + svn update \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + ${SIGNED_REPO_COPY} > /dev/null 2>&1 + + if [ ! -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then + # Retrieve notarization lock file + svn update \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} + + # Check status + STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //') + if [[ "${STATUS}" == "success" ]]; then + echo "Notarization successful!" + else + echo "Notarization failed!" + echo "----------------------- Contents of notarization logfile -----------------------" + cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} + echo "--------------------------------------------------------------------------------" + + exit 1 + fi + else + ((++SIGNING_CHECK_ATTEMPT)) + fi +done Index: /issm/trunk-jpl/packagers/mac/complete-issm-mac-binaries-matlab.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/complete-issm-mac-binaries-matlab.sh (revision 25892) +++ /issm/trunk-jpl/packagers/mac/complete-issm-mac-binaries-matlab.sh (revision 25893) @@ -112,5 +112,5 @@ # Transfer distributable package to ISSM Web site if [ ${transfer} -eq 1 ]; then - ./packagers/mac/transfer-issm-mac-binaries-matlab.sh + ./packagers/mac/transfer-issm-mac-binaries.sh fi Index: /issm/trunk-jpl/packagers/mac/complete-issm-mac-binaries-python.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/complete-issm-mac-binaries-python.sh (revision 25893) +++ /issm/trunk-jpl/packagers/mac/complete-issm-mac-binaries-python.sh (revision 25893) @@ -0,0 +1,116 @@ +#!/bin/bash + +################################################################################ +# Wrapper script to build, package, send for signing, and transfer to ISSM Web +# site ISSM distributable package for macOS with Python API. +# +# Normally, we would put this directly into the project configuration under +# 'Build' -> 'Excute shell', but becasue it is a bit more involved, it is a +# good idea to version it. +# +# When no failures/errors occur, performs the following: +# - Builds ISSM according to configuration. +# - Packages and compresses executables and libraries. +# - Runs test suite against package. +# - Commits compressed package to repository to be signed by JPL Cybersecurity. +# - Retrieves signed package and transmits it to ISSM Web site for +# distribution. +# +# Options: +# -b/--skipbuild Skip ISSM compilation. +# -r/--resign Skip ISSM compilation and packaging. Use to retrigger +# signing/notarization if it fails but build and package +# are valid. +# -s/--skiptests Skip ISSM compilation and testing during packaging +# step. Use if +# -t/--transferonly Transfer package to ISSM Web site only. Use if transfer +# fails for some reason to skip building, packaging, and +# signing. +# -u/--unlock Remove lock file from signed package repository. Use if +# build is aborted to allow for subsequent fresh build. +# +# Debugging: +# - Relies on a very tight handshake with project on remote JPL Cybersecurity +# Jenkins server. Debugging may be perfomed locally by running, +# +# packagers/mac/sign-issm-mac-binaries-python.sh +# +# with "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" hardcoded to Apple +# Developer credentials. +# - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can +# help debug potential SVN issues. +# +# NOTE: +# - Use only *one* of the above options at a time, and make sure it is removed +# again after a single run. +# - Builds will fail when any of the above options are used on a clean +# workspace. For example, if 'Source Code Management' -> 'Check-out Strategy' +# select menu is set to "Always check out a fresh copy". +# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in +# the 'Bindings' section under a 'Username and password (separated)' binding +# (requires 'Credentials Binding Plugin') with 'Credentials' select menu set +# to "jenkins/****** (SVN repository for ISSM binaries)". +################################################################################ + +## Constants +# +PKG="ISSM-macOS-Python" # Name of directory to copy distributable files to +SIGNED_REPO_COPY="./signed" +SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/signed" + +COMPRESSED_PKG="${PKG}.zip" + +## Environment +# +export COMPRESSED_PKG +export PKG +export SIGNED_REPO_COPY +export SIGNED_REPO_URL + +## Parse options +# +if [ $# -gt 1 ]; then + echo "Can use only one option at a time" + exit 1 +fi + +# NOTE: We could do this with binary switching (i.e. 0011 to sign and transfer, +# but the following is self-documenting). +# +build=1 +package=1 +sign=1 +transfer=1 + +if [ $# -eq 1 ]; then + case $1 in + -b|--skipbuild) build=0; shift ;; + -r|--resign) build=0; package=0; ;; + -s|--skiptests) build=0; ;; + -t|--transferonly) build=0; package=0; sign=0; ;; + -u|--unlock) build=0; package=0; transfer=0; ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac +fi + +# Build +if [ ${build} -eq 1 ]; then + ./jenkins/jenkins.sh ./jenkins/pine_island-mac-binaries-python +fi + +# Package +if [ ${package} -eq 1 ]; then + ./packagers/mac/package-issm-mac-binaries-python.sh $1 + shift # Clear $1 so that it is not passed to commit_for_signing script +fi + +# Commit for signing +if [ ${sign} -eq 1 ]; then + ./packagers/mac/commit_for_signing-issm-mac-binaries-python.sh $1 +fi + +# Transfer distributable package to ISSM Web site +if [ ${transfer} -eq 1 ]; then + ./packagers/mac/transfer-issm-mac-binaries.sh +fi + Index: /issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh (revision 25892) +++ /issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh (revision 25893) @@ -2,44 +2,19 @@ ################################################################################ -# To be used after running, -# -# ${ISSM_DIR}/jenkins/jenkins.sh ${ISSM_DIR}/jenkins/pine_island-mac-binaries-python -# -# in the context of a Jenkins project. -# -# When no runtime errors occur, performs the following: -# - Checks resulting executables and libraries against test suite. -# - Packages and compresses executables and libraries. -# - Commits compressed package to repository to be signed by JPL Cybersecurity. -# - Retrieves signed package and transmits it to ISSM Web site for -# distribution. +# Packages and tests ISSM distributable package for macOS with Python API. # # Options: -# -n/--notarizeonly Sign/notarize only (use if signing/notarization fails -# to skip tests/packaging) -# -s/--skiptests Skip tests (use if this script fails for some reason -# after tests have successfully passed to save time) -# -t/--transferonly Transfer package to ISSM Web site only (use if -# transfer fails for some reason to skip testing and -# signing) -# -u/--unlock Remove lock file from signed package repository (use if -# build is aborted to allow for subsequent fresh build) -# -# Debugging: -# - Relies on a very tight handshake with project on remote JPL Cybersecurity -# Jenkins server. Debugging may be perfomed locally by running, -# -# packagers/mac/sign-issm-mac-binaries-python.sh -# -# with Apple Developer credentials. -# - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can -# help debug potential SVN issues. +# -s/--skiptests Skip tests and package only. # # NOTE: -# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in -# the 'Bindings' section under a 'Username and password (separated)' binding -# (requires 'Credentials Binding Plugin'). -# - For local debugging, the aformentioned credentials can be hardcoded into -# the 'USERNAME' and 'PASSWORD' constants below. +# - Assumes that the following constants are defined, +# +# COMPRESSED_PKG +# ISSM_DIR +# PKG +# +# See also: +# - packagers/mac/complete-issm-mac-binaries-python.sh +# - packagers/mac/sign-issm-mac-binaries-python.sh ################################################################################ @@ -70,18 +45,5 @@ ## Constants # -NOTARIZATION_LOGFILE="notarization.log" -PASSWORD=${ISSM_BINARIES_PASS} -PKG="ISSM-macOS-Python" # Name of directory to copy distributable files to PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2006 2020 2021 2051 2052 2053 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded -RETRIGGER_SIGNING_FILE="retrigger.txt" -SIGNED_REPO_COPY="./signed" -SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/signed" -SIGNING_CHECK_PERIOD=60 # in seconds -SIGNING_LOCK_FILE="signing.lock" -UNSIGNED_REPO_COPY="./unsigned" -UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned" -USERNAME=${ISSM_BINARIES_USER} - -COMPRESSED_PKG="${PKG}.zip" ## Environment @@ -133,292 +95,115 @@ fi -notarize_only=0 skip_tests=0 -transfer_only=0 -unlock=0 -while [ $# -gt 0 ]; do - case $1 in - -n|--notarizeonly) notarize_only=1; shift ;; - -s|--skiptests) skip_tests=1; shift ;; - -t|--transferonly) transfer_only=1; shift ;; - -u|--unlock) unlock=1; shift ;; - *) echo "Unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -if [ ${unlock} -eq 1 ]; then - # Remove signing lock file from signed package repository so that a new - # build can run - echo "Removing lock file from repository for signed packages" - checkout_signed_repo_copy - svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} - svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} - svn commit \ - --trust-server-cert \ - --non-interactive \ - --username ${USERNAME} \ - --password ${PASSWORD} \ - --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} - svn cleanup ${SIGNED_REPO_COPY} - exit 1 -fi - -if [ ${transfer_only} -eq 0 ]; then - rm -rf ${SIGNED_REPO_COPY} - - checkout_signed_repo_copy - - # If lock file exists, a signing build is still in process by JPL Cybersecurity - svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} - if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then - echo "Previous signing job still in process by JPL Cybersecurity. Please try again later." + +if [ $# -eq 1 ]; then + case $1 in + -s|--skiptests) skip_tests=1; ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac +fi + +# Add required binaries and libraries to package and modify them where needed +cd ${ISSM_DIR}/bin + +echo "Modify generic" +cat generic_static.py | sed -e "s/generic_static/generic/g" > generic.py + +echo "Moving MPICH binaries to bin/" +if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then + cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec . + cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy . +elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then + cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec . + cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy . +else + echo "MPICH not found" + exit 1 +fi + +echo "Moving GDAL binaries to bin/" +if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then + cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo . + cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform . +else + echo "GDAL not found" + exit 1 +fi + +echo "Moving GMT binaries to bin/" +if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then + cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt . + cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect . +else + echo "GMT not found" + exit 1 +fi + +echo "Moving Gmsh binaries to bin/" +if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then + cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh . +else + echo "Gmsh not found" + exit 1 +fi + +# Run tests +if [ ${skip_tests} -eq 0 ]; then + echo "Running tests" + cd ${ISSM_DIR}/test/NightlyRun + rm python.log 2> /dev/null + + # Set Python environment + export PYTHONPATH="${ISSM_DIR}/src/m/dev" + export PYTHONSTARTUP="${PYTHONPATH}/devpath.py" + export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured + + # Run tests, redirecting output to logfile and suppressing output to console + ./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1 + + # Check that Python did not exit in error + pythonExitCode=`echo $?` + pythonExitedInError=`grep -E "Error|Standard exception|Traceback|bad interpreter" python.log | wc -l` + + if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then + echo "----------Python exited in error!----------" + cat python.log + echo "-----------End of python.log-----------" + + # Clean up execution directory + rm -rf ${ISSM_DIR}/execution/* + exit 1 fi - if [ ${notarize_only} -eq 0 ]; then - # Clean up from previous packaging - echo "Cleaning up existing assets" - cd ${ISSM_DIR} - rm -rf ${PKG} ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY} - mkdir ${PKG} - - # Check out copy of SVN repository for signed packages - # - # NOTE: Get empty copy because we do not want to have to check out package - # from previous signing. - # - echo "Checking out copy of repository for signed packages" - svn checkout \ - --trust-server-cert \ - --non-interactive \ - --username ${USERNAME} \ - --password ${PASSWORD} \ - --depth empty \ - ${SIGNED_REPO_URL} \ - ${SIGNED_REPO_COPY} - - # If lock file exists, a signing build is still in process by JPL Cybersecurity - svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} - if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then - echo "Previous signing job still in process by JPL Cybersecurity. Please try again later." - exit 1 - fi - - # Add required binaries and libraries to package and modify them where needed - cd ${ISSM_DIR}/bin - - echo "Modify generic" - cat generic_static.py | sed -e "s/generic_static/generic/g" > generic.py - - echo "Moving MPICH binaries to bin/" - if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then - cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec . - cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy . - elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then - cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec . - cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy . - else - echo "MPICH not found" - exit 1 - fi - - echo "Moving GDAL binaries to bin/" - if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then - cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo . - cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform . - else - echo "GDAL not found" - exit 1 - fi - - echo "Moving GMT binaries to bin/" - if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then - cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt . - cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect . - else - echo "GMT not found" - exit 1 - fi - - echo "Moving Gmsh binaries to bin/" - if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then - cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh . - else - echo "Gmsh not found" - exit 1 - fi - - # Run tests - if [ ${skip_tests} -eq 0 ]; then - echo "Running tests" - - cd ${ISSM_DIR}/test/NightlyRun - rm python.log 2> /dev/null - - # Set Python environment - export PYTHONPATH="${ISSM_DIR}/src/m/dev" - export PYTHONSTARTUP="${PYTHONPATH}/devpath.py" - export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured - - # Run tests, redirecting output to logfile and suppressing output to console - ./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1 - - # Check that Python did not exit in error - pythonExitCode=`echo $?` - pythonExitedInError=`grep -E "Error|Standard exception|Traceback|bad interpreter" python.log | wc -l` - - if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then - echo "----------Python exited in error!----------" - cat python.log - echo "-----------End of python.log-----------" - - # Clean up execution directory - rm -rf ${ISSM_DIR}/execution/* - - exit 1 - fi - - # Check that all tests passed - numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"` - - if [ ${numTestsFailed} -ne 0 ]; then - echo "One or more tests FAILED" - exit 1 - else - echo "All tests PASSED" - fi - else - echo "Skipping tests" - fi - - # Create package - cd ${ISSM_DIR} - svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package) - echo "Copying assets to package: ${PKG}" - cp -rf bin examples lib scripts test ${PKG}/ - mkdir ${PKG}/execution - cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist - ${ISSM_DIR}/scripts/py_to_pyc.sh ${PKG}/bin # Compile Python source files - echo "Cleaning up unneeded/unwanted files" - rm -f ${PKG}/bin/*.py # Remove all Python scripts - rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes - rm -f ${PKG}/lib/*.a # Remove static libraries from package - rm -f ${PKG}/lib/*.la # Remove libtool libraries from package - rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package - - # Compress package - echo "Compressing package" - ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG} + # Check that all tests passed + numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"` + + if [ ${numTestsFailed} -ne 0 ]; then + echo "One or more tests FAILED" + exit 1 else - # Assume that previous build was successful, but signing/notarization - # failed. - # - echo "Notarizing only" + echo "All tests PASSED" fi - - # Commit lock file to repository for signed packages - echo "Committing lock file to repository for signed packages" - touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} - svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} - svn commit \ - --trust-server-cert \ - --non-interactive \ - --username ${USERNAME} \ - --password ${PASSWORD} \ - --message "ADD: New lock file" ${SIGNED_REPO_COPY} - - # Check out copy of repository for unsigned packages - checkout_unsigned_repo_copy - - if [ ${notarize_only} -eq 0 ]; then - # Commit new compressed package to repository for unsigned binaries - # - # NOTE: This will not work if, for any reason, the checksum on the compressed - # package is unchanged. - # - echo "Committing package to repository for unsigned packages" - cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY} - svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} - svn commit \ - --trust-server-cert \ - --non-interactive \ - --username ${USERNAME} \ - --password ${PASSWORD} \ - --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} - else - # NOTE: If notarize_only == 1, we commit a dummy file as the signing - # build on the remote JPL Cybersecurity Jenkins server is - # triggered by polling SCM. - # - echo "Attempting to sign existing package again" - echo $(date +'%Y-%m-%d-%H-%M-%S') > ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} # Write datetime stamp to file to ensure modification is made - svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} - svn commit \ - --trust-server-cert \ - --non-interactive \ - --username ${USERNAME} \ - --password ${PASSWORD} \ - --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} - fi - - # Check status of signing - echo "Checking progress of signing..." - IN_PROCESS=1 - SUCCESS=0 - - while [ ${IN_PROCESS} -eq 1 ]; do - echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds" - sleep ${SIGNING_CHECK_PERIOD} - svn up ${SIGNED_REPO_COPY} - - if [ ! -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then - IN_PROCESS=0 - - # Retrieve notarization lock file - svn up ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} - - # Check status - STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //') - if [[ "${STATUS}" == "success" ]]; then - echo "Notarization successful!" - - # Set flag indicating notarization was successful - SUCCESS=1 - else - echo "Notarization failed!" - fi - fi - done -else - # Assume that previous build resulted in successful signing of package but - # that transfer to ISSM Web site failed and user built this project again - # with -t/--transferonly option. - # - - # Make sure copy of repository for signed packages exists - validate_signed_repo_copy - - SUCCESS=1 -fi - -# Handle result of signing -if [ ${SUCCESS} -eq 1 ]; then - # Retrieve signed and notarized package - svn up ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} - - # Transfer signed package to ISSM Web site - echo "Transferring signed package to ISSM Web site" - scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG} - - if [ $? -ne 0 ]; then - echo "Transfer failed! Verify connection then build this project again (with -t/--transferonly option to skip testing and signing)." - exit 1 - fi -else - echo "----------------------- Contents of notarization logfile -----------------------" - cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} - echo "--------------------------------------------------------------------------------" - - exit 1 -fi +else + echo "Skipping tests" +fi + +# Create package +cd ${ISSM_DIR} +svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package) +echo "Copying assets to package: ${PKG}" +cp -rf bin examples lib scripts test ${PKG}/ +mkdir ${PKG}/execution +cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist +${ISSM_DIR}/scripts/py_to_pyc.sh ${PKG}/bin # Compile Python source files +echo "Cleaning up unneeded/unwanted files" +rm -f ${PKG}/bin/*.py # Remove all Python scripts +rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes +rm -f ${PKG}/lib/*.a # Remove static libraries from package +rm -f ${PKG}/lib/*.la # Remove libtool libraries from package +rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package + +# Compress package +echo "Compressing package" +ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG} Index: sm/trunk-jpl/packagers/mac/transfer-issm-mac-binaries-matlab.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/transfer-issm-mac-binaries-matlab.sh (revision 25892) +++ (revision ) @@ -1,88 +1,0 @@ -#!/bin/bash - -################################################################################ -# Transfers ISSM distributable package for macOS with MATLAB API to ISSM Web -# site. -# -# NOTE: -# - Assumes that the following constants are defined, -# -# COMPRESSED_PKG -# ISSM_BINARIES_REPO_PASS -# ISSM_BINARIES_REPO_USER -# SIGNED_REPO_COPY -# SIGNED_REPO_URL -# -# See also: -# - packagers/mac/complete-issm-mac-binaries-matlab.sh -################################################################################ - -# Expand aliases within the context of this script -shopt -s expand_aliases - -# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes, -# -# Command line tool support for Subversion — including svn, git-svn, and -# related commands — is no longer provided by Xcode. (50266910) -# -# which results in, -# -# svn: error: The subversion command line tools are no longer provided by -# Xcode. -# -# when calling svn, even when subversion is installed via Homebrew and its path -# is available in PATH. -# -# NOTE: May be able to remove this after updating macOS. -# -alias svn='/usr/local/bin/svn' - -## Override certain other aliases -# -alias cp=$(which cp) -alias grep=$(which grep) - -## Functions -# -checkout_signed_repo_copy(){ - echo "Checking out copy of repository for signed packages" - - # NOTE: Get empty copy because we do not want to have to check out package - # from previous signing. - # - svn checkout \ - --trust-server-cert \ - --non-interactive \ - --depth empty \ - --username ${ISSM_BINARIES_REPO_USER} \ - --password ${ISSM_BINARIES_REPO_USER} \ - ${SIGNED_REPO_URL} \ - ${SIGNED_REPO_COPY} > /dev/null 2>&1 -} -validate_signed_repo_copy(){ - # Validate copy of repository for signed binaries (e.g. - # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible'; - # initial checkout failed) - if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then - rm -rf ${SIGNED_REPO_COPY} - checkout_signed_repo_copy - fi -} - -# Check if working copy of repository for signed packages is missing -validate_signed_repo_copy - -# Retrieve signed and notarized package -svn update \ - --username ${ISSM_BINARIES_REPO_USER} \ - --password ${ISSM_BINARIES_REPO_PASS} \ - ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} - -# Transfer signed package to ISSM Web site -echo "Transferring signed package to ISSM Web site" -scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG} - -if [ $? -ne 0 ]; then - echo "Transfer failed! Verify connection then build this project again (with -t/--transferonly option to skip building, packaging, and signing)." - exit 1 -fi Index: /issm/trunk-jpl/packagers/mac/transfer-issm-mac-binaries.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/transfer-issm-mac-binaries.sh (revision 25893) +++ /issm/trunk-jpl/packagers/mac/transfer-issm-mac-binaries.sh (revision 25893) @@ -0,0 +1,88 @@ +#!/bin/bash + +################################################################################ +# Transfers ISSM distributable package to ISSM Web site. +# +# NOTE: +# - Assumes that the following constants are defined, +# +# COMPRESSED_PKG +# ISSM_BINARIES_REPO_PASS +# ISSM_BINARIES_REPO_USER +# SIGNED_REPO_COPY +# SIGNED_REPO_URL +# +# See also: +# - packagers/mac/complete-issm-mac-binaries-matlab.sh +# - packagers/mac/complete-issm-mac-binaries-python.sh +################################################################################ + +# Expand aliases within the context of this script +shopt -s expand_aliases + +# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes, +# +# Command line tool support for Subversion — including svn, git-svn, and +# related commands — is no longer provided by Xcode. (50266910) +# +# which results in, +# +# svn: error: The subversion command line tools are no longer provided by +# Xcode. +# +# when calling svn, even when subversion is installed via Homebrew and its path +# is available in PATH. +# +# NOTE: May be able to remove this after updating macOS. +# +alias svn='/usr/local/bin/svn' + +## Override certain other aliases +# +alias cp=$(which cp) +alias grep=$(which grep) + +## Functions +# +checkout_signed_repo_copy(){ + echo "Checking out copy of repository for signed packages" + + # NOTE: Get empty copy because we do not want to have to check out package + # from previous signing. + # + svn checkout \ + --trust-server-cert \ + --non-interactive \ + --depth empty \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_USER} \ + ${SIGNED_REPO_URL} \ + ${SIGNED_REPO_COPY} > /dev/null 2>&1 +} +validate_signed_repo_copy(){ + # Validate copy of repository for signed binaries (e.g. + # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible'; + # initial checkout failed) + if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then + rm -rf ${SIGNED_REPO_COPY} + checkout_signed_repo_copy + fi +} + +# Check if working copy of repository for signed packages is missing +validate_signed_repo_copy + +# Retrieve signed and notarized package +svn update \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} + +# Transfer signed package to ISSM Web site +echo "Transferring signed package to ISSM Web site" +scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG} + +if [ $? -ne 0 ]; then + echo "Transfer failed! Verify connection then build this project again (with -t/--transferonly option to skip building, packaging, and signing)." + exit 1 +fi