Index: /issm/trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-matlab.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-matlab.sh (revision 25884) +++ /issm/trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-matlab.sh (revision 25884) @@ -0,0 +1,216 @@ +#!/bin/bash + +################################################################################ +# Commits ISSM distributable package for macOS with MATLAB API to repository +# for signing. This repository is polled by a project running on a JPL +# Cybersecurity Jenkins server and performs the actual signing and +# notarization. +# +# Options: +# -r/--resign Skip ISSM compilation and packaging. Use to retrigger +# signing/notarization if it fails but build and package +# are valid. +# -u/--unlock Remove lock file from signed package repository. Use if +# build is aborted to allow for subsequent fresh build. +# +# NOTE: +# - Assumes that the following constants are defined, +# +# COMPRESSED_PKG +# ISSM_BINARIES_REPO_PASS +# ISSM_BINARIES_REPO_USER +# SIGNED_REPO_COPY +# SIGNED_REPO_URL +# +# See also: +# - packagers/mac/complete-issm-mac-binaries-matlab.sh +# - packagers/mac/sign-issm-mac-binaries-matlab.sh +################################################################################ + +# Expand aliases within the context of this script +shopt -s expand_aliases + +# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes, +# +# Command line tool support for Subversion — including svn, git-svn, and +# related commands — is no longer provided by Xcode. (50266910) +# +# which results in, +# +# svn: error: The subversion command line tools are no longer provided by +# Xcode. +# +# when calling svn, even when subversion is installed via Homebrew and its path +# is available in PATH. +# +# NOTE: May be able to remove this after updating macOS. +# +alias svn='/usr/local/bin/svn' + +## Override certain other aliases +# +alias cp=$(which cp) +alias grep=$(which grep) + +## Constants +# +MAX_SIGNING_CHECK_ATTEMPTS=30 +NOTARIZATION_LOGFILE="notarization.log" +RETRIGGER_SIGNING_FILE="retrigger.txt" +SIGNING_CHECK_PERIOD=60 # in seconds +SIGNING_LOCK_FILE="signing.lock" +UNSIGNED_REPO_COPY="./unsigned" +UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned" + +## Functions +# +checkout_signed_repo_copy(){ + echo "Checking out copy of repository for signed packages" + + # NOTE: Get empty copy because we do not want to have to check out package + # from previous signing. + # + svn checkout \ + --trust-server-cert \ + --non-interactive \ + --depth empty \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + ${SIGNED_REPO_URL} \ + ${SIGNED_REPO_COPY} > /dev/null 2>&1 +} +checkout_unsigned_repo_copy(){ + echo "Checking out copy of repository for unsigned packages" + svn checkout \ + --trust-server-cert \ + --non-interactive \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + ${UNSIGNED_REPO_URL} \ + ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 +} +validate_signed_repo_copy(){ + # Validate copy of repository for signed binaries (e.g. + # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible'; + # initial checkout failed) + if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then + rm -rf ${SIGNED_REPO_COPY} + checkout_signed_repo_copy + fi +} + +## Parse options +# +if [ $# -gt 1 ]; then + echo "Can use only one option at a time" + exit 1 +fi + +retrigger_signing=0 +unlock=0 + +if [ $# -eq 1 ]; then + case $1 in + -r|--resign) retrigger_signing=1; ;; + -u|--unlock) unlock=1; ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac +fi + +validate_signed_repo_copy + +if [ ${unlock} -eq 1 ]; then + # Remove signing lock file from signed package repository so that a new + # build can run + echo "Removing lock file from repository for signed packages" + svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 + svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 + svn commit \ + --trust-server-cert \ + --non-interactive \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} > /dev/null 2>&1 + svn cleanup ${SIGNED_REPO_COPY} > /dev/null 2>&1 + + echo "Remove -u/--unlock option from configuration and run again" + exit 1 +fi + +# If lock file exists, a signing build is still in process by JPL Cybersecurity +svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} +if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then + echo "Previous signing job still in process by JPL Cybersecurity. Please try again later." + exit 1 +fi + +# Commit lock file to repository for signed packages +echo "Committing lock file to repository for signed packages" +touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} +svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 +svn commit \ + --trust-server-cert \ + --non-interactive \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + --message "ADD: New lock file" ${SIGNED_REPO_COPY} > /dev/null 2>&1 + +# Check out copy of repository for unsigned packages +checkout_unsigned_repo_copy + +if [ ${retrigger_signing} -eq 0 ]; then + # Commit new compressed package to repository for unsigned binaries + echo "Committing package to repository for unsigned packages" + cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY} + svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1 + svn commit \ + --trust-server-cert \ + --non-interactive \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 +else + # NOTE: If notarize_only == 1, we commit a dummy file as we do not want to + # have to commit the entire compressed package again simply to + # retrigger the signing build on the remote JPL Cybersecurity Jenkins + # server. + # + echo "Attempting to sign existing package again" + echo $(date +'%Y-%m-%d-%H-%M-%S') > ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} # Write datetime stamp to file to ensure modification is made + svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1 + svn commit \ + --trust-server-cert \ + --non-interactive \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 +fi + +# Check status of signing +echo "Checking progress of signing..." +SIGNING_CHECK_ATTEMPT=0 +while [ ${SIGNING_CHECK_ATTEMPT} -lt ${MAX_SIGNING_CHECK_ATTEMPTS} ]; do + echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds" + sleep ${SIGNING_CHECK_PERIOD} + svn up ${SIGNED_REPO_COPY} + + if [ ! -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then + # Retrieve notarization lock file + svn up ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} + + # Check status + STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //') + if [[ "${STATUS}" == "success" ]]; then + echo "Notarization successful!" + else + echo "Notarization failed!" + echo "----------------------- Contents of notarization logfile -----------------------" + cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} + echo "--------------------------------------------------------------------------------" + + exit 1 + fi + else + ((++SIGNING_CHECK_ATTEMPT)) + fi +done Index: /issm/trunk-jpl/packagers/mac/complete-issm-mac-binaries-matlab.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/complete-issm-mac-binaries-matlab.sh (revision 25884) +++ /issm/trunk-jpl/packagers/mac/complete-issm-mac-binaries-matlab.sh (revision 25884) @@ -0,0 +1,117 @@ +#!/bin/bash + +################################################################################ +# Wrapper script for Jenkins project to build, package, and commit ISSM +# distributable package for macOS with MATLAB API to repository for signing. +# Normally, we would put this directly into the project configuration under +# 'Build' -> 'Excute shell', but becasue it is a bit more involved, it is a +# good idea to version it. +# +# When no failures/errors occur, performs the following: +# - Builds ISSM according to configuration. +# - Packages and compresses executables and libraries. +# - Runs test suite against package. +# - Commits compressed package to repository to be signed by JPL Cybersecurity. +# - Retrieves signed package and transmits it to ISSM Web site for +# distribution. +# +# Options: +# -b/--skipbuild Skip ISSM compilation. +# -r/--resign Skip ISSM compilation and packaging. Use to retrigger +# signing/notarization if it fails but build and package +# are valid. +# -s/--skiptests Skip ISSM compilation and testing during packaging +# step. Use if +# -t/--transferonly Transfer package to ISSM Web site only. Use if transfer +# fails for some reason to skip building, packaging, and +# signing. +# -u/--unlock Remove lock file from signed package repository. Use if +# build is aborted to allow for subsequent fresh build. +# +# Debugging: +# - Relies on a very tight handshake with project on remote JPL Cybersecurity +# Jenkins server. Debugging may be perfomed locally by running, +# +# packagers/mac/sign-issm-mac-binaries-matlab.sh +# +# with "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" hardcoded to Apple +# Developer credentials. +# - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can +# help debug potential SVN issues. +# +# NOTE: +# - Use only *one* of the above options at a time, and make sure it is removed +# again after a single run. +# - Builds will fail when any of the above options are used on a clean +# workspace. For example, if 'Source Code Management' -> 'Check-out Strategy' +# select menu is set to "Always check out a fresh copy". +# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in +# the 'Bindings' section under a 'Username and password (separated)' binding +# (requires 'Credentials Binding Plugin') with 'Credentials' select menu set +# to "jenkins/****** (SVN repository for ISSM binaries)". +################################################################################ + +## Constants +# +PKG="ISSM-macOS-MATLAB" # Name of directory to copy distributable files to +SIGNED_REPO_COPY="./signed" +SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed" + +COMPRESSED_PKG="${PKG}.zip" + +## Environment +# +export COMPRESSED_PKG +export ISSM_BINARIES_REPO_PASS +export ISSM_BINARIES_REPO_USER +export PKG +export SIGNED_REPO_COPY +export SIGNED_REPO_URL + +## Parse options +# +if [ $# -gt 1 ]; then + echo "Can only use one option at a time" + exit 1 +fi + +# NOTE: We could do this with binary switching (i.e. 0011 to sign and transfer, +# but the following is self-documenting). +# +build=1 +package=1 +sign=1 +transfer=1 + +if [ $# -eq 1 ]; then + case $1 in + -b|--skipbuild) build=0; shift ;; + -r|--resign) build=0; package=0; ;; + -s|--skiptests) build=0; ;; + -t|--transferonly) build=0; package=0; sign=0; shift ;; + -u|--unlock) build=0; package=0; transfer=0; ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac +fi + +# Build +if [ ${build} -eq 1 ]; then + ./jenkins/jenkins.sh ./jenkins/pine_island-mac-binaries-matlab +fi + +# Package +if [ ${package} -eq 1 ]; then + ./packagers/mac/package-issm-mac-binaries-matlab.sh $1 + shift # Clear $1 so that it is not passed to commit_for_signing script +fi + +# Commit for signing +if [ ${sign} -eq 1 ]; then + ./packagers/mac/commit_for_signing-issm-mac-binaries-matlab.sh $1 +fi + +# Transfer distributable package to ISSM Web site +if [ ${transfer} -eq 1 ]; then + ./packagers/mac/transfer-issm-mac-binaries-matlab.sh +fi + Index: /issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-matlab.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-matlab.sh (revision 25883) +++ /issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-matlab.sh (revision 25884) @@ -2,44 +2,19 @@ ################################################################################ -# To be used after running, -# -# ${ISSM_DIR}/jenkins/jenkins.sh ${ISSM_DIR}/jenkins/pine_island-mac-binaries-matlab -# -# in the context of a Jenkins project. -# -# When no runtime errors occur, performs the following: -# - Checks resulting executables and libraries against test suite. -# - Packages and compresses executables and libraries. -# - Commits compressed package to repository to be signed by JPL Cybersecurity. -# - Retrieves signed package and transmits it to ISSM Web site for -# distribution. +# Packages and tests ISSM distributable package for macOS with MATLAB API. # # Options: -# -n/--notarizeonly Sign/notarize only (use if signing/notarization fails -# to skip tests/packaging) -# -s/--skiptests Skip tests (use if this script fails for some reason -# after tests have successfully passed to save time) -# -t/--transferonly Transfer package to ISSM Web site only (use if -# transfer fails for some reason to skip testing and -# signing) -# -u/--unlock Remove lock file from signed package repository (use if -# build is aborted to allow for subsequent fresh build) -# -# Debugging: -# - Relies on a very tight handshake with project on remote JPL Cybersecurity -# Jenkins server. Debugging may be perfomed locally by running, -# -# packagers/mac/sign-issm-mac-binaries-matlab.sh -# -# with Apple Developer credentials. -# - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can -# help debug potential SVN issues. +# -s/--skiptests Skip tests and package only. # # NOTE: -# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in -# the 'Bindings' section under a 'Username and password (separated)' binding -# (requires 'Credentials Binding Plugin'). -# - For local debugging, the aformentioned credentials can be hardcoded into -# the 'USERNAME' and 'PASSWORD' constants below. +# - Assumes that the following constants are defined, +# +# COMPRESSED_PKG +# ISSM_DIR +# PKG +# +# See also: +# - packagers/mac/complete-issm-mac-binaries-matlab.sh +# - packagers/mac/sign-issm-mac-binaries-matlab.sh ################################################################################ @@ -66,4 +41,5 @@ ## Override certain other aliases # +alias cp=$(which cp) alias grep=$(which grep) @@ -72,58 +48,8 @@ MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2006,2020,2021,2051,2052,2053,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded MATLAB_PATH="/Applications/MATLAB_R2018a.app" -NOTARIZATION_LOGFILE="notarization.log" -PASSWORD=${ISSM_BINARIES_PASS} -PKG="ISSM-macOS-MATLAB" # Name of directory to copy distributable files to -RETRIGGER_SIGNING_FILE="retrigger.txt" -SIGNED_REPO_COPY="./signed" -SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed" -SIGNING_CHECK_PERIOD=60 # in seconds -SIGNING_LOCK_FILE="signing.lock" -UNSIGNED_REPO_COPY="./unsigned" -UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned" -USERNAME=${ISSM_BINARIES_USER} - -COMPRESSED_PKG="${PKG}.zip" ## Environment # -export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages' - -## Functions -# -checkout_signed_repo_copy(){ - echo "Checking out copy of repository for signed packages" - - # NOTE: Get empty copy because we do not want to have to check out package - # from previous signing. - # - svn checkout \ - --trust-server-cert \ - --non-interactive \ - --depth empty \ - --username ${USERNAME} \ - --password ${PASSWORD} \ - ${SIGNED_REPO_URL} \ - ${SIGNED_REPO_COPY} > /dev/null 2>&1 -} -checkout_unsigned_repo_copy(){ - echo "Checking out copy of repository for unsigned packages" - svn checkout \ - --trust-server-cert \ - --non-interactive \ - --username ${USERNAME} \ - --password ${PASSWORD} \ - ${UNSIGNED_REPO_URL} \ - ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 -} -validate_signed_repo_copy(){ - # Validate copy of repository for signed binaries (e.g. - # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible'; - # initial checkout failed) - if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then - rm -rf ${SIGNED_REPO_COPY} - checkout_signed_repo_copy - fi -} +export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'; used when running tests ## Parse options @@ -134,268 +60,120 @@ fi -notarize_only=0 skip_tests=0 -transfer_only=0 -unlock=0 -while [ $# -gt 0 ]; do - case $1 in - -n|--notarizeonly) notarize_only=1; shift ;; - -s|--skiptests) skip_tests=1; shift ;; - -t|--transferonly) transfer_only=1; shift ;; - -u|--unlock) unlock=1; shift ;; - *) echo "Unknown parameter passed: $1"; exit 1 ;; - esac - shift -done -if [ ${unlock} -eq 1 ]; then - # Remove signing lock file from signed package repository so that a new - # build can run - echo "Removing lock file from repository for signed packages" - checkout_signed_repo_copy - svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} - svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 - svn commit \ - --trust-server-cert \ - --non-interactive \ - --username ${USERNAME} \ - --password ${PASSWORD} \ - --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} - svn cleanup ${SIGNED_REPO_COPY} +if [ $# -eq 1 ]; then + case $1 in + -s|--skiptests) skip_tests=1; ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac +fi + +# Check if MATLAB exists +if ! [ -d ${MATLAB_PATH} ]; then + echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again." exit 1 fi -if [ ${transfer_only} -eq 0 ]; then - rm -rf ${SIGNED_REPO_COPY} +# Clean up from previous packaging +echo "Cleaning up existing assets" +cd ${ISSM_DIR} +rm -rf ${PKG} ${COMPRESSED_PKG} +mkdir ${PKG} - checkout_signed_repo_copy +# Add required binaries and libraries to package and modify them where needed +cd ${ISSM_DIR}/bin - # If lock file exists, a signing build is still in process by JPL Cybersecurity - svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} - if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then - echo "Previous signing job still in process by JPL Cybersecurity. Please try again later." +echo "Modify generic" +cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m + +echo "Moving MPICH binaries to bin/" +if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then + cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec . + cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy . +elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then + cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec . + cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy . +else + echo "MPICH not found" + exit 1 +fi + +echo "Moving GDAL binaries to bin/" +if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then + cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo . + cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform . +else + echo "GDAL not found" + exit 1 +fi + +echo "Moving GMT binaries to bin/" +if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then + cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt . + cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect . +else + echo "GMT not found" + exit 1 +fi + +echo "Moving Gmsh binaries to bin/" +if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then + cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh . +else + echo "Gmsh not found" + exit 1 +fi + +# Run tests +if [ ${skip_tests} -eq 0 ]; then + echo "Running tests" + cd ${ISSM_DIR}/test/NightlyRun + rm matlab.log 2> /dev/null + + # Run tests, redirecting output to logfile and suppressing output to console + ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null + + # Check that MATLAB did not exit in error + matlabExitCode=`echo $?` + matlabExitedInError=`grep -E "Activation cannot proceed|Error in matlab_run" matlab.log | wc -l` + + if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then + echo "----------MATLAB exited in error!----------" + cat matlab.log + echo "-----------End of matlab.log-----------" + + # Clean up execution directory + rm -rf ${ISSM_DIR}/execution/* + exit 1 fi - if [ ${notarize_only} -eq 0 ]; then - # Check if MATLAB exists - if ! [ -d ${MATLAB_PATH} ]; then - echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again." - exit 1 - fi + # Check that all tests passed + numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"` - # Clean up from previous packaging - echo "Cleaning up existing assets" - cd ${ISSM_DIR} - rm -rf ${PKG} ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY} - mkdir ${PKG} - - # Add required binaries and libraries to package and modify them where needed - cd ${ISSM_DIR}/bin - - echo "Modify generic" - cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m - - echo "Moving MPICH binaries to bin/" - if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then - cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec . - cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy . - elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then - cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec . - cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy . - else - echo "MPICH not found" - exit 1 - fi - - echo "Moving GDAL binaries to bin/" - if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then - cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo . - cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform . - else - echo "GDAL not found" - exit 1 - fi - - echo "Moving GMT binaries to bin/" - if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then - cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt . - cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect . - else - echo "GMT not found" - exit 1 - fi - - echo "Moving Gmsh binaries to bin/" - if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then - cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh . - else - echo "Gmsh not found" - exit 1 - fi - - # Run tests - if [ ${skip_tests} -eq 0 ]; then - echo "Running tests" - cd ${ISSM_DIR}/test/NightlyRun - rm matlab.log 2> /dev/null - - # Run tests, redirecting output to logfile and suppressing output to console - ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null - - # Check that MATLAB did not exit in error - matlabExitCode=`echo $?` - matlabExitedInError=`grep -E "Activation cannot proceed|Error in matlab_run" matlab.log | wc -l` - - if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then - echo "----------MATLAB exited in error!----------" - cat matlab.log - echo "-----------End of matlab.log-----------" - - # Clean up execution directory - rm -rf ${ISSM_DIR}/execution/* - - exit 1 - fi - - # Check that all tests passed - numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"` - - if [ ${numTestsFailed} -ne 0 ]; then - echo "One or more tests FAILED" - exit 1 - else - echo "All tests PASSED" - fi - else - echo "Skipping tests" - fi - - # Create package - cd ${ISSM_DIR} - svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package) - echo "Copying assets to package: ${PKG}" - cp -rf bin examples lib scripts test ${PKG}/ - mkdir ${PKG}/execution - cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist - echo "Cleaning up unneeded/unwanted files" - rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes - rm -f ${PKG}/lib/*.a # Remove static libraries from package - rm -f ${PKG}/lib/*.la # Remove libtool libraries from package - rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package - - # Compress package - echo "Compressing package" - ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG} + if [ ${numTestsFailed} -ne 0 ]; then + echo "One or more tests FAILED" + exit 1 else - # Assume that previous build was successful, but signing/notarization - # failed. - # - echo "Notarizing only" + echo "All tests PASSED" fi - - # Commit lock file to repository for signed packages - echo "Committing lock file to repository for signed packages" - touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} - svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 - svn commit \ - --trust-server-cert \ - --non-interactive \ - --username ${USERNAME} \ - --password ${PASSWORD} \ - --message "ADD: New lock file" ${SIGNED_REPO_COPY} - - # Check out copy of repository for unsigned packages - checkout_unsigned_repo_copy - - if [ ${notarize_only} -eq 0 ]; then - # Commit new compressed package to repository for unsigned binaries - # - # NOTE: This will not work if, for any reason, the checksum on the compressed - # package is unchanged. - # - echo "Committing package to repository for unsigned packages" - cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY} - svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1 - svn commit \ - --trust-server-cert \ - --non-interactive \ - --username ${USERNAME} \ - --password ${PASSWORD} \ - --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} - else - # NOTE: If notarize_only == 1, we commit a dummy file as the signing - # build on the remote JPL Cybersecurity Jenkins server is - # triggered by polling SCM. - # - echo "Attempting to sign existing package again" - echo $(date +'%Y-%m-%d-%H-%M-%S') > ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} # Write datetime stamp to file to ensure modification is made - svcheckout_unsigned_repo_copyn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1 - svn commit \ - --trust-server-cert \ - --non-interactive \ - --username ${USERNAME} \ - --password ${PASSWORD} \ - --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} - fi - - # Check status of signing - echo "Checking progress of signing..." - IN_PROCESS=1 - SUCCESS=0 - - while [ ${IN_PROCESS} -eq 1 ]; do - echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds" - sleep ${SIGNING_CHECK_PERIOD} - svn up ${SIGNED_REPO_COPY} - - if [ ! -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then - IN_PROCESS=0 - - # Retrieve notarization lock file - svn up ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} - - # Check status - STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //') - if [[ "${STATUS}" == "success" ]]; then - echo "Notarization successful!" - - # Set flag indicating notarization was successful - SUCCESS=1 - else - echo "Notarization failed!" - fi - fi - done else - # Assume that previous build resulted in successful signing of package but - # that transfer to ISSM Web site failed and user built this project again - # with -t/--transferonly option. - # - - # Make sure copy of repository for signed packages exists - validate_signed_repo_copy - - SUCCESS=1 + echo "Skipping tests" fi -# Handle result of signing -if [ ${SUCCESS} -eq 1 ]; then - # Retrieve signed and notarized package - svn up ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} +# Create package +cd ${ISSM_DIR} +svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package) +echo "Copying assets to package: ${PKG}" +cp -rf bin examples lib scripts test ${PKG}/ +mkdir ${PKG}/execution +cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist +echo "Cleaning up unneeded/unwanted files" +rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes +rm -f ${PKG}/lib/*.a # Remove static libraries from package +rm -f ${PKG}/lib/*.la # Remove libtool libraries from package +rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package - # Transfer signed package to ISSM Web site - echo "Transferring signed package to ISSM Web site" - scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG} - - if [ $? -ne 0 ]; then - echo "Transfer failed! Verify connection then build this project again (with -t/--transferonly option to skip testing and signing)." - exit 1 - fi -else - echo "----------------------- Contents of notarization logfile -----------------------" - cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} - echo "--------------------------------------------------------------------------------" - - exit 1 -fi +# Compress package +echo "Compressing package" +ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG} Index: /issm/trunk-jpl/packagers/mac/transfer-issm-mac-binaries-matlab.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/transfer-issm-mac-binaries-matlab.sh (revision 25884) +++ /issm/trunk-jpl/packagers/mac/transfer-issm-mac-binaries-matlab.sh (revision 25884) @@ -0,0 +1,85 @@ +#!/bin/bash + +################################################################################ +# Transfers ISSM distributable package for macOS with MATLAB API to ISSM Web +# site. +# +# NOTE: +# - Assumes that the following constants are defined, +# +# COMPRESSED_PKG +# ISSM_BINARIES_REPO_PASS +# ISSM_BINARIES_REPO_USER +# SIGNED_REPO_COPY +# SIGNED_REPO_URL +# +# See also: +# - packagers/mac/complete-issm-mac-binaries-matlab.sh +################################################################################ + +# Expand aliases within the context of this script +shopt -s expand_aliases + +# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes, +# +# Command line tool support for Subversion — including svn, git-svn, and +# related commands — is no longer provided by Xcode. (50266910) +# +# which results in, +# +# svn: error: The subversion command line tools are no longer provided by +# Xcode. +# +# when calling svn, even when subversion is installed via Homebrew and its path +# is available in PATH. +# +# NOTE: May be able to remove this after updating macOS. +# +alias svn='/usr/local/bin/svn' + +## Override certain other aliases +# +alias cp=$(which cp) +alias grep=$(which grep) + +## Functions +# +checkout_signed_repo_copy(){ + echo "Checking out copy of repository for signed packages" + + # NOTE: Get empty copy because we do not want to have to check out package + # from previous signing. + # + svn checkout \ + --trust-server-cert \ + --non-interactive \ + --depth empty \ + --username ${ISSM_BINARIES_REPO_USER} \ + --password ${ISSM_BINARIES_REPO_PASS} \ + ${SIGNED_REPO_URL} \ + ${SIGNED_REPO_COPY} > /dev/null 2>&1 +} +validate_signed_repo_copy(){ + # Validate copy of repository for signed binaries (e.g. + # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible'; + # initial checkout failed) + if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then + rm -rf ${SIGNED_REPO_COPY} + checkout_signed_repo_copy + fi +} + +# Check if working copy of repository for signed packages is missing +validate_signed_repo_copy + +# Retrieve signed and notarized package +svn up ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} + +# Transfer signed package to ISSM Web site +echo "Transferring signed package to ISSM Web site" +scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG} + +if [ $? -ne 0 ]; then + echo "Transfer failed! Verify connection then build this project again (with -t/--transferonly option to skip building, packaging, and signing)." + exit 1 +fi