Index: /issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-matlab.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-matlab.sh (revision 25779) +++ /issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-matlab.sh (revision 25780) @@ -11,10 +11,18 @@ # - Checks resulting executables and libraries against test suite. # - Packages and compresses executables and libraries. -# - Commits compressed package to SVN repository to be signed by JPL -# Cybersecurity. +# - Commits compressed package to repository to be signed by JPL Cybersecurity. # - Retrieves signed package and transmits it to ISSM Web site for # distribution. # -# To skip tests, run with -s/--skiptests option. +# Options: +# -c/--cleanup Remove lock file from signed package repository (use if +# build is aborted to allow for subsequent fresh build) +# -n/--notarizeonly Sign/notarize only (use if signing/notarization fails +# to skip tests/packaging) +# -s/--skiptests Skip tests (use if this script fails for some reason +# after tests have successfully passed to save time) +# -t/--transferonly Transfer package to ISSM Web site only (use if +# transfer fails for some reason to skip testing and +# signing) # # Debugging: @@ -36,4 +44,7 @@ ################################################################################ +# Expand aliases within the context of this script +shopt -s expand_aliases + # From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes, # @@ -49,5 +60,6 @@ # is available in PATH. # -shopt -s expand_aliases +# NOTE: May be able to remove this after updating macOS. +# alias svn='/usr/local/bin/svn' @@ -58,13 +70,13 @@ ## Constants # - MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2006,2020,2021,2051,2052,2053,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded -MATLAB_PATH="/Applications/MATLAB_R2018a.app" +MATLAB_PATH="/Applications/MATLAB_R2019b.app" NOTARIZATION_LOGFILE="notarization.log" PASSWORD=$env:issm-binaries-pass PKG="ISSM-macOS-MATLAB" # Name of directory to copy distributable files to +RETRIGGER_SIGNING_FILE="retrigger.txt" SIGNED_REPO_COPY="./signed" SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed" -SIGNING_CHECK_PERIOD=60 # seconds +SIGNING_CHECK_PERIOD=60 # in seconds SIGNING_LOCK_FILE="signing.lock" UNSIGNED_REPO_COPY="./unsigned" @@ -80,8 +92,12 @@ ## Parse options # +cleanup=0 +notarize_only=0 skip_tests=0 transfer_only=0 -while [[ "$#" -gt 0 ]]; do +while [ $# -gt 0 ]; do case $1 in + -c|--cleanup) cleanup=1; shift ;; + -n|--notarizeonly) notarize_only=1; shift ;; -s|--skiptests) skip_tests=1; shift ;; -t|--transferonly) transfer_only=1; shift ;; @@ -91,24 +107,8 @@ done -if [ ${transfer_only} -eq 0 ]; then - # Check if MATLAB exists - if ! [ -d ${MATLAB_PATH} ]; then - echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again." - exit 1 - fi - - # Clean up from previous packaging (not necessary for single builds on Jenkins, - # but useful when testing packaging locally) - echo "Cleaning up existing assets" - cd ${ISSM_DIR} - rm -rf ${PKG} ${COMPRESSED_PKG} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY} - mkdir ${PKG} - - # Check out copy of SVN repository for signed macOS packages - # - # NOTE: Get empty copy because we do not want to have to check out package from - # previous signing. - # - echo "Checking out copy of repository for signed packages" +if [ ${cleanup} -eq 1 ]; then + # Remove signing lock file from signed package repository so that a new + # build can run + echo "Removing lock file from repository for signed packages" svn co \ --username ${USERNAME} \ @@ -117,107 +117,144 @@ ${SIGNED_REPO_URL} \ ${SIGNED_REPO_COPY} > /dev/null 2>&1 - - # If lock file exists, a signing build is still in process by JPL Cybersecurity svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 - if [[ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]]; then - echo "Previous signing job still in process by JPL Cybersecurity. Please try again later." - exit 1 + svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 + svn commit --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} > /dev/null 2>&1 + svn cleanup ${SIGNED_REPO_COPY} > /dev/null 2>&1 + exit 1 +fi + +if [ ${transfer_only} -eq 0 ]; then + if [ ${notarize_only} -eq 0 ]; then + # Check if MATLAB exists + if ! [ -d ${MATLAB_PATH} ]; then + echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again." + exit 1 + fi + + # Clean up from previous packaging (not necessary for single builds on Jenkins, + # but useful when testing packaging locally) + echo "Cleaning up existing assets" + cd ${ISSM_DIR} + rm -rf ${PKG} ${COMPRESSED_PKG} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY} + mkdir ${PKG} + + # Check out copy of SVN repository for signed packages + # + # NOTE: Get empty copy because we do not want to have to check out package from + # previous signing. + # + echo "Checking out copy of repository for signed packages" + svn co \ + --username ${USERNAME} \ + --password ${PASSWORD} \ + --depth empty \ + ${SIGNED_REPO_URL} \ + ${SIGNED_REPO_COPY} > /dev/null 2>&1 + + # If lock file exists, a signing build is still in process by JPL Cybersecurity + svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 + if [[ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]]; then + echo "Previous signing job still in process by JPL Cybersecurity. Please try again later." + exit 1 + fi + + # Add required binaries and libraries to package and modify them where needed + cd ${ISSM_DIR}/bin + + echo "Modify generic" + cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m + + echo "Moving MPICH binaries to bin/" + if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then + cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec . + cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy . + elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then + cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec . + cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy . + else + echo "MPICH not found" + exit 1 + fi + + echo "Moving GDAL binaries to bin/" + if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then + cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo . + cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform . + else + echo "GDAL not found" + exit 1 + fi + + echo "Moving GMT binaries to bin/" + if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then + cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt . + cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect . + else + echo "GMT not found" + exit 1 + fi + + echo "Moving Gmsh binaries to bin/" + if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then + cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh . + else + echo "Gmsh not found" + exit 1 + fi + + # Run tests + if [ ${skip_tests} -eq 0 ]; then + echo "Running tests" + cd ${ISSM_DIR}/test/NightlyRun + rm matlab.log 2> /dev/null + + # Run tests, redirecting output to logfile and suppressing output to console + ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null + + # Check that MATLAB did not exit in error + matlabExitCode=`echo $?` + matlabExitedInError=`grep -E "Activation cannot proceed|license|Error" matlab.log | wc -l` + + if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then + echo "----------MATLAB exited in error!----------" + cat matlab.log + echo "-----------End of matlab.log-----------" + + # Clean up execution directory + rm -rf ${ISSM_DIR}/execution/* + + exit 1 + fi + + # Check that all tests passed + numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"` + + if [[ ${numTestsFailed} -ne 0 ]]; then + echo "One or more tests FAILED" + exit 1 + else + echo "All tests PASSED" + fi + else + echo "Skipping tests" + fi + + # Create package + cd ${ISSM_DIR} + svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package) + echo "Copying assets to package: ${PKG}" + cp -rf bin examples lib scripts test ${PKG}/ + mkdir ${PKG}/execution + cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist + echo "Cleaning up unneeded/unwanted files" + rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes + rm -f ${PKG}/lib/*.a # Remove static libraries from package + rm -f ${PKG}/lib/*.la # Remove libtool libraries from package + rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package + + # Compress package + echo "Compressing package" + ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG} fi - - # Add required binaries and libraries to package and modify them where needed - cd ${ISSM_DIR}/bin - - echo "Modify generic" - cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m - - echo "Moving MPICH binaries to bin/" - if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then - cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec . - cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy . - elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then - cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec . - cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy . - else - echo "MPICH not found" - exit 1 - fi - - echo "Moving GDAL binaries to bin/" - if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then - cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo . - cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform . - else - echo "GDAL not found" - exit 1 - fi - - echo "Moving GMT binaries to bin/" - if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then - cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt . - cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect . - else - echo "GMT not found" - exit 1 - fi - - echo "Moving Gmsh binaries to bin/" - if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then - cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh . - else - echo "Gmsh not found" - exit 1 - fi - - # Run tests - if [ ${skip_tests} -eq 0 ]; then - echo "Running tests" - cd ${ISSM_DIR}/test/NightlyRun - rm matlab.log 2> /dev/null - - # Run tests, redirecting output to logfile and suppressing output to console - ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null - - # Check that MATLAB did not exit in error - matlabExitCode=`echo $?` - matlabExitedInError=`grep -E "Activation cannot proceed|license|Error" matlab.log | wc -l` - - if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then - echo "----------MATLAB exited in error!----------" - cat matlab.log - echo "-----------End of matlab.log-----------" - - # Clean up execution directory - rm -rf ${ISSM_DIR}/execution/* - - exit 1 - fi - - # Check that all tests passed - numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"` - - if [[ ${numTestsFailed} -ne 0 ]]; then - echo "One or more tests FAILED" - exit 1 - else - echo "All tests PASSED" - fi - else - echo "Skipping tests" - fi - - # Create package - cd ${ISSM_DIR} - svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package) - echo "Copying assets to package: ${PKG}" - cp -rf bin examples lib scripts test ${PKG}/ - echo "Cleaning up unneeded/unwanted files" - rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes - rm -f ${PKG}/lib/*.a # Remove static libraries from package - rm -f ${PKG}/lib/*.la # Remove libtool libraries from package - rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package - - # Compress package - echo "Compressing package" - ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG} # Commit lock file to repository for signed packages @@ -231,5 +268,5 @@ CURRENT_REV=$(svn info --show-item last-changed-revision ${SIGNED_REPO_COPY}) - # Check out copy of SVN repository for unsigned macOS packages + # Check out copy of SVN repository for unsigned packages echo "Checking out copy of repository for unsigned packages" svn co \ @@ -239,13 +276,24 @@ ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 - # Commit new compressed package to repository for unsigned binaries - # - # NOTE: This will not work if, for any reason, the checksum on the compressed - # package is unchanged. - # - echo "Committing package to repository for unsigned packages" - cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY} - svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1 - svn commit --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 + if [ ${notarize_only} -eq 0 ]; then + # Commit new compressed package to repository for unsigned binaries + # + # NOTE: This will not work if, for any reason, the checksum on the compressed + # package is unchanged. + # + echo "Committing package to repository for unsigned packages" + cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY} + svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1 + svn commit --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 + else + # NOTE: If notarize_only == 1, we commit a dummy file as the signing + # build on the remote JPL Cybersecurity Jenkins server is + # triggered by polling SCM. + # + echo "Attempting to sign existing package again" + touch ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} + svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1 + svn commit --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 + fi # Check status of signing @@ -255,5 +303,5 @@ while [ ${IN_PROCESS} -eq 1 ]; do - echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds." + echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds" sleep ${SIGNING_CHECK_PERIOD} svn up ${SIGNED_REPO_COPY} > /dev/null 2>&1 @@ -269,14 +317,14 @@ STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //') if [[ "${STATUS}" == "success" ]]; then - echo 'Notarization successful!' + echo "Notarization successful!" # Set flag indicating notarization was successful SUCCESS=1 else - echo 'Notarization failed!' + echo "Notarization failed!" fi fi done -else # transfer_only == 1 +else # Assume that previous build resulted in successful signing of package but # that transfer to ISSM Web site failed and user built this project again @@ -288,5 +336,5 @@ if [ ${SUCCESS} -eq 1 ]; then # Transfer signed package to ISSM Web site - echo 'Transferring signed package to ISSM Web site' + echo "Transferring signed package to ISSM Web site" scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG} @@ -296,7 +344,7 @@ fi else - echo '----------------------- Contents of notarization logfile -----------------------' + echo "----------------------- Contents of notarization logfile -----------------------" cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} - echo '--------------------------------------------------------------------------------' + echo "--------------------------------------------------------------------------------" exit 1 Index: /issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh (revision 25780) +++ /issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh (revision 25780) @@ -0,0 +1,352 @@ +#!/bin/bash + +################################################################################ +# To be used after running, +# +# ${ISSM_DIR}/jenkins/jenkins.sh ${ISSM_DIR}/jenkins/pine_island-mac-binaries-python +# +# in the context of a Jenkins project. +# +# When no runtime errors occur, performs the following: +# - Checks resulting executables and libraries against test suite. +# - Packages and compresses executables and libraries. +# - Commits compressed package to repository to be signed by JPL Cybersecurity. +# - Retrieves signed package and transmits it to ISSM Web site for +# distribution. +# +# Options: +# -c/--cleanup Remove lock file from signed package repository (use if +# build is aborted to allow for subsequent fresh build) +# -n/--notarizeonly Sign/notarize only (use if signing/notarization fails +# to skip tests/packaging) +# -s/--skiptests Skip tests (use if this script fails for some reason +# after tests have successfully passed to save time) +# -t/--transferonly Transfer package to ISSM Web site only (use if +# transfer fails for some reason to skip testing and +# signing) +# +# Debugging: +# - Relies on a very tight handshake with project on remote JPL Cybersecurity +# Jenkins server. Debugging may be perfomed locally by running, +# +# packagers/mac/sign-issm-mac-binaries-python.sh +# +# with Apple Developer credentials. +# - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can +# help debug potential SVN issues. +# +# NOTE: +# - Assumes that 'issm-binaries-user' and 'issm-binaries-pass' are set up in +# the 'Bindings' section under a 'Username and password (separated)' binding +# (requires 'Credentials Binding Plugin'). +# - For local debugging, the aformentioned credentials can be hardcoded into +# the 'USERNAME' and 'PASSWORD' constants below. +################################################################################ + +# Expand aliases within the context of this script +shopt -s expand_aliases + +# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes, +# +# Command line tool support for Subversion — including svn, git-svn, and +# related commands — is no longer provided by Xcode. (50266910) +# +# which results in, +# +# svn: error: The subversion command line tools are no longer provided by +# Xcode. +# +# when calling svn, even when subversion is installed via Homebrew and its path +# is available in PATH. +# +# NOTE: May be able to remove this after updating macOS. +# +alias svn='/usr/local/bin/svn' + +## Override certain other aliases +# +alias grep=$(which grep) + +## Constants +# +NOTARIZATION_LOGFILE="notarization.log" +PASSWORD=$env:issm-binaries-pass +PKG="ISSM-macOS-Python" # Name of directory to copy distributable files to +PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2006 2020 2021 2051 2052 2053 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded +RETRIGGER_SIGNING_FILE="retrigger.txt" +SIGNED_REPO_COPY="./signed" +SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/signed" +SIGNING_CHECK_PERIOD=60 # in seconds +SIGNING_LOCK_FILE="signing.lock" +UNSIGNED_REPO_COPY="./unsigned" +UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned" +USERNAME=$env:issm-binaries-user + +COMPRESSED_PKG="${PKG}.zip" + +## Environment +# +export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages' + +## Parse options +# +cleanup=0 +notarize_only=0 +skip_tests=0 +transfer_only=0 +while [ $# -gt 0 ]; do + case $1 in + -c|--cleanup) cleanup=1; shift ;; + -n|--notarizeonly) notarize_only=1; shift ;; + -s|--skiptests) skip_tests=1; shift ;; + -t|--transferonly) transfer_only=1; shift ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +if [ ${cleanup} -eq 1 ]; then + # Remove signing lock file from signed package repository so that a new + # build can run + echo "Removing lock file from repository for signed packages" + svn co \ + --username ${USERNAME} \ + --password ${PASSWORD} \ + --depth empty \ + ${SIGNED_REPO_URL} \ + ${SIGNED_REPO_COPY} > /dev/null 2>&1 + svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 + svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 + svn commit --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} > /dev/null 2>&1 + svn cleanup ${SIGNED_REPO_COPY} > /dev/null 2>&1 + exit 1 +fi + +if [ ${transfer_only} -eq 0 ]; then + if [ ${notarize_only} -eq 0 ]; then + # Clean up from previous packaging (not necessary for single builds on + # Jenkins, but useful when testing packaging locally) + echo "Cleaning up existing assets" + cd ${ISSM_DIR} + rm -rf ${PKG} ${COMPRESSED_PKG} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY} + mkdir ${PKG} + + # Check out copy of SVN repository for signed packages + # + # NOTE: Get empty copy because we do not want to have to check out package + # from previous signing. + # + echo "Checking out copy of repository for signed packages" + svn co \ + --username ${USERNAME} \ + --password ${PASSWORD} \ + --depth empty \ + ${SIGNED_REPO_URL} \ + ${SIGNED_REPO_COPY} > /dev/null 2>&1 + + # If lock file exists, a signing build is still in process by JPL Cybersecurity + svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 + if [[ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]]; then + echo "Previous signing job still in process by JPL Cybersecurity. Please try again later." + exit 1 + fi + + # Add required binaries and libraries to package and modify them where needed + cd ${ISSM_DIR}/bin + + echo "Modify generic" + cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m + + echo "Moving MPICH binaries to bin/" + if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then + cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec . + cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy . + elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then + cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec . + cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy . + else + echo "MPICH not found" + exit 1 + fi + + echo "Moving GDAL binaries to bin/" + if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then + cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo . + cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform . + else + echo "GDAL not found" + exit 1 + fi + + echo "Moving GMT binaries to bin/" + if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then + cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt . + cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect . + else + echo "GMT not found" + exit 1 + fi + + echo "Moving Gmsh binaries to bin/" + if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then + cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh . + else + echo "Gmsh not found" + exit 1 + fi + + # Run tests + if [ ${skip_tests} -eq 0 ]; then + echo "Running tests" + + cd ${ISSM_DIR}/test/NightlyRun + rm python.log 2> /dev/null + + # Set Python environment + export PYTHONPATH="${ISSM_DIR}/src/m/dev" + export PYTHONSTARTUP="${PYTHONPATH}/devpath.py" + export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured + + # Run tests, redirecting output to logfile and suppressing output to console + ./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1 + + # Check that Python did not exit in error + pythonExitCode=`echo $?` + pythonExitedInError=`grep -E "Error|Standard exception|Traceback|bad interpreter" python.log | wc -l` + + if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then + echo "----------Python exited in error!----------" + cat python.log + echo "-----------End of python.log-----------" + + # Clean up execution directory + rm -rf ${ISSM_DIR}/execution/* + + exit 1 + fi + + # Check that all tests passed + numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"` + + if [[ ${numTestsFailed} -ne 0 ]]; then + echo "One or more tests FAILED" + exit 1 + else + echo "All tests PASSED" + fi + else + echo "Skipping tests" + fi + + # Create package + cd ${ISSM_DIR} + svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package) + echo "Copying assets to package: ${PKG}" + cp -rf bin examples lib scripts test ${PKG}/ + mkdir ${PKG}/execution + cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist + ${ISSM_DIR}/scripts/py_to_pyc.sh ${PKG}/bin # Compile Python source files + echo "Cleaning up unneeded/unwanted files" + rm -f ${PKG}/bin/*.py # Remove all Python scripts + rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes + rm -f ${PKG}/lib/*.a # Remove static libraries from package + rm -f ${PKG}/lib/*.la # Remove libtool libraries from package + rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package + + # Compress package + echo "Compressing package" + ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG} + fi + + # Commit lock file to repository for signed packages + echo "Committing lock file to repository for signed packages" + touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} + svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 + svn commit --message "ADD: New lock file" ${SIGNED_REPO_COPY} > /dev/null 2>&1 + + # Save current working copy revision number + svn up ${SIGNED_REPO_COPY} > /dev/null 2>&1 + CURRENT_REV=$(svn info --show-item last-changed-revision ${SIGNED_REPO_COPY}) + + # Check out copy of SVN repository for unsigned packages + echo "Checking out copy of repository for unsigned packages" + svn co \ + --username ${USERNAME} \ + --password ${PASSWORD} \ + ${UNSIGNED_REPO_URL} \ + ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 + + if [ ${notarize_only} -eq 0 ]; then + # Commit new compressed package to repository for unsigned binaries + # + # NOTE: This will not work if, for any reason, the checksum on the compressed + # package is unchanged. + # + echo "Committing package to repository for unsigned packages" + cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY} + svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1 + svn commit --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 + else + # NOTE: If notarize_only == 1, we commit a dummy file as the signing + # build on the remote JPL Cybersecurity Jenkins server is + # triggered by polling SCM. + # + echo "Attempting to sign existing package again" + touch ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} + svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1 + svn commit --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 + fi + + # Check status of signing + echo "Checking progress of signing..." + IN_PROCESS=1 + SUCCESS=0 + + while [ ${IN_PROCESS} -eq 1 ]; do + echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds" + sleep ${SIGNING_CHECK_PERIOD} + svn up ${SIGNED_REPO_COPY} > /dev/null 2>&1 + NEW_REV=$(svn info --show-item last-changed-revision ${SIGNED_REPO_COPY}) + + if [ ${NEW_REV} -ne ${CURRENT_REV} ]; then + IN_PROCESS=0 + + svn up ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1 + svn up ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1 + + # No error, so check status + STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //') + if [[ "${STATUS}" == "success" ]]; then + echo "Notarization successful!" + + # Set flag indicating notarization was successful + SUCCESS=1 + else + echo "Notarization failed!" + fi + fi + done +else + # Assume that previous build resulted in successful signing of package but + # that transfer to ISSM Web site failed and user built this project again + # with -t/--transferonly option. + SUCCESS=1 +fi + +# Handle result of signing +if [ ${SUCCESS} -eq 1 ]; then + # Transfer signed package to ISSM Web site + echo "Transferring signed package to ISSM Web site" + scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG} + + if [ $? -ne 0 ]; then + echo "Transfer failed! Verify connection then build this project again (with -t/--transferonly option to skip testing and signing)." + exit 1 + fi +else + echo "----------------------- Contents of notarization logfile -----------------------" + cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} + echo "--------------------------------------------------------------------------------" + + exit 1 +fi Index: /issm/trunk-jpl/packagers/mac/sign-issm-mac-binaries-matlab.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/sign-issm-mac-binaries-matlab.sh (revision 25779) +++ /issm/trunk-jpl/packagers/mac/sign-issm-mac-binaries-matlab.sh (revision 25780) @@ -1,2 +1,53 @@ +#!/bin/bash + +################################################################################ +# Intended to be run in the context of a Jenkins project on a JPL +# Cybersecurity server for signing macOS applications. Polls SCM of the +# Subversion repository hosted at +# https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned to trigger new +# builds. +# +# In order to replicate the requried Jenkins project configuration: +# - First, navigate to 'Manage Jenkins' -> 'Manage Plugins' and install the +# 'Credentials Bindings Plugin' if it is not already installed. +# - Contact one of the members of the ISSM development team for crendentials +# for the ISSM binaries repository (mention that the credentials are stored +# in ISSM-Infrastructure.pdf). +# - Navigate to 'Manage Jenkins' -> 'Manage Credentials' -> -> +# 'Add Credentials' and enter the crendentials from above. +# - From the 'Dashboard', select 'New Item' -> 'Freestyle project'. +# - Under 'Source Code Management', select 'Subversion'. +# - The 'Repository URL' text field should be set to +# "https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned" +# - The 'Credentials' select menu should be set to the new credentials +# created previously. +# - Under 'Build Trigggers', check the box for 'Poll SCM' and set the +# 'Schedule' text area to "H/5 * * * *". +# - Under 'Build Environment', check the box for 'Use secret text(s) or +# file(s)', then under 'Bindings' click the 'Add...' button and select +# 'Username and password (separated)'. +# - Set 'Username Variable' to "issm-binaries-user”. +# - Set 'Password Variable' to "issm-binaries-pass”. +# - Under 'Credentials', select the same, new credentials that created +# previously. +# - The contents of this script can be copied/pasted directly into the ‘Build' +# -> 'Execute Shell' -> ‘Command' textarea of the project configuration (or +# you can simply store the script on disk and call it from there). +# - Make sure to click the 'Save' button. +# +# Current point of contact at JPL Cybersecurity: +# Alex Coward, alexander.g.coward@jpl.nasa.gov +# +# NOTE: +# - Assumes that 'issm-binaries-user' and 'issm-binaries-pass' are set up in +# the 'Bindings' section under a 'Username and password (separated)' binding +# (requires 'Credentials Binding Plugin'). +# - For local debugging, the aformentioned credentials can be hardcoded into +# the 'USERNAME' and 'PASSWORD' constants below. +################################################################################ + +# Expand aliases within the context of this script +shopt -s expand_aliases + # From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes, # @@ -12,5 +63,6 @@ # is available in PATH. # -shopt -s expand_aliases +# NOTE: May be able to remove this after updating macOS. +# alias svn='/usr/local/bin/svn' @@ -20,7 +72,9 @@ alias grep=$(which grep) -AD_IDENTITY="**********" -AD_USERNAME="**********" -ALTOOL_PASSWORD="@keychain:**********" +## Constants +# +AD_IDENTITY="**********" # Apple Developer identity +AD_USERNAME="**********" # Apple Developer username +ALTOOL_PASSWORD="@keychain:**********" # altool password (assumed to be stored in keychain) ASC_PROVIDER="**********" NOTARIZATION_CHECK_ATTEMPTS=60 @@ -29,5 +83,6 @@ NOTARIZATION_LOGFILE_PATH="." PKG="ISSM-macOS-MATLAB" -PRIMARY_BUNDLE_ID="**********.issm.matlab" +PRIMARY_BUNDLE_ID="**********.issm.matlab" # Maybe "nasa.jpl.issm.matlab"? +RETRIGGER_SIGNING_FILE="retrigger.txt" SIGNED_REPO_COPY="./signed" SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed" @@ -38,21 +93,22 @@ COMPRESSED_PKG="${PKG}.zip" +EXE_ENTITLEMENTS_PLIST="${PKG}/bin/entitlements.plist" # Clean up from previous packaging (not necessary for single builds on Jenkins, # but useful when testing packaging locally) -echo 'Cleaning up existing assets' +echo "Cleaning up existing assets" rm -rf ${PKG} ${COMPRESSED_PKG} ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY} mkdir ${PKG} -# Check out copy of SVN repository for unsigned macOS packages -echo 'Checking out copy of SVN respository for unsigned packages' +# Check out copy of repository for unsigned packages +echo "Checking out copy of respository for unsigned packages" svn co \ --username ${USERNAME} \ --password ${PASSWORD} \ ${UNSIGNED_REPO_URL} \ - ${UNSIGNED_REPO_COPY} - -# Uncompress package -echo 'Uncompressing package' + ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 + +# Extract package contents +echo "Extracting package contents" ditto -xk ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} . @@ -60,6 +116,12 @@ xattr -cr ${PKG} -# Build list of executables -EXECUTABLES=$(\ +# Build list of ISSM executables +ISSM_BINS=$(\ + find ${PKG}/bin -type f -name *.exe; \ + find ${PKG}/lib -type f -name *.mexmaci64; \ +) + +# Build list of third party executables +THIRD_PARTY_BINS=$(\ echo ${PKG}/bin/mpiexec; \ echo ${PKG}/bin/hydra_pmi_proxy; \ @@ -69,37 +131,36 @@ echo ${PKG}/bin/gmtselect; \ echo ${PKG}/bin/gmsh; \ - find ${PKG} -type f -name *.exe; \ - find ${PKG} -type f -name *.mexmaci64; \ ) # Sign all executables in package -echo 'Signing all executables in package' -codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${EXECUTABLES} +echo "Signing all executables in package" +codesign -s ${AD_IDENTITY} --timestamp --options=runtime --entitlements ${EXE_ENTITLEMENTS_PLIST} ${ISSM_BINS} +codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${THIRD_PARTY_BINS} # NOTE: Skipping signature validation because this is not a true package nor app # Compress signed package -echo 'Compressing signed package' +echo "Compressing signed package" ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG} # Submit compressed package for notarization -echo 'Submitting signed package for notarization' +echo "Submitting signed package to Apple for notarization" xcrun altool --notarize-app --primary-bundle-id ${PRIMARY_BUNDLE_ID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} --asc-provider ${ASC_PROVIDER} --file ${COMPRESSED_PKG} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} # Sleep until notarization request response is received -echo 'Waiting for notarizaion request response' +echo "Waiting for notarizaion request response" while [[ ! -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} || ! -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; do sleep 30 done -echo 'Notarizaion request response received' +echo "Notarizaion request response received" # Check if UUID exists in response HAS_UUID=$(grep 'RequestUUID = ' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}) # NOTE: Checking for "RequestUUID = " because "RequestUUID" shows up in some error messages if [[ -z "${HAS_UUID}" ]]; then - echo 'Notarization failed!' - echo '----------------------- Contents of notarization logfile -----------------------' + echo "Notarization failed!" + echo "----------------------- Contents of notarization logfile -----------------------" cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} - echo '--------------------------------------------------------------------------------' + echo "--------------------------------------------------------------------------------" # Clean up @@ -120,5 +181,5 @@ # will not be able to clear Gatekeeper if they are offline. # -echo 'Checking notarization status' +echo "Checking notarization status" SUCCESS=0 for ATTEMPT in $(seq 1 ${NOTARIZATION_CHECK_ATTEMPTS}); do @@ -156,10 +217,10 @@ fi else - if [ $ATTEMPT -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then + if [ ${ATTEMPT} -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then echo " ...not ready yet; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds." sleep ${NOTARIZATION_CHECK_PERIOD} else - echo ' ...maximum attempts reached, but no response, or something else went wrong.' - echo ' If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again.' + echo " ...maximum attempts reached, but no response, or something else went wrong." + echo " If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again." break fi @@ -167,36 +228,44 @@ done -# Check out copy of SVN repository for signed macOS packages -echo 'Checking out copy of SVN respository for signed packages' +if [ ${SUCCESS} -eq 1 ]; then + echo "Notarization successful!" +else + echo "Notarization failed!" + echo "----------------------- Contents of notarization logfile -----------------------" + cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} + echo "--------------------------------------------------------------------------------" +fi + +# Remove dummy file for retriggering signing/notarization (if it exists) +svn delete ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1 +svn commit --message "DEL: Removing dummy file for retriggering signing of same package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 + +# Check out copy of repository for signed packages +echo "Checking out copy of respository for signed packages" svn co \ --username ${USERNAME} \ --password ${PASSWORD} \ ${SIGNED_REPO_URL} \ - ${SIGNED_REPO_COPY} - -# Copy notarization file to repository for signed binaries + ${SIGNED_REPO_COPY} > /dev/null 2>&1 + +# Copy notarization file to repository for signed packages cp ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY} svn add ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1 -# Remove lock file from repository for signed binaries +# Remove lock file from repository for signed packages svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 if [ ${SUCCESS} -eq 1 ]; then - echo 'Notarization successful!' - - # Copy signed package to repository for signed binaries + # Copy signed package to repository for signed packages cp ${COMPRESSED_PKG} ${SIGNED_REPO_COPY} svn add ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1 # Commit changes - svn commit --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY} + echo "Committing changes to repository for signed packages" + svn commit --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY} > /dev/null 2>&1 else - echo 'Notarization failed!' - echo '----------------------- Contents of notarization logfile -----------------------' - cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} - echo '--------------------------------------------------------------------------------' - # Commit changes - svn commit --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY} + echo "Committing changes to repository for signed packages" + svn commit --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY} > /dev/null 2>&1 exit 1 Index: /issm/trunk-jpl/packagers/mac/sign-issm-mac-binaries-python.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/sign-issm-mac-binaries-python.sh (revision 25780) +++ /issm/trunk-jpl/packagers/mac/sign-issm-mac-binaries-python.sh (revision 25780) @@ -0,0 +1,277 @@ +#!/bin/bash + +################################################################################ +# Intended to be run in the context of a Jenkins project on a JPL +# Cybersecurity server for signing macOS applications. Polls SCM of the +# Subversion repository hosted at +# https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned to trigger new +# builds. +# +# In order to replicate the requried Jenkins project configuration: +# - First, navigate to 'Manage Jenkins' -> 'Manage Plugins' and install the +# 'Credentials Bindings Plugin' if it is not already installed. +# - Contact one of the members of the ISSM development team for crendentials +# for the ISSM binaries repository (mention that the credentials are stored +# in ISSM-Infrastructure.pdf). +# - Navigate to 'Manage Jenkins' -> 'Manage Credentials' -> -> +# 'Add Credentials' and enter the crendentials from above. +# - From the 'Dashboard', select 'New Item' -> 'Freestyle project'. +# - Under 'Source Code Management', select 'Subversion'. +# - The 'Repository URL' text field should be set to +# "https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned" +# - The 'Credentials' select menu should be set to the new credentials +# created previously. +# - Under 'Build Trigggers', check the box for 'Poll SCM' and set the +# 'Schedule' text area to "H/5 * * * *". +# - Under 'Build Environment', check the box for 'Use secret text(s) or +# file(s)', then under 'Bindings' click the 'Add...' button and select +# 'Username and password (separated)'. +# - Set 'Username Variable' to "issm-binaries-user”. +# - Set 'Password Variable' to "issm-binaries-pass”. +# - Under 'Credentials', select the same, new credentials that created +# previously. +# - The contents of this script can be copied/pasted directly into the ‘Build' +# -> 'Execute Shell' -> ‘Command' textarea of the project configuration (or +# you can simply store the script on disk and call it from there). +# - Make sure to click the 'Save' button. +# +# Current point of contact at JPL Cybersecurity: +# Alex Coward, alexander.g.coward@jpl.nasa.gov +# +# NOTE: +# - Assumes that 'issm-binaries-user' and 'issm-binaries-pass' are set up in +# the 'Bindings' section under a 'Username and password (separated)' binding +# (requires 'Credentials Binding Plugin'). +# - For local debugging, the aformentioned credentials can be hardcoded into +# the 'USERNAME' and 'PASSWORD' constants below. +################################################################################ + +# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes, +# +# Command line tool support for Subversion — including svn, git-svn, and +# related commands — is no longer provided by Xcode. (50266910) +# +# which results in, +# +# svn: error: The subversion command line tools are no longer provided by +# Xcode. +# +# when calling svn, even when subversion is installed via Homebrew and its path +# is available in PATH. +# +shopt -s expand_aliases +alias svn='/usr/local/bin/svn' + +## Override certain other aliases +# +alias cp=$(which cp) +alias grep=$(which grep) + +## Constants +# +AD_IDENTITY="**********" # Apple Developer identity +AD_USERNAME="**********" # Apple Developer username +ALTOOL_PASSWORD="@keychain:**********" # altool password (assumed to be stored in keychain) +ASC_PROVIDER="**********" +NOTARIZATION_CHECK_ATTEMPTS=60 +NOTARIZATION_CHECK_PERIOD=60 +NOTARIZATION_LOGFILE="notarization.log" +NOTARIZATION_LOGFILE_PATH="." +PKG="ISSM-macOS-Python" +PRIMARY_BUNDLE_ID="**********.issm.python" # Maybe "nasa.jpl.issm.matlab"? +RETRIGGER_SIGNING_FILE="retrigger.txt" +SIGNED_REPO_COPY="./signed" +SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/signed" +SIGNING_LOCK_FILE="signing.lock" +SUCCESS_LOGFILE="${SIGNED_REPO_COPY}/success.log" +UNSIGNED_REPO_COPY="./unsigned" +UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned" + +COMPRESSED_PKG="${PKG}.zip" +EXE_ENTITLEMENTS_PLIST="${PKG}/bin/entitlements.plist" + +# Clean up from previous packaging (not necessary for single builds on Jenkins, +# but useful when testing packaging locally) +echo "Cleaning up existing assets" +rm -rf ${PKG} ${COMPRESSED_PKG} ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY} +mkdir ${PKG} + +# Check out copy of repository for unsigned packages +echo "Checking out copy of respository for unsigned packages" +svn co \ + --username ${USERNAME} \ + --password ${PASSWORD} \ + ${UNSIGNED_REPO_URL} \ + ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 + +# Extract package contents +echo "Extracting package contents" +ditto -xk ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} . + +# Clear extended attributes on all files +xattr -cr ${PKG} + +# Build list of ISSM executables +ISSM_BINS=$(\ + find ${PKG}/bin -type f -name *.exe; \ + find ${PKG}/bin -type f -name *.pyc; \ +) + +# Build list of third party executables +THIRD_PARTY_BINS=$(\ + echo ${PKG}/bin/mpiexec; \ + echo ${PKG}/bin/hydra_pmi_proxy; \ + echo ${PKG}/bin/gdalsrsinfo; \ + echo ${PKG}/bin/gdaltransform; \ + echo ${PKG}/bin/gmt; \ + echo ${PKG}/bin/gmtselect; \ + echo ${PKG}/bin/gmsh; \ +) + +# Sign all executables in package +echo "Signing all executables in package" +codesign -s ${AD_IDENTITY} --timestamp --options=runtime --entitlements ${EXE_ENTITLEMENTS_PLIST} ${ISSM_BINS} +codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${THIRD_PARTY_BINS} + +# Build list of ISSM libraries +ISSM_LIBS=$(\ + find ${PKG}/lib -type f -name *.so; \ +) + +# Sign all libraries in package +echo "Signing all libraries in package" +codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${ISSM_LIBS} + +# NOTE: Skipping signature validation because this is not a true package nor app + +# Compress signed package +echo "Compressing signed package" +ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG} + +# Submit compressed package for notarization +echo "Submitting signed package to Apple for notarization" +xcrun altool --notarize-app --primary-bundle-id ${PRIMARY_BUNDLE_ID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} --asc-provider ${ASC_PROVIDER} --file ${COMPRESSED_PKG} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} + +# Sleep until notarization request response is received +echo "Waiting for notarizaion request response" +while [[ ! -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} || ! -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; do + sleep 30 +done + +echo "Notarizaion request response received" + +# Check if UUID exists in response +HAS_UUID=$(grep 'RequestUUID = ' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}) # NOTE: Checking for "RequestUUID = " because "RequestUUID" shows up in some error messages +if [[ -z "${HAS_UUID}" ]]; then + echo "Notarization failed!" + echo "----------------------- Contents of notarization logfile -----------------------" + cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} + echo "--------------------------------------------------------------------------------" + + # Clean up + rm -rf ${PKG} ${COMPRESSED_PKG} + + exit 1 +fi + +# Get UUID from notarization request response +UUID=$(echo ${HAS_UUID} | sed 's/[[:space:]]*RequestUUID = //') +echo "UUID: ${UUID}" + +# Check notarization status +# +# NOTE: Currently, this checks if notarization was successful, but we are not +# able to staple notarization as this is not a true package nor app and, +# at the very least, MATLAB Mex files cannot be stapled. As such, clients +# will not be able to clear Gatekeeper if they are offline. +# +echo "Checking notarization status" +SUCCESS=0 +for ATTEMPT in $(seq 1 ${NOTARIZATION_CHECK_ATTEMPTS}); do + echo " Attempt #${ATTEMPT}..." + xcrun altool --notarization-info ${UUID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} + if [[ -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} && -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; then + + # First, check if there is an error + ERROR_CHECK=$(grep 'Error' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}) + if [[ ! -z "${ERROR_CHECK}" ]]; then + break + fi + + # No error, so check status + STATUS=$(grep 'Status:' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //') + if [[ "${STATUS}" == "success" ]]; then + # Staple notarization to all elements of package that were previously signed + #xcrun stapler staple ${THIRD_PARTY_BINS} # NOTE: Fails with "Stapler is incapable of working with MATLAB Mex files." + + # Validate stapling of notarization + #xcrun stapler validation ${THIRD_PARTY_BINS} # NOTE: Skipping notarization stapling validation because this is not a true package nor app + + # Compress signed and notarized package + ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG} + + # Set flag indicating notarization was successful + SUCCESS=1 + + break + elif [[ "${STATUS}" == "in progress" ]]; then + echo " ...in progress still; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds." + sleep ${NOTARIZATION_CHECK_PERIOD} + elif [[ "${STATUS}" == "invalid" ]]; then + break + fi + else + if [ ${ATTEMPT} -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then + echo " ...not ready yet; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds" + sleep ${NOTARIZATION_CHECK_PERIOD} + else + echo " ...maximum attempts reached, but no response, or something else went wrong" + echo " If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again" + break + fi + fi +done + +if [ ${SUCCESS} -eq 1 ]; then + echo "Notarization successful!" +else + echo "Notarization failed!" + echo "----------------------- Contents of notarization logfile -----------------------" + cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} + echo "--------------------------------------------------------------------------------" +fi + +# Remove dummy file for retriggering signing/notarization (if it exists) +svn delete ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1 +svn commit --message "DEL: Removing dummy file for retriggering signing of same package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 + +# Check out copy of repository for signed packages +echo "Checking out copy of respository for signed packages" +svn co \ + --username ${USERNAME} \ + --password ${PASSWORD} \ + ${SIGNED_REPO_URL} \ + ${SIGNED_REPO_COPY} > /dev/null 2>&1 + +# Copy notarization file to repository for signed packages +cp ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY} +svn add ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1 + +# Remove lock file from repository for signed packages +svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 + +if [ ${SUCCESS} -eq 1 ]; then + # Copy signed package to repository for signed packages + cp ${COMPRESSED_PKG} ${SIGNED_REPO_COPY} + svn add ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1 + + # Commit changes + echo "Committing changes to repository for signed packages" + svn commit --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY} > /dev/null 2>&1 +else + # Commit changes + echo "Committing changes to repository for signed packages" + svn commit --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY} > /dev/null 2>&1 + + exit 1 +fi Index: /issm/trunk-jpl/packagers/mac/test-issm-mac-binaries-matlab.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/test-issm-mac-binaries-matlab.sh (revision 25779) +++ /issm/trunk-jpl/packagers/mac/test-issm-mac-binaries-matlab.sh (revision 25780) @@ -1,11 +1,14 @@ #!/bin/bash -# This script is intended to test ISSM macOS MATLAB binaries downloaded to a -# user-end machine. +################################################################################ +# This script is intended to test ISSM macOS MATLAB binaries on an end-user +# machine after successful packaging and signing. # # NOTE: Tarball must already exist in INSTALL_DIR +################################################################################ + +## Constants # - -INSTALL_DIR=~/Downloads +INSTALL_DIR=. MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2006,2020,2021,2051,2052,2053,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded MATLAB_PATH="/Applications/MATLAB_R2018a.app" @@ -14,4 +17,6 @@ COMPRESSED_PKG="${PKG}.zip" +## Environment +# export ISSM_DIR="${INSTALL_DIR}/${PKG}" export PATH="${PATH}:${ISSM_DIR}/bin:${ISSM_DIR}/scripts" @@ -19,12 +24,10 @@ cd ${INSTALL_DIR} rm -rf ${PKG} -tar -zxvf ${COMPRESSED_PKG} +ditto -xk ${COMPRESSED_PKG} . cd ${PKG}/test/NightlyRun -# Check that MATLAB tests run -echo "Running MATLAB tests" +# Run tests, redirecting output to logfile and suppressing output to console +echo "Running tests" rm matlab.log 2> /dev/null - -# Run MATLAB tests redirecting output to logfile and suppressing output to console ${MATLAB_PATH}/bin/matlab -nosplash -nodesktop -nojvm -r "try, addpath ../../bin; addpath ../../lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null @@ -41,11 +44,11 @@ fi -# Check that all MATLAB tests passed -numMatlabTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"` +# Check that all tests passed +numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"` -if [[ ${numMatlabTestsFailed} -ne 0 ]]; then - echo "One or more MATLAB tests FAILED" +if [[ ${numTestsFailed} -ne 0 ]]; then + echo "One or more tests FAILED" exit 1 else - echo "All MATLAB tests PASSED" + echo "All tests PASSED" fi Index: /issm/trunk-jpl/packagers/mac/test-issm-mac-binaries-python.sh =================================================================== --- /issm/trunk-jpl/packagers/mac/test-issm-mac-binaries-python.sh (revision 25780) +++ /issm/trunk-jpl/packagers/mac/test-issm-mac-binaries-python.sh (revision 25780) @@ -0,0 +1,53 @@ +#!/bin/bash + +################################################################################ +# This script is intended to test ISSM macOS MATLAB binaries on an end-user +# machine after successful packaging and signing. +# +# NOTE: Tarball must already exist in INSTALL_DIR +################################################################################ + +## Constants +# +INSTALL_DIR=. +PKG="ISSM-macOS-Python" +PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2006 2020 2021 2051 2052 2053 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded + +COMPRESSED_PKG="${PKG}.zip" + +export ISSM_DIR="${INSTALL_DIR}/${PKG}" +export PATH="${PATH}:${ISSM_DIR}/bin:${ISSM_DIR}/scripts" +export PYTHONPATH="${ISSM_DIR}/scripts" +export PYTHONSTARTUP="${PYTHONPATH}/devpath.py" +export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured + +cd ${INSTALL_DIR} +rm -rf ${PKG} +ditto -xk ${COMPRESSED_PKG} . +cd ${PKG}/test/NightlyRun + +# Run tests, redirecting output to logfile and suppressing output to console +echo "Running tests" +rm python.log 2> /dev/null +./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1 + +# Check that Python did not exit in error +pythonExitCode=`echo $?` +pythonExitedInError=`grep -E "runme.py: error" python.log | wc -l` + +if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then + echo "----------Python exited in error!----------" + cat python.log + echo "-----------End of python.log-----------" + exit 1 +fi + +# Check that all tests passed +numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"` + +if [[ ${numTestsFailed} -ne 0 ]]; then + echo "One or more tests FAILED" + exit 1 +else + echo "All tests PASSED" +fi Index: /issm/trunk-jpl/src/m/classes/results.py =================================================================== --- /issm/trunk-jpl/src/m/classes/results.py (revision 25779) +++ /issm/trunk-jpl/src/m/classes/results.py (revision 25780) @@ -107,4 +107,5 @@ def __init__(self, *args): #{{{ + self.steps = None if len(args) == 1: arg = args[0]