Changeset 25744

Timestamp:

11/12/20 16:43:32 (4 years ago)

Author:

jdquinn

Message:

CHG: Build, packaging, and testing scripts for reconfigured macOS binaries.

Location:

issm/trunk-jpl

Files:

• jenkins/pine_island-mac-binaries-matlab (moved) (moved from issm/trunk-jpl/jenkins/pine_island-mac-binaries ) (5 diffs)
• jenkins/pine_island-mac-binaries-solid_earth (deleted)
• jenkins/pine_island-mac-binaries-with_dakota (deleted)
• packagers/mac/package-issm-mac-binaries-matlab.sh (copied) (copied from issm/trunk-jpl/packagers/mac/package-issm-mac-binaries.sh ) (2 diffs)
• packagers/mac/package-issm-mac-binaries-solid_earth.sh (deleted)
• packagers/mac/package-issm-mac-binaries-with_dakota.sh (deleted)
• packagers/mac/sign-issm-mac-binaries-matlab.sh (copied) (copied from issm/trunk-jpl/packagers/mac/package-issm-mac-binaries.sh ) (1 diff)
• packagers/mac/test-issm-mac-binaries-matlab.sh (moved) (moved from issm/trunk-jpl/packagers/mac/package-issm-mac-binaries.sh ) (2 diffs)
• scripts/test-issm-mac-binaries-with_dakota.sh (deleted)
• scripts/test-issm-mac-binaries.sh (deleted)

issm/trunk-jpl/jenkins/pine_island-mac-binaries-matlab ¶

@@ -9 +9 @@
 #       should always be a one-time build.
 #
+
 ISSM_CONFIG='\
         --prefix=${ISSM_DIR} \
@@ -16 +17 @@
         --disable-dependency-tracking \
         --with-matlab-dir=${MATLAB_PATH} \
-        --with-fortran-lib="/usr/local/Cellar/gcc/10.2.0/lib/gcc/10/libgfortran.a /usr/local/Cellar/gcc/10.2.0/lib/gcc/10/libquadmath.a /usr/local/Cellar/gcc/10.2.0/lib/gcc/10/gcc/x86_64-apple-darwin15/10.2.0/libgcc.a" \
+        --with-fortran-lib="/usr/local/Cellar/gcc/10.2.0/lib/gcc/10/libgfortran.a /usr/local/Cellar/gcc/10.2.0/lib/gcc/10/libquadmath.a /usr/local/Cellar/gcc/10.2.0/lib/gcc/10/gcc/x86_64-apple-darwin19/10.2.0/libgcc.a" \
         --with-mpi-include=${ISSM_DIR}/externalpackages/petsc/install/include \
         --with-mpi-libflags="-L${ISSM_DIR}/externalpackages/petsc/install/lib -lmpi -lmpicxx -lmpifort" \
@@ -23 +24 @@
         --with-scalapack-dir=${ISSM_DIR}/externalpackages/petsc/install \
         --with-mumps-dir=${ISSM_DIR}/externalpackages/petsc/install \
+        --with-hdf5-dir=${ISSM_DIR}/externalpackages/petsc/install \
         --with-petsc-dir=${ISSM_DIR}/externalpackages/petsc/install \
+        --with-gsl-dir=${ISSM_DIR}/externalpackages/gsl/install \
+        --with-boost-dir=${ISSM_DIR}/externalpackages/boost/install \
+        --with-dakota-dir=${ISSM_DIR}/externalpackages/dakota/install \
         --with-triangle-dir=${ISSM_DIR}/externalpackages/triangle/install \
         --with-chaco-dir=${ISSM_DIR}/externalpackages/chaco/install \
@@ -43 +48 @@
         autotools       install.sh
         cmake           install.sh
-        petsc           install-3.14-mac-static.sh
+        petsc           install-3.12-mac-static.sh
+        gsl                     install-static.sh
+        boost           install-1.7-mac-static.sh
+        dakota          install-6.2-mac-static.sh
+        netcdf          install-4.7-mac-parallel-static.sh
+        proj            install-6.2-static.sh
+        gdal            install-3-netcdf-static.sh
+        gshhg           install.sh
+        gmt                     install-6-mac-static.sh
+        gmsh            install-4-mac-static.sh
         triangle        install-mac-static.sh
         chaco           install.sh
@@ -60 +74 @@
 #               compilation
 #
-NUMCPUS_INSTALL=1
+NUMCPUS_INSTALL=4
 
 # Number of CPUs used in the nightly runs

issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-matlab.sh ¶

@@ -1 +1 @@
 #!/bin/bash
 
-
-# Script to test, package, and transfer distributable to ISSM Web site.
-# Corresponds with build generated by configuration in 
-# $ISSM_DIR/jenkins/pine_island-mac-binaries.
-#
+################################################################################
+# To be used after running,
+#
+#       ${ISSM_DIR}/jenkins/jenkins.sh ${ISSM_DIR}/jenkins/pine_island-mac-binaries-matlab
+#
+# in the context of a Jenkins project.
+#
+# When no runtime errors occur, performs the following:
+# - Checks resulting executables and libraries against test suite.
+# - Packages and compresses executables and libraries.
+# - Commits compressed package to SVN repository to be signed by JPL 
+#       Cybersecurity.
+# - Retrieves signed package and transmits it to ISSM Web site for 
+#       distribution.
+#
+# To skip tests, run with -s/--skiptests option.
+#
+# Debugging:
+# - Relies on a very tight handshake with project on remote JPL Cybersecurity 
+#       Jenkins server. Debugging may be perfomed locally by running,
+#
+#               packagers/mac/sign-issm-mac-binaries-matlab.sh
+#
+#       with Apple Developer credentials.
+# - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can 
+#       help debug potential SVN issues.
+#
+# NOTE:
+# - Assumes that 'issm-binaries-user' and 'issm-binaries-pass' are set up in 
+#       the 'Bindings' section under a 'Username and password (separated)' binding 
+#       (requires 'Credentials Binding Plugin').
+# - For local debugging, the aformentioned credentials can be hardcoded into 
+#       the 'USERNAME' and 'PASSWORD' constants below.
+################################################################################
+
+# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
+#
+#       Command line tool support for Subversion — including svn, git-svn, and 
+#       related commands — is no longer provided by Xcode. (50266910)
+#
+# which results in,
+#
+#       svn: error: The subversion command line tools are no longer provided by 
+#       Xcode.
+#
+# when calling svn, even when subversion is installed via Homebrew and its path 
+# is available in PATH.
+#
+shopt -s expand_aliases
+alias svn='/usr/local/bin/svn'
+
+## Override certain other aliases
+#
+alias grep=$(which grep)
 
 ## Constants
 #
-MATLAB_NROPTIONS="'exclude',[IdFromString('Dakota'),125,126,435,701,702,703]"
+
+MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2006,2020,2021,2051,2052,2053,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
 MATLAB_PATH="/Applications/MATLAB_R2018a.app"
-PACKAGE="ISSM" # Name of directory to copy distributable files to
-TARBALL_NAME="issm-mac"
-TARBALL="${TARBALL_NAME}.tar.gz"
+NOTARIZATION_LOGFILE="notarization.log"
+PASSWORD=$env:issm-binaries-pass
+PKG="ISSM-macOS-MATLAB" # Name of directory to copy distributable files to
+SIGNED_REPO_COPY="./signed"
+SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed"
+SIGNING_CHECK_PERIOD=60 # seconds
+SIGNING_LOCK_FILE="signing.lock"
+UNSIGNED_REPO_COPY="./unsigned"
+UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned"
+USERNAME=$env:issm-binaries-user
+
+COMPRESSED_PKG="${PKG}.zip"
 
 ## Environment
@@ -19 +78 @@
 export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'
 
-# Check if MATLAB exists
-if ! [ -d ${MATLAB_PATH} ]; then
-        echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again."
+## Parse options
+#
+skip_tests=0
+transfer_only=0
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        -s|--skiptests) skip_tests=1; shift ;;
+        -t|--transferonly) transfer_only=1; shift ;;
+        *) echo "Unknown parameter passed: $1"; exit 1 ;;
+    esac
+    shift
+done
+
+if [ ${transfer_only} -eq 0 ]; then
+        # Check if MATLAB exists
+        if ! [ -d ${MATLAB_PATH} ]; then
+                echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again."
+                exit 1
+        fi
+
+        # Clean up from previous packaging (not necessary for single builds on Jenkins, 
+        # but useful when testing packaging locally)
+        echo "Cleaning up existing assets"
+        cd ${ISSM_DIR}
+        rm -rf ${PKG} ${COMPRESSED_PKG} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY}
+        mkdir ${PKG}
+
+        # Check out copy of SVN repository for signed macOS packages
+        #
+        # NOTE: Get empty copy because we do not want to have to check out package from 
+        #               previous signing.
+        #
+        echo "Checking out copy of repository for signed packages"
+        svn co \
+                --username ${USERNAME} \
+                --password ${PASSWORD} \
+                --depth empty \
+                ${SIGNED_REPO_URL} \
+                ${SIGNED_REPO_COPY} > /dev/null 2>&1
+
+        # If lock file exists, a signing build is still in process by JPL Cybersecurity
+        svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
+        if [[ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]]; then
+                echo "Previous signing job still in process by JPL Cybersecurity. Please try again later."
+                exit 1
+        fi
+
+        # Add required binaries and libraries to package and modify them where needed
+        cd ${ISSM_DIR}/bin
+
+        echo "Modify generic"
+        cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m
+
+        echo "Moving MPICH binaries to bin/"
+        if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
+                cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
+                cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
+        elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
+                cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
+                cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
+        else
+                echo "MPICH not found"
+                exit 1
+        fi
+
+        echo "Moving GDAL binaries to bin/"
+        if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
+                cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
+                cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
+        else
+                echo "GDAL not found"
+                exit 1
+        fi
+
+        echo "Moving GMT binaries to bin/"
+        if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
+                cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
+                cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
+        else
+                echo "GMT not found"
+                exit 1
+        fi
+
+        echo "Moving Gmsh binaries to bin/"
+        if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
+                cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
+        else
+                echo "Gmsh not found"
+                exit 1
+        fi
+
+        # Run tests
+        if [ ${skip_tests} -eq 0 ]; then
+                echo "Running tests"
+                cd ${ISSM_DIR}/test/NightlyRun
+                rm matlab.log 2> /dev/null
+
+                # Run tests, redirecting output to logfile and suppressing output to console
+                ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
+
+                # Check that MATLAB did not exit in error
+                matlabExitCode=`echo $?`
+                matlabExitedInError=`grep -E "Activation cannot proceed|license|Error" matlab.log | wc -l`
+
+                if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then
+                        echo "----------MATLAB exited in error!----------"
+                        cat matlab.log
+                        echo "-----------End of matlab.log-----------"
+
+                        # Clean up execution directory
+                        rm -rf ${ISSM_DIR}/execution/*
+
+                        exit 1
+                fi
+
+                # Check that all tests passed
+                numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
+
+                if [[ ${numTestsFailed} -ne 0 ]]; then
+                        echo "One or more tests FAILED"
+                        exit 1
+                else
+                        echo "All tests PASSED"
+                fi
+        else
+                echo "Skipping tests"
+        fi
+
+        # Create package
+        cd ${ISSM_DIR}
+        svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
+        echo "Copying assets to package: ${PKG}"
+        cp -rf bin examples lib scripts test ${PKG}/
+        echo "Cleaning up unneeded/unwanted files"
+        rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
+        rm -f ${PKG}/lib/*.a # Remove static libraries from package
+        rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
+        rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
+
+        # Compress package
+        echo "Compressing package"
+        ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
+
+        # Commit lock file to repository for signed packages
+        echo "Committing lock file to repository for signed packages"
+        touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
+        svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
+        svn commit --message "ADD: New lock file" ${SIGNED_REPO_COPY} > /dev/null 2>&1
+
+        # Save current working copy revision number
+        svn up ${SIGNED_REPO_COPY} > /dev/null 2>&1
+        CURRENT_REV=$(svn info --show-item last-changed-revision ${SIGNED_REPO_COPY})
+
+        # Check out copy of SVN repository for unsigned macOS packages
+        echo "Checking out copy of repository for unsigned packages"
+        svn co \
+                --username ${USERNAME} \
+                --password ${PASSWORD} \
+                ${UNSIGNED_REPO_URL} \
+                ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
+
+        # Commit new compressed package to repository for unsigned binaries
+        #
+        # NOTE: This will not work if, for any reason, the checksum on the compressed 
+        #               package is unchanged.
+        #
+        echo "Committing package to repository for unsigned packages"
+        cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
+        svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
+        svn commit --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
+
+        # Check status of signing
+        echo "Checking progress of signing..."
+        IN_PROCESS=1
+        SUCCESS=0
+
+        while [ ${IN_PROCESS} -eq 1 ]; do
+                echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds."
+                sleep ${SIGNING_CHECK_PERIOD}
+                svn up ${SIGNED_REPO_COPY} > /dev/null 2>&1
+                NEW_REV=$(svn info --show-item last-changed-revision ${SIGNED_REPO_COPY})
+
+                if [ ${NEW_REV} -ne ${CURRENT_REV} ]; then
+                        IN_PROCESS=0
+
+                        svn up ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1
+                        svn up ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
+
+                        # No error, so check status
+                        STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
+                        if [[ "${STATUS}" == "success" ]]; then
+                                echo 'Notarization successful!'
+
+                                # Set flag indicating notarization was successful
+                                SUCCESS=1
+                        else
+                                echo 'Notarization failed!'
+                        fi
+                fi
+        done
+else # transfer_only == 1
+        # Assume that previous build resulted in successful signing of package but 
+        # that transfer to ISSM Web site failed and user built this project again 
+        # with -t/--transferonly option.
+        SUCCESS=1
+fi
+
+# Handle result of signing
+if [ ${SUCCESS} -eq 1 ]; then
+        # Transfer signed package to ISSM Web site
+        echo 'Transferring signed package to ISSM Web site'
+        scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG}
+
+        if [ $? -ne 0 ]; then
+                echo "Transfer failed! Verify connection then build this project again (with -t/--transferonly option to skip testing and signing)."
+                exit 1
+        fi
+else
+        echo '----------------------- Contents of notarization logfile -----------------------'
+        cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
+        echo '--------------------------------------------------------------------------------'
+
         exit 1
 fi
-
-# Clean up from previous packaging
-echo "Cleaning up existing assets"
-cd ${ISSM_DIR}
-rm -rf ${PACKAGE}
-mkdir ${PACKAGE}
-
-# Add/modify required binaries and libraries
-cd ${ISSM_DIR}/bin
-
-echo "Modify generic"
-cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m
-
-echo "Moving MPICH binaries to bin/"
-if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
-        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
-        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
-elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
-        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
-        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
-else
-        echo "MPICH not found"
-        exit 1
-fi
-
-# Run tests
-echo "Running tests"
-cd ${ISSM_DIR}/test/NightlyRun
-
-# Check that MATLAB tests run
-echo "Running MATLAB tests"
-
-rm matlab.log 2> /dev/null
-
-# Run MATLAB tests redirecting output to logfile and suppressing output to console
-${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
-
-# Check that MATLAB did not exit in error
-matlabExitCode=`echo $?`
-matlabExitedInError=`grep -E "Activation cannot proceed|license|Error" matlab.log | wc -l`
-
-if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then
-        echo "----------MATLAB exited in error!----------"
-        cat matlab.log
-        echo "-----------End of matlab.log-----------"
-
-        # Clean up execution directory
-        rm -rf ${ISSM_DIR}/execution/*
-
-        exit 1
-fi
-
-# Check that all MATLAB tests passed
-numMatlabTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
-
-if [[ ${numMatlabTestsFailed} -ne 0 ]]; then
-        echo "One or more MATLAB tests FAILED"
-        exit 1
-else
-        echo "All MATLAB tests PASSED"
-fi
-
-# Create tarball
-cd ${ISSM_DIR}
-rm -f ${TARBALL}
-svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
-echo "Copying assets to package: ${PACKAGE}"
-cp -rf bin examples lib scripts test ${PACKAGE}/
-echo "Cleaning up unneeded/unwanted files"
-rm -f ${PACKAGE}/bin/generic_static.* # Remove static versions of generic cluster classes
-rm -f ${PACKAGE}/lib/*.a # Remove static libraries from package (we only need MEX-files)
-rm -f ${PACKAGE}/lib/*.la # Remove libtool libraries from package
-rm -rf ${PACKAGE}/test/SandBox # Remove testing sandbox from package
-echo "Creating tarball: ${TARBALL_NAME}"
-tar -czf ${TARBALL} ${PACKAGE}
-ls -lah ${ISSM_DIR}/${TARBALL}
-
-echo "Transferring binaries to ISSM Web site"
-scp -i ~/.ssh/pine_island_to_ross ${TARBALL} jenkins@ross.ics.uci.edu:/var/www/html/${TARBALL}
-
-if [ $? -ne 0 ]; then
-        echo "FAILED: Manually check connection"
-        exit 1
-fi

issm/trunk-jpl/packagers/mac/sign-issm-mac-binaries-matlab.sh ¶

@@ -1 @@
-#!/bin/bash
-
-
-# Script to test, package, and transfer distributable to ISSM Web site.
-# Corresponds with build generated by configuration in 
-# $ISSM_DIR/jenkins/pine_island-mac-binaries.
-#
-
-## Constants
-#
-MATLAB_NROPTIONS="'exclude',[IdFromString('Dakota'),125,126,435,701,702,703]"
-MATLAB_PATH="/Applications/MATLAB_R2018a.app"
-PACKAGE="ISSM" # Name of directory to copy distributable files to
-TARBALL_NAME="issm-mac"
-TARBALL="${TARBALL_NAME}.tar.gz"
-
-## Environment
-#
-export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'
-
-# Check if MATLAB exists
-if ! [ -d ${MATLAB_PATH} ]; then
-        echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again."
+# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
+#
+#       Command line tool support for Subversion — including svn, git-svn, and 
+#       related commands — is no longer provided by Xcode. (50266910)
+#
+# which results in,
+#
+#       svn: error: The subversion command line tools are no longer provided by 
+#       Xcode.
+#
+# when calling svn, even when subversion is installed via Homebrew and its path 
+# is available in PATH.
+#
+shopt -s expand_aliases
+alias svn='/usr/local/bin/svn'
+
+## Override certain other aliases
+#
+alias cp=$(which cp)
+alias grep=$(which grep)
+
+AD_IDENTITY="**********"
+AD_USERNAME="**********"
+ALTOOL_PASSWORD="@keychain:**********"
+ASC_PROVIDER="**********"
+NOTARIZATION_CHECK_ATTEMPTS=60
+NOTARIZATION_CHECK_PERIOD=60
+NOTARIZATION_LOGFILE="notarization.log"
+NOTARIZATION_LOGFILE_PATH="."
+PKG="ISSM-macOS-MATLAB"
+PRIMARY_BUNDLE_ID="**********.issm.matlab"
+SIGNED_REPO_COPY="./signed"
+SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed"
+SIGNING_LOCK_FILE="signing.lock"
+SUCCESS_LOGFILE="${SIGNED_REPO_COPY}/success.log"
+UNSIGNED_REPO_COPY="./unsigned"
+UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned"
+
+COMPRESSED_PKG="${PKG}.zip"
+
+# Clean up from previous packaging (not necessary for single builds on Jenkins, 
+# but useful when testing packaging locally)
+echo 'Cleaning up existing assets'
+rm -rf ${PKG} ${COMPRESSED_PKG} ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY}
+mkdir ${PKG}
+
+# Check out copy of SVN repository for unsigned macOS packages
+echo 'Checking out copy of SVN respository for unsigned packages'
+svn co \
+        --username ${USERNAME} \
+        --password ${PASSWORD} \
+        ${UNSIGNED_REPO_URL} \
+        ${UNSIGNED_REPO_COPY}
+
+# Uncompress package
+echo 'Uncompressing package'
+ditto -xk ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} .
+
+# Clear extended attributes on all files
+xattr -cr ${PKG}
+
+# Build list of executables
+EXECUTABLES=$(\
+        echo ${PKG}/bin/mpiexec; \
+        echo ${PKG}/bin/hydra_pmi_proxy; \
+        echo ${PKG}/bin/gdalsrsinfo; \
+        echo ${PKG}/bin/gdaltransform; \
+        echo ${PKG}/bin/gmt; \
+        echo ${PKG}/bin/gmtselect; \
+        echo ${PKG}/bin/gmsh; \
+        find ${PKG} -type f -name *.exe; \
+        find ${PKG} -type f -name *.mexmaci64; \
+)
+
+# Sign all executables in package
+echo 'Signing all executables in package'
+codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${EXECUTABLES}
+
+# NOTE: Skipping signature validation because this is not a true package nor app
+
+# Compress signed package
+echo 'Compressing signed package'
+ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
+
+# Submit compressed package for notarization
+echo 'Submitting signed package for notarization'
+xcrun altool --notarize-app --primary-bundle-id ${PRIMARY_BUNDLE_ID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} --asc-provider ${ASC_PROVIDER} --file ${COMPRESSED_PKG} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
+
+# Sleep until notarization request response is received
+echo 'Waiting for notarizaion request response'
+while [[ ! -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} || ! -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; do
+        sleep 30
+done
+
+echo 'Notarizaion request response received'
+
+# Check if UUID exists in response
+HAS_UUID=$(grep 'RequestUUID = ' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}) # NOTE: Checking for "RequestUUID = " because "RequestUUID" shows up in some error messages
+if [[ -z "${HAS_UUID}" ]]; then
+        echo 'Notarization failed!'
+        echo '----------------------- Contents of notarization logfile -----------------------'
+        cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
+        echo '--------------------------------------------------------------------------------'
+
+        # Clean up
+        rm -rf ${PKG} ${COMPRESSED_PKG}
+
         exit 1
 fi
 
-# Clean up from previous packaging
-echo "Cleaning up existing assets"
-cd ${ISSM_DIR}
-rm -rf ${PACKAGE}
-mkdir ${PACKAGE}
-
-# Add/modify required binaries and libraries
-cd ${ISSM_DIR}/bin
-
-echo "Modify generic"
-cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m
-
-echo "Moving MPICH binaries to bin/"
-if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
-        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
-        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
-elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
-        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
-        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
+# Get UUID from notarization request response
+UUID=$(echo ${HAS_UUID} | sed 's/[[:space:]]*RequestUUID = //')
+echo "UUID: ${UUID}" 
+
+# Check notarization status
+#
+# NOTE: Currently, this checks if notarization was successful, but we are not 
+#               able to staple notarization as this is not a true package nor app and, 
+#               at the very least, MATLAB Mex files cannot be stapled. As such, clients 
+#               will not be able to clear Gatekeeper if they are offline.
+#
+echo 'Checking notarization status'
+SUCCESS=0
+for ATTEMPT in $(seq 1 ${NOTARIZATION_CHECK_ATTEMPTS}); do
+        echo "    Attempt #${ATTEMPT}..."
+        xcrun altool --notarization-info ${UUID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
+        if [[ -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} && -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; then
+
+                # First, check if there is an error
+                ERROR_CHECK=$(grep 'Error' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE})
+                if [[ ! -z "${ERROR_CHECK}" ]]; then
+                        break
+                fi
+
+                # No error, so check status
+                STATUS=$(grep 'Status:' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
+                if [[ "${STATUS}" == "success" ]]; then
+                        # Staple notarization to all elements of package that were previously signed
+                        #xcrun stapler staple ${EXECUTABLES} # NOTE: Fails with "Stapler is incapable of working with MATLAB Mex files."
+
+                        # Validate stapling of notarization
+                        #xcrun stapler validation ${EXECUTABLES} # NOTE: Skipping notarization stapling validation because this is not a true package nor app
+
+                        # Compress signed and notarized package
+                        ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
+
+                        # Set flag indicating notarization was successful
+                        SUCCESS=1
+
+                        break
+                elif [[ "${STATUS}" == "in progress" ]]; then
+                        echo "    ...in progress still; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds."
+                        sleep ${NOTARIZATION_CHECK_PERIOD}
+                elif [[ "${STATUS}" == "invalid" ]]; then
+                        break
+                fi
+        else
+                if [ $ATTEMPT -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then
+                        echo "    ...not ready yet; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds."
+                        sleep ${NOTARIZATION_CHECK_PERIOD}
+                else
+                        echo '    ...maximum attempts reached, but no response, or something else went wrong.'
+                        echo '    If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again.'
+                        break
+                fi
+        fi
+done
+
+# Check out copy of SVN repository for signed macOS packages
+echo 'Checking out copy of SVN respository for signed packages'
+svn co \
+        --username ${USERNAME} \
+        --password ${PASSWORD} \
+        ${SIGNED_REPO_URL} \
+        ${SIGNED_REPO_COPY}
+
+# Copy notarization file to repository for signed binaries
+cp ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY}
+svn add ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1
+
+# Remove lock file from repository for signed binaries
+svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
+
+if [ ${SUCCESS} -eq 1 ]; then
+        echo 'Notarization successful!'
+
+        # Copy signed package to repository for signed binaries
+        cp ${COMPRESSED_PKG} ${SIGNED_REPO_COPY}
+        svn add ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
+
+        # Commit changes
+        svn commit --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY}
 else
-        echo "MPICH not found"
+        echo 'Notarization failed!'
+        echo '----------------------- Contents of notarization logfile -----------------------'
+        cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
+        echo '--------------------------------------------------------------------------------'
+
+        # Commit changes
+        svn commit --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY}
+
         exit 1
 fi
-
-# Run tests
-echo "Running tests"
-cd ${ISSM_DIR}/test/NightlyRun
-
-# Check that MATLAB tests run
-echo "Running MATLAB tests"
-
-rm matlab.log 2> /dev/null
-
-# Run MATLAB tests redirecting output to logfile and suppressing output to console
-${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
-
-# Check that MATLAB did not exit in error
-matlabExitCode=`echo $?`
-matlabExitedInError=`grep -E "Activation cannot proceed|license|Error" matlab.log | wc -l`
-
-if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then
-        echo "----------MATLAB exited in error!----------"
-        cat matlab.log
-        echo "-----------End of matlab.log-----------"
-
-        # Clean up execution directory
-        rm -rf ${ISSM_DIR}/execution/*
-
-        exit 1
-fi
-
-# Check that all MATLAB tests passed
-numMatlabTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
-
-if [[ ${numMatlabTestsFailed} -ne 0 ]]; then
-        echo "One or more MATLAB tests FAILED"
-        exit 1
-else
-        echo "All MATLAB tests PASSED"
-fi
-
-# Create tarball
-cd ${ISSM_DIR}
-rm -f ${TARBALL}
-svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
-echo "Copying assets to package: ${PACKAGE}"
-cp -rf bin examples lib scripts test ${PACKAGE}/
-echo "Cleaning up unneeded/unwanted files"
-rm -f ${PACKAGE}/bin/generic_static.* # Remove static versions of generic cluster classes
-rm -f ${PACKAGE}/lib/*.a # Remove static libraries from package (we only need MEX-files)
-rm -f ${PACKAGE}/lib/*.la # Remove libtool libraries from package
-rm -rf ${PACKAGE}/test/SandBox # Remove testing sandbox from package
-echo "Creating tarball: ${TARBALL_NAME}"
-tar -czf ${TARBALL} ${PACKAGE}
-ls -lah ${ISSM_DIR}/${TARBALL}
-
-echo "Transferring binaries to ISSM Web site"
-scp -i ~/.ssh/pine_island_to_ross ${TARBALL} jenkins@ross.ics.uci.edu:/var/www/html/${TARBALL}
-
-if [ $? -ne 0 ]; then
-        echo "FAILED: Manually check connection"
-        exit 1
-fi

issm/trunk-jpl/packagers/mac/test-issm-mac-binaries-matlab.sh ¶

@@ -1 +1 @@
 #!/bin/bash
 
-
-# Script to test, package, and transfer distributable to ISSM Web site.
-# Corresponds with build generated by configuration in 
-# $ISSM_DIR/jenkins/pine_island-mac-binaries.
+# This script is intended to test ISSM macOS MATLAB binaries downloaded to a 
+# user-end machine.
+#
+# NOTE: Tarball must already exist in INSTALL_DIR
 #
 
-## Constants
-#
-MATLAB_NROPTIONS="'exclude',[IdFromString('Dakota'),125,126,435,701,702,703]"
+INSTALL_DIR=~/Downloads
+MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2006,2020,2021,2051,2052,2053,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
 MATLAB_PATH="/Applications/MATLAB_R2018a.app"
-PACKAGE="ISSM" # Name of directory to copy distributable files to
-TARBALL_NAME="issm-mac"
-TARBALL="${TARBALL_NAME}.tar.gz"
+PKG="ISSM-macOS-MATLAB"
 
-## Environment
-#
-export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'
+COMPRESSED_PKG="${PKG}.zip"
 
-# Check if MATLAB exists
-if ! [ -d ${MATLAB_PATH} ]; then
-        echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again."
-        exit 1
-fi
+export ISSM_DIR="${INSTALL_DIR}/${PKG}"
+export PATH="${PATH}:${ISSM_DIR}/bin:${ISSM_DIR}/scripts"
 
-# Clean up from previous packaging
-echo "Cleaning up existing assets"
-cd ${ISSM_DIR}
-rm -rf ${PACKAGE}
-mkdir ${PACKAGE}
-
-# Add/modify required binaries and libraries
-cd ${ISSM_DIR}/bin
-
-echo "Modify generic"
-cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m
-
-echo "Moving MPICH binaries to bin/"
-if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
-        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
-        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
-elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
-        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
-        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
-else
-        echo "MPICH not found"
-        exit 1
-fi
-
-# Run tests
-echo "Running tests"
-cd ${ISSM_DIR}/test/NightlyRun
+cd ${INSTALL_DIR}
+rm -rf ${PKG}
+tar -zxvf ${COMPRESSED_PKG}
+cd ${PKG}/test/NightlyRun
 
 # Check that MATLAB tests run
 echo "Running MATLAB tests"
-
 rm matlab.log 2> /dev/null
 
 # Run MATLAB tests redirecting output to logfile and suppressing output to console
-${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
+${MATLAB_PATH}/bin/matlab -nosplash -nodesktop -nojvm -r "try, addpath ../../bin; addpath ../../lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
 
 # Check that MATLAB did not exit in error
 matlabExitCode=`echo $?`
-matlabExitedInError=`grep -E "Activation cannot proceed|license|Error" matlab.log | wc -l`
+matlabExitedInError=`grep -E "Activation cannot proceed|license|Error|Warning: Name is nonexistent or not a directory" matlab.log | wc -l`
 
 if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then
         echo "----------MATLAB exited in error!----------"
         cat matlab.log
+        echo
         echo "-----------End of matlab.log-----------"
-
-        # Clean up execution directory
-        rm -rf ${ISSM_DIR}/execution/*
-
         exit 1
 fi
@@ -85 +50 @@
         echo "All MATLAB tests PASSED"
 fi
-
-# Create tarball
-cd ${ISSM_DIR}
-rm -f ${TARBALL}
-svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
-echo "Copying assets to package: ${PACKAGE}"
-cp -rf bin examples lib scripts test ${PACKAGE}/
-echo "Cleaning up unneeded/unwanted files"
-rm -f ${PACKAGE}/bin/generic_static.* # Remove static versions of generic cluster classes
-rm -f ${PACKAGE}/lib/*.a # Remove static libraries from package (we only need MEX-files)
-rm -f ${PACKAGE}/lib/*.la # Remove libtool libraries from package
-rm -rf ${PACKAGE}/test/SandBox # Remove testing sandbox from package
-echo "Creating tarball: ${TARBALL_NAME}"
-tar -czf ${TARBALL} ${PACKAGE}
-ls -lah ${ISSM_DIR}/${TARBALL}
-
-echo "Transferring binaries to ISSM Web site"
-scp -i ~/.ssh/pine_island_to_ross ${TARBALL} jenkins@ross.ics.uci.edu:/var/www/html/${TARBALL}
-
-if [ $? -ne 0 ]; then
-        echo "FAILED: Manually check connection"
-        exit 1
-fi