Changeset 25780


Ignore:
Timestamp:
11/24/20 14:39:01 (4 years ago)
Author:
jdquinn
Message:

CHG: Addition of packaging scripts for ISSM macOS binaries with Python API; documentation; variable initialization

Location:
issm/trunk-jpl
Files:
4 edited
3 copied

Legend:

Unmodified
Added
Removed

  • issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-matlab.sh

    r25744 r25780  
    1111# - Checks resulting executables and libraries against test suite.
    1212# - Packages and compresses executables and libraries.
    13 # - Commits compressed package to SVN repository to be signed by JPL
    14 #       Cybersecurity.
     13# - Commits compressed package to repository to be signed by JPL Cybersecurity.
    1514# - Retrieves signed package and transmits it to ISSM Web site for
    1615#       distribution.
    1716#
    18 # To skip tests, run with -s/--skiptests option.
     17# Options:
     18# -c/--cleanup                  Remove lock file from signed package repository (use if
     19#                                               build is aborted to allow for subsequent fresh build)
     20# -n/--notarizeonly             Sign/notarize only (use if signing/notarization fails
     21#                                               to skip tests/packaging)
     22# -s/--skiptests                Skip tests (use if this script fails for some reason
     23#                                               after tests have successfully passed to save time)
     24# -t/--transferonly             Transfer package to ISSM Web site only (use if
     25#                                               transfer fails for some reason to skip testing and
     26#                                               signing)
    1927#
    2028# Debugging:
    … …  
    3644################################################################################
    3745
     46# Expand aliases within the context of this script
     47shopt -s expand_aliases
     48
    3849# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
    3950#
    … …  
    4960# is available in PATH.
    5061#
    51 shopt -s expand_aliases
     62# NOTE: May be able to remove this after updating macOS.
     63#
    5264alias svn='/usr/local/bin/svn'
    5365
    … …  
    5870## Constants
    5971#
    60 
    6172MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2006,2020,2021,2051,2052,2053,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
    62 MATLAB_PATH="/Applications/MATLAB_R2018a.app"
     73MATLAB_PATH="/Applications/MATLAB_R2019b.app"
    6374NOTARIZATION_LOGFILE="notarization.log"
    6475PASSWORD=$env:issm-binaries-pass
    6576PKG="ISSM-macOS-MATLAB" # Name of directory to copy distributable files to
     77RETRIGGER_SIGNING_FILE="retrigger.txt"
    6678SIGNED_REPO_COPY="./signed"
    6779SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed"
    68 SIGNING_CHECK_PERIOD=60 # seconds
     80SIGNING_CHECK_PERIOD=60 # in seconds
    6981SIGNING_LOCK_FILE="signing.lock"
    7082UNSIGNED_REPO_COPY="./unsigned"
    … …  
    8092## Parse options
    8193#
     94cleanup=0
     95notarize_only=0
    8296skip_tests=0
    8397transfer_only=0
    84 while [[ "$#" -gt 0 ]]; do
     98while [ $# -gt 0 ]; do
    8599    case $1 in
     100        -c|--cleanup) cleanup=1; shift ;;
     101        -n|--notarizeonly) notarize_only=1; shift ;;
    86102        -s|--skiptests) skip_tests=1; shift ;;
    87103        -t|--transferonly) transfer_only=1; shift ;;
    … …  
    91107done
    92108
    93 if [ ${transfer_only} -eq 0 ]; then
    94         # Check if MATLAB exists
    95         if ! [ -d ${MATLAB_PATH} ]; then
    96                 echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again."
    97                 exit 1
    98         fi
    99 
    100         # Clean up from previous packaging (not necessary for single builds on Jenkins,
    101         # but useful when testing packaging locally)
    102         echo "Cleaning up existing assets"
    103         cd ${ISSM_DIR}
    104         rm -rf ${PKG} ${COMPRESSED_PKG} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY}
    105         mkdir ${PKG}
    106 
    107         # Check out copy of SVN repository for signed macOS packages
    108         #
    109         # NOTE: Get empty copy because we do not want to have to check out package from
    110         #               previous signing.
    111         #
    112         echo "Checking out copy of repository for signed packages"
     109if [ ${cleanup} -eq 1 ]; then
     110        # Remove signing lock file from signed package repository so that a new
     111        # build can run
     112        echo "Removing lock file from repository for signed packages"
    113113        svn co \
    114114                --username ${USERNAME} \
    … …  
    117117                ${SIGNED_REPO_URL} \
    118118                ${SIGNED_REPO_COPY} > /dev/null 2>&1
    119 
    120         # If lock file exists, a signing build is still in process by JPL Cybersecurity
    121119        svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
    122         if [[ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]]; then
    123                 echo "Previous signing job still in process by JPL Cybersecurity. Please try again later."
    124                 exit 1
     120        svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
     121        svn commit --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} > /dev/null 2>&1
     122        svn cleanup ${SIGNED_REPO_COPY} > /dev/null 2>&1
     123        exit 1
     124fi
     125
     126if [ ${transfer_only} -eq 0 ]; then
     127        if [ ${notarize_only} -eq 0 ]; then
     128                # Check if MATLAB exists
     129                if ! [ -d ${MATLAB_PATH} ]; then
     130                        echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again."
     131                        exit 1
     132                fi
     133
     134                # Clean up from previous packaging (not necessary for single builds on Jenkins,
     135                # but useful when testing packaging locally)
     136                echo "Cleaning up existing assets"
     137                cd ${ISSM_DIR}
     138                rm -rf ${PKG} ${COMPRESSED_PKG} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY}
     139                mkdir ${PKG}
     140
     141                # Check out copy of SVN repository for signed packages
     142                #
     143                # NOTE: Get empty copy because we do not want to have to check out package from
     144                #               previous signing.
     145                #
     146                echo "Checking out copy of repository for signed packages"
     147                svn co \
     148                        --username ${USERNAME} \
     149                        --password ${PASSWORD} \
     150                        --depth empty \
     151                        ${SIGNED_REPO_URL} \
     152                        ${SIGNED_REPO_COPY} > /dev/null 2>&1
     153
     154                # If lock file exists, a signing build is still in process by JPL Cybersecurity
     155                svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
     156                if [[ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]]; then
     157                        echo "Previous signing job still in process by JPL Cybersecurity. Please try again later."
     158                        exit 1
     159                fi
     160
     161                # Add required binaries and libraries to package and modify them where needed
     162                cd ${ISSM_DIR}/bin
     163
     164                echo "Modify generic"
     165                cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m
     166
     167                echo "Moving MPICH binaries to bin/"
     168                if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
     169                        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
     170                        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
     171                elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
     172                        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
     173                        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
     174                else
     175                        echo "MPICH not found"
     176                        exit 1
     177                fi
     178
     179                echo "Moving GDAL binaries to bin/"
     180                if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
     181                        cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
     182                        cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
     183                else
     184                        echo "GDAL not found"
     185                        exit 1
     186                fi
     187
     188                echo "Moving GMT binaries to bin/"
     189                if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
     190                        cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
     191                        cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
     192                else
     193                        echo "GMT not found"
     194                        exit 1
     195                fi
     196
     197                echo "Moving Gmsh binaries to bin/"
     198                if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
     199                        cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
     200                else
     201                        echo "Gmsh not found"
     202                        exit 1
     203                fi
     204
     205                # Run tests
     206                if [ ${skip_tests} -eq 0 ]; then
     207                        echo "Running tests"
     208                        cd ${ISSM_DIR}/test/NightlyRun
     209                        rm matlab.log 2> /dev/null
     210
     211                        # Run tests, redirecting output to logfile and suppressing output to console
     212                        ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
     213
     214                        # Check that MATLAB did not exit in error
     215                        matlabExitCode=`echo $?`
     216                        matlabExitedInError=`grep -E "Activation cannot proceed|license|Error" matlab.log | wc -l`
     217
     218                        if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then
     219                                echo "----------MATLAB exited in error!----------"
     220                                cat matlab.log
     221                                echo "-----------End of matlab.log-----------"
     222
     223                                # Clean up execution directory
     224                                rm -rf ${ISSM_DIR}/execution/*
     225
     226                                exit 1
     227                        fi
     228
     229                        # Check that all tests passed
     230                        numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
     231
     232                        if [[ ${numTestsFailed} -ne 0 ]]; then
     233                                echo "One or more tests FAILED"
     234                                exit 1
     235                        else
     236                                echo "All tests PASSED"
     237                        fi
     238                else
     239                        echo "Skipping tests"
     240                fi
     241
     242                # Create package
     243                cd ${ISSM_DIR}
     244                svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
     245                echo "Copying assets to package: ${PKG}"
     246                cp -rf bin examples lib scripts test ${PKG}/
     247                mkdir ${PKG}/execution
     248                cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist
     249                echo "Cleaning up unneeded/unwanted files"
     250                rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
     251                rm -f ${PKG}/lib/*.a # Remove static libraries from package
     252                rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
     253                rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
     254
     255                # Compress package
     256                echo "Compressing package"
     257                ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
    125258        fi
    126 
    127         # Add required binaries and libraries to package and modify them where needed
    128         cd ${ISSM_DIR}/bin
    129 
    130         echo "Modify generic"
    131         cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m
    132 
    133         echo "Moving MPICH binaries to bin/"
    134         if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
    135                 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
    136                 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
    137         elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
    138                 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
    139                 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
    140         else
    141                 echo "MPICH not found"
    142                 exit 1
    143         fi
    144 
    145         echo "Moving GDAL binaries to bin/"
    146         if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
    147                 cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
    148                 cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
    149         else
    150                 echo "GDAL not found"
    151                 exit 1
    152         fi
    153 
    154         echo "Moving GMT binaries to bin/"
    155         if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
    156                 cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
    157                 cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
    158         else
    159                 echo "GMT not found"
    160                 exit 1
    161         fi
    162 
    163         echo "Moving Gmsh binaries to bin/"
    164         if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
    165                 cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
    166         else
    167                 echo "Gmsh not found"
    168                 exit 1
    169         fi
    170 
    171         # Run tests
    172         if [ ${skip_tests} -eq 0 ]; then
    173                 echo "Running tests"
    174                 cd ${ISSM_DIR}/test/NightlyRun
    175                 rm matlab.log 2> /dev/null
    176 
    177                 # Run tests, redirecting output to logfile and suppressing output to console
    178                 ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
    179 
    180                 # Check that MATLAB did not exit in error
    181                 matlabExitCode=`echo $?`
    182                 matlabExitedInError=`grep -E "Activation cannot proceed|license|Error" matlab.log | wc -l`
    183 
    184                 if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then
    185                         echo "----------MATLAB exited in error!----------"
    186                         cat matlab.log
    187                         echo "-----------End of matlab.log-----------"
    188 
    189                         # Clean up execution directory
    190                         rm -rf ${ISSM_DIR}/execution/*
    191 
    192                         exit 1
    193                 fi
    194 
    195                 # Check that all tests passed
    196                 numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
    197 
    198                 if [[ ${numTestsFailed} -ne 0 ]]; then
    199                         echo "One or more tests FAILED"
    200                         exit 1
    201                 else
    202                         echo "All tests PASSED"
    203                 fi
    204         else
    205                 echo "Skipping tests"
    206         fi
    207 
    208         # Create package
    209         cd ${ISSM_DIR}
    210         svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
    211         echo "Copying assets to package: ${PKG}"
    212         cp -rf bin examples lib scripts test ${PKG}/
    213         echo "Cleaning up unneeded/unwanted files"
    214         rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
    215         rm -f ${PKG}/lib/*.a # Remove static libraries from package
    216         rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
    217         rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
    218 
    219         # Compress package
    220         echo "Compressing package"
    221         ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
    222259
    223260        # Commit lock file to repository for signed packages
    … …  
    231268        CURRENT_REV=$(svn info --show-item last-changed-revision ${SIGNED_REPO_COPY})
    232269
    233         # Check out copy of SVN repository for unsigned macOS packages
     270        # Check out copy of SVN repository for unsigned packages
    234271        echo "Checking out copy of repository for unsigned packages"
    235272        svn co \
    … …  
    239276                ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
    240277
    241         # Commit new compressed package to repository for unsigned binaries
    242         #
    243         # NOTE: This will not work if, for any reason, the checksum on the compressed
    244         #               package is unchanged.
    245         #
    246         echo "Committing package to repository for unsigned packages"
    247         cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
    248         svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
    249         svn commit --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     278        if [ ${notarize_only} -eq 0 ]; then
     279                # Commit new compressed package to repository for unsigned binaries
     280                #
     281                # NOTE: This will not work if, for any reason, the checksum on the compressed
     282                #               package is unchanged.
     283                #
     284                echo "Committing package to repository for unsigned packages"
     285                cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
     286                svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
     287                svn commit --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     288        else
     289                # NOTE: If notarize_only == 1, we commit a dummy file as the signing
     290                #               build on the remote JPL Cybersecurity Jenkins server is
     291                #               triggered by polling SCM.
     292                #
     293                echo "Attempting to sign existing package again"
     294                touch ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE}
     295                svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1
     296                svn commit --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     297        fi
    250298
    251299        # Check status of signing
    … …  
    255303
    256304        while [ ${IN_PROCESS} -eq 1 ]; do
    257                 echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds."
     305                echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds"
    258306                sleep ${SIGNING_CHECK_PERIOD}
    259307                svn up ${SIGNED_REPO_COPY} > /dev/null 2>&1
    … …  
    269317                        STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
    270318                        if [[ "${STATUS}" == "success" ]]; then
    271                                 echo 'Notarization successful!'
     319                                echo "Notarization successful!"
    272320
    273321                                # Set flag indicating notarization was successful
    274322                                SUCCESS=1
    275323                        else
    276                                 echo 'Notarization failed!'
     324                                echo "Notarization failed!"
    277325                        fi
    278326                fi
    279327        done
    280 else # transfer_only == 1
     328else
    281329        # Assume that previous build resulted in successful signing of package but
    282330        # that transfer to ISSM Web site failed and user built this project again
    … …  
    288336if [ ${SUCCESS} -eq 1 ]; then
    289337        # Transfer signed package to ISSM Web site
    290         echo 'Transferring signed package to ISSM Web site'
     338        echo "Transferring signed package to ISSM Web site"
    291339        scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG}
    292340
    … …  
    296344        fi
    297345else
    298         echo '----------------------- Contents of notarization logfile -----------------------'
     346        echo "----------------------- Contents of notarization logfile -----------------------"
    299347        cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
    300         echo '--------------------------------------------------------------------------------'
     348        echo "--------------------------------------------------------------------------------"
    301349
    302350        exit 1

  • issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh

    r25769 r25780  
    44# To be used after running,
    55#
    6 #       ${ISSM_DIR}/jenkins/jenkins.sh ${ISSM_DIR}/jenkins/pine_island-mac-binaries-matlab
     6#       ${ISSM_DIR}/jenkins/jenkins.sh ${ISSM_DIR}/jenkins/pine_island-mac-binaries-python
    77#
    88# in the context of a Jenkins project.
    … …  
    1111# - Checks resulting executables and libraries against test suite.
    1212# - Packages and compresses executables and libraries.
    13 # - Commits compressed package to SVN repository to be signed by JPL
    14 #       Cybersecurity.
     13# - Commits compressed package to repository to be signed by JPL Cybersecurity.
    1514# - Retrieves signed package and transmits it to ISSM Web site for
    1615#       distribution.
    1716#
    18 # To skip tests, run with -s/--skiptests option.
     17# Options:
     18# -c/--cleanup                  Remove lock file from signed package repository (use if
     19#                                               build is aborted to allow for subsequent fresh build)
     20# -n/--notarizeonly             Sign/notarize only (use if signing/notarization fails
     21#                                               to skip tests/packaging)
     22# -s/--skiptests                Skip tests (use if this script fails for some reason
     23#                                               after tests have successfully passed to save time)
     24# -t/--transferonly             Transfer package to ISSM Web site only (use if
     25#                                               transfer fails for some reason to skip testing and
     26#                                               signing)
    1927#
    2028# Debugging:
    … …  
    2230#       Jenkins server. Debugging may be perfomed locally by running,
    2331#
    24 #               packagers/mac/sign-issm-mac-binaries-matlab.sh
     32#               packagers/mac/sign-issm-mac-binaries-python.sh
    2533#
    2634#       with Apple Developer credentials.
    … …  
    3644################################################################################
    3745
     46# Expand aliases within the context of this script
     47shopt -s expand_aliases
     48
    3849# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
    3950#
    … …  
    4960# is available in PATH.
    5061#
    51 shopt -s expand_aliases
     62# NOTE: May be able to remove this after updating macOS.
     63#
    5264alias svn='/usr/local/bin/svn'
    5365
    … …  
    5870## Constants
    5971#
    60 
    61 MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2006,2020,2021,2051,2052,2053,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
    62 MATLAB_PATH="/Applications/MATLAB_R2018a.app"
    6372NOTARIZATION_LOGFILE="notarization.log"
    6473PASSWORD=$env:issm-binaries-pass
    65 PKG="ISSM-macOS-MATLAB" # Name of directory to copy distributable files to
     74PKG="ISSM-macOS-Python" # Name of directory to copy distributable files to
     75PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2006 2020 2021 2051 2052 2053 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
     76RETRIGGER_SIGNING_FILE="retrigger.txt"
    6677SIGNED_REPO_COPY="./signed"
    67 SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed"
    68 SIGNING_CHECK_PERIOD=60 # seconds
     78SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/signed"
     79SIGNING_CHECK_PERIOD=60 # in seconds
    6980SIGNING_LOCK_FILE="signing.lock"
    7081UNSIGNED_REPO_COPY="./unsigned"
    71 UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned"
     82UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned"
    7283USERNAME=$env:issm-binaries-user
    7384
    … …  
    8091## Parse options
    8192#
     93cleanup=0
     94notarize_only=0
    8295skip_tests=0
    8396transfer_only=0
    84 while [[ "$#" -gt 0 ]]; do
     97while [ $# -gt 0 ]; do
    8598    case $1 in
     99        -c|--cleanup) cleanup=1; shift ;;
     100        -n|--notarizeonly) notarize_only=1; shift ;;
    86101        -s|--skiptests) skip_tests=1; shift ;;
    87102        -t|--transferonly) transfer_only=1; shift ;;
    … …  
    91106done
    92107
    93 if [ ${transfer_only} -eq 0 ]; then
    94         # Check if MATLAB exists
    95         if ! [ -d ${MATLAB_PATH} ]; then
    96                 echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again."
    97                 exit 1
    98         fi
    99 
    100         # Clean up from previous packaging (not necessary for single builds on Jenkins,
    101         # but useful when testing packaging locally)
    102         echo "Cleaning up existing assets"
    103         cd ${ISSM_DIR}
    104         rm -rf ${PKG} ${COMPRESSED_PKG} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY}
    105         mkdir ${PKG}
    106 
    107         # Check out copy of SVN repository for signed macOS packages
    108         #
    109         # NOTE: Get empty copy because we do not want to have to check out package from
    110         #               previous signing.
    111         #
    112         echo "Checking out copy of repository for signed packages"
     108if [ ${cleanup} -eq 1 ]; then
     109        # Remove signing lock file from signed package repository so that a new
     110        # build can run
     111        echo "Removing lock file from repository for signed packages"
    113112        svn co \
    114113                --username ${USERNAME} \
    … …  
    117116                ${SIGNED_REPO_URL} \
    118117                ${SIGNED_REPO_COPY} > /dev/null 2>&1
    119 
    120         # If lock file exists, a signing build is still in process by JPL Cybersecurity
    121118        svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
    122         if [[ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]]; then
    123                 echo "Previous signing job still in process by JPL Cybersecurity. Please try again later."
    124                 exit 1
     119        svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
     120        svn commit --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} > /dev/null 2>&1
     121        svn cleanup ${SIGNED_REPO_COPY} > /dev/null 2>&1
     122        exit 1
     123fi
     124
     125if [ ${transfer_only} -eq 0 ]; then
     126        if [ ${notarize_only} -eq 0 ]; then
     127                # Clean up from previous packaging (not necessary for single builds on
     128                # Jenkins, but useful when testing packaging locally)
     129                echo "Cleaning up existing assets"
     130                cd ${ISSM_DIR}
     131                rm -rf ${PKG} ${COMPRESSED_PKG} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY}
     132                mkdir ${PKG}
     133
     134                # Check out copy of SVN repository for signed packages
     135                #
     136                # NOTE: Get empty copy because we do not want to have to check out package
     137                #               from previous signing.
     138                #
     139                echo "Checking out copy of repository for signed packages"
     140                svn co \
     141                        --username ${USERNAME} \
     142                        --password ${PASSWORD} \
     143                        --depth empty \
     144                        ${SIGNED_REPO_URL} \
     145                        ${SIGNED_REPO_COPY} > /dev/null 2>&1
     146
     147                # If lock file exists, a signing build is still in process by JPL Cybersecurity
     148                svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
     149                if [[ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]]; then
     150                        echo "Previous signing job still in process by JPL Cybersecurity. Please try again later."
     151                        exit 1
     152                fi
     153
     154                # Add required binaries and libraries to package and modify them where needed
     155                cd ${ISSM_DIR}/bin
     156
     157                echo "Modify generic"
     158                cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m
     159
     160                echo "Moving MPICH binaries to bin/"
     161                if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
     162                        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
     163                        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
     164                elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
     165                        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
     166                        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
     167                else
     168                        echo "MPICH not found"
     169                        exit 1
     170                fi
     171
     172                echo "Moving GDAL binaries to bin/"
     173                if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
     174                        cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
     175                        cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
     176                else
     177                        echo "GDAL not found"
     178                        exit 1
     179                fi
     180
     181                echo "Moving GMT binaries to bin/"
     182                if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
     183                        cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
     184                        cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
     185                else
     186                        echo "GMT not found"
     187                        exit 1
     188                fi
     189
     190                echo "Moving Gmsh binaries to bin/"
     191                if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
     192                        cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
     193                else
     194                        echo "Gmsh not found"
     195                        exit 1
     196                fi
     197
     198                # Run tests
     199                if [ ${skip_tests} -eq 0 ]; then
     200                        echo "Running tests"
     201
     202                        cd ${ISSM_DIR}/test/NightlyRun
     203                        rm python.log 2> /dev/null
     204
     205                        # Set Python environment
     206                        export PYTHONPATH="${ISSM_DIR}/src/m/dev"
     207                        export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
     208                        export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
     209
     210                        # Run tests, redirecting output to logfile and suppressing output to console
     211                        ./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
     212
     213                        # Check that Python did not exit in error
     214                        pythonExitCode=`echo $?`
     215                        pythonExitedInError=`grep -E "Error|Standard exception|Traceback|bad interpreter" python.log | wc -l`
     216
     217                        if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
     218                                echo "----------Python exited in error!----------"
     219                                cat python.log
     220                                echo "-----------End of python.log-----------"
     221
     222                                # Clean up execution directory
     223                                rm -rf ${ISSM_DIR}/execution/*
     224
     225                                exit 1
     226                        fi
     227
     228                        # Check that all tests passed
     229                        numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
     230
     231                        if [[ ${numTestsFailed} -ne 0 ]]; then
     232                                echo "One or more tests FAILED"
     233                                exit 1
     234                        else
     235                                echo "All tests PASSED"
     236                        fi
     237                else
     238                        echo "Skipping tests"
     239                fi
     240
     241                # Create package
     242                cd ${ISSM_DIR}
     243                svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
     244                echo "Copying assets to package: ${PKG}"
     245                cp -rf bin examples lib scripts test ${PKG}/
     246                mkdir ${PKG}/execution
     247                cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist
     248                ${ISSM_DIR}/scripts/py_to_pyc.sh ${PKG}/bin # Compile Python source files
     249                echo "Cleaning up unneeded/unwanted files"
     250                rm -f ${PKG}/bin/*.py # Remove all Python scripts
     251                rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
     252                rm -f ${PKG}/lib/*.a # Remove static libraries from package
     253                rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
     254                rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
     255
     256                # Compress package
     257                echo "Compressing package"
     258                ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
    125259        fi
    126 
    127         # Add required binaries and libraries to package and modify them where needed
    128         cd ${ISSM_DIR}/bin
    129 
    130         echo "Modify generic"
    131         cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m
    132 
    133         echo "Moving MPICH binaries to bin/"
    134         if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
    135                 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
    136                 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
    137         elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
    138                 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
    139                 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
    140         else
    141                 echo "MPICH not found"
    142                 exit 1
    143         fi
    144 
    145         echo "Moving GDAL binaries to bin/"
    146         if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
    147                 cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
    148                 cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
    149         else
    150                 echo "GDAL not found"
    151                 exit 1
    152         fi
    153 
    154         echo "Moving GMT binaries to bin/"
    155         if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
    156                 cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
    157                 cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
    158         else
    159                 echo "GMT not found"
    160                 exit 1
    161         fi
    162 
    163         echo "Moving Gmsh binaries to bin/"
    164         if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
    165                 cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
    166         else
    167                 echo "Gmsh not found"
    168                 exit 1
    169         fi
    170 
    171         # Run tests
    172         if [ ${skip_tests} -eq 0 ]; then
    173                 echo "Running tests"
    174                 cd ${ISSM_DIR}/test/NightlyRun
    175                 rm matlab.log 2> /dev/null
    176 
    177                 # Run tests, redirecting output to logfile and suppressing output to console
    178                 ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
    179 
    180                 # Check that MATLAB did not exit in error
    181                 matlabExitCode=`echo $?`
    182                 matlabExitedInError=`grep -E "Activation cannot proceed|license|Error" matlab.log | wc -l`
    183 
    184                 if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then
    185                         echo "----------MATLAB exited in error!----------"
    186                         cat matlab.log
    187                         echo "-----------End of matlab.log-----------"
    188 
    189                         # Clean up execution directory
    190                         rm -rf ${ISSM_DIR}/execution/*
    191 
    192                         exit 1
    193                 fi
    194 
    195                 # Check that all tests passed
    196                 numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
    197 
    198                 if [[ ${numTestsFailed} -ne 0 ]]; then
    199                         echo "One or more tests FAILED"
    200                         exit 1
    201                 else
    202                         echo "All tests PASSED"
    203                 fi
    204         else
    205                 echo "Skipping tests"
    206         fi
    207 
    208         # Create package
    209         cd ${ISSM_DIR}
    210         svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
    211         echo "Copying assets to package: ${PKG}"
    212         cp -rf bin examples lib scripts test ${PKG}/
    213         echo "Cleaning up unneeded/unwanted files"
    214         rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
    215         rm -f ${PKG}/lib/*.a # Remove static libraries from package
    216         rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
    217         rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
    218 
    219         # Compress package
    220         echo "Compressing package"
    221         ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
    222260
    223261        # Commit lock file to repository for signed packages
    … …  
    231269        CURRENT_REV=$(svn info --show-item last-changed-revision ${SIGNED_REPO_COPY})
    232270
    233         # Check out copy of SVN repository for unsigned macOS packages
     271        # Check out copy of SVN repository for unsigned packages
    234272        echo "Checking out copy of repository for unsigned packages"
    235273        svn co \
    … …  
    239277                ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
    240278
    241         # Commit new compressed package to repository for unsigned binaries
    242         #
    243         # NOTE: This will not work if, for any reason, the checksum on the compressed
    244         #               package is unchanged.
    245         #
    246         echo "Committing package to repository for unsigned packages"
    247         cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
    248         svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
    249         svn commit --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     279        if [ ${notarize_only} -eq 0 ]; then
     280                # Commit new compressed package to repository for unsigned binaries
     281                #
     282                # NOTE: This will not work if, for any reason, the checksum on the compressed
     283                #               package is unchanged.
     284                #
     285                echo "Committing package to repository for unsigned packages"
     286                cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
     287                svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
     288                svn commit --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     289        else
     290                # NOTE: If notarize_only == 1, we commit a dummy file as the signing
     291                #               build on the remote JPL Cybersecurity Jenkins server is
     292                #               triggered by polling SCM.
     293                #
     294                echo "Attempting to sign existing package again"
     295                touch ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE}
     296                svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1
     297                svn commit --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     298        fi
    250299
    251300        # Check status of signing
    … …  
    255304
    256305        while [ ${IN_PROCESS} -eq 1 ]; do
    257                 echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds."
     306                echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds"
    258307                sleep ${SIGNING_CHECK_PERIOD}
    259308                svn up ${SIGNED_REPO_COPY} > /dev/null 2>&1
    … …  
    269318                        STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
    270319                        if [[ "${STATUS}" == "success" ]]; then
    271                                 echo 'Notarization successful!'
    272 
     320                                echo "Notarization successful!"
     321                               
    273322                                # Set flag indicating notarization was successful
    274323                                SUCCESS=1
    275324                        else
    276                                 echo 'Notarization failed!'
     325                                echo "Notarization failed!"
    277326                        fi
    278327                fi
    279328        done
    280 else # transfer_only == 1
     329else
    281330        # Assume that previous build resulted in successful signing of package but
    282331        # that transfer to ISSM Web site failed and user built this project again
    … …  
    288337if [ ${SUCCESS} -eq 1 ]; then
    289338        # Transfer signed package to ISSM Web site
    290         echo 'Transferring signed package to ISSM Web site'
     339        echo "Transferring signed package to ISSM Web site"
    291340        scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG}
    292341
    … …  
    296345        fi
    297346else
    298         echo '----------------------- Contents of notarization logfile -----------------------'
     347        echo "----------------------- Contents of notarization logfile -----------------------"
    299348        cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
    300         echo '--------------------------------------------------------------------------------'
     349        echo "--------------------------------------------------------------------------------"
    301350
    302351        exit 1

  • issm/trunk-jpl/packagers/mac/sign-issm-mac-binaries-matlab.sh

    r25744 r25780  
     1#!/bin/bash
     2
     3################################################################################
     4# Intended to be run in the context of a Jenkins project on a JPL
     5# Cybersecurity server for signing macOS applications. Polls SCM of the
     6# Subversion repository hosted at
     7# https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned to trigger new
     8# builds.
     9#
     10# In order to replicate the requried Jenkins project configuration:
     11# - First, navigate to 'Manage Jenkins' -> 'Manage Plugins' and install the
     12#       'Credentials Bindings Plugin' if it is not already installed.
     13# - Contact one of the members of the ISSM development team for crendentials
     14#       for the ISSM binaries repository (mention that the credentials are stored
     15#       in ISSM-Infrastructure.pdf).
     16# - Navigate to 'Manage Jenkins' -> 'Manage Credentials' -> <domain> ->
     17#       'Add Credentials' and enter the crendentials from above.
     18# - From the 'Dashboard', select 'New Item' -> 'Freestyle project'.
     19# - Under 'Source Code Management', select 'Subversion'.
     20#               - The 'Repository URL' text field should be set to
     21#               "https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned"
     22#               - The 'Credentials' select menu should be set to the new credentials
     23#               created previously.
     24# - Under 'Build Trigggers', check the box for 'Poll SCM' and set the
     25#       'Schedule' text area to "H/5 * * * *".
     26# - Under 'Build Environment', check the box for 'Use secret text(s) or
     27#       file(s)', then under 'Bindings' click the 'Add...' button and select
     28#       'Username and password (separated)'.
     29#               - Set 'Username Variable' to "issm-binaries-user”.
     30#               - Set 'Password Variable' to "issm-binaries-pass”.
     31# - Under 'Credentials', select the same, new credentials that created
     32#       previously.
     33# - The contents of this script can be copied/pasted directly into the ‘Build'
     34#       -> 'Execute Shell' -> ‘Command' textarea of the project configuration (or
     35#       you can simply store the script on disk and call it from there).
     36# - Make sure to click the 'Save' button.
     37#
     38# Current point of contact at JPL Cybersecurity:
     39#       Alex Coward, alexander.g.coward@jpl.nasa.gov
     40#
     41# NOTE:
     42# - Assumes that 'issm-binaries-user' and 'issm-binaries-pass' are set up in
     43#       the 'Bindings' section under a 'Username and password (separated)' binding
     44#       (requires 'Credentials Binding Plugin').
     45# - For local debugging, the aformentioned credentials can be hardcoded into
     46#       the 'USERNAME' and 'PASSWORD' constants below.
     47################################################################################
     48
     49# Expand aliases within the context of this script
     50shopt -s expand_aliases
     51
    152# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
    253#
    … …  
    1263# is available in PATH.
    1364#
    14 shopt -s expand_aliases
     65# NOTE: May be able to remove this after updating macOS.
     66#
    1567alias svn='/usr/local/bin/svn'
    1668
    … …  
    2072alias grep=$(which grep)
    2173
    22 AD_IDENTITY="**********"
    23 AD_USERNAME="**********"
    24 ALTOOL_PASSWORD="@keychain:**********"
     74## Constants
     75#
     76AD_IDENTITY="**********" # Apple Developer identity
     77AD_USERNAME="**********" # Apple Developer username
     78ALTOOL_PASSWORD="@keychain:**********" # altool password (assumed to be stored in keychain)
    2579ASC_PROVIDER="**********"
    2680NOTARIZATION_CHECK_ATTEMPTS=60
    … …  
    2983NOTARIZATION_LOGFILE_PATH="."
    3084PKG="ISSM-macOS-MATLAB"
    31 PRIMARY_BUNDLE_ID="**********.issm.matlab"
     85PRIMARY_BUNDLE_ID="**********.issm.matlab" # Maybe "nasa.jpl.issm.matlab"?
     86RETRIGGER_SIGNING_FILE="retrigger.txt"
    3287SIGNED_REPO_COPY="./signed"
    3388SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed"
    … …  
    3893
    3994COMPRESSED_PKG="${PKG}.zip"
     95EXE_ENTITLEMENTS_PLIST="${PKG}/bin/entitlements.plist"
    4096
    4197# Clean up from previous packaging (not necessary for single builds on Jenkins,
    4298# but useful when testing packaging locally)
    43 echo 'Cleaning up existing assets'
     99echo "Cleaning up existing assets"
    44100rm -rf ${PKG} ${COMPRESSED_PKG} ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY}
    45101mkdir ${PKG}
    46102
    47 # Check out copy of SVN repository for unsigned macOS packages
    48 echo 'Checking out copy of SVN respository for unsigned packages'
     103# Check out copy of repository for unsigned packages
     104echo "Checking out copy of respository for unsigned packages"
    49105svn co \
    50106        --username ${USERNAME} \
    51107        --password ${PASSWORD} \
    52108        ${UNSIGNED_REPO_URL} \
    53         ${UNSIGNED_REPO_COPY}
    54 
    55 # Uncompress package
    56 echo 'Uncompressing package'
     109        ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     110
     111# Extract package contents
     112echo "Extracting package contents"
    57113ditto -xk ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} .
    58114
    … …  
    60116xattr -cr ${PKG}
    61117
    62 # Build list of executables
    63 EXECUTABLES=$(\
     118# Build list of ISSM executables
     119ISSM_BINS=$(\
     120        find ${PKG}/bin -type f -name *.exe; \
     121        find ${PKG}/lib -type f -name *.mexmaci64; \
     122)
     123
     124# Build list of third party executables
     125THIRD_PARTY_BINS=$(\
    64126        echo ${PKG}/bin/mpiexec; \
    65127        echo ${PKG}/bin/hydra_pmi_proxy; \
    … …  
    69131        echo ${PKG}/bin/gmtselect; \
    70132        echo ${PKG}/bin/gmsh; \
    71         find ${PKG} -type f -name *.exe; \
    72         find ${PKG} -type f -name *.mexmaci64; \
    73133)
    74134
    75135# Sign all executables in package
    76 echo 'Signing all executables in package'
    77 codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${EXECUTABLES}
     136echo "Signing all executables in package"
     137codesign -s ${AD_IDENTITY} --timestamp --options=runtime --entitlements ${EXE_ENTITLEMENTS_PLIST} ${ISSM_BINS}
     138codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${THIRD_PARTY_BINS}
    78139
    79140# NOTE: Skipping signature validation because this is not a true package nor app
    80141
    81142# Compress signed package
    82 echo 'Compressing signed package'
     143echo "Compressing signed package"
    83144ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
    84145
    85146# Submit compressed package for notarization
    86 echo 'Submitting signed package for notarization'
     147echo "Submitting signed package to Apple for notarization"
    87148xcrun altool --notarize-app --primary-bundle-id ${PRIMARY_BUNDLE_ID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} --asc-provider ${ASC_PROVIDER} --file ${COMPRESSED_PKG} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
    88149
    89150# Sleep until notarization request response is received
    90 echo 'Waiting for notarizaion request response'
     151echo "Waiting for notarizaion request response"
    91152while [[ ! -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} || ! -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; do
    92153        sleep 30
    93154done
    94155
    95 echo 'Notarizaion request response received'
     156echo "Notarizaion request response received"
    96157
    97158# Check if UUID exists in response
    98159HAS_UUID=$(grep 'RequestUUID = ' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}) # NOTE: Checking for "RequestUUID = " because "RequestUUID" shows up in some error messages
    99160if [[ -z "${HAS_UUID}" ]]; then
    100         echo 'Notarization failed!'
    101         echo '----------------------- Contents of notarization logfile -----------------------'
     161        echo "Notarization failed!"
     162        echo "----------------------- Contents of notarization logfile -----------------------"
    102163        cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
    103         echo '--------------------------------------------------------------------------------'
     164        echo "--------------------------------------------------------------------------------"
    104165
    105166        # Clean up
    … …  
    120181#               will not be able to clear Gatekeeper if they are offline.
    121182#
    122 echo 'Checking notarization status'
     183echo "Checking notarization status"
    123184SUCCESS=0
    124185for ATTEMPT in $(seq 1 ${NOTARIZATION_CHECK_ATTEMPTS}); do
    … …  
    156217                fi
    157218        else
    158                 if [ $ATTEMPT -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then
     219                if [ ${ATTEMPT} -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then
    159220                        echo "    ...not ready yet; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds."
    160221                        sleep ${NOTARIZATION_CHECK_PERIOD}
    161222                else
    162                         echo '    ...maximum attempts reached, but no response, or something else went wrong.'
    163                         echo '    If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again.'
     223                        echo "    ...maximum attempts reached, but no response, or something else went wrong."
     224                        echo "    If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again."
    164225                        break
    165226                fi
    … …  
    167228done
    168229
    169 # Check out copy of SVN repository for signed macOS packages
    170 echo 'Checking out copy of SVN respository for signed packages'
     230if [ ${SUCCESS} -eq 1 ]; then
     231        echo "Notarization successful!"
     232else
     233        echo "Notarization failed!"
     234        echo "----------------------- Contents of notarization logfile -----------------------"
     235        cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
     236        echo "--------------------------------------------------------------------------------"
     237fi
     238
     239# Remove dummy file for retriggering signing/notarization (if it exists)
     240svn delete ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1
     241svn commit --message "DEL: Removing dummy file for retriggering signing of same package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     242
     243# Check out copy of repository for signed packages
     244echo "Checking out copy of respository for signed packages"
    171245svn co \
    172246        --username ${USERNAME} \
    173247        --password ${PASSWORD} \
    174248        ${SIGNED_REPO_URL} \
    175         ${SIGNED_REPO_COPY}
    176 
    177 # Copy notarization file to repository for signed binaries
     249        ${SIGNED_REPO_COPY} > /dev/null 2>&1
     250
     251# Copy notarization file to repository for signed packages
    178252cp ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY}
    179253svn add ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1
    180254
    181 # Remove lock file from repository for signed binaries
     255# Remove lock file from repository for signed packages
    182256svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
    183257
    184258if [ ${SUCCESS} -eq 1 ]; then
    185         echo 'Notarization successful!'
    186 
    187         # Copy signed package to repository for signed binaries
     259        # Copy signed package to repository for signed packages
    188260        cp ${COMPRESSED_PKG} ${SIGNED_REPO_COPY}
    189261        svn add ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
    190262
    191263        # Commit changes
    192         svn commit --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY}
     264        echo "Committing changes to repository for signed packages"
     265        svn commit --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
    193266else
    194         echo 'Notarization failed!'
    195         echo '----------------------- Contents of notarization logfile -----------------------'
    196         cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
    197         echo '--------------------------------------------------------------------------------'
    198 
    199267        # Commit changes
    200         svn commit --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY}
     268        echo "Committing changes to repository for signed packages"
     269        svn commit --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
    201270
    202271        exit 1

  • issm/trunk-jpl/packagers/mac/sign-issm-mac-binaries-python.sh

    r25751 r25780  
     1#!/bin/bash
     2
     3################################################################################
     4# Intended to be run in the context of a Jenkins project on a JPL
     5# Cybersecurity server for signing macOS applications. Polls SCM of the
     6# Subversion repository hosted at
     7# https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned to trigger new
     8# builds.
     9#
     10# In order to replicate the requried Jenkins project configuration:
     11# - First, navigate to 'Manage Jenkins' -> 'Manage Plugins' and install the
     12#       'Credentials Bindings Plugin' if it is not already installed.
     13# - Contact one of the members of the ISSM development team for crendentials
     14#       for the ISSM binaries repository (mention that the credentials are stored
     15#       in ISSM-Infrastructure.pdf).
     16# - Navigate to 'Manage Jenkins' -> 'Manage Credentials' -> <domain> ->
     17#       'Add Credentials' and enter the crendentials from above.
     18# - From the 'Dashboard', select 'New Item' -> 'Freestyle project'.
     19# - Under 'Source Code Management', select 'Subversion'.
     20#               - The 'Repository URL' text field should be set to
     21#               "https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned"
     22#               - The 'Credentials' select menu should be set to the new credentials
     23#               created previously.
     24# - Under 'Build Trigggers', check the box for 'Poll SCM' and set the
     25#       'Schedule' text area to "H/5 * * * *".
     26# - Under 'Build Environment', check the box for 'Use secret text(s) or
     27#       file(s)', then under 'Bindings' click the 'Add...' button and select
     28#       'Username and password (separated)'.
     29#               - Set 'Username Variable' to "issm-binaries-user”.
     30#               - Set 'Password Variable' to "issm-binaries-pass”.
     31# - Under 'Credentials', select the same, new credentials that created
     32#       previously.
     33# - The contents of this script can be copied/pasted directly into the ‘Build'
     34#       -> 'Execute Shell' -> ‘Command' textarea of the project configuration (or
     35#       you can simply store the script on disk and call it from there).
     36# - Make sure to click the 'Save' button.
     37#
     38# Current point of contact at JPL Cybersecurity:
     39#       Alex Coward, alexander.g.coward@jpl.nasa.gov
     40#
     41# NOTE:
     42# - Assumes that 'issm-binaries-user' and 'issm-binaries-pass' are set up in
     43#       the 'Bindings' section under a 'Username and password (separated)' binding
     44#       (requires 'Credentials Binding Plugin').
     45# - For local debugging, the aformentioned credentials can be hardcoded into
     46#       the 'USERNAME' and 'PASSWORD' constants below.
     47################################################################################
     48
    149# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
    250#
    … …  
    2068alias grep=$(which grep)
    2169
    22 AD_IDENTITY="**********"
    23 AD_USERNAME="**********"
    24 ALTOOL_PASSWORD="@keychain:**********"
     70## Constants
     71#
     72AD_IDENTITY="**********" # Apple Developer identity
     73AD_USERNAME="**********" # Apple Developer username
     74ALTOOL_PASSWORD="@keychain:**********" # altool password (assumed to be stored in keychain)
    2575ASC_PROVIDER="**********"
    2676NOTARIZATION_CHECK_ATTEMPTS=60
    … …  
    2878NOTARIZATION_LOGFILE="notarization.log"
    2979NOTARIZATION_LOGFILE_PATH="."
    30 PKG="ISSM-macOS-MATLAB"
    31 PRIMARY_BUNDLE_ID="**********.issm.matlab"
     80PKG="ISSM-macOS-Python"
     81PRIMARY_BUNDLE_ID="**********.issm.python" # Maybe "nasa.jpl.issm.matlab"?
     82RETRIGGER_SIGNING_FILE="retrigger.txt"
    3283SIGNED_REPO_COPY="./signed"
    33 SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed"
     84SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/signed"
    3485SIGNING_LOCK_FILE="signing.lock"
    3586SUCCESS_LOGFILE="${SIGNED_REPO_COPY}/success.log"
    3687UNSIGNED_REPO_COPY="./unsigned"
    37 UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned"
     88UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned"
    3889
    3990COMPRESSED_PKG="${PKG}.zip"
     91EXE_ENTITLEMENTS_PLIST="${PKG}/bin/entitlements.plist"
    4092
    4193# Clean up from previous packaging (not necessary for single builds on Jenkins,
    4294# but useful when testing packaging locally)
    43 echo 'Cleaning up existing assets'
     95echo "Cleaning up existing assets"
    4496rm -rf ${PKG} ${COMPRESSED_PKG} ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY}
    4597mkdir ${PKG}
    4698
    47 # Check out copy of SVN repository for unsigned macOS packages
    48 echo 'Checking out copy of SVN respository for unsigned packages'
     99# Check out copy of repository for unsigned packages
     100echo "Checking out copy of respository for unsigned packages"
    49101svn co \
    50102        --username ${USERNAME} \
    51103        --password ${PASSWORD} \
    52104        ${UNSIGNED_REPO_URL} \
    53         ${UNSIGNED_REPO_COPY}
    54 
    55 # Uncompress package
    56 echo 'Uncompressing package'
     105        ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     106
     107# Extract package contents
     108echo "Extracting package contents"
    57109ditto -xk ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} .
    58110
    … …  
    60112xattr -cr ${PKG}
    61113
    62 # Build list of executables
    63 EXECUTABLES=$(\
     114# Build list of ISSM executables
     115ISSM_BINS=$(\
     116        find ${PKG}/bin -type f -name *.exe; \
     117        find ${PKG}/bin -type f -name *.pyc; \
     118)
     119
     120# Build list of third party executables
     121THIRD_PARTY_BINS=$(\
    64122        echo ${PKG}/bin/mpiexec; \
    65123        echo ${PKG}/bin/hydra_pmi_proxy; \
    … …  
    69127        echo ${PKG}/bin/gmtselect; \
    70128        echo ${PKG}/bin/gmsh; \
    71         find ${PKG} -type f -name *.exe; \
    72         find ${PKG} -type f -name *.mexmaci64; \
    73129)
    74130
    75131# Sign all executables in package
    76 echo 'Signing all executables in package'
    77 codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${EXECUTABLES}
     132echo "Signing all executables in package"
     133codesign -s ${AD_IDENTITY} --timestamp --options=runtime --entitlements ${EXE_ENTITLEMENTS_PLIST} ${ISSM_BINS}
     134codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${THIRD_PARTY_BINS}
     135
     136# Build list of ISSM libraries
     137ISSM_LIBS=$(\
     138        find ${PKG}/lib -type f -name *.so; \
     139)
     140
     141# Sign all libraries in package
     142echo "Signing all libraries in package"
     143codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${ISSM_LIBS}
    78144
    79145# NOTE: Skipping signature validation because this is not a true package nor app
    80146
    81147# Compress signed package
    82 echo 'Compressing signed package'
     148echo "Compressing signed package"
    83149ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
    84150
    85151# Submit compressed package for notarization
    86 echo 'Submitting signed package for notarization'
     152echo "Submitting signed package to Apple for notarization"
    87153xcrun altool --notarize-app --primary-bundle-id ${PRIMARY_BUNDLE_ID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} --asc-provider ${ASC_PROVIDER} --file ${COMPRESSED_PKG} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
    88154
    89155# Sleep until notarization request response is received
    90 echo 'Waiting for notarizaion request response'
     156echo "Waiting for notarizaion request response"
    91157while [[ ! -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} || ! -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; do
    92158        sleep 30
    93159done
    94160
    95 echo 'Notarizaion request response received'
     161echo "Notarizaion request response received"
    96162
    97163# Check if UUID exists in response
    98164HAS_UUID=$(grep 'RequestUUID = ' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}) # NOTE: Checking for "RequestUUID = " because "RequestUUID" shows up in some error messages
    99165if [[ -z "${HAS_UUID}" ]]; then
    100         echo 'Notarization failed!'
    101         echo '----------------------- Contents of notarization logfile -----------------------'
     166        echo "Notarization failed!"
     167        echo "----------------------- Contents of notarization logfile -----------------------"
    102168        cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
    103         echo '--------------------------------------------------------------------------------'
     169        echo "--------------------------------------------------------------------------------"
    104170
    105171        # Clean up
    … …  
    120186#               will not be able to clear Gatekeeper if they are offline.
    121187#
    122 echo 'Checking notarization status'
     188echo "Checking notarization status"
    123189SUCCESS=0
    124190for ATTEMPT in $(seq 1 ${NOTARIZATION_CHECK_ATTEMPTS}); do
    … …  
    137203                if [[ "${STATUS}" == "success" ]]; then
    138204                        # Staple notarization to all elements of package that were previously signed
    139                         #xcrun stapler staple ${EXECUTABLES} # NOTE: Fails with "Stapler is incapable of working with MATLAB Mex files."
     205                        #xcrun stapler staple ${THIRD_PARTY_BINS} # NOTE: Fails with "Stapler is incapable of working with MATLAB Mex files."
    140206
    141207                        # Validate stapling of notarization
    142                         #xcrun stapler validation ${EXECUTABLES} # NOTE: Skipping notarization stapling validation because this is not a true package nor app
     208                        #xcrun stapler validation ${THIRD_PARTY_BINS} # NOTE: Skipping notarization stapling validation because this is not a true package nor app
    143209
    144210                        # Compress signed and notarized package
    … …  
    156222                fi
    157223        else
    158                 if [ $ATTEMPT -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then
    159                         echo "    ...not ready yet; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds."
     224                if [ ${ATTEMPT} -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then
     225                        echo "    ...not ready yet; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds"
    160226                        sleep ${NOTARIZATION_CHECK_PERIOD}
    161227                else
    162                         echo '    ...maximum attempts reached, but no response, or something else went wrong.'
    163                         echo '    If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again.'
     228                        echo "    ...maximum attempts reached, but no response, or something else went wrong"
     229                        echo "    If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again"
    164230                        break
    165231                fi
    … …  
    167233done
    168234
    169 # Check out copy of SVN repository for signed macOS packages
    170 echo 'Checking out copy of SVN respository for signed packages'
     235if [ ${SUCCESS} -eq 1 ]; then
     236        echo "Notarization successful!"
     237else
     238        echo "Notarization failed!"
     239        echo "----------------------- Contents of notarization logfile -----------------------"
     240        cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
     241        echo "--------------------------------------------------------------------------------"
     242fi
     243
     244# Remove dummy file for retriggering signing/notarization (if it exists)
     245svn delete ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1
     246svn commit --message "DEL: Removing dummy file for retriggering signing of same package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     247
     248# Check out copy of repository for signed packages
     249echo "Checking out copy of respository for signed packages"
    171250svn co \
    172251        --username ${USERNAME} \
    173252        --password ${PASSWORD} \
    174253        ${SIGNED_REPO_URL} \
    175         ${SIGNED_REPO_COPY}
    176 
    177 # Copy notarization file to repository for signed binaries
     254        ${SIGNED_REPO_COPY} > /dev/null 2>&1
     255
     256# Copy notarization file to repository for signed packages
    178257cp ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY}
    179258svn add ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1
    180259
    181 # Remove lock file from repository for signed binaries
     260# Remove lock file from repository for signed packages
    182261svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
    183262
    184263if [ ${SUCCESS} -eq 1 ]; then
    185         echo 'Notarization successful!'
    186 
    187         # Copy signed package to repository for signed binaries
     264        # Copy signed package to repository for signed packages
    188265        cp ${COMPRESSED_PKG} ${SIGNED_REPO_COPY}
    189266        svn add ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
    190267
    191268        # Commit changes
    192         svn commit --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY}
     269        echo "Committing changes to repository for signed packages"
     270        svn commit --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
    193271else
    194         echo 'Notarization failed!'
    195         echo '----------------------- Contents of notarization logfile -----------------------'
    196         cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
    197         echo '--------------------------------------------------------------------------------'
    198 
    199272        # Commit changes
    200         svn commit --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY}
     273        echo "Committing changes to repository for signed packages"
     274        svn commit --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
    201275
    202276        exit 1

  • issm/trunk-jpl/packagers/mac/test-issm-mac-binaries-matlab.sh

    r25744 r25780  
    11#!/bin/bash
    22
    3 # This script is intended to test ISSM macOS MATLAB binaries downloaded to a
    4 # user-end machine.
     3################################################################################
     4# This script is intended to test ISSM macOS MATLAB binaries on an end-user
     5# machine after successful packaging and signing.
    56#
    67# NOTE: Tarball must already exist in INSTALL_DIR
     8################################################################################
     9
     10## Constants
    711#
    8 
    9 INSTALL_DIR=~/Downloads
     12INSTALL_DIR=.
    1013MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2006,2020,2021,2051,2052,2053,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
    1114MATLAB_PATH="/Applications/MATLAB_R2018a.app"
    … …  
    1417COMPRESSED_PKG="${PKG}.zip"
    1518
     19## Environment
     20#
    1621export ISSM_DIR="${INSTALL_DIR}/${PKG}"
    1722export PATH="${PATH}:${ISSM_DIR}/bin:${ISSM_DIR}/scripts"
    … …  
    1924cd ${INSTALL_DIR}
    2025rm -rf ${PKG}
    21 tar -zxvf ${COMPRESSED_PKG}
     26ditto -xk ${COMPRESSED_PKG} .
    2227cd ${PKG}/test/NightlyRun
    2328
    24 # Check that MATLAB tests run
    25 echo "Running MATLAB tests"
     29# Run tests, redirecting output to logfile and suppressing output to console
     30echo "Running tests"
    2631rm matlab.log 2> /dev/null
    27 
    28 # Run MATLAB tests redirecting output to logfile and suppressing output to console
    2932${MATLAB_PATH}/bin/matlab -nosplash -nodesktop -nojvm -r "try, addpath ../../bin; addpath ../../lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
    3033
    … …  
    4144fi
    4245
    43 # Check that all MATLAB tests passed
    44 numMatlabTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
     46# Check that all tests passed
     47numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
    4548
    46 if [[ ${numMatlabTestsFailed} -ne 0 ]]; then
    47         echo "One or more MATLAB tests FAILED"
     49if [[ ${numTestsFailed} -ne 0 ]]; then
     50        echo "One or more tests FAILED"
    4851        exit 1
    4952else
    50         echo "All MATLAB tests PASSED"
     53        echo "All tests PASSED"
    5154fi

  • issm/trunk-jpl/packagers/mac/test-issm-mac-binaries-python.sh

    r25769 r25780  
    11#!/bin/bash
    22
    3 # This script is intended to test ISSM macOS MATLAB binaries downloaded to a
    4 # user-end machine.
     3################################################################################
     4# This script is intended to test ISSM macOS MATLAB binaries on an end-user
     5# machine after successful packaging and signing.
    56#
    67# NOTE: Tarball must already exist in INSTALL_DIR
     8################################################################################
     9
     10## Constants
    711#
    8 
    9 INSTALL_DIR=~/Downloads
    10 MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2006,2020,2021,2051,2052,2053,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
    11 MATLAB_PATH="/Applications/MATLAB_R2018a.app"
    12 PKG="ISSM-macOS-MATLAB"
     12INSTALL_DIR=.
     13PKG="ISSM-macOS-Python"
     14PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2006 2020 2021 2051 2052 2053 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
    1315
    1416COMPRESSED_PKG="${PKG}.zip"
    … …  
    1618export ISSM_DIR="${INSTALL_DIR}/${PKG}"
    1719export PATH="${PATH}:${ISSM_DIR}/bin:${ISSM_DIR}/scripts"
     20export PYTHONPATH="${ISSM_DIR}/scripts"
     21export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
     22export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
    1823
    1924cd ${INSTALL_DIR}
    2025rm -rf ${PKG}
    21 tar -zxvf ${COMPRESSED_PKG}
     26ditto -xk ${COMPRESSED_PKG} .
    2227cd ${PKG}/test/NightlyRun
    2328
    24 # Check that MATLAB tests run
    25 echo "Running MATLAB tests"
    26 rm matlab.log 2> /dev/null
     29# Run tests, redirecting output to logfile and suppressing output to console
     30echo "Running tests"
     31rm python.log 2> /dev/null
     32./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
    2733
    28 # Run MATLAB tests redirecting output to logfile and suppressing output to console
    29 ${MATLAB_PATH}/bin/matlab -nosplash -nodesktop -nojvm -r "try, addpath ../../bin; addpath ../../lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
     34# Check that Python did not exit in error
     35pythonExitCode=`echo $?`
     36pythonExitedInError=`grep -E "runme.py: error" python.log | wc -l`
    3037
    31 # Check that MATLAB did not exit in error
    32 matlabExitCode=`echo $?`
    33 matlabExitedInError=`grep -E "Activation cannot proceed|license|Error|Warning: Name is nonexistent or not a directory" matlab.log | wc -l`
    34 
    35 if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then
    36         echo "----------MATLAB exited in error!----------"
    37         cat matlab.log
    38         echo
    39         echo "-----------End of matlab.log-----------"
     38if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
     39        echo "----------Python exited in error!----------"
     40        cat python.log
     41        echo "-----------End of python.log-----------"
    4042        exit 1
    4143fi
    4244
    43 # Check that all MATLAB tests passed
    44 numMatlabTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
     45# Check that all tests passed
     46numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
    4547
    46 if [[ ${numMatlabTestsFailed} -ne 0 ]]; then
    47         echo "One or more MATLAB tests FAILED"
     48if [[ ${numTestsFailed} -ne 0 ]]; then
     49        echo "One or more tests FAILED"
    4850        exit 1
    4951else
    50         echo "All MATLAB tests PASSED"
     52        echo "All tests PASSED"
    5153fi

  • issm/trunk-jpl/src/m/classes/results.py

    r25726 r25780  
    107107
    108108    def __init__(self, *args): #{{{
     109        self.steps = None
    109110        if len(args) == 1:
    110111            arg = args[0]
Note: See TracChangeset for help on using the changeset viewer.