![(please configure the [header_logo] section in trac.ini)](/trac/issm/chrome/common/trac_banner.png)
- Timestamp:
- 11/12/20 16:43:32 (4 years ago)
- File:
-
- 1 copied
Legend:
- Unmodified
- Added
- Removed
-
issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-matlab.sh
r25741 r25744 1 1 #!/bin/bash 2 2 3 4 # Script to test, package, and transfer distributable to ISSM Web site. 5 # Corresponds with build generated by configuration in 6 # $ISSM_DIR/jenkins/pine_island-mac-binaries. 7 # 3 ################################################################################ 4 # To be used after running, 5 # 6 # ${ISSM_DIR}/jenkins/jenkins.sh ${ISSM_DIR}/jenkins/pine_island-mac-binaries-matlab 7 # 8 # in the context of a Jenkins project. 9 # 10 # When no runtime errors occur, performs the following: 11 # - Checks resulting executables and libraries against test suite. 12 # - Packages and compresses executables and libraries. 13 # - Commits compressed package to SVN repository to be signed by JPL 14 # Cybersecurity. 15 # - Retrieves signed package and transmits it to ISSM Web site for 16 # distribution. 17 # 18 # To skip tests, run with -s/--skiptests option. 19 # 20 # Debugging: 21 # - Relies on a very tight handshake with project on remote JPL Cybersecurity 22 # Jenkins server. Debugging may be perfomed locally by running, 23 # 24 # packagers/mac/sign-issm-mac-binaries-matlab.sh 25 # 26 # with Apple Developer credentials. 27 # - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can 28 # help debug potential SVN issues. 29 # 30 # NOTE: 31 # - Assumes that 'issm-binaries-user' and 'issm-binaries-pass' are set up in 32 # the 'Bindings' section under a 'Username and password (separated)' binding 33 # (requires 'Credentials Binding Plugin'). 34 # - For local debugging, the aformentioned credentials can be hardcoded into 35 # the 'USERNAME' and 'PASSWORD' constants below. 36 ################################################################################ 37 38 # From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes, 39 # 40 # Command line tool support for Subversion — including svn, git-svn, and 41 # related commands — is no longer provided by Xcode. (50266910) 42 # 43 # which results in, 44 # 45 # svn: error: The subversion command line tools are no longer provided by 46 # Xcode. 47 # 48 # when calling svn, even when subversion is installed via Homebrew and its path 49 # is available in PATH. 50 # 51 shopt -s expand_aliases 52 alias svn='/usr/local/bin/svn' 53 54 ## Override certain other aliases 55 # 56 alias grep=$(which grep) 8 57 9 58 ## Constants 10 59 # 11 MATLAB_NROPTIONS="'exclude',[IdFromString('Dakota'),125,126,435,701,702,703]" 60 61 MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2006,2020,2021,2051,2052,2053,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded 12 62 MATLAB_PATH="/Applications/MATLAB_R2018a.app" 13 PACKAGE="ISSM" # Name of directory to copy distributable files to 14 TARBALL_NAME="issm-mac" 15 TARBALL="${TARBALL_NAME}.tar.gz" 63 NOTARIZATION_LOGFILE="notarization.log" 64 PASSWORD=$env:issm-binaries-pass 65 PKG="ISSM-macOS-MATLAB" # Name of directory to copy distributable files to 66 SIGNED_REPO_COPY="./signed" 67 SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed" 68 SIGNING_CHECK_PERIOD=60 # seconds 69 SIGNING_LOCK_FILE="signing.lock" 70 UNSIGNED_REPO_COPY="./unsigned" 71 UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned" 72 USERNAME=$env:issm-binaries-user 73 74 COMPRESSED_PKG="${PKG}.zip" 16 75 17 76 ## Environment … … 19 78 export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages' 20 79 21 # Check if MATLAB exists 22 if ! [ -d ${MATLAB_PATH} ]; then 23 echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again." 80 ## Parse options 81 # 82 skip_tests=0 83 transfer_only=0 84 while [[ "$#" -gt 0 ]]; do 85 case $1 in 86 -s|--skiptests) skip_tests=1; shift ;; 87 -t|--transferonly) transfer_only=1; shift ;; 88 *) echo "Unknown parameter passed: $1"; exit 1 ;; 89 esac 90 shift 91 done 92 93 if [ ${transfer_only} -eq 0 ]; then 94 # Check if MATLAB exists 95 if ! [ -d ${MATLAB_PATH} ]; then 96 echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again." 97 exit 1 98 fi 99 100 # Clean up from previous packaging (not necessary for single builds on Jenkins, 101 # but useful when testing packaging locally) 102 echo "Cleaning up existing assets" 103 cd ${ISSM_DIR} 104 rm -rf ${PKG} ${COMPRESSED_PKG} ${SIGNED_REPO_COPY} ${UNSIGNED_REPO_COPY} 105 mkdir ${PKG} 106 107 # Check out copy of SVN repository for signed macOS packages 108 # 109 # NOTE: Get empty copy because we do not want to have to check out package from 110 # previous signing. 111 # 112 echo "Checking out copy of repository for signed packages" 113 svn co \ 114 --username ${USERNAME} \ 115 --password ${PASSWORD} \ 116 --depth empty \ 117 ${SIGNED_REPO_URL} \ 118 ${SIGNED_REPO_COPY} > /dev/null 2>&1 119 120 # If lock file exists, a signing build is still in process by JPL Cybersecurity 121 svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 122 if [[ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]]; then 123 echo "Previous signing job still in process by JPL Cybersecurity. Please try again later." 124 exit 1 125 fi 126 127 # Add required binaries and libraries to package and modify them where needed 128 cd ${ISSM_DIR}/bin 129 130 echo "Modify generic" 131 cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m 132 133 echo "Moving MPICH binaries to bin/" 134 if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then 135 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec . 136 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy . 137 elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then 138 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec . 139 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy . 140 else 141 echo "MPICH not found" 142 exit 1 143 fi 144 145 echo "Moving GDAL binaries to bin/" 146 if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then 147 cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo . 148 cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform . 149 else 150 echo "GDAL not found" 151 exit 1 152 fi 153 154 echo "Moving GMT binaries to bin/" 155 if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then 156 cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt . 157 cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect . 158 else 159 echo "GMT not found" 160 exit 1 161 fi 162 163 echo "Moving Gmsh binaries to bin/" 164 if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then 165 cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh . 166 else 167 echo "Gmsh not found" 168 exit 1 169 fi 170 171 # Run tests 172 if [ ${skip_tests} -eq 0 ]; then 173 echo "Running tests" 174 cd ${ISSM_DIR}/test/NightlyRun 175 rm matlab.log 2> /dev/null 176 177 # Run tests, redirecting output to logfile and suppressing output to console 178 ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null 179 180 # Check that MATLAB did not exit in error 181 matlabExitCode=`echo $?` 182 matlabExitedInError=`grep -E "Activation cannot proceed|license|Error" matlab.log | wc -l` 183 184 if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then 185 echo "----------MATLAB exited in error!----------" 186 cat matlab.log 187 echo "-----------End of matlab.log-----------" 188 189 # Clean up execution directory 190 rm -rf ${ISSM_DIR}/execution/* 191 192 exit 1 193 fi 194 195 # Check that all tests passed 196 numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"` 197 198 if [[ ${numTestsFailed} -ne 0 ]]; then 199 echo "One or more tests FAILED" 200 exit 1 201 else 202 echo "All tests PASSED" 203 fi 204 else 205 echo "Skipping tests" 206 fi 207 208 # Create package 209 cd ${ISSM_DIR} 210 svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package) 211 echo "Copying assets to package: ${PKG}" 212 cp -rf bin examples lib scripts test ${PKG}/ 213 echo "Cleaning up unneeded/unwanted files" 214 rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes 215 rm -f ${PKG}/lib/*.a # Remove static libraries from package 216 rm -f ${PKG}/lib/*.la # Remove libtool libraries from package 217 rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package 218 219 # Compress package 220 echo "Compressing package" 221 ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG} 222 223 # Commit lock file to repository for signed packages 224 echo "Committing lock file to repository for signed packages" 225 touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} 226 svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 227 svn commit --message "ADD: New lock file" ${SIGNED_REPO_COPY} > /dev/null 2>&1 228 229 # Save current working copy revision number 230 svn up ${SIGNED_REPO_COPY} > /dev/null 2>&1 231 CURRENT_REV=$(svn info --show-item last-changed-revision ${SIGNED_REPO_COPY}) 232 233 # Check out copy of SVN repository for unsigned macOS packages 234 echo "Checking out copy of repository for unsigned packages" 235 svn co \ 236 --username ${USERNAME} \ 237 --password ${PASSWORD} \ 238 ${UNSIGNED_REPO_URL} \ 239 ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 240 241 # Commit new compressed package to repository for unsigned binaries 242 # 243 # NOTE: This will not work if, for any reason, the checksum on the compressed 244 # package is unchanged. 245 # 246 echo "Committing package to repository for unsigned packages" 247 cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY} 248 svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1 249 svn commit --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 250 251 # Check status of signing 252 echo "Checking progress of signing..." 253 IN_PROCESS=1 254 SUCCESS=0 255 256 while [ ${IN_PROCESS} -eq 1 ]; do 257 echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds." 258 sleep ${SIGNING_CHECK_PERIOD} 259 svn up ${SIGNED_REPO_COPY} > /dev/null 2>&1 260 NEW_REV=$(svn info --show-item last-changed-revision ${SIGNED_REPO_COPY}) 261 262 if [ ${NEW_REV} -ne ${CURRENT_REV} ]; then 263 IN_PROCESS=0 264 265 svn up ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1 266 svn up ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1 267 268 # No error, so check status 269 STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //') 270 if [[ "${STATUS}" == "success" ]]; then 271 echo 'Notarization successful!' 272 273 # Set flag indicating notarization was successful 274 SUCCESS=1 275 else 276 echo 'Notarization failed!' 277 fi 278 fi 279 done 280 else # transfer_only == 1 281 # Assume that previous build resulted in successful signing of package but 282 # that transfer to ISSM Web site failed and user built this project again 283 # with -t/--transferonly option. 284 SUCCESS=1 285 fi 286 287 # Handle result of signing 288 if [ ${SUCCESS} -eq 1 ]; then 289 # Transfer signed package to ISSM Web site 290 echo 'Transferring signed package to ISSM Web site' 291 scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG} 292 293 if [ $? -ne 0 ]; then 294 echo "Transfer failed! Verify connection then build this project again (with -t/--transferonly option to skip testing and signing)." 295 exit 1 296 fi 297 else 298 echo '----------------------- Contents of notarization logfile -----------------------' 299 cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} 300 echo '--------------------------------------------------------------------------------' 301 24 302 exit 1 25 303 fi 26 27 # Clean up from previous packaging28 echo "Cleaning up existing assets"29 cd ${ISSM_DIR}30 rm -rf ${PACKAGE}31 mkdir ${PACKAGE}32 33 # Add/modify required binaries and libraries34 cd ${ISSM_DIR}/bin35 36 echo "Modify generic"37 cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m38 39 echo "Moving MPICH binaries to bin/"40 if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then41 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .42 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .43 elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then44 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .45 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .46 else47 echo "MPICH not found"48 exit 149 fi50 51 # Run tests52 echo "Running tests"53 cd ${ISSM_DIR}/test/NightlyRun54 55 # Check that MATLAB tests run56 echo "Running MATLAB tests"57 58 rm matlab.log 2> /dev/null59 60 # Run MATLAB tests redirecting output to logfile and suppressing output to console61 ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null62 63 # Check that MATLAB did not exit in error64 matlabExitCode=`echo $?`65 matlabExitedInError=`grep -E "Activation cannot proceed|license|Error" matlab.log | wc -l`66 67 if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then68 echo "----------MATLAB exited in error!----------"69 cat matlab.log70 echo "-----------End of matlab.log-----------"71 72 # Clean up execution directory73 rm -rf ${ISSM_DIR}/execution/*74 75 exit 176 fi77 78 # Check that all MATLAB tests passed79 numMatlabTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`80 81 if [[ ${numMatlabTestsFailed} -ne 0 ]]; then82 echo "One or more MATLAB tests FAILED"83 exit 184 else85 echo "All MATLAB tests PASSED"86 fi87 88 # Create tarball89 cd ${ISSM_DIR}90 rm -f ${TARBALL}91 svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)92 echo "Copying assets to package: ${PACKAGE}"93 cp -rf bin examples lib scripts test ${PACKAGE}/94 echo "Cleaning up unneeded/unwanted files"95 rm -f ${PACKAGE}/bin/generic_static.* # Remove static versions of generic cluster classes96 rm -f ${PACKAGE}/lib/*.a # Remove static libraries from package (we only need MEX-files)97 rm -f ${PACKAGE}/lib/*.la # Remove libtool libraries from package98 rm -rf ${PACKAGE}/test/SandBox # Remove testing sandbox from package99 echo "Creating tarball: ${TARBALL_NAME}"100 tar -czf ${TARBALL} ${PACKAGE}101 ls -lah ${ISSM_DIR}/${TARBALL}102 103 echo "Transferring binaries to ISSM Web site"104 scp -i ~/.ssh/pine_island_to_ross ${TARBALL} jenkins@ross.ics.uci.edu:/var/www/html/${TARBALL}105 106 if [ $? -ne 0 ]; then107 echo "FAILED: Manually check connection"108 exit 1109 fi
Note:
See TracChangeset
for help on using the changeset viewer.
