Changeset 25884
- Timestamp:
- 12/17/20 19:45:30 (4 years ago)
- Location:
- issm/trunk-jpl/packagers/mac
- Files:
-
- 3 added
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-matlab.sh
r25879 r25884 2 2 3 3 ################################################################################ 4 # To be used after running, 5 # 6 # ${ISSM_DIR}/jenkins/jenkins.sh ${ISSM_DIR}/jenkins/pine_island-mac-binaries-matlab 7 # 8 # in the context of a Jenkins project. 9 # 10 # When no runtime errors occur, performs the following: 11 # - Checks resulting executables and libraries against test suite. 12 # - Packages and compresses executables and libraries. 13 # - Commits compressed package to repository to be signed by JPL Cybersecurity. 14 # - Retrieves signed package and transmits it to ISSM Web site for 15 # distribution. 4 # Packages and tests ISSM distributable package for macOS with MATLAB API. 16 5 # 17 6 # Options: 18 # -n/--notarizeonly Sign/notarize only (use if signing/notarization fails 19 # to skip tests/packaging) 20 # -s/--skiptests Skip tests (use if this script fails for some reason 21 # after tests have successfully passed to save time) 22 # -t/--transferonly Transfer package to ISSM Web site only (use if 23 # transfer fails for some reason to skip testing and 24 # signing) 25 # -u/--unlock Remove lock file from signed package repository (use if 26 # build is aborted to allow for subsequent fresh build) 27 # 28 # Debugging: 29 # - Relies on a very tight handshake with project on remote JPL Cybersecurity 30 # Jenkins server. Debugging may be perfomed locally by running, 31 # 32 # packagers/mac/sign-issm-mac-binaries-matlab.sh 33 # 34 # with Apple Developer credentials. 35 # - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can 36 # help debug potential SVN issues. 7 # -s/--skiptests Skip tests and package only. 37 8 # 38 9 # NOTE: 39 # - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in 40 # the 'Bindings' section under a 'Username and password (separated)' binding 41 # (requires 'Credentials Binding Plugin'). 42 # - For local debugging, the aformentioned credentials can be hardcoded into 43 # the 'USERNAME' and 'PASSWORD' constants below. 10 # - Assumes that the following constants are defined, 11 # 12 # COMPRESSED_PKG 13 # ISSM_DIR 14 # PKG 15 # 16 # See also: 17 # - packagers/mac/complete-issm-mac-binaries-matlab.sh 18 # - packagers/mac/sign-issm-mac-binaries-matlab.sh 44 19 ################################################################################ 45 20 … … 66 41 ## Override certain other aliases 67 42 # 43 alias cp=$(which cp) 68 44 alias grep=$(which grep) 69 45 … … 72 48 MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2006,2020,2021,2051,2052,2053,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded 73 49 MATLAB_PATH="/Applications/MATLAB_R2018a.app" 74 NOTARIZATION_LOGFILE="notarization.log"75 PASSWORD=${ISSM_BINARIES_PASS}76 PKG="ISSM-macOS-MATLAB" # Name of directory to copy distributable files to77 RETRIGGER_SIGNING_FILE="retrigger.txt"78 SIGNED_REPO_COPY="./signed"79 SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed"80 SIGNING_CHECK_PERIOD=60 # in seconds81 SIGNING_LOCK_FILE="signing.lock"82 UNSIGNED_REPO_COPY="./unsigned"83 UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned"84 USERNAME=${ISSM_BINARIES_USER}85 86 COMPRESSED_PKG="${PKG}.zip"87 50 88 51 ## Environment 89 52 # 90 export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages' 91 92 ## Functions 93 # 94 checkout_signed_repo_copy(){ 95 echo "Checking out copy of repository for signed packages" 96 97 # NOTE: Get empty copy because we do not want to have to check out package 98 # from previous signing. 99 # 100 svn checkout \ 101 --trust-server-cert \ 102 --non-interactive \ 103 --depth empty \ 104 --username ${USERNAME} \ 105 --password ${PASSWORD} \ 106 ${SIGNED_REPO_URL} \ 107 ${SIGNED_REPO_COPY} > /dev/null 2>&1 108 } 109 checkout_unsigned_repo_copy(){ 110 echo "Checking out copy of repository for unsigned packages" 111 svn checkout \ 112 --trust-server-cert \ 113 --non-interactive \ 114 --username ${USERNAME} \ 115 --password ${PASSWORD} \ 116 ${UNSIGNED_REPO_URL} \ 117 ${UNSIGNED_REPO_COPY} > /dev/null 2>&1 118 } 119 validate_signed_repo_copy(){ 120 # Validate copy of repository for signed binaries (e.g. 121 # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible'; 122 # initial checkout failed) 123 if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then 124 rm -rf ${SIGNED_REPO_COPY} 125 checkout_signed_repo_copy 126 fi 127 } 53 export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'; used when running tests 128 54 129 55 ## Parse options … … 134 60 fi 135 61 136 notarize_only=0137 62 skip_tests=0 138 transfer_only=0139 unlock=0140 while [ $# -gt 0 ]; do141 case $1 in142 -n|--notarizeonly) notarize_only=1; shift ;;143 -s|--skiptests) skip_tests=1; shift ;;144 -t|--transferonly) transfer_only=1; shift ;;145 -u|--unlock) unlock=1; shift ;;146 *) echo "Unknown parameter passed: $1"; exit 1 ;;147 esac148 shift149 done150 63 151 if [ ${unlock} -eq 1 ]; then 152 # Remove signing lock file from signed package repository so that a new 153 # build can run 154 echo "Removing lock file from repository for signed packages" 155 checkout_signed_repo_copy 156 svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} 157 svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1 158 svn commit \ 159 --trust-server-cert \ 160 --non-interactive \ 161 --username ${USERNAME} \ 162 --password ${PASSWORD} \ 163 --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} 164 svn cleanup ${SIGNED_REPO_COPY} 64 if [ $# -eq 1 ]; then 65 case $1 in 66 -s|--skiptests) skip_tests=1; ;; 67 *) echo "Unknown parameter passed: $1"; exit 1 ;; 68 esac 69 fi 70 71 # Check if MATLAB exists 72 if ! [ -d ${MATLAB_PATH} ]; then 73 echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again." 165 74 exit 1 166 75 fi 167 76 168 if [ ${transfer_only} -eq 0 ]; then 169 rm -rf ${SIGNED_REPO_COPY} 77 # Clean up from previous packaging 78 echo "Cleaning up existing assets" 79 cd ${ISSM_DIR} 80 rm -rf ${PKG} ${COMPRESSED_PKG} 81 mkdir ${PKG} 170 82 171 checkout_signed_repo_copy 83 # Add required binaries and libraries to package and modify them where needed 84 cd ${ISSM_DIR}/bin 172 85 173 # If lock file exists, a signing build is still in process by JPL Cybersecurity 174 svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} 175 if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then 176 echo "Previous signing job still in process by JPL Cybersecurity. Please try again later." 86 echo "Modify generic" 87 cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m 88 89 echo "Moving MPICH binaries to bin/" 90 if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then 91 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec . 92 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy . 93 elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then 94 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec . 95 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy . 96 else 97 echo "MPICH not found" 98 exit 1 99 fi 100 101 echo "Moving GDAL binaries to bin/" 102 if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then 103 cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo . 104 cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform . 105 else 106 echo "GDAL not found" 107 exit 1 108 fi 109 110 echo "Moving GMT binaries to bin/" 111 if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then 112 cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt . 113 cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect . 114 else 115 echo "GMT not found" 116 exit 1 117 fi 118 119 echo "Moving Gmsh binaries to bin/" 120 if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then 121 cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh . 122 else 123 echo "Gmsh not found" 124 exit 1 125 fi 126 127 # Run tests 128 if [ ${skip_tests} -eq 0 ]; then 129 echo "Running tests" 130 cd ${ISSM_DIR}/test/NightlyRun 131 rm matlab.log 2> /dev/null 132 133 # Run tests, redirecting output to logfile and suppressing output to console 134 ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null 135 136 # Check that MATLAB did not exit in error 137 matlabExitCode=`echo $?` 138 matlabExitedInError=`grep -E "Activation cannot proceed|Error in matlab_run" matlab.log | wc -l` 139 140 if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then 141 echo "----------MATLAB exited in error!----------" 142 cat matlab.log 143 echo "-----------End of matlab.log-----------" 144 145 # Clean up execution directory 146 rm -rf ${ISSM_DIR}/execution/* 147 177 148 exit 1 178 149 fi 179 150 180 if [ ${notarize_only} -eq 0 ]; then 181 # Check if MATLAB exists 182 if ! [ -d ${MATLAB_PATH} ]; then 183 echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again." 184 exit 1 185 fi 151 # Check that all tests passed 152 numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"` 186 153 187 # Clean up from previous packaging 188 echo "Cleaning up existing assets" 189 cd ${ISSM_DIR} 190 rm -rf ${PKG} ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY} 191 mkdir ${PKG} 192 193 # Add required binaries and libraries to package and modify them where needed 194 cd ${ISSM_DIR}/bin 195 196 echo "Modify generic" 197 cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m 198 199 echo "Moving MPICH binaries to bin/" 200 if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then 201 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec . 202 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy . 203 elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then 204 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec . 205 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy . 206 else 207 echo "MPICH not found" 208 exit 1 209 fi 210 211 echo "Moving GDAL binaries to bin/" 212 if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then 213 cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo . 214 cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform . 215 else 216 echo "GDAL not found" 217 exit 1 218 fi 219 220 echo "Moving GMT binaries to bin/" 221 if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then 222 cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt . 223 cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect . 224 else 225 echo "GMT not found" 226 exit 1 227 fi 228 229 echo "Moving Gmsh binaries to bin/" 230 if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then 231 cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh . 232 else 233 echo "Gmsh not found" 234 exit 1 235 fi 236 237 # Run tests 238 if [ ${skip_tests} -eq 0 ]; then 239 echo "Running tests" 240 cd ${ISSM_DIR}/test/NightlyRun 241 rm matlab.log 2> /dev/null 242 243 # Run tests, redirecting output to logfile and suppressing output to console 244 ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null 245 246 # Check that MATLAB did not exit in error 247 matlabExitCode=`echo $?` 248 matlabExitedInError=`grep -E "Activation cannot proceed|Error in matlab_run" matlab.log | wc -l` 249 250 if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then 251 echo "----------MATLAB exited in error!----------" 252 cat matlab.log 253 echo "-----------End of matlab.log-----------" 254 255 # Clean up execution directory 256 rm -rf ${ISSM_DIR}/execution/* 257 258 exit 1 259 fi 260 261 # Check that all tests passed 262 numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"` 263 264 if [ ${numTestsFailed} -ne 0 ]; then 265 echo "One or more tests FAILED" 266 exit 1 267 else 268 echo "All tests PASSED" 269 fi 270 else 271 echo "Skipping tests" 272 fi 273 274 # Create package 275 cd ${ISSM_DIR} 276 svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package) 277 echo "Copying assets to package: ${PKG}" 278 cp -rf bin examples lib scripts test ${PKG}/ 279 mkdir ${PKG}/execution 280 cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist 281 echo "Cleaning up unneeded/unwanted files" 282 rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes 283 rm -f ${PKG}/lib/*.a # Remove static libraries from package 284 rm -f ${PKG}/lib/*.la # Remove libtool libraries from package 285 rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package 286 287 # Compress package 288 echo "Compressing package" 289 ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG} 154 if [ ${numTestsFailed} -ne 0 ]; then 155 echo "One or more tests FAILED" 156 exit 1 290 157 else 291 # Assume that previous build was successful, but signing/notarization 292 # failed. 293 # 294 echo "Notarizing only" 158 echo "All tests PASSED" 295 159 fi 296 297 # Commit lock file to repository for signed packages298 echo "Committing lock file to repository for signed packages"299 touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}300 svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1301 svn commit \302 --trust-server-cert \303 --non-interactive \304 --username ${USERNAME} \305 --password ${PASSWORD} \306 --message "ADD: New lock file" ${SIGNED_REPO_COPY}307 308 # Check out copy of repository for unsigned packages309 checkout_unsigned_repo_copy310 311 if [ ${notarize_only} -eq 0 ]; then312 # Commit new compressed package to repository for unsigned binaries313 #314 # NOTE: This will not work if, for any reason, the checksum on the compressed315 # package is unchanged.316 #317 echo "Committing package to repository for unsigned packages"318 cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}319 svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1320 svn commit \321 --trust-server-cert \322 --non-interactive \323 --username ${USERNAME} \324 --password ${PASSWORD} \325 --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY}326 else327 # NOTE: If notarize_only == 1, we commit a dummy file as the signing328 # build on the remote JPL Cybersecurity Jenkins server is329 # triggered by polling SCM.330 #331 echo "Attempting to sign existing package again"332 echo $(date +'%Y-%m-%d-%H-%M-%S') > ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} # Write datetime stamp to file to ensure modification is made333 svcheckout_unsigned_repo_copyn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1334 svn commit \335 --trust-server-cert \336 --non-interactive \337 --username ${USERNAME} \338 --password ${PASSWORD} \339 --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY}340 fi341 342 # Check status of signing343 echo "Checking progress of signing..."344 IN_PROCESS=1345 SUCCESS=0346 347 while [ ${IN_PROCESS} -eq 1 ]; do348 echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds"349 sleep ${SIGNING_CHECK_PERIOD}350 svn up ${SIGNED_REPO_COPY}351 352 if [ ! -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then353 IN_PROCESS=0354 355 # Retrieve notarization lock file356 svn up ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}357 358 # Check status359 STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')360 if [[ "${STATUS}" == "success" ]]; then361 echo "Notarization successful!"362 363 # Set flag indicating notarization was successful364 SUCCESS=1365 else366 echo "Notarization failed!"367 fi368 fi369 done370 160 else 371 # Assume that previous build resulted in successful signing of package but 372 # that transfer to ISSM Web site failed and user built this project again 373 # with -t/--transferonly option. 374 # 375 376 # Make sure copy of repository for signed packages exists 377 validate_signed_repo_copy 378 379 SUCCESS=1 161 echo "Skipping tests" 380 162 fi 381 163 382 # Handle result of signing 383 if [ ${SUCCESS} -eq 1 ]; then 384 # Retrieve signed and notarized package 385 svn up ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} 164 # Create package 165 cd ${ISSM_DIR} 166 svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package) 167 echo "Copying assets to package: ${PKG}" 168 cp -rf bin examples lib scripts test ${PKG}/ 169 mkdir ${PKG}/execution 170 cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist 171 echo "Cleaning up unneeded/unwanted files" 172 rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes 173 rm -f ${PKG}/lib/*.a # Remove static libraries from package 174 rm -f ${PKG}/lib/*.la # Remove libtool libraries from package 175 rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package 386 176 387 # Transfer signed package to ISSM Web site 388 echo "Transferring signed package to ISSM Web site" 389 scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG} 390 391 if [ $? -ne 0 ]; then 392 echo "Transfer failed! Verify connection then build this project again (with -t/--transferonly option to skip testing and signing)." 393 exit 1 394 fi 395 else 396 echo "----------------------- Contents of notarization logfile -----------------------" 397 cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} 398 echo "--------------------------------------------------------------------------------" 399 400 exit 1 401 fi 177 # Compress package 178 echo "Compressing package" 179 ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
Note:
See TracChangeset
for help on using the changeset viewer.