source: issm/oecreview/Archive/26740-27031/ISSM-27001-27002.diff@ 27032

Last change on this file since 27032 was 27032, checked in by Mathieu Morlighem, 3 years ago

CHG: added 26740-27031
File size: 109.5 KB
RevLine 
[27032]1Index: ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python.sh
2===================================================================
3--- ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python.sh (revision 27001)
4+++ ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python.sh (nonexistent)
5@@ -1,53 +0,0 @@
6-#!/bin/bash
7-
8-################################################################################
9-# This script is intended to test ISSM macOS MATLAB binaries on an end-user
10-# machine after successful packaging and signing.
11-#
12-# NOTE: Tarball must already exist in INSTALL_DIR
13-################################################################################
14-
15-## Constants
16-#
17-INSTALL_DIR=.
18-PKG="ISSM-macOS-Python"
19-PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2002 2003 2004 2005 2006 2007 2008 2010 2011 2021 2051 2052 2053 2084 2085 2090 2101 2424 2425 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
20-
21-COMPRESSED_PKG="${PKG}.zip"
22-
23-export ISSM_DIR="${INSTALL_DIR}/${PKG}"
24-export PATH="${PATH}:${ISSM_DIR}/bin:${ISSM_DIR}/scripts"
25-export PYTHONPATH="${ISSM_DIR}/scripts"
26-export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
27-export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
28-
29-cd ${INSTALL_DIR}
30-rm -rf ${PKG}
31-ditto -xk ${COMPRESSED_PKG} .
32-cd ${PKG}/test/NightlyRun
33-
34-# Run tests, redirecting output to logfile and suppressing output to console
35-echo "Running tests"
36-rm python.log 2> /dev/null
37-./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
38-
39-# Check that Python did not exit in error
40-pythonExitCode=`echo $?`
41-pythonExitedInError=`grep -E "runme.py: error" python.log | wc -l`
42-
43-if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
44- echo "----------Python exited in error!----------"
45- cat python.log
46- echo "-----------End of python.log-----------"
47- exit 1
48-fi
49-
50-# Check that all tests passed
51-numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
52-
53-if [[ ${numTestsFailed} -ne 0 ]]; then
54- echo "One or more tests FAILED"
55- exit 1
56-else
57- echo "All tests PASSED"
58-fi
59
60Property changes on: ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python.sh
61___________________________________________________________________
62Deleted: svn:executable
63## -1 +0,0 ##
64-*
65\ No newline at end of property
66Index: ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python.sh
67===================================================================
68--- ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python.sh (revision 27001)
69+++ ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python.sh (nonexistent)
70@@ -1,134 +0,0 @@
71-#!/bin/bash
72-
73-################################################################################
74-# Wrapper script to build, package, send for signing, and transfer to ISSM Web
75-# site ISSM distributable package for macOS with Python API.
76-#
77-# Normally, we would put this directly into the project configuration under
78-# 'Build' -> 'Excute shell', but becasue it is a bit more involved, it is a
79-# good idea to version it.
80-#
81-# When no failures/errors occur, performs the following:
82-# - Builds ISSM according to configuration.
83-# - Packages executables and libraries.
84-# - Runs test suite against package.
85-# - Commits compressed package to repository to be signed by JPL Cybersecurity.
86-# - Retrieves signed package and transmits it to ISSM Web site for
87-# distribution.
88-#
89-# Options:
90-# -b/--skipbuild Skip ISSM compilation.
91-# -r/--resign Skip ISSM compilation and packaging. Use to retrigger
92-# signing/notarization if it fails but build and package
93-# are valid.
94-# -s/--skiptests Skip ISSM compilation and testing during packaging
95-# step. Use if packaging fails for some reason but build
96-# is valid.
97-# -t/--transferonly Transfer package to ISSM Web site only. Use if transfer
98-# fails for some reason to skip building, packaging, and
99-# signing.
100-# -u/--unlock Remove lock file from signed package repository. Use if
101-# build is aborted to allow for subsequent fresh build.
102-#
103-# Debugging:
104-# - Relies on a very tight handshake with project on remote JPL Cybersecurity
105-# Jenkins server. Debugging may be perfomed locally by running,
106-#
107-# packagers/mac/sign-issm-mac-binaries-python.sh
108-#
109-# with "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" hardcoded to Apple
110-# Developer credentials.
111-# - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can
112-# help debug potential SVN issues.
113-#
114-# NOTE:
115-# - Use only *one* of the above options at a time, and make sure it is removed
116-# again after a single run.
117-# - Builds will fail when any of the above options are used on a clean
118-# workspace. For example, if 'Source Code Management' -> 'Check-out Strategy'
119-# select menu is set to "Always check out a fresh copy".
120-# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
121-# the 'Bindings' section under a 'Username and password (separated)' binding
122-# (requires 'Credentials Binding Plugin') with 'Credentials' select menu set
123-# to "jenkins/****** (SVN repository for ISSM binaries)".
124-################################################################################
125-
126-## Constants
127-#
128-PKG="ISSM-macOS-Python" # Name of directory to copy distributable files to
129-SIGNED_REPO_COPY="./signed"
130-SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/signed"
131-
132-COMPRESSED_PKG="${PKG}.zip"
133-
134-## Environment
135-#
136-export COMPRESSED_PKG
137-export PKG
138-export SIGNED_REPO_COPY
139-export SIGNED_REPO_URL
140-
141-## Parse options
142-#
143-if [ $# -gt 1 ]; then
144- echo "Can use only one option at a time"
145- exit 1
146-fi
147-
148-# NOTE: We could do this with binary switching (i.e. 0011 to sign and transfer,
149-# but the following is self-documenting).
150-#
151-build=1
152-package=1
153-sign=1
154-transfer=1
155-
156-if [ $# -eq 1 ]; then
157- case $1 in
158- -b|--skipbuild) build=0; shift ;;
159- -r|--resign) build=0; package=0; ;;
160- -s|--skiptests) build=0; ;;
161- -t|--transferonly) build=0; package=0; sign=0; ;;
162- -u|--unlock) build=0; package=0; transfer=0; ;;
163- *) echo "Unknown parameter passed: $1"; exit 1 ;;
164- esac
165-fi
166-
167-# Build
168-if [ ${build} -eq 1 ]; then
169- ./jenkins/jenkins.sh ./jenkins/pine_island-mac-binaries-python
170-
171- if [ $? -ne 0 ]; then
172- exit 1
173- fi
174-fi
175-
176-# Package
177-if [ ${package} -eq 1 ]; then
178- ./packagers/mac/package-issm-mac-binaries-python.sh $1
179-
180- if [ $? -ne 0 ]; then
181- exit 1
182- fi
183-
184- shift # Clear $1 so that it is not passed to commit_for_signing script
185-fi
186-
187-# Commit for signing
188-if [ ${sign} -eq 1 ]; then
189- ./packagers/mac/commit_for_signing-issm-mac-binaries-python.sh $1
190-
191- if [ $? -ne 0 ]; then
192- exit 1
193- fi
194-fi
195-
196-# Transfer distributable package to ISSM Web site
197-if [ ${transfer} -eq 1 ]; then
198- ./packagers/mac/transfer-issm-mac-binaries.sh
199-
200- if [ $? -ne 0 ]; then
201- exit 1
202- fi
203-fi
204-
205
206Property changes on: ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python.sh
207___________________________________________________________________
208Deleted: svn:executable
209## -1 +0,0 ##
210-*
211\ No newline at end of property
212Index: ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh
213===================================================================
214--- ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh (revision 27001)
215+++ ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh (nonexistent)
216@@ -1,179 +0,0 @@
217-#!/bin/bash
218-
219-################################################################################
220-# Packages and tests ISSM distributable package for macOS with Python API.
221-#
222-# Options:
223-# -s/--skiptests Skip testing during packaging Use if packaging fails
224-# for some reason but build is valid.
225-#
226-# NOTE:
227-# - Assumes that the following constants are defined,
228-#
229-# COMPRESSED_PKG
230-# ISSM_DIR
231-# PKG
232-#
233-# See also:
234-# - packagers/mac/complete-issm-mac-binaries-python.sh
235-# - packagers/mac/sign-issm-mac-binaries-python.sh
236-################################################################################
237-
238-# Expand aliases within the context of this script
239-shopt -s expand_aliases
240-
241-# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
242-#
243-# Command line tool support for Subversion — including svn, git-svn, and
244-# related commands — is no longer provided by Xcode. (50266910)
245-#
246-# which results in,
247-#
248-# svn: error: The subversion command line tools are no longer provided by
249-# Xcode.
250-#
251-# when calling svn, even when subversion is installed via Homebrew and its path
252-# is available in PATH.
253-#
254-# NOTE: May be able to remove this after updating macOS.
255-#
256-#alias svn='/usr/local/bin/svn'
257-
258-## Override certain other aliases
259-#
260-alias grep=$(which grep)
261-
262-## Constants
263-#
264-PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2002 2003 2004 2005 2006 2007 2008 2010 2011 2021 2051 2052 2053 2084 2085 2090 2101 2424 2425 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
265-
266-## Environment
267-#
268-export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'
269-
270-## Parse options
271-#
272-if [ $# -gt 1 ]; then
273- echo "Can use only one option at a time"
274- exit 1
275-fi
276-
277-skip_tests=0
278-
279-if [ $# -eq 1 ]; then
280- case $1 in
281- -s|--skiptests) skip_tests=1; ;;
282- *) echo "Unknown parameter passed: $1"; exit 1 ;;
283- esac
284-fi
285-
286-# Clean up from previous packaging
287-echo "Cleaning up existing assets"
288-cd ${ISSM_DIR}
289-rm -rf ${PKG} ${COMPRESSED_PKG}
290-mkdir ${PKG}
291-
292-# Add required binaries and libraries to package and modify them where needed
293-cd ${ISSM_DIR}/bin
294-
295-echo "Modify generic"
296-cat generic_static.py | sed -e "s/generic_static/generic/g" > generic.py
297-
298-echo "Moving MPICH binaries to bin/"
299-if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
300- cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
301- cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
302-elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
303- cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
304- cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
305-else
306- echo "MPICH not found"
307- exit 1
308-fi
309-
310-echo "Moving GDAL binaries to bin/"
311-if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
312- cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
313- cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
314-else
315- echo "GDAL not found"
316- exit 1
317-fi
318-
319-echo "Moving GMT binaries to bin/"
320-if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
321- cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
322- cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
323-else
324- echo "GMT not found"
325- exit 1
326-fi
327-
328-echo "Moving Gmsh binaries to bin/"
329-if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
330- cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
331-else
332- echo "Gmsh not found"
333- exit 1
334-fi
335-
336-# Run tests
337-if [ ${skip_tests} -eq 0 ]; then
338- echo "Running tests"
339- cd ${ISSM_DIR}/test/NightlyRun
340- rm python.log 2> /dev/null
341-
342- # Set Python environment
343- export PYTHONPATH="${ISSM_DIR}/src/m/dev"
344- export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
345- export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
346-
347- # Run tests, redirecting output to logfile and suppressing output to console
348- ./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
349-
350- # Check that Python did not exit in error
351- pythonExitCode=`echo $?`
352- pythonExitedInError=`grep -E "Error|Standard exception|Traceback|bad interpreter" python.log | wc -l`
353-
354- if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
355- echo "----------Python exited in error!----------"
356- cat python.log
357- echo "-----------End of python.log-----------"
358-
359- # Clean up execution directory
360- rm -rf ${ISSM_DIR}/execution/*
361-
362- exit 1
363- fi
364-
365- # Check that all tests passed
366- numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
367-
368- if [ ${numTestsFailed} -ne 0 ]; then
369- echo "One or more tests FAILED"
370- exit 1
371- else
372- echo "All tests PASSED"
373- fi
374-else
375- echo "Skipping tests"
376-fi
377-
378-# Create package
379-cd ${ISSM_DIR}
380-svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
381-echo "Copying assets to package: ${PKG}"
382-cp -rf bin examples lib scripts test ${PKG}
383-mkdir ${PKG}/execution
384-cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist
385-${ISSM_DIR}/scripts/py_to_pyc.sh ${PKG}/bin # Compile Python source files
386-echo "Cleaning up unneeded/unwanted files"
387-rm -f ${PKG}/bin/*.py # Remove all Python scripts
388-rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
389-rm -f ${PKG}/lib/*.a # Remove static libraries from package
390-rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
391-rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
392-
393-# Compress package
394-echo "Compressing package"
395-ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
396
397Property changes on: ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh
398___________________________________________________________________
399Deleted: svn:executable
400## -1 +0,0 ##
401-*
402\ No newline at end of property
403Index: ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python.sh
404===================================================================
405--- ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python.sh (revision 27001)
406+++ ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python.sh (nonexistent)
407@@ -1,348 +0,0 @@
408-#!/bin/bash
409-
410-################################################################################
411-# Intended to be run in the context of a Jenkins project on a JPL
412-# Cybersecurity server for signing macOS applications. Polls SCM of the
413-# Subversion repository hosted at
414-# https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned to trigger new
415-# builds.
416-#
417-# In order to replicate the requried Jenkins project configuration:
418-# - First, navigate to 'Manage Jenkins' -> 'Manage Plugins' and install the
419-# 'Credentials Bindings Plugin' if it is not already installed.
420-# - Contact one of the members of the ISSM development team for crendentials
421-# for the ISSM binaries repository (mention that the credentials are stored
422-# in ISSM-Infrastructure.pdf).
423-# - Navigate to 'Manage Jenkins' -> 'Manage Credentials' -> <domain> ->
424-# 'Add Credentials' and enter the crendentials from above.
425-# - From the 'Dashboard', select 'New Item' -> 'Freestyle project'.
426-# - Under 'Source Code Management', select 'Subversion'.
427-# - The 'Repository URL' text field should be set to
428-# "https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned".
429-# - The 'Credentials' select menu should be set to the new credentials
430-# created previously.
431-# - The 'Local module directory' text field should be set to the same
432-# value as the constant UNSIGNED_REPO_COPY (set below to './unsigned').
433-# - Under 'Build Trigggers', check the box for 'Poll SCM' and set the
434-# 'Schedule' text area to "H/5 * * * *".
435-# - Under 'Build Environment', check the box for 'Use secret text(s) or
436-# file(s)', then under 'Bindings' click the 'Add...' button and select
437-# 'Username and password (separated)'.
438-# - Set 'Username Variable' to "ISSM_BINARIES_USER".
439-# - Set 'Password Variable' to "ISSM_BINARIES_PASS".
440-# - Under 'Credentials', select the same, new credentials that created
441-# previously.
442-# - The contents of this script can be copied/pasted directly into the ‘Build'
443-# -> 'Execute Shell' -> ‘Command' textarea of the project configuration (or
444-# you can simply store the script on disk and call it from there).
445-# - Make sure to click the 'Save' button.
446-#
447-# Current point of contact at JPL Cybersecurity:
448-# Alex Coward, alexander.g.coward@jpl.nasa.gov
449-#
450-# NOTE:
451-# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
452-# the 'Bindings' section under a 'Username and password (separated)' binding
453-# (requires 'Credentials Binding Plugin').
454-# - For local debugging, the aformentioned credentials can be hardcoded into
455-# the 'USERNAME' and 'PASSWORD' constants below.
456-################################################################################
457-
458-# Expand aliases within the context of this script
459-shopt -s expand_aliases
460-
461-# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
462-#
463-# Command line tool support for Subversion — including svn, git-svn, and
464-# related commands — is no longer provided by Xcode. (50266910)
465-#
466-# which results in,
467-#
468-# svn: error: The subversion command line tools are no longer provided by
469-# Xcode.
470-#
471-# when calling svn, even when subversion is installed via Homebrew and its path
472-# is available in PATH.
473-#
474-# NOTE: May be able to remove this after updating macOS.
475-#
476-#alias svn='/usr/local/bin/svn'
477-
478-## Override certain other aliases
479-#
480-alias cp=$(which cp)
481-alias grep=$(which grep)
482-
483-## Constants
484-#
485-AD_IDENTITY="**********" # Apple Developer identity
486-AD_USERNAME="**********" # Apple Developer username
487-ALTOOL_PASSWORD="@keychain:**********" # altool password (assumed to be stored in keychain)
488-ASC_PROVIDER="**********"
489-MAX_SVN_ATTEMPTS=10
490-NOTARIZATION_CHECK_ATTEMPTS=20
491-NOTARIZATION_CHECK_PERIOD=60
492-NOTARIZATION_LOGFILE="notarization.log"
493-NOTARIZATION_LOGFILE_PATH="."
494-PASSWORD=${ISSM_BINARIES_PASS}
495-PKG="ISSM-macOS-Python"
496-PRIMARY_BUNDLE_ID="gov.nasa.jpl.issm.python"
497-SIGNED_REPO_COPY="./signed"
498-SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/signed"
499-SIGNING_LOCK_FILE="signing.lock"
500-SUCCESS_LOGFILE="${SIGNED_REPO_COPY}/success.log"
501-UNSIGNED_REPO_COPY="./unsigned"
502-UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned"
503-USERNAME=${ISSM_BINARIES_USER}
504-
505-COMPRESSED_PKG="${PKG}.zip"
506-EXE_ENTITLEMENTS_PLIST="${PKG}/bin/entitlements.plist"
507-
508-# NOTE: Uncomment the following for local testing (Jenkins checks out copy of
509-# repository for unsigned packages to working directory)
510-#
511-
512-# # Clean up from previous packaging (not necessary for single builds on Jenkins,
513-# # but useful when testing packaging locally)
514-# echo "Cleaning up existing assets"
515-# rm -rf ${COMPRESSED_PKG} ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${UNSIGNED_REPO_COPY}
516-
517-# # Check out copy of repository for unsigned packages
518-# echo "Checking out copy of respository for unsigned packages"
519-# svn checkout \
520-# --trust-server-cert \
521-# --non-interactive \
522-# --username ${USERNAME} \
523-# --password ${PASSWORD} \
524-# ${UNSIGNED_REPO_URL} \
525-# ${UNSIGNED_REPO_COPY}
526-
527-rm -rf ${PKG} ${SIGNED_REPO_COPY}
528-
529-
530-# Extract package contents
531-echo "Extracting package contents"
532-ditto -xk ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} .
533-
534-# Clear extended attributes on all files
535-xattr -cr ${PKG}
536-
537-# Build list of ISSM executables
538-ISSM_BINS=$(\
539- find ${PKG}/bin -type f -name *.exe; \
540- find ${PKG}/bin -type f -name *.pyc; \
541-)
542-
543-# Build list of third party executables
544-THIRD_PARTY_BINS=$(\
545- echo ${PKG}/bin/mpiexec; \
546- echo ${PKG}/bin/hydra_pmi_proxy; \
547- echo ${PKG}/bin/gdalsrsinfo; \
548- echo ${PKG}/bin/gdaltransform; \
549- echo ${PKG}/bin/gmt; \
550- echo ${PKG}/bin/gmtselect; \
551- echo ${PKG}/bin/gmsh; \
552-)
553-
554-# Sign all executables in package
555-echo "Signing all executables in package"
556-codesign -s ${AD_IDENTITY} --timestamp --options=runtime --entitlements ${EXE_ENTITLEMENTS_PLIST} ${ISSM_BINS}
557-codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${THIRD_PARTY_BINS}
558-
559-# Build list of ISSM libraries
560-ISSM_LIBS=$(\
561- find ${PKG}/lib -type f -name *.so; \
562-)
563-
564-# Sign all libraries in package
565-echo "Signing all libraries in package"
566-codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${ISSM_LIBS}
567-
568-# NOTE: Skipping signature validation because this is not a true package nor app
569-
570-# Compress signed package
571-echo "Compressing signed package"
572-ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
573-
574-# Submit compressed package for notarization
575-echo "Submitting signed package to Apple for notarization"
576-xcrun altool --notarize-app --primary-bundle-id ${PRIMARY_BUNDLE_ID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} --asc-provider ${ASC_PROVIDER} --file ${COMPRESSED_PKG} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
577-
578-# Sleep until notarization request response is received
579-echo "Waiting for notarizaion request response"
580-while [[ ! -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} || ! -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; do
581- sleep 30
582-done
583-
584-echo "Notarizaion request response received"
585-
586-# Check if UUID exists in response
587-HAS_UUID=$(grep 'RequestUUID = ' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}) # NOTE: Checking for "RequestUUID = " because "RequestUUID" shows up in some error messages
588-if [ -z "${HAS_UUID}" ]; then
589- echo "Notarization failed!"
590- echo "----------------------- Contents of notarization logfile -----------------------"
591- cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
592- echo "--------------------------------------------------------------------------------"
593-
594- # Clean up
595- rm -rf ${PKG} ${COMPRESSED_PKG}
596-
597- exit 1
598-fi
599-
600-# Get UUID from notarization request response
601-UUID=$(echo ${HAS_UUID} | sed 's/[[:space:]]*RequestUUID = //')
602-echo "UUID: ${UUID}"
603-
604-# Check notarization status
605-#
606-# NOTE: Currently, this checks if notarization was successful, but we are not
607-# able to staple notarization as this is not a true package nor app and,
608-# at the very least, MATLAB Mex files cannot be stapled. As such, clients
609-# will not be able to clear Gatekeeper if they are offline.
610-#
611-echo "Checking notarization status"
612-SUCCESS=0
613-for ATTEMPT in $(seq 1 ${NOTARIZATION_CHECK_ATTEMPTS}); do
614- echo " Attempt #${ATTEMPT}..."
615- xcrun altool --notarization-info ${UUID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
616- if [[ -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} && -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; then
617-
618- # First, check if there is an error
619- ERROR_CHECK=$(grep 'Error' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE})
620- if [ ! -z "${ERROR_CHECK}" ]; then
621- break
622- fi
623-
624- # No error, so check status
625- STATUS=$(grep 'Status:' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
626- if [[ "${STATUS}" == "success" ]]; then
627- # Staple notarization to all elements of package that were previously signed
628- #xcrun stapler staple ${THIRD_PARTY_BINS} # NOTE: Fails with "Stapler is incapable of working with MATLAB Mex files."
629-
630- # Validate stapling of notarization
631- #xcrun stapler validation ${THIRD_PARTY_BINS} # NOTE: Skipping notarization stapling validation because this is not a true package nor app
632-
633- # Compress signed and notarized package
634- ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
635-
636- # Set flag indicating notarization was successful
637- SUCCESS=1
638-
639- break
640- elif [[ "${STATUS}" == "in progress" ]]; then
641- echo " ...in progress still; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds."
642- sleep ${NOTARIZATION_CHECK_PERIOD}
643- elif [[ "${STATUS}" == "invalid" ]]; then
644- break
645- fi
646- else
647- if [ ${ATTEMPT} -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then
648- echo " ...not ready yet; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds"
649- sleep ${NOTARIZATION_CHECK_PERIOD}
650- else
651- echo " ...maximum attempts reached, but no response, or something else went wrong"
652- echo " If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again"
653- break
654- fi
655- fi
656-done
657-
658-if [ ${SUCCESS} -eq 1 ]; then
659- echo "Notarization successful!"
660-else
661- echo "Notarization failed!"
662- echo "----------------------- Contents of notarization logfile -----------------------"
663- cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
664- echo "--------------------------------------------------------------------------------"
665-fi
666-
667-# Check out copy of repository for signed packages
668-echo "Checking out copy of respository for signed packages"
669-SVN_ATTEMPT=0
670-SVN_SUCCESS=0
671-while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
672- rm -rf ${SIGNED_REPO_COPY}
673- svn checkout \
674- --trust-server-cert \
675- --non-interactive \
676- --username ${USERNAME} \
677- --password ${PASSWORD} \
678- ${SIGNED_REPO_URL} \
679- ${SIGNED_REPO_COPY} > /dev/null 2>&1
680- if [ $? -eq 0 ]; then
681- SVN_SUCCESS=1
682- break
683- else
684- ((++SVN_ATTEMPT))
685- sleep 5
686- fi
687-done
688-
689-if [ ${SVN_SUCCESS} -eq 0 ]; then
690- echo "Checkout of respository for signed packages failed"
691- exit 1
692-fi
693-
694-# Copy notarization file to repository for signed packages
695-cp ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY}
696-svn add ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1
697-
698-# Remove lock file from repository for signed packages
699-svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
700-
701-SVN_ATTEMPT=0
702-SVN_SUCCESS=0
703-if [ ${SUCCESS} -eq 1 ]; then
704- # Copy signed package to repository for signed packages
705- cp ${COMPRESSED_PKG} ${SIGNED_REPO_COPY}
706- svn add ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
707-
708- # Commit changes
709- echo "Committing changes to repository for signed packages"
710- while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
711- svn commit \
712- --trust-server-cert \
713- --non-interactive \
714- --username ${USERNAME} \
715- --password ${PASSWORD} \
716- --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
717- if [ $? -eq 0 ]; then
718- SVN_SUCCESS=1
719- break
720- else
721- ((++SVN_ATTEMPT))
722- sleep 5
723- fi
724- done
725-
726- if [ ${SVN_SUCCESS} -eq 0 ]; then
727- echo "Commit to respository for signed packages failed"
728- exit 1
729- fi
730-else
731- # Commit changes
732- echo "Committing changes to repository for signed packages"
733- while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
734- svn commit \
735- --trust-server-cert \
736- --non-interactive \
737- --username ${USERNAME} \
738- --password ${PASSWORD} \
739- --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
740- if [ $? -eq 0 ]; then
741- SVN_SUCCESS=1
742- break
743- else
744- ((++SVN_ATTEMPT))
745- sleep 5
746- fi
747- done
748-
749- if [ ${SVN_SUCCESS} -eq 0 ]; then
750- echo "Commit to respository for signed packages failed"
751- exit 1
752- fi
753-
754- exit 1
755-fi
756
757Property changes on: ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python.sh
758___________________________________________________________________
759Deleted: svn:executable
760## -1 +0,0 ##
761-*
762\ No newline at end of property
763Index: ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python.sh
764===================================================================
765--- ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python.sh (revision 27001)
766+++ ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python.sh (nonexistent)
767@@ -1,248 +0,0 @@
768-#!/bin/bash
769-
770-################################################################################
771-# Commits ISSM distributable package for macOS with Python API to repository
772-# for signing. This repository is polled by a project running on a JPL
773-# Cybersecurity Jenkins server and performs the actual signing and
774-# notarization.
775-#
776-# Options:
777-# -r/--resign Skip ISSM compilation and packaging. Use to retrigger
778-# signing/notarization if it fails but build and package
779-# are valid.
780-# -u/--unlock Remove lock file from signed package repository. Use if
781-# build is aborted to allow for subsequent fresh build.
782-#
783-# NOTE:
784-# - Assumes that the following constants are defined,
785-#
786-# COMPRESSED_PKG
787-# ISSM_BINARIES_REPO_PASS
788-# ISSM_BINARIES_REPO_USER
789-# SIGNED_REPO_COPY
790-# SIGNED_REPO_URL
791-#
792-# See also:
793-# - packagers/mac/complete-issm-mac-binaries-python.sh
794-# - packagers/mac/sign-issm-mac-binaries-python.sh
795-#
796-# TODO:
797-# - Generalize checkout_*_repo_copy and validate_*_repo_copy functions (e.g.
798-# pass 'signed' or 'unsigned' as argument)
799-################################################################################
800-
801-# Expand aliases within the context of this script
802-shopt -s expand_aliases
803-
804-# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
805-#
806-# Command line tool support for Subversion — including svn, git-svn, and
807-# related commands — is no longer provided by Xcode. (50266910)
808-#
809-# which results in,
810-#
811-# svn: error: The subversion command line tools are no longer provided by
812-# Xcode.
813-#
814-# when calling svn, even when subversion is installed via Homebrew and its path
815-# is available in PATH.
816-#
817-# NOTE: May be able to remove this after updating macOS.
818-#
819-#alias svn='/usr/local/bin/svn'
820-
821-## Override certain other aliases
822-#
823-alias cp=$(which cp)
824-alias grep=$(which grep)
825-
826-## Constants
827-#
828-MAX_SIGNING_CHECK_ATTEMPTS=30
829-NOTARIZATION_LOGFILE="notarization.log"
830-RETRIGGER_SIGNING_FILE="retrigger.txt"
831-SIGNING_CHECK_PERIOD=60 # in seconds
832-SIGNING_LOCK_FILE="signing.lock"
833-UNSIGNED_REPO_COPY="./unsigned"
834-UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned"
835-
836-## Functions
837-#
838-checkout_signed_repo_copy(){
839- echo "Checking out copy of repository for signed packages"
840-
841- # NOTE: Get empty copy because we do not want to have to check out package
842- # from previous signing.
843- #
844- svn checkout \
845- --trust-server-cert \
846- --non-interactive \
847- --depth empty \
848- --username ${ISSM_BINARIES_REPO_USER} \
849- --password ${ISSM_BINARIES_REPO_PASS} \
850- ${SIGNED_REPO_URL} \
851- ${SIGNED_REPO_COPY} > /dev/null 2>&1
852-}
853-checkout_unsigned_repo_copy(){
854- echo "Checking out copy of repository for unsigned packages"
855- svn checkout \
856- --trust-server-cert \
857- --non-interactive \
858- --username ${ISSM_BINARIES_REPO_USER} \
859- --password ${ISSM_BINARIES_REPO_PASS} \
860- ${UNSIGNED_REPO_URL} \
861- ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
862-}
863-validate_signed_repo_copy(){
864- # Validate copy of repository for signed binaries (e.g.
865- # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
866- # initial checkout failed)
867- if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then
868- rm -rf ${SIGNED_REPO_COPY}
869- checkout_signed_repo_copy
870- fi
871-}
872-validate_unsigned_repo_copy(){
873- # Validate copy of repository for unsigned binaries (e.g.
874- # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
875- # initial checkout failed)
876- if [[ ! -d ${UNSIGNED_REPO_COPY} || ! -d ${UNSIGNED_REPO_COPY}/.svn ]]; then
877- rm -rf ${UNSIGNED_REPO_COPY}
878- checkout_unsigned_repo_copy
879- fi
880-}
881-
882-## Parse options
883-#
884-if [ $# -gt 1 ]; then
885- echo "Can use only one option at a time"
886- exit 1
887-fi
888-
889-retrigger_signing=0
890-unlock=0
891-
892-if [ $# -eq 1 ]; then
893- case $1 in
894- -r|--resign) retrigger_signing=1; ;;
895- -u|--unlock) unlock=1; ;;
896- *) echo "Unknown parameter passed: $1"; exit 1 ;;
897- esac
898-fi
899-
900-validate_signed_repo_copy
901-
902-if [ ${unlock} -eq 1 ]; then
903- # Remove signing lock file from signed package repository so that a new
904- # build can run
905- echo "Removing lock file from repository for signed packages"
906- svn update \
907- --username ${ISSM_BINARIES_REPO_USER} \
908- --password ${ISSM_BINARIES_REPO_PASS} \
909- ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
910- svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
911- svn commit \
912- --trust-server-cert \
913- --non-interactive \
914- --username ${ISSM_BINARIES_REPO_USER} \
915- --password ${ISSM_BINARIES_REPO_PASS} \
916- --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} > /dev/null 2>&1
917- svn cleanup ${SIGNED_REPO_COPY} > /dev/null 2>&1
918-
919- echo "Remove -u/--unlock option from configuration and run again"
920- exit 1
921-fi
922-
923-# If lock file exists, a signing build is still in process by JPL Cybersecurity
924-svn update \
925- --username ${ISSM_BINARIES_REPO_USER} \
926- --password ${ISSM_BINARIES_REPO_PASS} \
927- ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
928-
929-if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
930- echo "Previous signing job still in process by JPL Cybersecurity. Please try again later."
931- exit 1
932-fi
933-
934-# Commit lock file to repository for signed packages
935-echo "Committing lock file to repository for signed packages"
936-touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
937-svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
938-svn commit \
939- --trust-server-cert \
940- --non-interactive \
941- --username ${ISSM_BINARIES_REPO_USER} \
942- --password ${ISSM_BINARIES_REPO_PASS} \
943- --message "ADD: New lock file" ${SIGNED_REPO_COPY} > /dev/null 2>&1
944-
945-# Check out copy of repository for unsigned packages
946-validate_unsigned_repo_copy
947-
948-if [ ${retrigger_signing} -eq 0 ]; then
949- # Commit new compressed package to repository for unsigned binaries
950- echo "Committing package to repository for unsigned packages"
951- cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
952- svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
953- svn commit \
954- --trust-server-cert \
955- --non-interactive \
956- --username ${ISSM_BINARIES_REPO_USER} \
957- --password ${ISSM_BINARIES_REPO_PASS} \
958- --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
959-else
960- # NOTE: If notarize_only == 1, we commit a dummy file as we do not want to
961- # have to commit the entire compressed package again simply to
962- # retrigger the signing build on the remote JPL Cybersecurity Jenkins
963- # server.
964- #
965- echo "Attempting to sign existing package again"
966- echo $(date +'%Y-%m-%d-%H-%M-%S') > ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} # Write datetime stamp to file to ensure modification is made
967- svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1
968- svn commit \
969- --trust-server-cert \
970- --non-interactive \
971- --username ${ISSM_BINARIES_REPO_USER} \
972- --password ${ISSM_BINARIES_REPO_PASS} \
973- --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
974-fi
975-
976-# Check status of signing
977-echo "Checking progress of signing..."
978-SIGNING_CHECK_ATTEMPT=0
979-while [ ${SIGNING_CHECK_ATTEMPT} -lt ${MAX_SIGNING_CHECK_ATTEMPTS} ]; do
980- echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds"
981- sleep ${SIGNING_CHECK_PERIOD}
982- svn update \
983- --username ${ISSM_BINARIES_REPO_USER} \
984- --password ${ISSM_BINARIES_REPO_PASS} \
985- ${SIGNED_REPO_COPY} > /dev/null 2>&1
986-
987- if [ ! -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
988- # Retrieve notarization lock file
989- svn update \
990- --username ${ISSM_BINARIES_REPO_USER} \
991- --password ${ISSM_BINARIES_REPO_PASS} \
992- ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
993-
994- # Check status
995- STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
996- if [[ "${STATUS}" == "success" ]]; then
997- echo "Notarization successful!"
998- break
999- else
1000- echo "Notarization failed!"
1001- echo "----------------------- Contents of notarization logfile -----------------------"
1002- cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
1003- echo "--------------------------------------------------------------------------------"
1004-
1005- exit 1
1006- fi
1007- else
1008- ((++SIGNING_CHECK_ATTEMPT))
1009- fi
1010-done
1011-
1012-if [ ! -f ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} ]; then
1013- echo "Signing timed out!"
1014- exit 1
1015-fi
1016
1017Property changes on: ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python.sh
1018___________________________________________________________________
1019Deleted: svn:executable
1020## -1 +0,0 ##
1021-*
1022\ No newline at end of property
1023Index: ../trunk-jpl/packagers/linux/transfer-issm-linux-binaries.sh
1024===================================================================
1025--- ../trunk-jpl/packagers/linux/transfer-issm-linux-binaries.sh (revision 27001)
1026+++ ../trunk-jpl/packagers/linux/transfer-issm-linux-binaries.sh (revision 27002)
1027@@ -1,8 +1,7 @@
1028 #!/bin/bash
1029
1030 ################################################################################
1031-# Transfers ISSM distributable package for Linux with MATLAB API to ISSM Web
1032-# site.
1033+# Transfers ISSM distributable package for Linux to ISSM website.
1034 #
1035 # NOTE:
1036 # - Assumes that the following constants are defined,
1037@@ -11,7 +10,8 @@
1038 #
1039 # See also:
1040 # - packagers/linux/complete-issm-linux-binaries-matlab.sh
1041-# - packagers/linux/complete-issm-linux-binaries-python.sh
1042+# - packagers/linux/complete-issm-linux-binaries-python-2.sh
1043+# - packagers/linux/complete-issm-linux-binaries-python-3.sh
1044 ################################################################################
1045
1046 # Transfer package to ISSM Web site
1047Index: ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python-2.sh
1048===================================================================
1049--- ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python-2.sh (nonexistent)
1050+++ ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python-2.sh (revision 27002)
1051@@ -0,0 +1,248 @@
1052+#!/bin/bash
1053+
1054+################################################################################
1055+# Commits ISSM distributable package for macOS with Python 2 API to repository
1056+# for signing. This repository is polled by a project running on a JPL
1057+# Cybersecurity Jenkins server and performs the actual signing and
1058+# notarization.
1059+#
1060+# Options:
1061+# -r/--resign Skip ISSM compilation and packaging. Use to retrigger
1062+# signing/notarization if it fails but build and package
1063+# are valid.
1064+# -u/--unlock Remove lock file from signed package repository. Use if
1065+# build is aborted to allow for subsequent fresh build.
1066+#
1067+# NOTE:
1068+# - Assumes that the following constants are defined,
1069+#
1070+# COMPRESSED_PKG
1071+# ISSM_BINARIES_REPO_PASS
1072+# ISSM_BINARIES_REPO_USER
1073+# SIGNED_REPO_COPY
1074+# SIGNED_REPO_URL
1075+#
1076+# See also:
1077+# - packagers/mac/complete-issm-mac-binaries-python-2.sh
1078+# - packagers/mac/sign-issm-mac-binaries-python-2.sh
1079+#
1080+# TODO:
1081+# - Generalize checkout_*_repo_copy and validate_*_repo_copy functions (e.g.
1082+# pass 'signed' or 'unsigned' as argument)
1083+################################################################################
1084+
1085+# Expand aliases within the context of this script
1086+shopt -s expand_aliases
1087+
1088+# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
1089+#
1090+# Command line tool support for Subversion — including svn, git-svn, and
1091+# related commands — is no longer provided by Xcode. (50266910)
1092+#
1093+# which results in,
1094+#
1095+# svn: error: The subversion command line tools are no longer provided by
1096+# Xcode.
1097+#
1098+# when calling svn, even when subversion is installed via Homebrew and its path
1099+# is available in PATH.
1100+#
1101+# NOTE: May be able to remove this after updating macOS.
1102+#
1103+#alias svn='/usr/local/bin/svn'
1104+
1105+## Override certain other aliases
1106+#
1107+alias cp=$(which cp)
1108+alias grep=$(which grep)
1109+
1110+## Constants
1111+#
1112+MAX_SIGNING_CHECK_ATTEMPTS=30
1113+NOTARIZATION_LOGFILE="notarization.log"
1114+RETRIGGER_SIGNING_FILE="retrigger.txt"
1115+SIGNING_CHECK_PERIOD=60 # in seconds
1116+SIGNING_LOCK_FILE="signing.lock"
1117+UNSIGNED_REPO_COPY="./unsigned"
1118+UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/2/unsigned"
1119+
1120+## Functions
1121+#
1122+checkout_signed_repo_copy(){
1123+ echo "Checking out copy of repository for signed packages"
1124+
1125+ # NOTE: Get empty copy because we do not want to have to check out package
1126+ # from previous signing.
1127+ #
1128+ svn checkout \
1129+ --trust-server-cert \
1130+ --non-interactive \
1131+ --depth empty \
1132+ --username ${ISSM_BINARIES_REPO_USER} \
1133+ --password ${ISSM_BINARIES_REPO_PASS} \
1134+ ${SIGNED_REPO_URL} \
1135+ ${SIGNED_REPO_COPY} > /dev/null 2>&1
1136+}
1137+checkout_unsigned_repo_copy(){
1138+ echo "Checking out copy of repository for unsigned packages"
1139+ svn checkout \
1140+ --trust-server-cert \
1141+ --non-interactive \
1142+ --username ${ISSM_BINARIES_REPO_USER} \
1143+ --password ${ISSM_BINARIES_REPO_PASS} \
1144+ ${UNSIGNED_REPO_URL} \
1145+ ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
1146+}
1147+validate_signed_repo_copy(){
1148+ # Validate copy of repository for signed binaries (e.g.
1149+ # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
1150+ # initial checkout failed)
1151+ if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then
1152+ rm -rf ${SIGNED_REPO_COPY}
1153+ checkout_signed_repo_copy
1154+ fi
1155+}
1156+validate_unsigned_repo_copy(){
1157+ # Validate copy of repository for unsigned binaries (e.g.
1158+ # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
1159+ # initial checkout failed)
1160+ if [[ ! -d ${UNSIGNED_REPO_COPY} || ! -d ${UNSIGNED_REPO_COPY}/.svn ]]; then
1161+ rm -rf ${UNSIGNED_REPO_COPY}
1162+ checkout_unsigned_repo_copy
1163+ fi
1164+}
1165+
1166+## Parse options
1167+#
1168+if [ $# -gt 1 ]; then
1169+ echo "Can use only one option at a time"
1170+ exit 1
1171+fi
1172+
1173+retrigger_signing=0
1174+unlock=0
1175+
1176+if [ $# -eq 1 ]; then
1177+ case $1 in
1178+ -r|--resign) retrigger_signing=1; ;;
1179+ -u|--unlock) unlock=1; ;;
1180+ *) echo "Unknown parameter passed: $1"; exit 1 ;;
1181+ esac
1182+fi
1183+
1184+validate_signed_repo_copy
1185+
1186+if [ ${unlock} -eq 1 ]; then
1187+ # Remove signing lock file from signed package repository so that a new
1188+ # build can run
1189+ echo "Removing lock file from repository for signed packages"
1190+ svn update \
1191+ --username ${ISSM_BINARIES_REPO_USER} \
1192+ --password ${ISSM_BINARIES_REPO_PASS} \
1193+ ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
1194+ svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
1195+ svn commit \
1196+ --trust-server-cert \
1197+ --non-interactive \
1198+ --username ${ISSM_BINARIES_REPO_USER} \
1199+ --password ${ISSM_BINARIES_REPO_PASS} \
1200+ --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} > /dev/null 2>&1
1201+ svn cleanup ${SIGNED_REPO_COPY} > /dev/null 2>&1
1202+
1203+ echo "Remove -u/--unlock option from configuration and run again"
1204+ exit 1
1205+fi
1206+
1207+# If lock file exists, a signing build is still in process by JPL Cybersecurity
1208+svn update \
1209+ --username ${ISSM_BINARIES_REPO_USER} \
1210+ --password ${ISSM_BINARIES_REPO_PASS} \
1211+ ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
1212+
1213+if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
1214+ echo "Previous signing job still in process by JPL Cybersecurity. Please try again later."
1215+ exit 1
1216+fi
1217+
1218+# Commit lock file to repository for signed packages
1219+echo "Committing lock file to repository for signed packages"
1220+touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
1221+svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
1222+svn commit \
1223+ --trust-server-cert \
1224+ --non-interactive \
1225+ --username ${ISSM_BINARIES_REPO_USER} \
1226+ --password ${ISSM_BINARIES_REPO_PASS} \
1227+ --message "ADD: New lock file" ${SIGNED_REPO_COPY} > /dev/null 2>&1
1228+
1229+# Check out copy of repository for unsigned packages
1230+validate_unsigned_repo_copy
1231+
1232+if [ ${retrigger_signing} -eq 0 ]; then
1233+ # Commit new compressed package to repository for unsigned binaries
1234+ echo "Committing package to repository for unsigned packages"
1235+ cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
1236+ svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
1237+ svn commit \
1238+ --trust-server-cert \
1239+ --non-interactive \
1240+ --username ${ISSM_BINARIES_REPO_USER} \
1241+ --password ${ISSM_BINARIES_REPO_PASS} \
1242+ --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
1243+else
1244+ # NOTE: If notarize_only == 1, we commit a dummy file as we do not want to
1245+ # have to commit the entire compressed package again simply to
1246+ # retrigger the signing build on the remote JPL Cybersecurity Jenkins
1247+ # server.
1248+ #
1249+ echo "Attempting to sign existing package again"
1250+ echo $(date +'%Y-%m-%d-%H-%M-%S') > ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} # Write datetime stamp to file to ensure modification is made
1251+ svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1
1252+ svn commit \
1253+ --trust-server-cert \
1254+ --non-interactive \
1255+ --username ${ISSM_BINARIES_REPO_USER} \
1256+ --password ${ISSM_BINARIES_REPO_PASS} \
1257+ --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
1258+fi
1259+
1260+# Check status of signing
1261+echo "Checking progress of signing..."
1262+SIGNING_CHECK_ATTEMPT=0
1263+while [ ${SIGNING_CHECK_ATTEMPT} -lt ${MAX_SIGNING_CHECK_ATTEMPTS} ]; do
1264+ echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds"
1265+ sleep ${SIGNING_CHECK_PERIOD}
1266+ svn update \
1267+ --username ${ISSM_BINARIES_REPO_USER} \
1268+ --password ${ISSM_BINARIES_REPO_PASS} \
1269+ ${SIGNED_REPO_COPY} > /dev/null 2>&1
1270+
1271+ if [ ! -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
1272+ # Retrieve notarization lock file
1273+ svn update \
1274+ --username ${ISSM_BINARIES_REPO_USER} \
1275+ --password ${ISSM_BINARIES_REPO_PASS} \
1276+ ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
1277+
1278+ # Check status
1279+ STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
1280+ if [[ "${STATUS}" == "success" ]]; then
1281+ echo "Notarization successful!"
1282+ break
1283+ else
1284+ echo "Notarization failed!"
1285+ echo "----------------------- Contents of notarization logfile -----------------------"
1286+ cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
1287+ echo "--------------------------------------------------------------------------------"
1288+
1289+ exit 1
1290+ fi
1291+ else
1292+ ((++SIGNING_CHECK_ATTEMPT))
1293+ fi
1294+done
1295+
1296+if [ ! -f ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} ]; then
1297+ echo "Signing timed out!"
1298+ exit 1
1299+fi
1300
1301Property changes on: ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python-2.sh
1302___________________________________________________________________
1303Added: svn:executable
1304## -0,0 +1 ##
1305+*
1306\ No newline at end of property
1307Index: ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python-3.sh
1308===================================================================
1309--- ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python-3.sh (nonexistent)
1310+++ ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python-3.sh (revision 27002)
1311@@ -0,0 +1,248 @@
1312+#!/bin/bash
1313+
1314+################################################################################
1315+# Commits ISSM distributable package for macOS with Python 3 API to repository
1316+# for signing. This repository is polled by a project running on a JPL
1317+# Cybersecurity Jenkins server and performs the actual signing and
1318+# notarization.
1319+#
1320+# Options:
1321+# -r/--resign Skip ISSM compilation and packaging. Use to retrigger
1322+# signing/notarization if it fails but build and package
1323+# are valid.
1324+# -u/--unlock Remove lock file from signed package repository. Use if
1325+# build is aborted to allow for subsequent fresh build.
1326+#
1327+# NOTE:
1328+# - Assumes that the following constants are defined,
1329+#
1330+# COMPRESSED_PKG
1331+# ISSM_BINARIES_REPO_PASS
1332+# ISSM_BINARIES_REPO_USER
1333+# SIGNED_REPO_COPY
1334+# SIGNED_REPO_URL
1335+#
1336+# See also:
1337+# - packagers/mac/complete-issm-mac-binaries-python-3.sh
1338+# - packagers/mac/sign-issm-mac-binaries-python-3.sh
1339+#
1340+# TODO:
1341+# - Generalize checkout_*_repo_copy and validate_*_repo_copy functions (e.g.
1342+# pass 'signed' or 'unsigned' as argument)
1343+################################################################################
1344+
1345+# Expand aliases within the context of this script
1346+shopt -s expand_aliases
1347+
1348+# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
1349+#
1350+# Command line tool support for Subversion — including svn, git-svn, and
1351+# related commands — is no longer provided by Xcode. (50266910)
1352+#
1353+# which results in,
1354+#
1355+# svn: error: The subversion command line tools are no longer provided by
1356+# Xcode.
1357+#
1358+# when calling svn, even when subversion is installed via Homebrew and its path
1359+# is available in PATH.
1360+#
1361+# NOTE: May be able to remove this after updating macOS.
1362+#
1363+#alias svn='/usr/local/bin/svn'
1364+
1365+## Override certain other aliases
1366+#
1367+alias cp=$(which cp)
1368+alias grep=$(which grep)
1369+
1370+## Constants
1371+#
1372+MAX_SIGNING_CHECK_ATTEMPTS=30
1373+NOTARIZATION_LOGFILE="notarization.log"
1374+RETRIGGER_SIGNING_FILE="retrigger.txt"
1375+SIGNING_CHECK_PERIOD=60 # in seconds
1376+SIGNING_LOCK_FILE="signing.lock"
1377+UNSIGNED_REPO_COPY="./unsigned"
1378+UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/3/unsigned"
1379+
1380+## Functions
1381+#
1382+checkout_signed_repo_copy(){
1383+ echo "Checking out copy of repository for signed packages"
1384+
1385+ # NOTE: Get empty copy because we do not want to have to check out package
1386+ # from previous signing.
1387+ #
1388+ svn checkout \
1389+ --trust-server-cert \
1390+ --non-interactive \
1391+ --depth empty \
1392+ --username ${ISSM_BINARIES_REPO_USER} \
1393+ --password ${ISSM_BINARIES_REPO_PASS} \
1394+ ${SIGNED_REPO_URL} \
1395+ ${SIGNED_REPO_COPY} > /dev/null 2>&1
1396+}
1397+checkout_unsigned_repo_copy(){
1398+ echo "Checking out copy of repository for unsigned packages"
1399+ svn checkout \
1400+ --trust-server-cert \
1401+ --non-interactive \
1402+ --username ${ISSM_BINARIES_REPO_USER} \
1403+ --password ${ISSM_BINARIES_REPO_PASS} \
1404+ ${UNSIGNED_REPO_URL} \
1405+ ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
1406+}
1407+validate_signed_repo_copy(){
1408+ # Validate copy of repository for signed binaries (e.g.
1409+ # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
1410+ # initial checkout failed)
1411+ if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then
1412+ rm -rf ${SIGNED_REPO_COPY}
1413+ checkout_signed_repo_copy
1414+ fi
1415+}
1416+validate_unsigned_repo_copy(){
1417+ # Validate copy of repository for unsigned binaries (e.g.
1418+ # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
1419+ # initial checkout failed)
1420+ if [[ ! -d ${UNSIGNED_REPO_COPY} || ! -d ${UNSIGNED_REPO_COPY}/.svn ]]; then
1421+ rm -rf ${UNSIGNED_REPO_COPY}
1422+ checkout_unsigned_repo_copy
1423+ fi
1424+}
1425+
1426+## Parse options
1427+#
1428+if [ $# -gt 1 ]; then
1429+ echo "Can use only one option at a time"
1430+ exit 1
1431+fi
1432+
1433+retrigger_signing=0
1434+unlock=0
1435+
1436+if [ $# -eq 1 ]; then
1437+ case $1 in
1438+ -r|--resign) retrigger_signing=1; ;;
1439+ -u|--unlock) unlock=1; ;;
1440+ *) echo "Unknown parameter passed: $1"; exit 1 ;;
1441+ esac
1442+fi
1443+
1444+validate_signed_repo_copy
1445+
1446+if [ ${unlock} -eq 1 ]; then
1447+ # Remove signing lock file from signed package repository so that a new
1448+ # build can run
1449+ echo "Removing lock file from repository for signed packages"
1450+ svn update \
1451+ --username ${ISSM_BINARIES_REPO_USER} \
1452+ --password ${ISSM_BINARIES_REPO_PASS} \
1453+ ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
1454+ svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
1455+ svn commit \
1456+ --trust-server-cert \
1457+ --non-interactive \
1458+ --username ${ISSM_BINARIES_REPO_USER} \
1459+ --password ${ISSM_BINARIES_REPO_PASS} \
1460+ --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} > /dev/null 2>&1
1461+ svn cleanup ${SIGNED_REPO_COPY} > /dev/null 2>&1
1462+
1463+ echo "Remove -u/--unlock option from configuration and run again"
1464+ exit 1
1465+fi
1466+
1467+# If lock file exists, a signing build is still in process by JPL Cybersecurity
1468+svn update \
1469+ --username ${ISSM_BINARIES_REPO_USER} \
1470+ --password ${ISSM_BINARIES_REPO_PASS} \
1471+ ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
1472+
1473+if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
1474+ echo "Previous signing job still in process by JPL Cybersecurity. Please try again later."
1475+ exit 1
1476+fi
1477+
1478+# Commit lock file to repository for signed packages
1479+echo "Committing lock file to repository for signed packages"
1480+touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
1481+svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
1482+svn commit \
1483+ --trust-server-cert \
1484+ --non-interactive \
1485+ --username ${ISSM_BINARIES_REPO_USER} \
1486+ --password ${ISSM_BINARIES_REPO_PASS} \
1487+ --message "ADD: New lock file" ${SIGNED_REPO_COPY} > /dev/null 2>&1
1488+
1489+# Check out copy of repository for unsigned packages
1490+validate_unsigned_repo_copy
1491+
1492+if [ ${retrigger_signing} -eq 0 ]; then
1493+ # Commit new compressed package to repository for unsigned binaries
1494+ echo "Committing package to repository for unsigned packages"
1495+ cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
1496+ svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
1497+ svn commit \
1498+ --trust-server-cert \
1499+ --non-interactive \
1500+ --username ${ISSM_BINARIES_REPO_USER} \
1501+ --password ${ISSM_BINARIES_REPO_PASS} \
1502+ --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
1503+else
1504+ # NOTE: If notarize_only == 1, we commit a dummy file as we do not want to
1505+ # have to commit the entire compressed package again simply to
1506+ # retrigger the signing build on the remote JPL Cybersecurity Jenkins
1507+ # server.
1508+ #
1509+ echo "Attempting to sign existing package again"
1510+ echo $(date +'%Y-%m-%d-%H-%M-%S') > ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} # Write datetime stamp to file to ensure modification is made
1511+ svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1
1512+ svn commit \
1513+ --trust-server-cert \
1514+ --non-interactive \
1515+ --username ${ISSM_BINARIES_REPO_USER} \
1516+ --password ${ISSM_BINARIES_REPO_PASS} \
1517+ --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
1518+fi
1519+
1520+# Check status of signing
1521+echo "Checking progress of signing..."
1522+SIGNING_CHECK_ATTEMPT=0
1523+while [ ${SIGNING_CHECK_ATTEMPT} -lt ${MAX_SIGNING_CHECK_ATTEMPTS} ]; do
1524+ echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds"
1525+ sleep ${SIGNING_CHECK_PERIOD}
1526+ svn update \
1527+ --username ${ISSM_BINARIES_REPO_USER} \
1528+ --password ${ISSM_BINARIES_REPO_PASS} \
1529+ ${SIGNED_REPO_COPY} > /dev/null 2>&1
1530+
1531+ if [ ! -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
1532+ # Retrieve notarization lock file
1533+ svn update \
1534+ --username ${ISSM_BINARIES_REPO_USER} \
1535+ --password ${ISSM_BINARIES_REPO_PASS} \
1536+ ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
1537+
1538+ # Check status
1539+ STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
1540+ if [[ "${STATUS}" == "success" ]]; then
1541+ echo "Notarization successful!"
1542+ break
1543+ else
1544+ echo "Notarization failed!"
1545+ echo "----------------------- Contents of notarization logfile -----------------------"
1546+ cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
1547+ echo "--------------------------------------------------------------------------------"
1548+
1549+ exit 1
1550+ fi
1551+ else
1552+ ((++SIGNING_CHECK_ATTEMPT))
1553+ fi
1554+done
1555+
1556+if [ ! -f ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} ]; then
1557+ echo "Signing timed out!"
1558+ exit 1
1559+fi
1560
1561Property changes on: ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python-3.sh
1562___________________________________________________________________
1563Added: svn:executable
1564## -0,0 +1 ##
1565+*
1566\ No newline at end of property
1567Index: ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python-2.sh
1568===================================================================
1569--- ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python-2.sh (nonexistent)
1570+++ ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python-2.sh (revision 27002)
1571@@ -0,0 +1,134 @@
1572+#!/bin/bash
1573+
1574+################################################################################
1575+# Wrapper script to build, package, send for signing, and transfer to ISSM Web
1576+# site ISSM distributable package for macOS with Python 2 API.
1577+#
1578+# Normally, we would put this directly into the project configuration under
1579+# 'Build' -> 'Excute shell', but becasue it is a bit more involved, it is a
1580+# good idea to version it.
1581+#
1582+# When no failures/errors occur, performs the following:
1583+# - Builds ISSM according to configuration.
1584+# - Packages executables and libraries.
1585+# - Runs test suite against package.
1586+# - Commits compressed package to repository to be signed by JPL Cybersecurity.
1587+# - Retrieves signed package and transmits it to ISSM Web site for
1588+# distribution.
1589+#
1590+# Options:
1591+# -b/--skipbuild Skip ISSM compilation.
1592+# -r/--resign Skip ISSM compilation and packaging. Use to retrigger
1593+# signing/notarization if it fails but build and package
1594+# are valid.
1595+# -s/--skiptests Skip ISSM compilation and testing during packaging
1596+# step. Use if packaging fails for some reason but build
1597+# is valid.
1598+# -t/--transferonly Transfer package to ISSM Web site only. Use if transfer
1599+# fails for some reason to skip building, packaging, and
1600+# signing.
1601+# -u/--unlock Remove lock file from signed package repository. Use if
1602+# build is aborted to allow for subsequent fresh build.
1603+#
1604+# Debugging:
1605+# - Relies on a very tight handshake with project on remote JPL Cybersecurity
1606+# Jenkins server. Debugging may be perfomed locally by running,
1607+#
1608+# packagers/mac/sign-issm-mac-binaries-python-2.sh
1609+#
1610+# with "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" hardcoded to Apple
1611+# Developer credentials.
1612+# - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can
1613+# help debug potential SVN issues.
1614+#
1615+# NOTE:
1616+# - Use only *one* of the above options at a time, and make sure it is removed
1617+# again after a single run.
1618+# - Builds will fail when any of the above options are used on a clean
1619+# workspace. For example, if 'Source Code Management' -> 'Check-out Strategy'
1620+# select menu is set to "Always check out a fresh copy".
1621+# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
1622+# the 'Bindings' section under a 'Username and password (separated)' binding
1623+# (requires 'Credentials Binding Plugin') with 'Credentials' select menu set
1624+# to "jenkins/****** (SVN repository for ISSM binaries)".
1625+################################################################################
1626+
1627+## Constants
1628+#
1629+PKG="ISSM-macOS-Python-2" # Name of directory to copy distributable files to
1630+SIGNED_REPO_COPY="./signed"
1631+SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/2/signed"
1632+
1633+COMPRESSED_PKG="${PKG}.zip"
1634+
1635+## Environment
1636+#
1637+export COMPRESSED_PKG
1638+export PKG
1639+export SIGNED_REPO_COPY
1640+export SIGNED_REPO_URL
1641+
1642+## Parse options
1643+#
1644+if [ $# -gt 1 ]; then
1645+ echo "Can use only one option at a time"
1646+ exit 1
1647+fi
1648+
1649+# NOTE: We could do this with binary switching (i.e. 0011 to sign and transfer,
1650+# but the following is self-documenting).
1651+#
1652+build=1
1653+package=1
1654+sign=1
1655+transfer=1
1656+
1657+if [ $# -eq 1 ]; then
1658+ case $1 in
1659+ -b|--skipbuild) build=0; shift ;;
1660+ -r|--resign) build=0; package=0; ;;
1661+ -s|--skiptests) build=0; ;;
1662+ -t|--transferonly) build=0; package=0; sign=0; ;;
1663+ -u|--unlock) build=0; package=0; transfer=0; ;;
1664+ *) echo "Unknown parameter passed: $1"; exit 1 ;;
1665+ esac
1666+fi
1667+
1668+# Build
1669+if [ ${build} -eq 1 ]; then
1670+ ./jenkins/jenkins.sh ./jenkins/pine_island-mac-binaries-python-2
1671+
1672+ if [ $? -ne 0 ]; then
1673+ exit 1
1674+ fi
1675+fi
1676+
1677+# Package
1678+if [ ${package} -eq 1 ]; then
1679+ ./packagers/mac/package-issm-mac-binaries-python-2.sh $1
1680+
1681+ if [ $? -ne 0 ]; then
1682+ exit 1
1683+ fi
1684+
1685+ shift # Clear $1 so that it is not passed to commit_for_signing script
1686+fi
1687+
1688+# Commit for signing
1689+if [ ${sign} -eq 1 ]; then
1690+ ./packagers/mac/commit_for_signing-issm-mac-binaries-python-2.sh $1
1691+
1692+ if [ $? -ne 0 ]; then
1693+ exit 1
1694+ fi
1695+fi
1696+
1697+# Transfer distributable package to ISSM Web site
1698+if [ ${transfer} -eq 1 ]; then
1699+ ./packagers/mac/transfer-issm-mac-binaries.sh
1700+
1701+ if [ $? -ne 0 ]; then
1702+ exit 1
1703+ fi
1704+fi
1705+
1706
1707Property changes on: ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python-2.sh
1708___________________________________________________________________
1709Added: svn:executable
1710## -0,0 +1 ##
1711+*
1712\ No newline at end of property
1713Index: ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python-3.sh
1714===================================================================
1715--- ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python-3.sh (nonexistent)
1716+++ ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python-3.sh (revision 27002)
1717@@ -0,0 +1,134 @@
1718+#!/bin/bash
1719+
1720+################################################################################
1721+# Wrapper script to build, package, send for signing, and transfer to ISSM Web
1722+# site ISSM distributable package for macOS with Python 3 API.
1723+#
1724+# Normally, we would put this directly into the project configuration under
1725+# 'Build' -> 'Excute shell', but becasue it is a bit more involved, it is a
1726+# good idea to version it.
1727+#
1728+# When no failures/errors occur, performs the following:
1729+# - Builds ISSM according to configuration.
1730+# - Packages executables and libraries.
1731+# - Runs test suite against package.
1732+# - Commits compressed package to repository to be signed by JPL Cybersecurity.
1733+# - Retrieves signed package and transmits it to ISSM Web site for
1734+# distribution.
1735+#
1736+# Options:
1737+# -b/--skipbuild Skip ISSM compilation.
1738+# -r/--resign Skip ISSM compilation and packaging. Use to retrigger
1739+# signing/notarization if it fails but build and package
1740+# are valid.
1741+# -s/--skiptests Skip ISSM compilation and testing during packaging
1742+# step. Use if packaging fails for some reason but build
1743+# is valid.
1744+# -t/--transferonly Transfer package to ISSM Web site only. Use if transfer
1745+# fails for some reason to skip building, packaging, and
1746+# signing.
1747+# -u/--unlock Remove lock file from signed package repository. Use if
1748+# build is aborted to allow for subsequent fresh build.
1749+#
1750+# Debugging:
1751+# - Relies on a very tight handshake with project on remote JPL Cybersecurity
1752+# Jenkins server. Debugging may be perfomed locally by running,
1753+#
1754+# packagers/mac/sign-issm-mac-binaries-python-3.sh
1755+#
1756+# with "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" hardcoded to Apple
1757+# Developer credentials.
1758+# - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can
1759+# help debug potential SVN issues.
1760+#
1761+# NOTE:
1762+# - Use only *one* of the above options at a time, and make sure it is removed
1763+# again after a single run.
1764+# - Builds will fail when any of the above options are used on a clean
1765+# workspace. For example, if 'Source Code Management' -> 'Check-out Strategy'
1766+# select menu is set to "Always check out a fresh copy".
1767+# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
1768+# the 'Bindings' section under a 'Username and password (separated)' binding
1769+# (requires 'Credentials Binding Plugin') with 'Credentials' select menu set
1770+# to "jenkins/****** (SVN repository for ISSM binaries)".
1771+################################################################################
1772+
1773+## Constants
1774+#
1775+PKG="ISSM-macOS-Python-3" # Name of directory to copy distributable files to
1776+SIGNED_REPO_COPY="./signed"
1777+SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/3/signed"
1778+
1779+COMPRESSED_PKG="${PKG}.zip"
1780+
1781+## Environment
1782+#
1783+export COMPRESSED_PKG
1784+export PKG
1785+export SIGNED_REPO_COPY
1786+export SIGNED_REPO_URL
1787+
1788+## Parse options
1789+#
1790+if [ $# -gt 1 ]; then
1791+ echo "Can use only one option at a time"
1792+ exit 1
1793+fi
1794+
1795+# NOTE: We could do this with binary switching (i.e. 0011 to sign and transfer,
1796+# but the following is self-documenting).
1797+#
1798+build=1
1799+package=1
1800+sign=1
1801+transfer=1
1802+
1803+if [ $# -eq 1 ]; then
1804+ case $1 in
1805+ -b|--skipbuild) build=0; shift ;;
1806+ -r|--resign) build=0; package=0; ;;
1807+ -s|--skiptests) build=0; ;;
1808+ -t|--transferonly) build=0; package=0; sign=0; ;;
1809+ -u|--unlock) build=0; package=0; transfer=0; ;;
1810+ *) echo "Unknown parameter passed: $1"; exit 1 ;;
1811+ esac
1812+fi
1813+
1814+# Build
1815+if [ ${build} -eq 1 ]; then
1816+ ./jenkins/jenkins.sh ./jenkins/pine_island-mac-binaries-python-3
1817+
1818+ if [ $? -ne 0 ]; then
1819+ exit 1
1820+ fi
1821+fi
1822+
1823+# Package
1824+if [ ${package} -eq 1 ]; then
1825+ ./packagers/mac/package-issm-mac-binaries-python-3.sh $1
1826+
1827+ if [ $? -ne 0 ]; then
1828+ exit 1
1829+ fi
1830+
1831+ shift # Clear $1 so that it is not passed to commit_for_signing script
1832+fi
1833+
1834+# Commit for signing
1835+if [ ${sign} -eq 1 ]; then
1836+ ./packagers/mac/commit_for_signing-issm-mac-binaries-python-3.sh $1
1837+
1838+ if [ $? -ne 0 ]; then
1839+ exit 1
1840+ fi
1841+fi
1842+
1843+# Transfer distributable package to ISSM Web site
1844+if [ ${transfer} -eq 1 ]; then
1845+ ./packagers/mac/transfer-issm-mac-binaries.sh
1846+
1847+ if [ $? -ne 0 ]; then
1848+ exit 1
1849+ fi
1850+fi
1851+
1852
1853Property changes on: ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python-3.sh
1854___________________________________________________________________
1855Added: svn:executable
1856## -0,0 +1 ##
1857+*
1858\ No newline at end of property
1859Index: ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python-2.sh
1860===================================================================
1861--- ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python-2.sh (nonexistent)
1862+++ ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python-2.sh (revision 27002)
1863@@ -0,0 +1,179 @@
1864+#!/bin/bash
1865+
1866+################################################################################
1867+# Packages and tests ISSM distributable package for macOS with Python 2 API.
1868+#
1869+# Options:
1870+# -s/--skiptests Skip testing during packaging Use if packaging fails
1871+# for some reason but build is valid.
1872+#
1873+# NOTE:
1874+# - Assumes that the following constants are defined,
1875+#
1876+# COMPRESSED_PKG
1877+# ISSM_DIR
1878+# PKG
1879+#
1880+# See also:
1881+# - packagers/mac/complete-issm-mac-binaries-python-2.sh
1882+# - packagers/mac/sign-issm-mac-binaries-python-2.sh
1883+################################################################################
1884+
1885+# Expand aliases within the context of this script
1886+shopt -s expand_aliases
1887+
1888+# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
1889+#
1890+# Command line tool support for Subversion — including svn, git-svn, and
1891+# related commands — is no longer provided by Xcode. (50266910)
1892+#
1893+# which results in,
1894+#
1895+# svn: error: The subversion command line tools are no longer provided by
1896+# Xcode.
1897+#
1898+# when calling svn, even when subversion is installed via Homebrew and its path
1899+# is available in PATH.
1900+#
1901+# NOTE: May be able to remove this after updating macOS.
1902+#
1903+#alias svn='/usr/local/bin/svn'
1904+
1905+## Override certain other aliases
1906+#
1907+alias grep=$(which grep)
1908+
1909+## Constants
1910+#
1911+PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2002 2003 2004 2005 2006 2007 2008 2010 2011 2021 2051 2052 2053 2084 2085 2090 2101 2424 2425 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
1912+
1913+## Environment
1914+#
1915+export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'
1916+
1917+## Parse options
1918+#
1919+if [ $# -gt 1 ]; then
1920+ echo "Can use only one option at a time"
1921+ exit 1
1922+fi
1923+
1924+skip_tests=0
1925+
1926+if [ $# -eq 1 ]; then
1927+ case $1 in
1928+ -s|--skiptests) skip_tests=1; ;;
1929+ *) echo "Unknown parameter passed: $1"; exit 1 ;;
1930+ esac
1931+fi
1932+
1933+# Clean up from previous packaging
1934+echo "Cleaning up existing assets"
1935+cd ${ISSM_DIR}
1936+rm -rf ${PKG} ${COMPRESSED_PKG}
1937+mkdir ${PKG}
1938+
1939+# Add required binaries and libraries to package and modify them where needed
1940+cd ${ISSM_DIR}/bin
1941+
1942+echo "Modify generic"
1943+cat generic_static.py | sed -e "s/generic_static/generic/g" > generic.py
1944+
1945+echo "Moving MPICH binaries to bin/"
1946+if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
1947+ cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
1948+ cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
1949+elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
1950+ cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
1951+ cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
1952+else
1953+ echo "MPICH not found"
1954+ exit 1
1955+fi
1956+
1957+echo "Moving GDAL binaries to bin/"
1958+if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
1959+ cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
1960+ cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
1961+else
1962+ echo "GDAL not found"
1963+ exit 1
1964+fi
1965+
1966+echo "Moving GMT binaries to bin/"
1967+if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
1968+ cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
1969+ cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
1970+else
1971+ echo "GMT not found"
1972+ exit 1
1973+fi
1974+
1975+echo "Moving Gmsh binaries to bin/"
1976+if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
1977+ cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
1978+else
1979+ echo "Gmsh not found"
1980+ exit 1
1981+fi
1982+
1983+# Run tests
1984+if [ ${skip_tests} -eq 0 ]; then
1985+ echo "Running tests"
1986+ cd ${ISSM_DIR}/test/NightlyRun
1987+ rm python.log 2> /dev/null
1988+
1989+ # Set Python environment
1990+ export PYTHONPATH="${ISSM_DIR}/src/m/dev"
1991+ export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
1992+ export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
1993+
1994+ # Run tests, redirecting output to logfile and suppressing output to console
1995+ ./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
1996+
1997+ # Check that Python did not exit in error
1998+ pythonExitCode=`echo $?`
1999+ pythonExitedInError=`grep -E "Error|Standard exception|Traceback|bad interpreter" python.log | wc -l`
2000+
2001+ if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
2002+ echo "----------Python exited in error!----------"
2003+ cat python.log
2004+ echo "-----------End of python.log-----------"
2005+
2006+ # Clean up execution directory
2007+ rm -rf ${ISSM_DIR}/execution/*
2008+
2009+ exit 1
2010+ fi
2011+
2012+ # Check that all tests passed
2013+ numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
2014+
2015+ if [ ${numTestsFailed} -ne 0 ]; then
2016+ echo "One or more tests FAILED"
2017+ exit 1
2018+ else
2019+ echo "All tests PASSED"
2020+ fi
2021+else
2022+ echo "Skipping tests"
2023+fi
2024+
2025+# Create package
2026+cd ${ISSM_DIR}
2027+svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
2028+echo "Copying assets to package: ${PKG}"
2029+cp -rf bin examples lib scripts test ${PKG}
2030+mkdir ${PKG}/execution
2031+cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist
2032+${ISSM_DIR}/scripts/py_to_pyc.sh ${PKG}/bin # Compile Python source files
2033+echo "Cleaning up unneeded/unwanted files"
2034+rm -f ${PKG}/bin/*.py # Remove all Python scripts
2035+rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
2036+rm -f ${PKG}/lib/*.a # Remove static libraries from package
2037+rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
2038+rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
2039+
2040+# Compress package
2041+echo "Compressing package"
2042+ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
2043
2044Property changes on: ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python-2.sh
2045___________________________________________________________________
2046Added: svn:executable
2047## -0,0 +1 ##
2048+*
2049\ No newline at end of property
2050Index: ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python-3.sh
2051===================================================================
2052--- ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python-3.sh (nonexistent)
2053+++ ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python-3.sh (revision 27002)
2054@@ -0,0 +1,182 @@
2055+#!/bin/bash
2056+
2057+################################################################################
2058+# Packages and tests ISSM distributable package for macOS with Python 3 API.
2059+#
2060+# Options:
2061+# -s/--skiptests Skip testing during packaging Use if packaging fails
2062+# for some reason but build is valid.
2063+#
2064+# NOTE:
2065+# - Assumes that the following constants are defined,
2066+#
2067+# COMPRESSED_PKG
2068+# ISSM_DIR
2069+# PKG
2070+#
2071+# See also:
2072+# - packagers/mac/complete-issm-mac-binaries-python-3.sh
2073+# - packagers/mac/sign-issm-mac-binaries-python-3.sh
2074+################################################################################
2075+
2076+# Expand aliases within the context of this script
2077+shopt -s expand_aliases
2078+
2079+# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
2080+#
2081+# Command line tool support for Subversion — including svn, git-svn, and
2082+# related commands — is no longer provided by Xcode. (50266910)
2083+#
2084+# which results in,
2085+#
2086+# svn: error: The subversion command line tools are no longer provided by
2087+# Xcode.
2088+#
2089+# when calling svn, even when subversion is installed via Homebrew and its path
2090+# is available in PATH.
2091+#
2092+# NOTE: May be able to remove this after updating macOS.
2093+#
2094+#alias svn='/usr/local/bin/svn'
2095+
2096+## Override certain other aliases
2097+#
2098+alias grep=$(which grep)
2099+
2100+## Constants
2101+#
2102+PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2002 2003 2004 2005 2006 2007 2008 2010 2011 2021 2051 2052 2053 2084 2085 2090 2101 2424 2425 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
2103+
2104+## Environment
2105+#
2106+export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'
2107+
2108+## Parse options
2109+#
2110+if [ $# -gt 1 ]; then
2111+ echo "Can use only one option at a time"
2112+ exit 1
2113+fi
2114+
2115+skip_tests=0
2116+
2117+if [ $# -eq 1 ]; then
2118+ case $1 in
2119+ -s|--skiptests) skip_tests=1; ;;
2120+ *) echo "Unknown parameter passed: $1"; exit 1 ;;
2121+ esac
2122+fi
2123+
2124+# Clean up from previous packaging
2125+echo "Cleaning up existing assets"
2126+cd ${ISSM_DIR}
2127+rm -rf ${PKG} ${COMPRESSED_PKG}
2128+mkdir ${PKG}
2129+
2130+# Add required binaries and libraries to package and modify them where needed
2131+cd ${ISSM_DIR}/bin
2132+
2133+echo "Modify generic"
2134+cat generic_static.py | sed -e "s/generic_static/generic/g" > generic.py
2135+
2136+echo "Moving MPICH binaries to bin/"
2137+if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
2138+ cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
2139+ cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
2140+elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
2141+ cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
2142+ cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
2143+else
2144+ echo "MPICH not found"
2145+ exit 1
2146+fi
2147+
2148+echo "Moving GDAL binaries to bin/"
2149+if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
2150+ cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
2151+ cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
2152+else
2153+ echo "GDAL not found"
2154+ exit 1
2155+fi
2156+
2157+echo "Moving GMT binaries to bin/"
2158+if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
2159+ cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
2160+ cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
2161+else
2162+ echo "GMT not found"
2163+ exit 1
2164+fi
2165+
2166+echo "Moving Gmsh binaries to bin/"
2167+if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
2168+ cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
2169+else
2170+ echo "Gmsh not found"
2171+ exit 1
2172+fi
2173+
2174+# Run tests
2175+if [ ${skip_tests} -eq 0 ]; then
2176+ echo "Running tests"
2177+ cd ${ISSM_DIR}/test/NightlyRun
2178+ rm python.log 2> /dev/null
2179+
2180+ # Set Python environment
2181+ export PYTHONPATH="${ISSM_DIR}/src/m/dev"
2182+ export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
2183+ export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
2184+
2185+ # Ensure that runme.py uses Python 3 interpreter
2186+ sed -i "s|/usr/bin/env python|/usr/local/bin/python3|g" ./runme.py
2187+
2188+ # Run tests, redirecting output to logfile and suppressing output to console
2189+ ./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
2190+
2191+ # Check that Python did not exit in error
2192+ pythonExitCode=`echo $?`
2193+ pythonExitedInError=`grep -E "Error|Standard exception|Traceback|bad interpreter" python.log | wc -l`
2194+
2195+ if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
2196+ echo "----------Python exited in error!----------"
2197+ cat python.log
2198+ echo "-----------End of python.log-----------"
2199+
2200+ # Clean up execution directory
2201+ rm -rf ${ISSM_DIR}/execution/*
2202+
2203+ exit 1
2204+ fi
2205+
2206+ # Check that all tests passed
2207+ numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
2208+
2209+ if [ ${numTestsFailed} -ne 0 ]; then
2210+ echo "One or more tests FAILED"
2211+ exit 1
2212+ else
2213+ echo "All tests PASSED"
2214+ fi
2215+else
2216+ echo "Skipping tests"
2217+fi
2218+
2219+# Create package
2220+cd ${ISSM_DIR}
2221+svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
2222+echo "Copying assets to package: ${PKG}"
2223+cp -rf bin examples lib scripts test ${PKG}
2224+mkdir ${PKG}/execution
2225+cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist
2226+${ISSM_DIR}/scripts/py_to_pyc.sh ${PKG}/bin # Compile Python source files
2227+echo "Cleaning up unneeded/unwanted files"
2228+rm -f ${PKG}/bin/*.py # Remove all Python scripts
2229+rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
2230+rm -f ${PKG}/lib/*.a # Remove static libraries from package
2231+rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
2232+rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
2233+
2234+# Compress package
2235+echo "Compressing package"
2236+ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
2237
2238Property changes on: ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python-3.sh
2239___________________________________________________________________
2240Added: svn:executable
2241## -0,0 +1 ##
2242+*
2243\ No newline at end of property
2244Index: ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python-2.sh
2245===================================================================
2246--- ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python-2.sh (nonexistent)
2247+++ ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python-2.sh (revision 27002)
2248@@ -0,0 +1,348 @@
2249+#!/bin/bash
2250+
2251+################################################################################
2252+# Intended to be run in the context of a Jenkins project on a JPL
2253+# Cybersecurity server for signing macOS applications. Polls SCM of the
2254+# Subversion repository hosted at
2255+# https://issm.ess.uci.edu/svn/issm-binaries/mac/python/2/unsigned to trigger
2256+# new builds.
2257+#
2258+# In order to replicate the requried Jenkins project configuration:
2259+# - First, navigate to 'Manage Jenkins' -> 'Manage Plugins' and install the
2260+# 'Credentials Bindings Plugin' if it is not already installed.
2261+# - Contact one of the members of the ISSM development team for crendentials
2262+# for the ISSM binaries repository (mention that the credentials are stored
2263+# in ISSM-Infrastructure.pdf).
2264+# - Navigate to 'Manage Jenkins' -> 'Manage Credentials' -> <domain> ->
2265+# 'Add Credentials' and enter the crendentials from above.
2266+# - From the 'Dashboard', select 'New Item' -> 'Freestyle project'.
2267+# - Under 'Source Code Management', select 'Subversion'.
2268+# - The 'Repository URL' text field should be set to
2269+# "https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned".
2270+# - The 'Credentials' select menu should be set to the new credentials
2271+# created previously.
2272+# - The 'Local module directory' text field should be set to the same
2273+# value as the constant UNSIGNED_REPO_COPY (set below to './unsigned').
2274+# - Under 'Build Trigggers', check the box for 'Poll SCM' and set the
2275+# 'Schedule' text area to "H/5 * * * *".
2276+# - Under 'Build Environment', check the box for 'Use secret text(s) or
2277+# file(s)', then under 'Bindings' click the 'Add...' button and select
2278+# 'Username and password (separated)'.
2279+# - Set 'Username Variable' to "ISSM_BINARIES_USER".
2280+# - Set 'Password Variable' to "ISSM_BINARIES_PASS".
2281+# - Under 'Credentials', select the same, new credentials that created
2282+# previously.
2283+# - The contents of this script can be copied/pasted directly into the ‘Build'
2284+# -> 'Execute Shell' -> ‘Command' textarea of the project configuration (or
2285+# you can simply store the script on disk and call it from there).
2286+# - Make sure to click the 'Save' button.
2287+#
2288+# Current point of contact at JPL Cybersecurity:
2289+# Alex Coward, alexander.g.coward@jpl.nasa.gov
2290+#
2291+# NOTE:
2292+# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
2293+# the 'Bindings' section under a 'Username and password (separated)' binding
2294+# (requires 'Credentials Binding Plugin').
2295+# - For local debugging, the aformentioned credentials can be hardcoded into
2296+# the 'USERNAME' and 'PASSWORD' constants below.
2297+################################################################################
2298+
2299+# Expand aliases within the context of this script
2300+shopt -s expand_aliases
2301+
2302+# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
2303+#
2304+# Command line tool support for Subversion — including svn, git-svn, and
2305+# related commands — is no longer provided by Xcode. (50266910)
2306+#
2307+# which results in,
2308+#
2309+# svn: error: The subversion command line tools are no longer provided by
2310+# Xcode.
2311+#
2312+# when calling svn, even when subversion is installed via Homebrew and its path
2313+# is available in PATH.
2314+#
2315+# NOTE: May be able to remove this after updating macOS.
2316+#
2317+#alias svn='/usr/local/bin/svn'
2318+
2319+## Override certain other aliases
2320+#
2321+alias cp=$(which cp)
2322+alias grep=$(which grep)
2323+
2324+## Constants
2325+#
2326+AD_IDENTITY="**********" # Apple Developer identity
2327+AD_USERNAME="**********" # Apple Developer username
2328+ALTOOL_PASSWORD="@keychain:**********" # altool password (assumed to be stored in keychain)
2329+ASC_PROVIDER="**********"
2330+MAX_SVN_ATTEMPTS=10
2331+NOTARIZATION_CHECK_ATTEMPTS=20
2332+NOTARIZATION_CHECK_PERIOD=60
2333+NOTARIZATION_LOGFILE="notarization.log"
2334+NOTARIZATION_LOGFILE_PATH="."
2335+PASSWORD=${ISSM_BINARIES_PASS}
2336+PKG="ISSM-macOS-Python-2"
2337+PRIMARY_BUNDLE_ID="gov.nasa.jpl.issm.python"
2338+SIGNED_REPO_COPY="./signed"
2339+SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/2/signed"
2340+SIGNING_LOCK_FILE="signing.lock"
2341+SUCCESS_LOGFILE="${SIGNED_REPO_COPY}/success.log"
2342+UNSIGNED_REPO_COPY="./unsigned"
2343+UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/2/unsigned"
2344+USERNAME=${ISSM_BINARIES_USER}
2345+
2346+COMPRESSED_PKG="${PKG}.zip"
2347+EXE_ENTITLEMENTS_PLIST="${PKG}/bin/entitlements.plist"
2348+
2349+# NOTE: Uncomment the following for local testing (Jenkins checks out copy of
2350+# repository for unsigned packages to working directory)
2351+#
2352+
2353+# # Clean up from previous packaging (not necessary for single builds on Jenkins,
2354+# # but useful when testing packaging locally)
2355+# echo "Cleaning up existing assets"
2356+# rm -rf ${COMPRESSED_PKG} ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${UNSIGNED_REPO_COPY}
2357+
2358+# # Check out copy of repository for unsigned packages
2359+# echo "Checking out copy of respository for unsigned packages"
2360+# svn checkout \
2361+# --trust-server-cert \
2362+# --non-interactive \
2363+# --username ${USERNAME} \
2364+# --password ${PASSWORD} \
2365+# ${UNSIGNED_REPO_URL} \
2366+# ${UNSIGNED_REPO_COPY}
2367+
2368+rm -rf ${PKG} ${SIGNED_REPO_COPY}
2369+
2370+
2371+# Extract package contents
2372+echo "Extracting package contents"
2373+ditto -xk ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} .
2374+
2375+# Clear extended attributes on all files
2376+xattr -cr ${PKG}
2377+
2378+# Build list of ISSM executables
2379+ISSM_BINS=$(\
2380+ find ${PKG}/bin -type f -name *.exe; \
2381+ find ${PKG}/bin -type f -name *.pyc; \
2382+)
2383+
2384+# Build list of third party executables
2385+THIRD_PARTY_BINS=$(\
2386+ echo ${PKG}/bin/mpiexec; \
2387+ echo ${PKG}/bin/hydra_pmi_proxy; \
2388+ echo ${PKG}/bin/gdalsrsinfo; \
2389+ echo ${PKG}/bin/gdaltransform; \
2390+ echo ${PKG}/bin/gmt; \
2391+ echo ${PKG}/bin/gmtselect; \
2392+ echo ${PKG}/bin/gmsh; \
2393+)
2394+
2395+# Sign all executables in package
2396+echo "Signing all executables in package"
2397+codesign -s ${AD_IDENTITY} --timestamp --options=runtime --entitlements ${EXE_ENTITLEMENTS_PLIST} ${ISSM_BINS}
2398+codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${THIRD_PARTY_BINS}
2399+
2400+# Build list of ISSM libraries
2401+ISSM_LIBS=$(\
2402+ find ${PKG}/lib -type f -name *.so; \
2403+)
2404+
2405+# Sign all libraries in package
2406+echo "Signing all libraries in package"
2407+codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${ISSM_LIBS}
2408+
2409+# NOTE: Skipping signature validation because this is not a true package nor app
2410+
2411+# Compress signed package
2412+echo "Compressing signed package"
2413+ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
2414+
2415+# Submit compressed package for notarization
2416+echo "Submitting signed package to Apple for notarization"
2417+xcrun altool --notarize-app --primary-bundle-id ${PRIMARY_BUNDLE_ID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} --asc-provider ${ASC_PROVIDER} --file ${COMPRESSED_PKG} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
2418+
2419+# Sleep until notarization request response is received
2420+echo "Waiting for notarizaion request response"
2421+while [[ ! -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} || ! -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; do
2422+ sleep 30
2423+done
2424+
2425+echo "Notarizaion request response received"
2426+
2427+# Check if UUID exists in response
2428+HAS_UUID=$(grep 'RequestUUID = ' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}) # NOTE: Checking for "RequestUUID = " because "RequestUUID" shows up in some error messages
2429+if [ -z "${HAS_UUID}" ]; then
2430+ echo "Notarization failed!"
2431+ echo "----------------------- Contents of notarization logfile -----------------------"
2432+ cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
2433+ echo "--------------------------------------------------------------------------------"
2434+
2435+ # Clean up
2436+ rm -rf ${PKG} ${COMPRESSED_PKG}
2437+
2438+ exit 1
2439+fi
2440+
2441+# Get UUID from notarization request response
2442+UUID=$(echo ${HAS_UUID} | sed 's/[[:space:]]*RequestUUID = //')
2443+echo "UUID: ${UUID}"
2444+
2445+# Check notarization status
2446+#
2447+# NOTE: Currently, this checks if notarization was successful, but we are not
2448+# able to staple notarization as this is not a true package nor app and,
2449+# at the very least, MATLAB Mex files cannot be stapled. As such, clients
2450+# will not be able to clear Gatekeeper if they are offline.
2451+#
2452+echo "Checking notarization status"
2453+SUCCESS=0
2454+for ATTEMPT in $(seq 1 ${NOTARIZATION_CHECK_ATTEMPTS}); do
2455+ echo " Attempt #${ATTEMPT}..."
2456+ xcrun altool --notarization-info ${UUID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
2457+ if [[ -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} && -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; then
2458+
2459+ # First, check if there is an error
2460+ ERROR_CHECK=$(grep 'Error' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE})
2461+ if [ ! -z "${ERROR_CHECK}" ]; then
2462+ break
2463+ fi
2464+
2465+ # No error, so check status
2466+ STATUS=$(grep 'Status:' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
2467+ if [[ "${STATUS}" == "success" ]]; then
2468+ # Staple notarization to all elements of package that were previously signed
2469+ #xcrun stapler staple ${THIRD_PARTY_BINS} # NOTE: Fails with "Stapler is incapable of working with MATLAB Mex files."
2470+
2471+ # Validate stapling of notarization
2472+ #xcrun stapler validation ${THIRD_PARTY_BINS} # NOTE: Skipping notarization stapling validation because this is not a true package nor app
2473+
2474+ # Compress signed and notarized package
2475+ ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
2476+
2477+ # Set flag indicating notarization was successful
2478+ SUCCESS=1
2479+
2480+ break
2481+ elif [[ "${STATUS}" == "in progress" ]]; then
2482+ echo " ...in progress still; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds."
2483+ sleep ${NOTARIZATION_CHECK_PERIOD}
2484+ elif [[ "${STATUS}" == "invalid" ]]; then
2485+ break
2486+ fi
2487+ else
2488+ if [ ${ATTEMPT} -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then
2489+ echo " ...not ready yet; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds"
2490+ sleep ${NOTARIZATION_CHECK_PERIOD}
2491+ else
2492+ echo " ...maximum attempts reached, but no response, or something else went wrong"
2493+ echo " If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again"
2494+ break
2495+ fi
2496+ fi
2497+done
2498+
2499+if [ ${SUCCESS} -eq 1 ]; then
2500+ echo "Notarization successful!"
2501+else
2502+ echo "Notarization failed!"
2503+ echo "----------------------- Contents of notarization logfile -----------------------"
2504+ cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
2505+ echo "--------------------------------------------------------------------------------"
2506+fi
2507+
2508+# Check out copy of repository for signed packages
2509+echo "Checking out copy of respository for signed packages"
2510+SVN_ATTEMPT=0
2511+SVN_SUCCESS=0
2512+while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
2513+ rm -rf ${SIGNED_REPO_COPY}
2514+ svn checkout \
2515+ --trust-server-cert \
2516+ --non-interactive \
2517+ --username ${USERNAME} \
2518+ --password ${PASSWORD} \
2519+ ${SIGNED_REPO_URL} \
2520+ ${SIGNED_REPO_COPY} > /dev/null 2>&1
2521+ if [ $? -eq 0 ]; then
2522+ SVN_SUCCESS=1
2523+ break
2524+ else
2525+ ((++SVN_ATTEMPT))
2526+ sleep 5
2527+ fi
2528+done
2529+
2530+if [ ${SVN_SUCCESS} -eq 0 ]; then
2531+ echo "Checkout of respository for signed packages failed"
2532+ exit 1
2533+fi
2534+
2535+# Copy notarization file to repository for signed packages
2536+cp ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY}
2537+svn add ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1
2538+
2539+# Remove lock file from repository for signed packages
2540+svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
2541+
2542+SVN_ATTEMPT=0
2543+SVN_SUCCESS=0
2544+if [ ${SUCCESS} -eq 1 ]; then
2545+ # Copy signed package to repository for signed packages
2546+ cp ${COMPRESSED_PKG} ${SIGNED_REPO_COPY}
2547+ svn add ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
2548+
2549+ # Commit changes
2550+ echo "Committing changes to repository for signed packages"
2551+ while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
2552+ svn commit \
2553+ --trust-server-cert \
2554+ --non-interactive \
2555+ --username ${USERNAME} \
2556+ --password ${PASSWORD} \
2557+ --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
2558+ if [ $? -eq 0 ]; then
2559+ SVN_SUCCESS=1
2560+ break
2561+ else
2562+ ((++SVN_ATTEMPT))
2563+ sleep 5
2564+ fi
2565+ done
2566+
2567+ if [ ${SVN_SUCCESS} -eq 0 ]; then
2568+ echo "Commit to respository for signed packages failed"
2569+ exit 1
2570+ fi
2571+else
2572+ # Commit changes
2573+ echo "Committing changes to repository for signed packages"
2574+ while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
2575+ svn commit \
2576+ --trust-server-cert \
2577+ --non-interactive \
2578+ --username ${USERNAME} \
2579+ --password ${PASSWORD} \
2580+ --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
2581+ if [ $? -eq 0 ]; then
2582+ SVN_SUCCESS=1
2583+ break
2584+ else
2585+ ((++SVN_ATTEMPT))
2586+ sleep 5
2587+ fi
2588+ done
2589+
2590+ if [ ${SVN_SUCCESS} -eq 0 ]; then
2591+ echo "Commit to respository for signed packages failed"
2592+ exit 1
2593+ fi
2594+
2595+ exit 1
2596+fi
2597
2598Property changes on: ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python-2.sh
2599___________________________________________________________________
2600Added: svn:executable
2601## -0,0 +1 ##
2602+*
2603\ No newline at end of property
2604Index: ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python-3.sh
2605===================================================================
2606--- ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python-3.sh (nonexistent)
2607+++ ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python-3.sh (revision 27002)
2608@@ -0,0 +1,348 @@
2609+#!/bin/bash
2610+
2611+################################################################################
2612+# Intended to be run in the context of a Jenkins project on a JPL
2613+# Cybersecurity server for signing macOS applications. Polls SCM of the
2614+# Subversion repository hosted at
2615+# https://issm.ess.uci.edu/svn/issm-binaries/mac/python/3/unsigned to trigger
2616+# new builds.
2617+#
2618+# In order to replicate the requried Jenkins project configuration:
2619+# - First, navigate to 'Manage Jenkins' -> 'Manage Plugins' and install the
2620+# 'Credentials Bindings Plugin' if it is not already installed.
2621+# - Contact one of the members of the ISSM development team for crendentials
2622+# for the ISSM binaries repository (mention that the credentials are stored
2623+# in ISSM-Infrastructure.pdf).
2624+# - Navigate to 'Manage Jenkins' -> 'Manage Credentials' -> <domain> ->
2625+# 'Add Credentials' and enter the crendentials from above.
2626+# - From the 'Dashboard', select 'New Item' -> 'Freestyle project'.
2627+# - Under 'Source Code Management', select 'Subversion'.
2628+# - The 'Repository URL' text field should be set to
2629+# "https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned".
2630+# - The 'Credentials' select menu should be set to the new credentials
2631+# created previously.
2632+# - The 'Local module directory' text field should be set to the same
2633+# value as the constant UNSIGNED_REPO_COPY (set below to './unsigned').
2634+# - Under 'Build Trigggers', check the box for 'Poll SCM' and set the
2635+# 'Schedule' text area to "H/5 * * * *".
2636+# - Under 'Build Environment', check the box for 'Use secret text(s) or
2637+# file(s)', then under 'Bindings' click the 'Add...' button and select
2638+# 'Username and password (separated)'.
2639+# - Set 'Username Variable' to "ISSM_BINARIES_USER".
2640+# - Set 'Password Variable' to "ISSM_BINARIES_PASS".
2641+# - Under 'Credentials', select the same, new credentials that created
2642+# previously.
2643+# - The contents of this script can be copied/pasted directly into the ‘Build'
2644+# -> 'Execute Shell' -> ‘Command' textarea of the project configuration (or
2645+# you can simply store the script on disk and call it from there).
2646+# - Make sure to click the 'Save' button.
2647+#
2648+# Current point of contact at JPL Cybersecurity:
2649+# Alex Coward, alexander.g.coward@jpl.nasa.gov
2650+#
2651+# NOTE:
2652+# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
2653+# the 'Bindings' section under a 'Username and password (separated)' binding
2654+# (requires 'Credentials Binding Plugin').
2655+# - For local debugging, the aformentioned credentials can be hardcoded into
2656+# the 'USERNAME' and 'PASSWORD' constants below.
2657+################################################################################
2658+
2659+# Expand aliases within the context of this script
2660+shopt -s expand_aliases
2661+
2662+# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
2663+#
2664+# Command line tool support for Subversion — including svn, git-svn, and
2665+# related commands — is no longer provided by Xcode. (50266910)
2666+#
2667+# which results in,
2668+#
2669+# svn: error: The subversion command line tools are no longer provided by
2670+# Xcode.
2671+#
2672+# when calling svn, even when subversion is installed via Homebrew and its path
2673+# is available in PATH.
2674+#
2675+# NOTE: May be able to remove this after updating macOS.
2676+#
2677+#alias svn='/usr/local/bin/svn'
2678+
2679+## Override certain other aliases
2680+#
2681+alias cp=$(which cp)
2682+alias grep=$(which grep)
2683+
2684+## Constants
2685+#
2686+AD_IDENTITY="**********" # Apple Developer identity
2687+AD_USERNAME="**********" # Apple Developer username
2688+ALTOOL_PASSWORD="@keychain:**********" # altool password (assumed to be stored in keychain)
2689+ASC_PROVIDER="**********"
2690+MAX_SVN_ATTEMPTS=10
2691+NOTARIZATION_CHECK_ATTEMPTS=20
2692+NOTARIZATION_CHECK_PERIOD=60
2693+NOTARIZATION_LOGFILE="notarization.log"
2694+NOTARIZATION_LOGFILE_PATH="."
2695+PASSWORD=${ISSM_BINARIES_PASS}
2696+PKG="ISSM-macOS-Python-3"
2697+PRIMARY_BUNDLE_ID="gov.nasa.jpl.issm.python"
2698+SIGNED_REPO_COPY="./signed"
2699+SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/3/signed"
2700+SIGNING_LOCK_FILE="signing.lock"
2701+SUCCESS_LOGFILE="${SIGNED_REPO_COPY}/success.log"
2702+UNSIGNED_REPO_COPY="./unsigned"
2703+UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/3/unsigned"
2704+USERNAME=${ISSM_BINARIES_USER}
2705+
2706+COMPRESSED_PKG="${PKG}.zip"
2707+EXE_ENTITLEMENTS_PLIST="${PKG}/bin/entitlements.plist"
2708+
2709+# NOTE: Uncomment the following for local testing (Jenkins checks out copy of
2710+# repository for unsigned packages to working directory)
2711+#
2712+
2713+# # Clean up from previous packaging (not necessary for single builds on Jenkins,
2714+# # but useful when testing packaging locally)
2715+# echo "Cleaning up existing assets"
2716+# rm -rf ${COMPRESSED_PKG} ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${UNSIGNED_REPO_COPY}
2717+
2718+# # Check out copy of repository for unsigned packages
2719+# echo "Checking out copy of respository for unsigned packages"
2720+# svn checkout \
2721+# --trust-server-cert \
2722+# --non-interactive \
2723+# --username ${USERNAME} \
2724+# --password ${PASSWORD} \
2725+# ${UNSIGNED_REPO_URL} \
2726+# ${UNSIGNED_REPO_COPY}
2727+
2728+rm -rf ${PKG} ${SIGNED_REPO_COPY}
2729+
2730+
2731+# Extract package contents
2732+echo "Extracting package contents"
2733+ditto -xk ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} .
2734+
2735+# Clear extended attributes on all files
2736+xattr -cr ${PKG}
2737+
2738+# Build list of ISSM executables
2739+ISSM_BINS=$(\
2740+ find ${PKG}/bin -type f -name *.exe; \
2741+ find ${PKG}/bin -type f -name *.pyc; \
2742+)
2743+
2744+# Build list of third party executables
2745+THIRD_PARTY_BINS=$(\
2746+ echo ${PKG}/bin/mpiexec; \
2747+ echo ${PKG}/bin/hydra_pmi_proxy; \
2748+ echo ${PKG}/bin/gdalsrsinfo; \
2749+ echo ${PKG}/bin/gdaltransform; \
2750+ echo ${PKG}/bin/gmt; \
2751+ echo ${PKG}/bin/gmtselect; \
2752+ echo ${PKG}/bin/gmsh; \
2753+)
2754+
2755+# Sign all executables in package
2756+echo "Signing all executables in package"
2757+codesign -s ${AD_IDENTITY} --timestamp --options=runtime --entitlements ${EXE_ENTITLEMENTS_PLIST} ${ISSM_BINS}
2758+codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${THIRD_PARTY_BINS}
2759+
2760+# Build list of ISSM libraries
2761+ISSM_LIBS=$(\
2762+ find ${PKG}/lib -type f -name *.so; \
2763+)
2764+
2765+# Sign all libraries in package
2766+echo "Signing all libraries in package"
2767+codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${ISSM_LIBS}
2768+
2769+# NOTE: Skipping signature validation because this is not a true package nor app
2770+
2771+# Compress signed package
2772+echo "Compressing signed package"
2773+ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
2774+
2775+# Submit compressed package for notarization
2776+echo "Submitting signed package to Apple for notarization"
2777+xcrun altool --notarize-app --primary-bundle-id ${PRIMARY_BUNDLE_ID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} --asc-provider ${ASC_PROVIDER} --file ${COMPRESSED_PKG} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
2778+
2779+# Sleep until notarization request response is received
2780+echo "Waiting for notarizaion request response"
2781+while [[ ! -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} || ! -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; do
2782+ sleep 30
2783+done
2784+
2785+echo "Notarizaion request response received"
2786+
2787+# Check if UUID exists in response
2788+HAS_UUID=$(grep 'RequestUUID = ' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}) # NOTE: Checking for "RequestUUID = " because "RequestUUID" shows up in some error messages
2789+if [ -z "${HAS_UUID}" ]; then
2790+ echo "Notarization failed!"
2791+ echo "----------------------- Contents of notarization logfile -----------------------"
2792+ cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
2793+ echo "--------------------------------------------------------------------------------"
2794+
2795+ # Clean up
2796+ rm -rf ${PKG} ${COMPRESSED_PKG}
2797+
2798+ exit 1
2799+fi
2800+
2801+# Get UUID from notarization request response
2802+UUID=$(echo ${HAS_UUID} | sed 's/[[:space:]]*RequestUUID = //')
2803+echo "UUID: ${UUID}"
2804+
2805+# Check notarization status
2806+#
2807+# NOTE: Currently, this checks if notarization was successful, but we are not
2808+# able to staple notarization as this is not a true package nor app and,
2809+# at the very least, MATLAB Mex files cannot be stapled. As such, clients
2810+# will not be able to clear Gatekeeper if they are offline.
2811+#
2812+echo "Checking notarization status"
2813+SUCCESS=0
2814+for ATTEMPT in $(seq 1 ${NOTARIZATION_CHECK_ATTEMPTS}); do
2815+ echo " Attempt #${ATTEMPT}..."
2816+ xcrun altool --notarization-info ${UUID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
2817+ if [[ -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} && -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; then
2818+
2819+ # First, check if there is an error
2820+ ERROR_CHECK=$(grep 'Error' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE})
2821+ if [ ! -z "${ERROR_CHECK}" ]; then
2822+ break
2823+ fi
2824+
2825+ # No error, so check status
2826+ STATUS=$(grep 'Status:' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
2827+ if [[ "${STATUS}" == "success" ]]; then
2828+ # Staple notarization to all elements of package that were previously signed
2829+ #xcrun stapler staple ${THIRD_PARTY_BINS} # NOTE: Fails with "Stapler is incapable of working with MATLAB Mex files."
2830+
2831+ # Validate stapling of notarization
2832+ #xcrun stapler validation ${THIRD_PARTY_BINS} # NOTE: Skipping notarization stapling validation because this is not a true package nor app
2833+
2834+ # Compress signed and notarized package
2835+ ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
2836+
2837+ # Set flag indicating notarization was successful
2838+ SUCCESS=1
2839+
2840+ break
2841+ elif [[ "${STATUS}" == "in progress" ]]; then
2842+ echo " ...in progress still; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds."
2843+ sleep ${NOTARIZATION_CHECK_PERIOD}
2844+ elif [[ "${STATUS}" == "invalid" ]]; then
2845+ break
2846+ fi
2847+ else
2848+ if [ ${ATTEMPT} -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then
2849+ echo " ...not ready yet; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds"
2850+ sleep ${NOTARIZATION_CHECK_PERIOD}
2851+ else
2852+ echo " ...maximum attempts reached, but no response, or something else went wrong"
2853+ echo " If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again"
2854+ break
2855+ fi
2856+ fi
2857+done
2858+
2859+if [ ${SUCCESS} -eq 1 ]; then
2860+ echo "Notarization successful!"
2861+else
2862+ echo "Notarization failed!"
2863+ echo "----------------------- Contents of notarization logfile -----------------------"
2864+ cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
2865+ echo "--------------------------------------------------------------------------------"
2866+fi
2867+
2868+# Check out copy of repository for signed packages
2869+echo "Checking out copy of respository for signed packages"
2870+SVN_ATTEMPT=0
2871+SVN_SUCCESS=0
2872+while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
2873+ rm -rf ${SIGNED_REPO_COPY}
2874+ svn checkout \
2875+ --trust-server-cert \
2876+ --non-interactive \
2877+ --username ${USERNAME} \
2878+ --password ${PASSWORD} \
2879+ ${SIGNED_REPO_URL} \
2880+ ${SIGNED_REPO_COPY} > /dev/null 2>&1
2881+ if [ $? -eq 0 ]; then
2882+ SVN_SUCCESS=1
2883+ break
2884+ else
2885+ ((++SVN_ATTEMPT))
2886+ sleep 5
2887+ fi
2888+done
2889+
2890+if [ ${SVN_SUCCESS} -eq 0 ]; then
2891+ echo "Checkout of respository for signed packages failed"
2892+ exit 1
2893+fi
2894+
2895+# Copy notarization file to repository for signed packages
2896+cp ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY}
2897+svn add ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1
2898+
2899+# Remove lock file from repository for signed packages
2900+svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
2901+
2902+SVN_ATTEMPT=0
2903+SVN_SUCCESS=0
2904+if [ ${SUCCESS} -eq 1 ]; then
2905+ # Copy signed package to repository for signed packages
2906+ cp ${COMPRESSED_PKG} ${SIGNED_REPO_COPY}
2907+ svn add ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
2908+
2909+ # Commit changes
2910+ echo "Committing changes to repository for signed packages"
2911+ while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
2912+ svn commit \
2913+ --trust-server-cert \
2914+ --non-interactive \
2915+ --username ${USERNAME} \
2916+ --password ${PASSWORD} \
2917+ --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
2918+ if [ $? -eq 0 ]; then
2919+ SVN_SUCCESS=1
2920+ break
2921+ else
2922+ ((++SVN_ATTEMPT))
2923+ sleep 5
2924+ fi
2925+ done
2926+
2927+ if [ ${SVN_SUCCESS} -eq 0 ]; then
2928+ echo "Commit to respository for signed packages failed"
2929+ exit 1
2930+ fi
2931+else
2932+ # Commit changes
2933+ echo "Committing changes to repository for signed packages"
2934+ while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
2935+ svn commit \
2936+ --trust-server-cert \
2937+ --non-interactive \
2938+ --username ${USERNAME} \
2939+ --password ${PASSWORD} \
2940+ --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
2941+ if [ $? -eq 0 ]; then
2942+ SVN_SUCCESS=1
2943+ break
2944+ else
2945+ ((++SVN_ATTEMPT))
2946+ sleep 5
2947+ fi
2948+ done
2949+
2950+ if [ ${SVN_SUCCESS} -eq 0 ]; then
2951+ echo "Commit to respository for signed packages failed"
2952+ exit 1
2953+ fi
2954+
2955+ exit 1
2956+fi
2957
2958Property changes on: ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python-3.sh
2959___________________________________________________________________
2960Added: svn:executable
2961## -0,0 +1 ##
2962+*
2963\ No newline at end of property
2964Index: ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python-2.sh
2965===================================================================
2966--- ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python-2.sh (nonexistent)
2967+++ ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python-2.sh (revision 27002)
2968@@ -0,0 +1,53 @@
2969+#!/bin/bash
2970+
2971+################################################################################
2972+# This script is intended to test ISSM macOS Python 2 binaries on an end-user
2973+# machine after successful packaging and signing.
2974+#
2975+# NOTE: Tarball must already exist in INSTALL_DIR
2976+################################################################################
2977+
2978+## Constants
2979+#
2980+INSTALL_DIR=.
2981+PKG="ISSM-macOS-Python-2"
2982+PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2002 2003 2004 2005 2006 2007 2008 2010 2011 2021 2051 2052 2053 2084 2085 2090 2101 2424 2425 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
2983+
2984+COMPRESSED_PKG="${PKG}.zip"
2985+
2986+export ISSM_DIR="${INSTALL_DIR}/${PKG}"
2987+export PATH="${PATH}:${ISSM_DIR}/bin:${ISSM_DIR}/scripts"
2988+export PYTHONPATH="${ISSM_DIR}/scripts"
2989+export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
2990+export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
2991+
2992+cd ${INSTALL_DIR}
2993+rm -rf ${PKG}
2994+ditto -xk ${COMPRESSED_PKG} .
2995+cd ${PKG}/test/NightlyRun
2996+
2997+# Run tests, redirecting output to logfile and suppressing output to console
2998+echo "Running tests"
2999+rm python.log 2> /dev/null
3000+./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
3001+
3002+# Check that Python did not exit in error
3003+pythonExitCode=`echo $?`
3004+pythonExitedInError=`grep -E "runme.py: error" python.log | wc -l`
3005+
3006+if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
3007+ echo "----------Python exited in error!----------"
3008+ cat python.log
3009+ echo "-----------End of python.log-----------"
3010+ exit 1
3011+fi
3012+
3013+# Check that all tests passed
3014+numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
3015+
3016+if [[ ${numTestsFailed} -ne 0 ]]; then
3017+ echo "One or more tests FAILED"
3018+ exit 1
3019+else
3020+ echo "All tests PASSED"
3021+fi
3022
3023Property changes on: ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python-2.sh
3024___________________________________________________________________
3025Added: svn:executable
3026## -0,0 +1 ##
3027+*
3028\ No newline at end of property
3029Index: ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python-3.sh
3030===================================================================
3031--- ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python-3.sh (nonexistent)
3032+++ ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python-3.sh (revision 27002)
3033@@ -0,0 +1,53 @@
3034+#!/bin/bash
3035+
3036+################################################################################
3037+# This script is intended to test ISSM macOS Python 3 binaries on an end-user
3038+# machine after successful packaging and signing.
3039+#
3040+# NOTE: Tarball must already exist in INSTALL_DIR
3041+################################################################################
3042+
3043+## Constants
3044+#
3045+INSTALL_DIR=.
3046+PKG="ISSM-macOS-Python-3"
3047+PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2002 2003 2004 2005 2006 2007 2008 2010 2011 2021 2051 2052 2053 2084 2085 2090 2101 2424 2425 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
3048+
3049+COMPRESSED_PKG="${PKG}.zip"
3050+
3051+export ISSM_DIR="${INSTALL_DIR}/${PKG}"
3052+export PATH="${PATH}:${ISSM_DIR}/bin:${ISSM_DIR}/scripts"
3053+export PYTHONPATH="${ISSM_DIR}/scripts"
3054+export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
3055+export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
3056+
3057+cd ${INSTALL_DIR}
3058+rm -rf ${PKG}
3059+ditto -xk ${COMPRESSED_PKG} .
3060+cd ${PKG}/test/NightlyRun
3061+
3062+# Run tests, redirecting output to logfile and suppressing output to console
3063+echo "Running tests"
3064+rm python.log 2> /dev/null
3065+./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
3066+
3067+# Check that Python did not exit in error
3068+pythonExitCode=`echo $?`
3069+pythonExitedInError=`grep -E "runme.py: error" python.log | wc -l`
3070+
3071+if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
3072+ echo "----------Python exited in error!----------"
3073+ cat python.log
3074+ echo "-----------End of python.log-----------"
3075+ exit 1
3076+fi
3077+
3078+# Check that all tests passed
3079+numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
3080+
3081+if [[ ${numTestsFailed} -ne 0 ]]; then
3082+ echo "One or more tests FAILED"
3083+ exit 1
3084+else
3085+ echo "All tests PASSED"
3086+fi
3087
3088Property changes on: ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python-3.sh
3089___________________________________________________________________
3090Added: svn:executable
3091## -0,0 +1 ##
3092+*
3093\ No newline at end of property
3094Index: ../trunk-jpl/packagers/mac/transfer-issm-mac-binaries.sh
3095===================================================================
3096--- ../trunk-jpl/packagers/mac/transfer-issm-mac-binaries.sh (revision 27001)
3097+++ ../trunk-jpl/packagers/mac/transfer-issm-mac-binaries.sh (revision 27002)
3098@@ -1,8 +1,7 @@
3099 #!/bin/bash
3100
3101 ################################################################################
3102-# Transfers ISSM distributable package for macOS with MATLAB API to ISSM Web
3103-# site.
3104+# Transfers ISSM distributable package for macOS to ISSM website.
3105 #
3106 # NOTE:
3107 # - Assumes that the following constants are defined,
3108@@ -15,7 +14,8 @@
3109 #
3110 # See also:
3111 # - packagers/mac/complete-issm-mac-binaries-matlab.sh
3112-# - packagers/mac/complete-issm-mac-binaries-python.sh
3113+# - packagers/mac/complete-issm-mac-binaries-python-2.sh
3114+# - packagers/mac/complete-issm-mac-binaries-python-3.sh
3115 ################################################################################
3116
3117 # Expand aliases within the context of this script
Note: See TracBrowser for help on using the repository browser.