source: issm/oecreview/Archive/26740-27031/ISSM-27001-27002.diff@ 27032

Last change on this file since 27032 was 27032, checked in by Mathieu Morlighem, 3 years ago

CHG: added 26740-27031
File size: 109.5 KB

  • TabularUnified ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python.sh

     
    1 #!/bin/bash
    2 
    3 ################################################################################
    4 # This script is intended to test ISSM macOS MATLAB binaries on an end-user
    5 # machine after successful packaging and signing.
    6 #
    7 # NOTE: Tarball must already exist in INSTALL_DIR
    8 ################################################################################
    9 
    10 ## Constants
    11 #
    12 INSTALL_DIR=.
    13 PKG="ISSM-macOS-Python"
    14 PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2002 2003 2004 2005 2006 2007 2008 2010 2011 2021 2051 2052 2053 2084 2085 2090 2101 2424 2425 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
    15 
    16 COMPRESSED_PKG="${PKG}.zip"
    17 
    18 export ISSM_DIR="${INSTALL_DIR}/${PKG}"
    19 export PATH="${PATH}:${ISSM_DIR}/bin:${ISSM_DIR}/scripts"
    20 export PYTHONPATH="${ISSM_DIR}/scripts"
    21 export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
    22 export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
    23 
    24 cd ${INSTALL_DIR}
    25 rm -rf ${PKG}
    26 ditto -xk ${COMPRESSED_PKG} .
    27 cd ${PKG}/test/NightlyRun
    28 
    29 # Run tests, redirecting output to logfile and suppressing output to console
    30 echo "Running tests"
    31 rm python.log 2> /dev/null
    32 ./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
    33 
    34 # Check that Python did not exit in error
    35 pythonExitCode=`echo $?`
    36 pythonExitedInError=`grep -E "runme.py: error" python.log | wc -l`
    37 
    38 if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
    39         echo "----------Python exited in error!----------"
    40         cat python.log
    41         echo "-----------End of python.log-----------"
    42         exit 1
    43 fi
    44 
    45 # Check that all tests passed
    46 numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
    47 
    48 if [[ ${numTestsFailed} -ne 0 ]]; then
    49         echo "One or more tests FAILED"
    50         exit 1
    51 else
    52         echo "All tests PASSED"
    53 fi

  • TabularUnified ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python.sh 

    Property changes on: ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python.sh
___________________________________________________________________
Deleted: svn:executable
## -1 +0,0 ##
-*
\ No newline at end of property
     
    1 #!/bin/bash
    2 
    3 ################################################################################
    4 # Wrapper script to build, package, send for signing, and transfer to ISSM Web
    5 # site ISSM distributable package for macOS with Python API.
    6 #
    7 # Normally, we would put this directly into the project configuration under
    8 # 'Build' -> 'Excute shell', but becasue it is a bit more involved, it is a
    9 # good idea to version it.
    10 #
    11 # When no failures/errors occur, performs the following:
    12 # - Builds ISSM according to configuration.
    13 # - Packages executables and libraries.
    14 # - Runs test suite against package.
    15 # - Commits compressed package to repository to be signed by JPL Cybersecurity.
    16 # - Retrieves signed package and transmits it to ISSM Web site for
    17 #       distribution.
    18 #
    19 # Options:
    20 # -b/--skipbuild                Skip ISSM compilation.
    21 # -r/--resign                   Skip ISSM compilation and packaging. Use to retrigger
    22 #                                               signing/notarization if it fails but build and package
    23 #                                               are valid.
    24 # -s/--skiptests                Skip ISSM compilation and testing during packaging
    25 #                                               step. Use if packaging fails for some reason but build
    26 #                                               is valid.
    27 # -t/--transferonly             Transfer package to ISSM Web site only. Use if transfer
    28 #                                               fails for some reason to skip building, packaging, and
    29 #                                               signing.
    30 # -u/--unlock                   Remove lock file from signed package repository. Use if
    31 #                                               build is aborted to allow for subsequent fresh build.
    32 #
    33 # Debugging:
    34 # - Relies on a very tight handshake with project on remote JPL Cybersecurity
    35 #       Jenkins server. Debugging may be perfomed locally by running,
    36 #
    37 #               packagers/mac/sign-issm-mac-binaries-python.sh
    38 #
    39 #       with "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" hardcoded to Apple
    40 #       Developer credentials.
    41 # - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can
    42 #       help debug potential SVN issues.
    43 #
    44 # NOTE:
    45 # - Use only *one* of the above options at a time, and make sure it is removed
    46 #       again after a single run.
    47 # - Builds will fail when any of the above options are used on a clean
    48 #       workspace. For example, if 'Source Code Management' -> 'Check-out Strategy'
    49 #       select menu is set to "Always check out a fresh copy".
    50 # - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
    51 #       the 'Bindings' section under a 'Username and password (separated)' binding
    52 #       (requires 'Credentials Binding Plugin') with 'Credentials' select menu set
    53 #       to "jenkins/****** (SVN repository for ISSM binaries)".
    54 ################################################################################
    55 
    56 ## Constants
    57 #
    58 PKG="ISSM-macOS-Python" # Name of directory to copy distributable files to
    59 SIGNED_REPO_COPY="./signed"
    60 SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/signed"
    61 
    62 COMPRESSED_PKG="${PKG}.zip"
    63 
    64 ## Environment
    65 #
    66 export COMPRESSED_PKG
    67 export PKG
    68 export SIGNED_REPO_COPY
    69 export SIGNED_REPO_URL
    70 
    71 ## Parse options
    72 #
    73 if [ $# -gt 1 ]; then
    74         echo "Can use only one option at a time"
    75         exit 1
    76 fi
    77 
    78 # NOTE: We could do this with binary switching (i.e. 0011 to sign and transfer,
    79 #               but the following is self-documenting).
    80 #
    81 build=1
    82 package=1
    83 sign=1
    84 transfer=1
    85 
    86 if [ $# -eq 1 ]; then
    87         case $1 in
    88                 -b|--skipbuild)         build=0;                                                        shift   ;;
    89                 -r|--resign)            build=0;        package=0;                                              ;;
    90                 -s|--skiptests)         build=0;                                                                        ;;
    91                 -t|--transferonly)      build=0;        package=0;      sign=0;                         ;;
    92                 -u|--unlock)            build=0;        package=0;      transfer=0;                     ;;
    93                 *) echo "Unknown parameter passed: $1"; exit 1                                  ;;
    94         esac
    95 fi
    96 
    97 # Build
    98 if [ ${build} -eq 1 ]; then
    99         ./jenkins/jenkins.sh ./jenkins/pine_island-mac-binaries-python
    100 
    101         if [ $? -ne 0 ]; then
    102                 exit 1
    103         fi
    104 fi
    105 
    106 # Package
    107 if [ ${package} -eq 1 ]; then
    108         ./packagers/mac/package-issm-mac-binaries-python.sh $1
    109 
    110         if [ $? -ne 0 ]; then
    111                 exit 1
    112         fi
    113 
    114         shift # Clear $1 so that it is not passed to commit_for_signing script
    115 fi
    116 
    117 # Commit for signing
    118 if [ ${sign} -eq 1 ]; then
    119         ./packagers/mac/commit_for_signing-issm-mac-binaries-python.sh $1
    120 
    121         if [ $? -ne 0 ]; then
    122                 exit 1
    123         fi
    124 fi
    125 
    126 # Transfer distributable package to ISSM Web site
    127 if [ ${transfer} -eq 1 ]; then
    128         ./packagers/mac/transfer-issm-mac-binaries.sh
    129 
    130         if [ $? -ne 0 ]; then
    131                 exit 1
    132         fi
    133 fi
    134 

  • TabularUnified ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh 

    Property changes on: ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python.sh
___________________________________________________________________
Deleted: svn:executable
## -1 +0,0 ##
-*
\ No newline at end of property
     
    1 #!/bin/bash
    2 
    3 ################################################################################
    4 # Packages and tests ISSM distributable package for macOS with Python API.
    5 #
    6 # Options:
    7 # -s/--skiptests                Skip testing during packaging Use if packaging fails
    8 #                                               for some reason but build is valid.
    9 #
    10 # NOTE:
    11 # - Assumes that the following constants are defined,
    12 #
    13 #               COMPRESSED_PKG
    14 #               ISSM_DIR
    15 #               PKG
    16 #
    17 # See also:
    18 # - packagers/mac/complete-issm-mac-binaries-python.sh
    19 # - packagers/mac/sign-issm-mac-binaries-python.sh
    20 ################################################################################
    21 
    22 # Expand aliases within the context of this script
    23 shopt -s expand_aliases
    24 
    25 # From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
    26 #
    27 #       Command line tool support for Subversion — including svn, git-svn, and
    28 #       related commands — is no longer provided by Xcode. (50266910)
    29 #
    30 # which results in,
    31 #
    32 #       svn: error: The subversion command line tools are no longer provided by
    33 #       Xcode.
    34 #
    35 # when calling svn, even when subversion is installed via Homebrew and its path
    36 # is available in PATH.
    37 #
    38 # NOTE: May be able to remove this after updating macOS.
    39 #
    40 #alias svn='/usr/local/bin/svn'
    41 
    42 ## Override certain other aliases
    43 #
    44 alias grep=$(which grep)
    45 
    46 ## Constants
    47 #
    48 PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2002 2003 2004 2005 2006 2007 2008 2010 2011 2021 2051 2052 2053 2084 2085 2090 2101 2424 2425 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
    49 
    50 ## Environment
    51 #
    52 export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'
    53 
    54 ## Parse options
    55 #
    56 if [ $# -gt 1 ]; then
    57         echo "Can use only one option at a time"
    58         exit 1
    59 fi
    60 
    61 skip_tests=0
    62 
    63 if [ $# -eq 1 ]; then
    64         case $1 in
    65                 -s|--skiptests) skip_tests=1;                                   ;;
    66                 *) echo "Unknown parameter passed: $1"; exit 1  ;;
    67         esac
    68 fi
    69 
    70 # Clean up from previous packaging
    71 echo "Cleaning up existing assets"
    72 cd ${ISSM_DIR}
    73 rm -rf ${PKG} ${COMPRESSED_PKG}
    74 mkdir ${PKG}
    75 
    76 # Add required binaries and libraries to package and modify them where needed
    77 cd ${ISSM_DIR}/bin
    78 
    79 echo "Modify generic"
    80 cat generic_static.py | sed -e "s/generic_static/generic/g" > generic.py
    81 
    82 echo "Moving MPICH binaries to bin/"
    83 if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
    84         cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
    85         cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
    86 elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
    87         cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
    88         cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
    89 else
    90         echo "MPICH not found"
    91         exit 1
    92 fi
    93 
    94 echo "Moving GDAL binaries to bin/"
    95 if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
    96         cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
    97         cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
    98 else
    99         echo "GDAL not found"
    100         exit 1
    101 fi
    102 
    103 echo "Moving GMT binaries to bin/"
    104 if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
    105         cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
    106         cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
    107 else
    108         echo "GMT not found"
    109         exit 1
    110 fi
    111 
    112 echo "Moving Gmsh binaries to bin/"
    113 if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
    114         cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
    115 else
    116         echo "Gmsh not found"
    117         exit 1
    118 fi
    119 
    120 # Run tests
    121 if [ ${skip_tests} -eq 0 ]; then
    122         echo "Running tests"
    123         cd ${ISSM_DIR}/test/NightlyRun
    124         rm python.log 2> /dev/null
    125 
    126         # Set Python environment
    127         export PYTHONPATH="${ISSM_DIR}/src/m/dev"
    128         export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
    129         export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
    130 
    131         # Run tests, redirecting output to logfile and suppressing output to console
    132         ./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
    133 
    134         # Check that Python did not exit in error
    135         pythonExitCode=`echo $?`
    136         pythonExitedInError=`grep -E "Error|Standard exception|Traceback|bad interpreter" python.log | wc -l`
    137 
    138         if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
    139                 echo "----------Python exited in error!----------"
    140                 cat python.log
    141                 echo "-----------End of python.log-----------"
    142 
    143                 # Clean up execution directory
    144                 rm -rf ${ISSM_DIR}/execution/*
    145 
    146                 exit 1
    147         fi
    148 
    149         # Check that all tests passed
    150         numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
    151 
    152         if [ ${numTestsFailed} -ne 0 ]; then
    153                 echo "One or more tests FAILED"
    154                 exit 1
    155         else
    156                 echo "All tests PASSED"
    157         fi
    158 else
    159         echo "Skipping tests"
    160 fi
    161 
    162 # Create package
    163 cd ${ISSM_DIR}
    164 svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
    165 echo "Copying assets to package: ${PKG}"
    166 cp -rf bin examples lib scripts test ${PKG}
    167 mkdir ${PKG}/execution
    168 cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist
    169 ${ISSM_DIR}/scripts/py_to_pyc.sh ${PKG}/bin # Compile Python source files
    170 echo "Cleaning up unneeded/unwanted files"
    171 rm -f ${PKG}/bin/*.py # Remove all Python scripts
    172 rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
    173 rm -f ${PKG}/lib/*.a # Remove static libraries from package
    174 rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
    175 rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
    176 
    177 # Compress package
    178 echo "Compressing package"
    179 ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}

  • TabularUnified ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python.sh 

    Property changes on: ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python.sh
___________________________________________________________________
Deleted: svn:executable
## -1 +0,0 ##
-*
\ No newline at end of property
     
    1 #!/bin/bash
    2 
    3 ################################################################################
    4 # Intended to be run in the context of a Jenkins project on a JPL
    5 # Cybersecurity server for signing macOS applications. Polls SCM of the
    6 # Subversion repository hosted at
    7 # https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned to trigger new
    8 # builds.
    9 #
    10 # In order to replicate the requried Jenkins project configuration:
    11 # - First, navigate to 'Manage Jenkins' -> 'Manage Plugins' and install the
    12 #       'Credentials Bindings Plugin' if it is not already installed.
    13 # - Contact one of the members of the ISSM development team for crendentials
    14 #       for the ISSM binaries repository (mention that the credentials are stored
    15 #       in ISSM-Infrastructure.pdf).
    16 # - Navigate to 'Manage Jenkins' -> 'Manage Credentials' -> <domain> ->
    17 #       'Add Credentials' and enter the crendentials from above.
    18 # - From the 'Dashboard', select 'New Item' -> 'Freestyle project'.
    19 # - Under 'Source Code Management', select 'Subversion'.
    20 #               - The 'Repository URL' text field should be set to
    21 #               "https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned".
    22 #               - The 'Credentials' select menu should be set to the new credentials
    23 #               created previously.
    24 #               - The 'Local module directory' text field should be set to the same
    25 #               value as the constant UNSIGNED_REPO_COPY (set below to './unsigned').
    26 # - Under 'Build Trigggers', check the box for 'Poll SCM' and set the
    27 #       'Schedule' text area to "H/5 * * * *".
    28 # - Under 'Build Environment', check the box for 'Use secret text(s) or
    29 #       file(s)', then under 'Bindings' click the 'Add...' button and select
    30 #       'Username and password (separated)'.
    31 #               - Set 'Username Variable' to "ISSM_BINARIES_USER".
    32 #               - Set 'Password Variable' to "ISSM_BINARIES_PASS".
    33 # - Under 'Credentials', select the same, new credentials that created
    34 #       previously.
    35 # - The contents of this script can be copied/pasted directly into the ‘Build'
    36 #       -> 'Execute Shell' -> ‘Command' textarea of the project configuration (or
    37 #       you can simply store the script on disk and call it from there).
    38 # - Make sure to click the 'Save' button.
    39 #
    40 # Current point of contact at JPL Cybersecurity:
    41 #       Alex Coward, alexander.g.coward@jpl.nasa.gov
    42 #
    43 # NOTE:
    44 # - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
    45 #       the 'Bindings' section under a 'Username and password (separated)' binding
    46 #       (requires 'Credentials Binding Plugin').
    47 # - For local debugging, the aformentioned credentials can be hardcoded into
    48 #       the 'USERNAME' and 'PASSWORD' constants below.
    49 ################################################################################
    50 
    51 # Expand aliases within the context of this script
    52 shopt -s expand_aliases
    53 
    54 # From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
    55 #
    56 #       Command line tool support for Subversion — including svn, git-svn, and
    57 #       related commands — is no longer provided by Xcode. (50266910)
    58 #
    59 # which results in,
    60 #
    61 #       svn: error: The subversion command line tools are no longer provided by
    62 #       Xcode.
    63 #
    64 # when calling svn, even when subversion is installed via Homebrew and its path
    65 # is available in PATH.
    66 #
    67 # NOTE: May be able to remove this after updating macOS.
    68 #
    69 #alias svn='/usr/local/bin/svn'
    70 
    71 ## Override certain other aliases
    72 #
    73 alias cp=$(which cp)
    74 alias grep=$(which grep)
    75 
    76 ## Constants
    77 #
    78 AD_IDENTITY="**********" # Apple Developer identity
    79 AD_USERNAME="**********" # Apple Developer username
    80 ALTOOL_PASSWORD="@keychain:**********" # altool password (assumed to be stored in keychain)
    81 ASC_PROVIDER="**********"
    82 MAX_SVN_ATTEMPTS=10
    83 NOTARIZATION_CHECK_ATTEMPTS=20
    84 NOTARIZATION_CHECK_PERIOD=60
    85 NOTARIZATION_LOGFILE="notarization.log"
    86 NOTARIZATION_LOGFILE_PATH="."
    87 PASSWORD=${ISSM_BINARIES_PASS}
    88 PKG="ISSM-macOS-Python"
    89 PRIMARY_BUNDLE_ID="gov.nasa.jpl.issm.python"
    90 SIGNED_REPO_COPY="./signed"
    91 SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/signed"
    92 SIGNING_LOCK_FILE="signing.lock"
    93 SUCCESS_LOGFILE="${SIGNED_REPO_COPY}/success.log"
    94 UNSIGNED_REPO_COPY="./unsigned"
    95 UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned"
    96 USERNAME=${ISSM_BINARIES_USER}
    97 
    98 COMPRESSED_PKG="${PKG}.zip"
    99 EXE_ENTITLEMENTS_PLIST="${PKG}/bin/entitlements.plist"
    100 
    101 # NOTE: Uncomment the following for local testing (Jenkins checks out copy of
    102 #               repository for unsigned packages to working directory)
    103 #
    104 
    105 # # Clean up from previous packaging (not necessary for single builds on Jenkins,
    106 # # but useful when testing packaging locally)
    107 # echo "Cleaning up existing assets"
    108 # rm -rf ${COMPRESSED_PKG} ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${UNSIGNED_REPO_COPY}
    109 
    110 # # Check out copy of repository for unsigned packages
    111 # echo "Checking out copy of respository for unsigned packages"
    112 # svn checkout \
    113 #       --trust-server-cert \
    114 #       --non-interactive \
    115 #       --username ${USERNAME} \
    116 #       --password ${PASSWORD} \
    117 #       ${UNSIGNED_REPO_URL} \
    118 #       ${UNSIGNED_REPO_COPY}
    119 
    120 rm -rf ${PKG} ${SIGNED_REPO_COPY}
    121 
    122 
    123 # Extract package contents
    124 echo "Extracting package contents"
    125 ditto -xk ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} .
    126 
    127 # Clear extended attributes on all files
    128 xattr -cr ${PKG}
    129 
    130 # Build list of ISSM executables
    131 ISSM_BINS=$(\
    132         find ${PKG}/bin -type f -name *.exe; \
    133         find ${PKG}/bin -type f -name *.pyc; \
    134 )
    135 
    136 # Build list of third party executables
    137 THIRD_PARTY_BINS=$(\
    138         echo ${PKG}/bin/mpiexec; \
    139         echo ${PKG}/bin/hydra_pmi_proxy; \
    140         echo ${PKG}/bin/gdalsrsinfo; \
    141         echo ${PKG}/bin/gdaltransform; \
    142         echo ${PKG}/bin/gmt; \
    143         echo ${PKG}/bin/gmtselect; \
    144         echo ${PKG}/bin/gmsh; \
    145 )
    146 
    147 # Sign all executables in package
    148 echo "Signing all executables in package"
    149 codesign -s ${AD_IDENTITY} --timestamp --options=runtime --entitlements ${EXE_ENTITLEMENTS_PLIST} ${ISSM_BINS}
    150 codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${THIRD_PARTY_BINS}
    151 
    152 # Build list of ISSM libraries
    153 ISSM_LIBS=$(\
    154         find ${PKG}/lib -type f -name *.so; \
    155 )
    156 
    157 # Sign all libraries in package
    158 echo "Signing all libraries in package"
    159 codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${ISSM_LIBS}
    160 
    161 # NOTE: Skipping signature validation because this is not a true package nor app
    162 
    163 # Compress signed package
    164 echo "Compressing signed package"
    165 ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
    166 
    167 # Submit compressed package for notarization
    168 echo "Submitting signed package to Apple for notarization"
    169 xcrun altool --notarize-app --primary-bundle-id ${PRIMARY_BUNDLE_ID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} --asc-provider ${ASC_PROVIDER} --file ${COMPRESSED_PKG} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
    170 
    171 # Sleep until notarization request response is received
    172 echo "Waiting for notarizaion request response"
    173 while [[ ! -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} || ! -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; do
    174         sleep 30
    175 done
    176 
    177 echo "Notarizaion request response received"
    178 
    179 # Check if UUID exists in response
    180 HAS_UUID=$(grep 'RequestUUID = ' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}) # NOTE: Checking for "RequestUUID = " because "RequestUUID" shows up in some error messages
    181 if [ -z "${HAS_UUID}" ]; then
    182         echo "Notarization failed!"
    183         echo "----------------------- Contents of notarization logfile -----------------------"
    184         cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
    185         echo "--------------------------------------------------------------------------------"
    186 
    187         # Clean up
    188         rm -rf ${PKG} ${COMPRESSED_PKG}
    189 
    190         exit 1
    191 fi
    192 
    193 # Get UUID from notarization request response
    194 UUID=$(echo ${HAS_UUID} | sed 's/[[:space:]]*RequestUUID = //')
    195 echo "UUID: ${UUID}"
    196 
    197 # Check notarization status
    198 #
    199 # NOTE: Currently, this checks if notarization was successful, but we are not
    200 #               able to staple notarization as this is not a true package nor app and,
    201 #               at the very least, MATLAB Mex files cannot be stapled. As such, clients
    202 #               will not be able to clear Gatekeeper if they are offline.
    203 #
    204 echo "Checking notarization status"
    205 SUCCESS=0
    206 for ATTEMPT in $(seq 1 ${NOTARIZATION_CHECK_ATTEMPTS}); do
    207         echo "    Attempt #${ATTEMPT}..."
    208         xcrun altool --notarization-info ${UUID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
    209         if [[ -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} && -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; then
    210 
    211                 # First, check if there is an error
    212                 ERROR_CHECK=$(grep 'Error' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE})
    213                 if [ ! -z "${ERROR_CHECK}" ]; then
    214                         break
    215                 fi
    216 
    217                 # No error, so check status
    218                 STATUS=$(grep 'Status:' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
    219                 if [[ "${STATUS}" == "success" ]]; then
    220                         # Staple notarization to all elements of package that were previously signed
    221                         #xcrun stapler staple ${THIRD_PARTY_BINS} # NOTE: Fails with "Stapler is incapable of working with MATLAB Mex files."
    222 
    223                         # Validate stapling of notarization
    224                         #xcrun stapler validation ${THIRD_PARTY_BINS} # NOTE: Skipping notarization stapling validation because this is not a true package nor app
    225 
    226                         # Compress signed and notarized package
    227                         ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
    228 
    229                         # Set flag indicating notarization was successful
    230                         SUCCESS=1
    231 
    232                         break
    233                 elif [[ "${STATUS}" == "in progress" ]]; then
    234                         echo "    ...in progress still; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds."
    235                         sleep ${NOTARIZATION_CHECK_PERIOD}
    236                 elif [[ "${STATUS}" == "invalid" ]]; then
    237                         break
    238                 fi
    239         else
    240                 if [ ${ATTEMPT} -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then
    241                         echo "    ...not ready yet; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds"
    242                         sleep ${NOTARIZATION_CHECK_PERIOD}
    243                 else
    244                         echo "    ...maximum attempts reached, but no response, or something else went wrong"
    245                         echo "    If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again"
    246                         break
    247                 fi
    248         fi
    249 done
    250 
    251 if [ ${SUCCESS} -eq 1 ]; then
    252         echo "Notarization successful!"
    253 else
    254         echo "Notarization failed!"
    255         echo "----------------------- Contents of notarization logfile -----------------------"
    256         cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
    257         echo "--------------------------------------------------------------------------------"
    258 fi
    259 
    260 # Check out copy of repository for signed packages
    261 echo "Checking out copy of respository for signed packages"
    262 SVN_ATTEMPT=0
    263 SVN_SUCCESS=0
    264 while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
    265         rm -rf ${SIGNED_REPO_COPY}
    266         svn checkout \
    267                 --trust-server-cert \
    268                 --non-interactive \
    269                 --username ${USERNAME} \
    270                 --password ${PASSWORD} \
    271                 ${SIGNED_REPO_URL} \
    272                 ${SIGNED_REPO_COPY} > /dev/null 2>&1
    273         if [ $? -eq 0 ]; then
    274                 SVN_SUCCESS=1
    275                 break
    276         else
    277                 ((++SVN_ATTEMPT))
    278                 sleep 5
    279         fi
    280 done
    281 
    282 if [ ${SVN_SUCCESS} -eq 0 ]; then
    283         echo "Checkout of respository for signed packages failed"
    284         exit 1
    285 fi
    286 
    287 # Copy notarization file to repository for signed packages
    288 cp ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY}
    289 svn add ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1
    290 
    291 # Remove lock file from repository for signed packages
    292 svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
    293 
    294 SVN_ATTEMPT=0
    295 SVN_SUCCESS=0
    296 if [ ${SUCCESS} -eq 1 ]; then
    297         # Copy signed package to repository for signed packages
    298         cp ${COMPRESSED_PKG} ${SIGNED_REPO_COPY}
    299         svn add ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
    300 
    301         # Commit changes
    302         echo "Committing changes to repository for signed packages"
    303         while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
    304                 svn commit \
    305                         --trust-server-cert \
    306                         --non-interactive \
    307                         --username ${USERNAME} \
    308                         --password ${PASSWORD} \
    309                         --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
    310                 if [ $? -eq 0 ]; then
    311                         SVN_SUCCESS=1
    312                         break
    313                 else
    314                         ((++SVN_ATTEMPT))
    315                         sleep 5
    316                 fi
    317         done
    318 
    319         if [ ${SVN_SUCCESS} -eq 0 ]; then
    320                 echo "Commit to respository for signed packages failed"
    321                 exit 1
    322         fi
    323 else
    324         # Commit changes
    325         echo "Committing changes to repository for signed packages"
    326         while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
    327                 svn commit \
    328                         --trust-server-cert \
    329                         --non-interactive \
    330                         --username ${USERNAME} \
    331                         --password ${PASSWORD} \
    332                         --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
    333                 if [ $? -eq 0 ]; then
    334                         SVN_SUCCESS=1
    335                         break
    336                 else
    337                         ((++SVN_ATTEMPT))
    338                         sleep 5
    339                 fi
    340         done
    341 
    342         if [ ${SVN_SUCCESS} -eq 0 ]; then
    343                 echo "Commit to respository for signed packages failed"
    344                 exit 1
    345         fi
    346 
    347         exit 1
    348 fi

  • TabularUnified ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python.sh 

    Property changes on: ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python.sh
___________________________________________________________________
Deleted: svn:executable
## -1 +0,0 ##
-*
\ No newline at end of property
     
    1 #!/bin/bash
    2 
    3 ################################################################################
    4 # Commits ISSM distributable package for macOS with Python API to repository
    5 # for signing. This repository is polled by a project running on a JPL
    6 # Cybersecurity Jenkins server and performs the actual signing and
    7 # notarization.
    8 #
    9 # Options:
    10 # -r/--resign                   Skip ISSM compilation and packaging. Use to retrigger
    11 #                                               signing/notarization if it fails but build and package
    12 #                                               are valid.
    13 # -u/--unlock                   Remove lock file from signed package repository. Use if
    14 #                                               build is aborted to allow for subsequent fresh build.
    15 #
    16 # NOTE:
    17 # - Assumes that the following constants are defined,
    18 #
    19 #               COMPRESSED_PKG
    20 #               ISSM_BINARIES_REPO_PASS
    21 #               ISSM_BINARIES_REPO_USER
    22 #               SIGNED_REPO_COPY
    23 #               SIGNED_REPO_URL
    24 #
    25 # See also:
    26 # - packagers/mac/complete-issm-mac-binaries-python.sh
    27 # - packagers/mac/sign-issm-mac-binaries-python.sh
    28 #
    29 # TODO:
    30 # - Generalize checkout_*_repo_copy and validate_*_repo_copy functions (e.g.
    31 #       pass 'signed' or 'unsigned' as argument)
    32 ################################################################################
    33 
    34 # Expand aliases within the context of this script
    35 shopt -s expand_aliases
    36 
    37 # From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
    38 #
    39 #       Command line tool support for Subversion — including svn, git-svn, and
    40 #       related commands — is no longer provided by Xcode. (50266910)
    41 #
    42 # which results in,
    43 #
    44 #       svn: error: The subversion command line tools are no longer provided by
    45 #       Xcode.
    46 #
    47 # when calling svn, even when subversion is installed via Homebrew and its path
    48 # is available in PATH.
    49 #
    50 # NOTE: May be able to remove this after updating macOS.
    51 #
    52 #alias svn='/usr/local/bin/svn'
    53 
    54 ## Override certain other aliases
    55 #
    56 alias cp=$(which cp)
    57 alias grep=$(which grep)
    58 
    59 ## Constants
    60 #
    61 MAX_SIGNING_CHECK_ATTEMPTS=30
    62 NOTARIZATION_LOGFILE="notarization.log"
    63 RETRIGGER_SIGNING_FILE="retrigger.txt"
    64 SIGNING_CHECK_PERIOD=60 # in seconds
    65 SIGNING_LOCK_FILE="signing.lock"
    66 UNSIGNED_REPO_COPY="./unsigned"
    67 UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/unsigned"
    68 
    69 ## Functions
    70 #
    71 checkout_signed_repo_copy(){
    72         echo "Checking out copy of repository for signed packages"
    73 
    74         # NOTE: Get empty copy because we do not want to have to check out package
    75         #               from previous signing.
    76         #
    77         svn checkout \
    78                 --trust-server-cert \
    79                 --non-interactive \
    80                 --depth empty \
    81                 --username ${ISSM_BINARIES_REPO_USER} \
    82                 --password ${ISSM_BINARIES_REPO_PASS} \
    83                 ${SIGNED_REPO_URL} \
    84                 ${SIGNED_REPO_COPY} > /dev/null 2>&1
    85 }
    86 checkout_unsigned_repo_copy(){
    87         echo "Checking out copy of repository for unsigned packages"
    88         svn checkout \
    89                 --trust-server-cert \
    90                 --non-interactive \
    91                 --username ${ISSM_BINARIES_REPO_USER} \
    92                 --password ${ISSM_BINARIES_REPO_PASS} \
    93                 ${UNSIGNED_REPO_URL} \
    94                 ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
    95 }
    96 validate_signed_repo_copy(){
    97         # Validate copy of repository for signed binaries (e.g.
    98         # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
    99         # initial checkout failed)
    100         if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then
    101                 rm -rf ${SIGNED_REPO_COPY}
    102                 checkout_signed_repo_copy
    103         fi
    104 }
    105 validate_unsigned_repo_copy(){
    106         # Validate copy of repository for unsigned binaries (e.g.
    107         # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
    108         # initial checkout failed)
    109         if [[ ! -d ${UNSIGNED_REPO_COPY} || ! -d ${UNSIGNED_REPO_COPY}/.svn ]]; then
    110                 rm -rf ${UNSIGNED_REPO_COPY}
    111                 checkout_unsigned_repo_copy
    112         fi
    113 }
    114 
    115 ## Parse options
    116 #
    117 if [ $# -gt 1 ]; then
    118         echo "Can use only one option at a time"
    119         exit 1
    120 fi
    121 
    122 retrigger_signing=0
    123 unlock=0
    124 
    125 if [ $# -eq 1 ]; then
    126         case $1 in
    127                 -r|--resign)    retrigger_signing=1;    ;;
    128                 -u|--unlock)    unlock=1;                               ;;
    129                 *) echo "Unknown parameter passed: $1"; exit 1  ;;
    130         esac
    131 fi
    132 
    133 validate_signed_repo_copy
    134 
    135 if [ ${unlock} -eq 1 ]; then
    136         # Remove signing lock file from signed package repository so that a new
    137         # build can run
    138         echo "Removing lock file from repository for signed packages"
    139         svn update \
    140                 --username ${ISSM_BINARIES_REPO_USER} \
    141                 --password ${ISSM_BINARIES_REPO_PASS} \
    142                 ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
    143         svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
    144         svn commit \
    145                 --trust-server-cert \
    146                 --non-interactive \
    147                 --username ${ISSM_BINARIES_REPO_USER} \
    148                 --password ${ISSM_BINARIES_REPO_PASS} \
    149                 --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} > /dev/null 2>&1
    150         svn cleanup ${SIGNED_REPO_COPY} > /dev/null 2>&1
    151 
    152         echo "Remove -u/--unlock option from configuration and run again"
    153         exit 1
    154 fi
    155 
    156 # If lock file exists, a signing build is still in process by JPL Cybersecurity
    157 svn update \
    158         --username ${ISSM_BINARIES_REPO_USER} \
    159         --password ${ISSM_BINARIES_REPO_PASS} \
    160         ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
    161 
    162 if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
    163         echo "Previous signing job still in process by JPL Cybersecurity. Please try again later."
    164         exit 1
    165 fi
    166 
    167 # Commit lock file to repository for signed packages
    168 echo "Committing lock file to repository for signed packages"
    169 touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
    170 svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
    171 svn commit \
    172         --trust-server-cert \
    173         --non-interactive \
    174         --username ${ISSM_BINARIES_REPO_USER} \
    175         --password ${ISSM_BINARIES_REPO_PASS} \
    176         --message "ADD: New lock file" ${SIGNED_REPO_COPY} > /dev/null 2>&1
    177 
    178 # Check out copy of repository for unsigned packages
    179 validate_unsigned_repo_copy
    180 
    181 if [ ${retrigger_signing} -eq 0 ]; then
    182         # Commit new compressed package to repository for unsigned binaries
    183         echo "Committing package to repository for unsigned packages"
    184         cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
    185         svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
    186         svn commit \
    187                 --trust-server-cert \
    188                 --non-interactive \
    189                 --username ${ISSM_BINARIES_REPO_USER} \
    190                 --password ${ISSM_BINARIES_REPO_PASS} \
    191                 --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
    192 else
    193         # NOTE: If notarize_only == 1, we commit a dummy file as we do not want to
    194         #               have to commit the entire compressed package again simply to
    195         #               retrigger the signing build on the remote JPL Cybersecurity Jenkins
    196         #               server.
    197         #
    198         echo "Attempting to sign existing package again"
    199         echo $(date +'%Y-%m-%d-%H-%M-%S') > ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} # Write datetime stamp to file to ensure modification is made
    200         svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1
    201         svn commit \
    202                 --trust-server-cert \
    203                 --non-interactive \
    204                 --username ${ISSM_BINARIES_REPO_USER} \
    205                 --password ${ISSM_BINARIES_REPO_PASS} \
    206                 --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
    207 fi
    208 
    209 # Check status of signing
    210 echo "Checking progress of signing..."
    211 SIGNING_CHECK_ATTEMPT=0
    212 while [ ${SIGNING_CHECK_ATTEMPT} -lt ${MAX_SIGNING_CHECK_ATTEMPTS} ]; do
    213         echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds"
    214         sleep ${SIGNING_CHECK_PERIOD}
    215         svn update \
    216                 --username ${ISSM_BINARIES_REPO_USER} \
    217                 --password ${ISSM_BINARIES_REPO_PASS} \
    218                 ${SIGNED_REPO_COPY} > /dev/null 2>&1
    219 
    220         if [ ! -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
    221                 # Retrieve notarization lock file
    222                 svn update \
    223                         --username ${ISSM_BINARIES_REPO_USER} \
    224                         --password ${ISSM_BINARIES_REPO_PASS} \
    225                         ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
    226 
    227                 # Check status
    228                 STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
    229                 if [[ "${STATUS}" == "success" ]]; then
    230                         echo "Notarization successful!"
    231                         break
    232                 else
    233                         echo "Notarization failed!"
    234                         echo "----------------------- Contents of notarization logfile -----------------------"
    235                         cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
    236                         echo "--------------------------------------------------------------------------------"
    237 
    238                         exit 1
    239                 fi
    240         else
    241                 ((++SIGNING_CHECK_ATTEMPT))
    242         fi
    243 done
    244 
    245 if [ ! -f ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} ]; then
    246         echo "Signing timed out!"
    247         exit 1
    248 fi

  • TabularUnified ../trunk-jpl/packagers/linux/transfer-issm-linux-binaries.sh 

    Property changes on: ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python.sh
___________________________________________________________________
Deleted: svn:executable
## -1 +0,0 ##
-*
\ No newline at end of property
     
    11#!/bin/bash
    22
    33################################################################################
    4 # Transfers ISSM distributable package for Linux with MATLAB API to ISSM Web
    5 # site.
     4# Transfers ISSM distributable package for Linux to ISSM website.
    65#
    76# NOTE:
    87# - Assumes that the following constants are defined,
    … …  
    1110#
    1211# See also:
    1312# - packagers/linux/complete-issm-linux-binaries-matlab.sh
    14 # - packagers/linux/complete-issm-linux-binaries-python.sh
     13# - packagers/linux/complete-issm-linux-binaries-python-2.sh
     14# - packagers/linux/complete-issm-linux-binaries-python-3.sh
    1515################################################################################
    1616
    1717# Transfer package to ISSM Web site

  • TabularUnified ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python-2.sh

     
     1#!/bin/bash
     2
     3################################################################################
     4# Commits ISSM distributable package for macOS with Python 2 API to repository
     5# for signing. This repository is polled by a project running on a JPL
     6# Cybersecurity Jenkins server and performs the actual signing and
     7# notarization.
     8#
     9# Options:
     10# -r/--resign                   Skip ISSM compilation and packaging. Use to retrigger
     11#                                               signing/notarization if it fails but build and package
     12#                                               are valid.
     13# -u/--unlock                   Remove lock file from signed package repository. Use if
     14#                                               build is aborted to allow for subsequent fresh build.
     15#
     16# NOTE:
     17# - Assumes that the following constants are defined,
     18#
     19#               COMPRESSED_PKG
     20#               ISSM_BINARIES_REPO_PASS
     21#               ISSM_BINARIES_REPO_USER
     22#               SIGNED_REPO_COPY
     23#               SIGNED_REPO_URL
     24#
     25# See also:
     26# - packagers/mac/complete-issm-mac-binaries-python-2.sh
     27# - packagers/mac/sign-issm-mac-binaries-python-2.sh
     28#
     29# TODO:
     30# - Generalize checkout_*_repo_copy and validate_*_repo_copy functions (e.g.
     31#       pass 'signed' or 'unsigned' as argument)
     32################################################################################
     33
     34# Expand aliases within the context of this script
     35shopt -s expand_aliases
     36
     37# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
     38#
     39#       Command line tool support for Subversion — including svn, git-svn, and
     40#       related commands — is no longer provided by Xcode. (50266910)
     41#
     42# which results in,
     43#
     44#       svn: error: The subversion command line tools are no longer provided by
     45#       Xcode.
     46#
     47# when calling svn, even when subversion is installed via Homebrew and its path
     48# is available in PATH.
     49#
     50# NOTE: May be able to remove this after updating macOS.
     51#
     52#alias svn='/usr/local/bin/svn'
     53
     54## Override certain other aliases
     55#
     56alias cp=$(which cp)
     57alias grep=$(which grep)
     58
     59## Constants
     60#
     61MAX_SIGNING_CHECK_ATTEMPTS=30
     62NOTARIZATION_LOGFILE="notarization.log"
     63RETRIGGER_SIGNING_FILE="retrigger.txt"
     64SIGNING_CHECK_PERIOD=60 # in seconds
     65SIGNING_LOCK_FILE="signing.lock"
     66UNSIGNED_REPO_COPY="./unsigned"
     67UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/2/unsigned"
     68
     69## Functions
     70#
     71checkout_signed_repo_copy(){
     72        echo "Checking out copy of repository for signed packages"
     73
     74        # NOTE: Get empty copy because we do not want to have to check out package
     75        #               from previous signing.
     76        #
     77        svn checkout \
     78                --trust-server-cert \
     79                --non-interactive \
     80                --depth empty \
     81                --username ${ISSM_BINARIES_REPO_USER} \
     82                --password ${ISSM_BINARIES_REPO_PASS} \
     83                ${SIGNED_REPO_URL} \
     84                ${SIGNED_REPO_COPY} > /dev/null 2>&1
     85}
     86checkout_unsigned_repo_copy(){
     87        echo "Checking out copy of repository for unsigned packages"
     88        svn checkout \
     89                --trust-server-cert \
     90                --non-interactive \
     91                --username ${ISSM_BINARIES_REPO_USER} \
     92                --password ${ISSM_BINARIES_REPO_PASS} \
     93                ${UNSIGNED_REPO_URL} \
     94                ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     95}
     96validate_signed_repo_copy(){
     97        # Validate copy of repository for signed binaries (e.g.
     98        # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
     99        # initial checkout failed)
     100        if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then
     101                rm -rf ${SIGNED_REPO_COPY}
     102                checkout_signed_repo_copy
     103        fi
     104}
     105validate_unsigned_repo_copy(){
     106        # Validate copy of repository for unsigned binaries (e.g.
     107        # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
     108        # initial checkout failed)
     109        if [[ ! -d ${UNSIGNED_REPO_COPY} || ! -d ${UNSIGNED_REPO_COPY}/.svn ]]; then
     110                rm -rf ${UNSIGNED_REPO_COPY}
     111                checkout_unsigned_repo_copy
     112        fi
     113}
     114
     115## Parse options
     116#
     117if [ $# -gt 1 ]; then
     118        echo "Can use only one option at a time"
     119        exit 1
     120fi
     121
     122retrigger_signing=0
     123unlock=0
     124
     125if [ $# -eq 1 ]; then
     126        case $1 in
     127                -r|--resign)    retrigger_signing=1;    ;;
     128                -u|--unlock)    unlock=1;                               ;;
     129                *) echo "Unknown parameter passed: $1"; exit 1  ;;
     130        esac
     131fi
     132
     133validate_signed_repo_copy
     134
     135if [ ${unlock} -eq 1 ]; then
     136        # Remove signing lock file from signed package repository so that a new
     137        # build can run
     138        echo "Removing lock file from repository for signed packages"
     139        svn update \
     140                --username ${ISSM_BINARIES_REPO_USER} \
     141                --password ${ISSM_BINARIES_REPO_PASS} \
     142                ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
     143        svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
     144        svn commit \
     145                --trust-server-cert \
     146                --non-interactive \
     147                --username ${ISSM_BINARIES_REPO_USER} \
     148                --password ${ISSM_BINARIES_REPO_PASS} \
     149                --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} > /dev/null 2>&1
     150        svn cleanup ${SIGNED_REPO_COPY} > /dev/null 2>&1
     151
     152        echo "Remove -u/--unlock option from configuration and run again"
     153        exit 1
     154fi
     155
     156# If lock file exists, a signing build is still in process by JPL Cybersecurity
     157svn update \
     158        --username ${ISSM_BINARIES_REPO_USER} \
     159        --password ${ISSM_BINARIES_REPO_PASS} \
     160        ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
     161
     162if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
     163        echo "Previous signing job still in process by JPL Cybersecurity. Please try again later."
     164        exit 1
     165fi
     166
     167# Commit lock file to repository for signed packages
     168echo "Committing lock file to repository for signed packages"
     169touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
     170svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
     171svn commit \
     172        --trust-server-cert \
     173        --non-interactive \
     174        --username ${ISSM_BINARIES_REPO_USER} \
     175        --password ${ISSM_BINARIES_REPO_PASS} \
     176        --message "ADD: New lock file" ${SIGNED_REPO_COPY} > /dev/null 2>&1
     177
     178# Check out copy of repository for unsigned packages
     179validate_unsigned_repo_copy
     180
     181if [ ${retrigger_signing} -eq 0 ]; then
     182        # Commit new compressed package to repository for unsigned binaries
     183        echo "Committing package to repository for unsigned packages"
     184        cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
     185        svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
     186        svn commit \
     187                --trust-server-cert \
     188                --non-interactive \
     189                --username ${ISSM_BINARIES_REPO_USER} \
     190                --password ${ISSM_BINARIES_REPO_PASS} \
     191                --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     192else
     193        # NOTE: If notarize_only == 1, we commit a dummy file as we do not want to
     194        #               have to commit the entire compressed package again simply to
     195        #               retrigger the signing build on the remote JPL Cybersecurity Jenkins
     196        #               server.
     197        #
     198        echo "Attempting to sign existing package again"
     199        echo $(date +'%Y-%m-%d-%H-%M-%S') > ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} # Write datetime stamp to file to ensure modification is made
     200        svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1
     201        svn commit \
     202                --trust-server-cert \
     203                --non-interactive \
     204                --username ${ISSM_BINARIES_REPO_USER} \
     205                --password ${ISSM_BINARIES_REPO_PASS} \
     206                --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     207fi
     208
     209# Check status of signing
     210echo "Checking progress of signing..."
     211SIGNING_CHECK_ATTEMPT=0
     212while [ ${SIGNING_CHECK_ATTEMPT} -lt ${MAX_SIGNING_CHECK_ATTEMPTS} ]; do
     213        echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds"
     214        sleep ${SIGNING_CHECK_PERIOD}
     215        svn update \
     216                --username ${ISSM_BINARIES_REPO_USER} \
     217                --password ${ISSM_BINARIES_REPO_PASS} \
     218                ${SIGNED_REPO_COPY} > /dev/null 2>&1
     219
     220        if [ ! -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
     221                # Retrieve notarization lock file
     222                svn update \
     223                        --username ${ISSM_BINARIES_REPO_USER} \
     224                        --password ${ISSM_BINARIES_REPO_PASS} \
     225                        ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
     226
     227                # Check status
     228                STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
     229                if [[ "${STATUS}" == "success" ]]; then
     230                        echo "Notarization successful!"
     231                        break
     232                else
     233                        echo "Notarization failed!"
     234                        echo "----------------------- Contents of notarization logfile -----------------------"
     235                        cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
     236                        echo "--------------------------------------------------------------------------------"
     237
     238                        exit 1
     239                fi
     240        else
     241                ((++SIGNING_CHECK_ATTEMPT))
     242        fi
     243done
     244
     245if [ ! -f ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} ]; then
     246        echo "Signing timed out!"
     247        exit 1
     248fi

  • TabularUnified ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python-3.sh 

    Property changes on: ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python-2.sh
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
     
     1#!/bin/bash
     2
     3################################################################################
     4# Commits ISSM distributable package for macOS with Python 3 API to repository
     5# for signing. This repository is polled by a project running on a JPL
     6# Cybersecurity Jenkins server and performs the actual signing and
     7# notarization.
     8#
     9# Options:
     10# -r/--resign                   Skip ISSM compilation and packaging. Use to retrigger
     11#                                               signing/notarization if it fails but build and package
     12#                                               are valid.
     13# -u/--unlock                   Remove lock file from signed package repository. Use if
     14#                                               build is aborted to allow for subsequent fresh build.
     15#
     16# NOTE:
     17# - Assumes that the following constants are defined,
     18#
     19#               COMPRESSED_PKG
     20#               ISSM_BINARIES_REPO_PASS
     21#               ISSM_BINARIES_REPO_USER
     22#               SIGNED_REPO_COPY
     23#               SIGNED_REPO_URL
     24#
     25# See also:
     26# - packagers/mac/complete-issm-mac-binaries-python-3.sh
     27# - packagers/mac/sign-issm-mac-binaries-python-3.sh
     28#
     29# TODO:
     30# - Generalize checkout_*_repo_copy and validate_*_repo_copy functions (e.g.
     31#       pass 'signed' or 'unsigned' as argument)
     32################################################################################
     33
     34# Expand aliases within the context of this script
     35shopt -s expand_aliases
     36
     37# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
     38#
     39#       Command line tool support for Subversion — including svn, git-svn, and
     40#       related commands — is no longer provided by Xcode. (50266910)
     41#
     42# which results in,
     43#
     44#       svn: error: The subversion command line tools are no longer provided by
     45#       Xcode.
     46#
     47# when calling svn, even when subversion is installed via Homebrew and its path
     48# is available in PATH.
     49#
     50# NOTE: May be able to remove this after updating macOS.
     51#
     52#alias svn='/usr/local/bin/svn'
     53
     54## Override certain other aliases
     55#
     56alias cp=$(which cp)
     57alias grep=$(which grep)
     58
     59## Constants
     60#
     61MAX_SIGNING_CHECK_ATTEMPTS=30
     62NOTARIZATION_LOGFILE="notarization.log"
     63RETRIGGER_SIGNING_FILE="retrigger.txt"
     64SIGNING_CHECK_PERIOD=60 # in seconds
     65SIGNING_LOCK_FILE="signing.lock"
     66UNSIGNED_REPO_COPY="./unsigned"
     67UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/3/unsigned"
     68
     69## Functions
     70#
     71checkout_signed_repo_copy(){
     72        echo "Checking out copy of repository for signed packages"
     73
     74        # NOTE: Get empty copy because we do not want to have to check out package
     75        #               from previous signing.
     76        #
     77        svn checkout \
     78                --trust-server-cert \
     79                --non-interactive \
     80                --depth empty \
     81                --username ${ISSM_BINARIES_REPO_USER} \
     82                --password ${ISSM_BINARIES_REPO_PASS} \
     83                ${SIGNED_REPO_URL} \
     84                ${SIGNED_REPO_COPY} > /dev/null 2>&1
     85}
     86checkout_unsigned_repo_copy(){
     87        echo "Checking out copy of repository for unsigned packages"
     88        svn checkout \
     89                --trust-server-cert \
     90                --non-interactive \
     91                --username ${ISSM_BINARIES_REPO_USER} \
     92                --password ${ISSM_BINARIES_REPO_PASS} \
     93                ${UNSIGNED_REPO_URL} \
     94                ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     95}
     96validate_signed_repo_copy(){
     97        # Validate copy of repository for signed binaries (e.g.
     98        # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
     99        # initial checkout failed)
     100        if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then
     101                rm -rf ${SIGNED_REPO_COPY}
     102                checkout_signed_repo_copy
     103        fi
     104}
     105validate_unsigned_repo_copy(){
     106        # Validate copy of repository for unsigned binaries (e.g.
     107        # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
     108        # initial checkout failed)
     109        if [[ ! -d ${UNSIGNED_REPO_COPY} || ! -d ${UNSIGNED_REPO_COPY}/.svn ]]; then
     110                rm -rf ${UNSIGNED_REPO_COPY}
     111                checkout_unsigned_repo_copy
     112        fi
     113}
     114
     115## Parse options
     116#
     117if [ $# -gt 1 ]; then
     118        echo "Can use only one option at a time"
     119        exit 1
     120fi
     121
     122retrigger_signing=0
     123unlock=0
     124
     125if [ $# -eq 1 ]; then
     126        case $1 in
     127                -r|--resign)    retrigger_signing=1;    ;;
     128                -u|--unlock)    unlock=1;                               ;;
     129                *) echo "Unknown parameter passed: $1"; exit 1  ;;
     130        esac
     131fi
     132
     133validate_signed_repo_copy
     134
     135if [ ${unlock} -eq 1 ]; then
     136        # Remove signing lock file from signed package repository so that a new
     137        # build can run
     138        echo "Removing lock file from repository for signed packages"
     139        svn update \
     140                --username ${ISSM_BINARIES_REPO_USER} \
     141                --password ${ISSM_BINARIES_REPO_PASS} \
     142                ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
     143        svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
     144        svn commit \
     145                --trust-server-cert \
     146                --non-interactive \
     147                --username ${ISSM_BINARIES_REPO_USER} \
     148                --password ${ISSM_BINARIES_REPO_PASS} \
     149                --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY} > /dev/null 2>&1
     150        svn cleanup ${SIGNED_REPO_COPY} > /dev/null 2>&1
     151
     152        echo "Remove -u/--unlock option from configuration and run again"
     153        exit 1
     154fi
     155
     156# If lock file exists, a signing build is still in process by JPL Cybersecurity
     157svn update \
     158        --username ${ISSM_BINARIES_REPO_USER} \
     159        --password ${ISSM_BINARIES_REPO_PASS} \
     160        ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
     161
     162if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
     163        echo "Previous signing job still in process by JPL Cybersecurity. Please try again later."
     164        exit 1
     165fi
     166
     167# Commit lock file to repository for signed packages
     168echo "Committing lock file to repository for signed packages"
     169touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
     170svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
     171svn commit \
     172        --trust-server-cert \
     173        --non-interactive \
     174        --username ${ISSM_BINARIES_REPO_USER} \
     175        --password ${ISSM_BINARIES_REPO_PASS} \
     176        --message "ADD: New lock file" ${SIGNED_REPO_COPY} > /dev/null 2>&1
     177
     178# Check out copy of repository for unsigned packages
     179validate_unsigned_repo_copy
     180
     181if [ ${retrigger_signing} -eq 0 ]; then
     182        # Commit new compressed package to repository for unsigned binaries
     183        echo "Committing package to repository for unsigned packages"
     184        cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
     185        svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
     186        svn commit \
     187                --trust-server-cert \
     188                --non-interactive \
     189                --username ${ISSM_BINARIES_REPO_USER} \
     190                --password ${ISSM_BINARIES_REPO_PASS} \
     191                --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     192else
     193        # NOTE: If notarize_only == 1, we commit a dummy file as we do not want to
     194        #               have to commit the entire compressed package again simply to
     195        #               retrigger the signing build on the remote JPL Cybersecurity Jenkins
     196        #               server.
     197        #
     198        echo "Attempting to sign existing package again"
     199        echo $(date +'%Y-%m-%d-%H-%M-%S') > ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} # Write datetime stamp to file to ensure modification is made
     200        svn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1
     201        svn commit \
     202                --trust-server-cert \
     203                --non-interactive \
     204                --username ${ISSM_BINARIES_REPO_USER} \
     205                --password ${ISSM_BINARIES_REPO_PASS} \
     206                --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
     207fi
     208
     209# Check status of signing
     210echo "Checking progress of signing..."
     211SIGNING_CHECK_ATTEMPT=0
     212while [ ${SIGNING_CHECK_ATTEMPT} -lt ${MAX_SIGNING_CHECK_ATTEMPTS} ]; do
     213        echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds"
     214        sleep ${SIGNING_CHECK_PERIOD}
     215        svn update \
     216                --username ${ISSM_BINARIES_REPO_USER} \
     217                --password ${ISSM_BINARIES_REPO_PASS} \
     218                ${SIGNED_REPO_COPY} > /dev/null 2>&1
     219
     220        if [ ! -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
     221                # Retrieve notarization lock file
     222                svn update \
     223                        --username ${ISSM_BINARIES_REPO_USER} \
     224                        --password ${ISSM_BINARIES_REPO_PASS} \
     225                        ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
     226
     227                # Check status
     228                STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
     229                if [[ "${STATUS}" == "success" ]]; then
     230                        echo "Notarization successful!"
     231                        break
     232                else
     233                        echo "Notarization failed!"
     234                        echo "----------------------- Contents of notarization logfile -----------------------"
     235                        cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
     236                        echo "--------------------------------------------------------------------------------"
     237
     238                        exit 1
     239                fi
     240        else
     241                ((++SIGNING_CHECK_ATTEMPT))
     242        fi
     243done
     244
     245if [ ! -f ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} ]; then
     246        echo "Signing timed out!"
     247        exit 1
     248fi

  • TabularUnified ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python-2.sh 

    Property changes on: ../trunk-jpl/packagers/mac/commit_for_signing-issm-mac-binaries-python-3.sh
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
     
     1#!/bin/bash
     2
     3################################################################################
     4# Wrapper script to build, package, send for signing, and transfer to ISSM Web
     5# site ISSM distributable package for macOS with Python 2 API.
     6#
     7# Normally, we would put this directly into the project configuration under
     8# 'Build' -> 'Excute shell', but becasue it is a bit more involved, it is a
     9# good idea to version it.
     10#
     11# When no failures/errors occur, performs the following:
     12# - Builds ISSM according to configuration.
     13# - Packages executables and libraries.
     14# - Runs test suite against package.
     15# - Commits compressed package to repository to be signed by JPL Cybersecurity.
     16# - Retrieves signed package and transmits it to ISSM Web site for
     17#       distribution.
     18#
     19# Options:
     20# -b/--skipbuild                Skip ISSM compilation.
     21# -r/--resign                   Skip ISSM compilation and packaging. Use to retrigger
     22#                                               signing/notarization if it fails but build and package
     23#                                               are valid.
     24# -s/--skiptests                Skip ISSM compilation and testing during packaging
     25#                                               step. Use if packaging fails for some reason but build
     26#                                               is valid.
     27# -t/--transferonly             Transfer package to ISSM Web site only. Use if transfer
     28#                                               fails for some reason to skip building, packaging, and
     29#                                               signing.
     30# -u/--unlock                   Remove lock file from signed package repository. Use if
     31#                                               build is aborted to allow for subsequent fresh build.
     32#
     33# Debugging:
     34# - Relies on a very tight handshake with project on remote JPL Cybersecurity
     35#       Jenkins server. Debugging may be perfomed locally by running,
     36#
     37#               packagers/mac/sign-issm-mac-binaries-python-2.sh
     38#
     39#       with "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" hardcoded to Apple
     40#       Developer credentials.
     41# - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can
     42#       help debug potential SVN issues.
     43#
     44# NOTE:
     45# - Use only *one* of the above options at a time, and make sure it is removed
     46#       again after a single run.
     47# - Builds will fail when any of the above options are used on a clean
     48#       workspace. For example, if 'Source Code Management' -> 'Check-out Strategy'
     49#       select menu is set to "Always check out a fresh copy".
     50# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
     51#       the 'Bindings' section under a 'Username and password (separated)' binding
     52#       (requires 'Credentials Binding Plugin') with 'Credentials' select menu set
     53#       to "jenkins/****** (SVN repository for ISSM binaries)".
     54################################################################################
     55
     56## Constants
     57#
     58PKG="ISSM-macOS-Python-2" # Name of directory to copy distributable files to
     59SIGNED_REPO_COPY="./signed"
     60SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/2/signed"
     61
     62COMPRESSED_PKG="${PKG}.zip"
     63
     64## Environment
     65#
     66export COMPRESSED_PKG
     67export PKG
     68export SIGNED_REPO_COPY
     69export SIGNED_REPO_URL
     70
     71## Parse options
     72#
     73if [ $# -gt 1 ]; then
     74        echo "Can use only one option at a time"
     75        exit 1
     76fi
     77
     78# NOTE: We could do this with binary switching (i.e. 0011 to sign and transfer,
     79#               but the following is self-documenting).
     80#
     81build=1
     82package=1
     83sign=1
     84transfer=1
     85
     86if [ $# -eq 1 ]; then
     87        case $1 in
     88                -b|--skipbuild)         build=0;                                                        shift   ;;
     89                -r|--resign)            build=0;        package=0;                                              ;;
     90                -s|--skiptests)         build=0;                                                                        ;;
     91                -t|--transferonly)      build=0;        package=0;      sign=0;                         ;;
     92                -u|--unlock)            build=0;        package=0;      transfer=0;                     ;;
     93                *) echo "Unknown parameter passed: $1"; exit 1                                  ;;
     94        esac
     95fi
     96
     97# Build
     98if [ ${build} -eq 1 ]; then
     99        ./jenkins/jenkins.sh ./jenkins/pine_island-mac-binaries-python-2
     100
     101        if [ $? -ne 0 ]; then
     102                exit 1
     103        fi
     104fi
     105
     106# Package
     107if [ ${package} -eq 1 ]; then
     108        ./packagers/mac/package-issm-mac-binaries-python-2.sh $1
     109
     110        if [ $? -ne 0 ]; then
     111                exit 1
     112        fi
     113
     114        shift # Clear $1 so that it is not passed to commit_for_signing script
     115fi
     116
     117# Commit for signing
     118if [ ${sign} -eq 1 ]; then
     119        ./packagers/mac/commit_for_signing-issm-mac-binaries-python-2.sh $1
     120
     121        if [ $? -ne 0 ]; then
     122                exit 1
     123        fi
     124fi
     125
     126# Transfer distributable package to ISSM Web site
     127if [ ${transfer} -eq 1 ]; then
     128        ./packagers/mac/transfer-issm-mac-binaries.sh
     129
     130        if [ $? -ne 0 ]; then
     131                exit 1
     132        fi
     133fi
     134

  • TabularUnified ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python-3.sh 

    Property changes on: ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python-2.sh
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
     
     1#!/bin/bash
     2
     3################################################################################
     4# Wrapper script to build, package, send for signing, and transfer to ISSM Web
     5# site ISSM distributable package for macOS with Python 3 API.
     6#
     7# Normally, we would put this directly into the project configuration under
     8# 'Build' -> 'Excute shell', but becasue it is a bit more involved, it is a
     9# good idea to version it.
     10#
     11# When no failures/errors occur, performs the following:
     12# - Builds ISSM according to configuration.
     13# - Packages executables and libraries.
     14# - Runs test suite against package.
     15# - Commits compressed package to repository to be signed by JPL Cybersecurity.
     16# - Retrieves signed package and transmits it to ISSM Web site for
     17#       distribution.
     18#
     19# Options:
     20# -b/--skipbuild                Skip ISSM compilation.
     21# -r/--resign                   Skip ISSM compilation and packaging. Use to retrigger
     22#                                               signing/notarization if it fails but build and package
     23#                                               are valid.
     24# -s/--skiptests                Skip ISSM compilation and testing during packaging
     25#                                               step. Use if packaging fails for some reason but build
     26#                                               is valid.
     27# -t/--transferonly             Transfer package to ISSM Web site only. Use if transfer
     28#                                               fails for some reason to skip building, packaging, and
     29#                                               signing.
     30# -u/--unlock                   Remove lock file from signed package repository. Use if
     31#                                               build is aborted to allow for subsequent fresh build.
     32#
     33# Debugging:
     34# - Relies on a very tight handshake with project on remote JPL Cybersecurity
     35#       Jenkins server. Debugging may be perfomed locally by running,
     36#
     37#               packagers/mac/sign-issm-mac-binaries-python-3.sh
     38#
     39#       with "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" hardcoded to Apple
     40#       Developer credentials.
     41# - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can
     42#       help debug potential SVN issues.
     43#
     44# NOTE:
     45# - Use only *one* of the above options at a time, and make sure it is removed
     46#       again after a single run.
     47# - Builds will fail when any of the above options are used on a clean
     48#       workspace. For example, if 'Source Code Management' -> 'Check-out Strategy'
     49#       select menu is set to "Always check out a fresh copy".
     50# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
     51#       the 'Bindings' section under a 'Username and password (separated)' binding
     52#       (requires 'Credentials Binding Plugin') with 'Credentials' select menu set
     53#       to "jenkins/****** (SVN repository for ISSM binaries)".
     54################################################################################
     55
     56## Constants
     57#
     58PKG="ISSM-macOS-Python-3" # Name of directory to copy distributable files to
     59SIGNED_REPO_COPY="./signed"
     60SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/3/signed"
     61
     62COMPRESSED_PKG="${PKG}.zip"
     63
     64## Environment
     65#
     66export COMPRESSED_PKG
     67export PKG
     68export SIGNED_REPO_COPY
     69export SIGNED_REPO_URL
     70
     71## Parse options
     72#
     73if [ $# -gt 1 ]; then
     74        echo "Can use only one option at a time"
     75        exit 1
     76fi
     77
     78# NOTE: We could do this with binary switching (i.e. 0011 to sign and transfer,
     79#               but the following is self-documenting).
     80#
     81build=1
     82package=1
     83sign=1
     84transfer=1
     85
     86if [ $# -eq 1 ]; then
     87        case $1 in
     88                -b|--skipbuild)         build=0;                                                        shift   ;;
     89                -r|--resign)            build=0;        package=0;                                              ;;
     90                -s|--skiptests)         build=0;                                                                        ;;
     91                -t|--transferonly)      build=0;        package=0;      sign=0;                         ;;
     92                -u|--unlock)            build=0;        package=0;      transfer=0;                     ;;
     93                *) echo "Unknown parameter passed: $1"; exit 1                                  ;;
     94        esac
     95fi
     96
     97# Build
     98if [ ${build} -eq 1 ]; then
     99        ./jenkins/jenkins.sh ./jenkins/pine_island-mac-binaries-python-3
     100
     101        if [ $? -ne 0 ]; then
     102                exit 1
     103        fi
     104fi
     105
     106# Package
     107if [ ${package} -eq 1 ]; then
     108        ./packagers/mac/package-issm-mac-binaries-python-3.sh $1
     109
     110        if [ $? -ne 0 ]; then
     111                exit 1
     112        fi
     113
     114        shift # Clear $1 so that it is not passed to commit_for_signing script
     115fi
     116
     117# Commit for signing
     118if [ ${sign} -eq 1 ]; then
     119        ./packagers/mac/commit_for_signing-issm-mac-binaries-python-3.sh $1
     120
     121        if [ $? -ne 0 ]; then
     122                exit 1
     123        fi
     124fi
     125
     126# Transfer distributable package to ISSM Web site
     127if [ ${transfer} -eq 1 ]; then
     128        ./packagers/mac/transfer-issm-mac-binaries.sh
     129
     130        if [ $? -ne 0 ]; then
     131                exit 1
     132        fi
     133fi
     134

  • TabularUnified ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python-2.sh 

    Property changes on: ../trunk-jpl/packagers/mac/complete-issm-mac-binaries-python-3.sh
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
     
     1#!/bin/bash
     2
     3################################################################################
     4# Packages and tests ISSM distributable package for macOS with Python 2 API.
     5#
     6# Options:
     7# -s/--skiptests                Skip testing during packaging Use if packaging fails
     8#                                               for some reason but build is valid.
     9#
     10# NOTE:
     11# - Assumes that the following constants are defined,
     12#
     13#               COMPRESSED_PKG
     14#               ISSM_DIR
     15#               PKG
     16#
     17# See also:
     18# - packagers/mac/complete-issm-mac-binaries-python-2.sh
     19# - packagers/mac/sign-issm-mac-binaries-python-2.sh
     20################################################################################
     21
     22# Expand aliases within the context of this script
     23shopt -s expand_aliases
     24
     25# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
     26#
     27#       Command line tool support for Subversion — including svn, git-svn, and
     28#       related commands — is no longer provided by Xcode. (50266910)
     29#
     30# which results in,
     31#
     32#       svn: error: The subversion command line tools are no longer provided by
     33#       Xcode.
     34#
     35# when calling svn, even when subversion is installed via Homebrew and its path
     36# is available in PATH.
     37#
     38# NOTE: May be able to remove this after updating macOS.
     39#
     40#alias svn='/usr/local/bin/svn'
     41
     42## Override certain other aliases
     43#
     44alias grep=$(which grep)
     45
     46## Constants
     47#
     48PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2002 2003 2004 2005 2006 2007 2008 2010 2011 2021 2051 2052 2053 2084 2085 2090 2101 2424 2425 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
     49
     50## Environment
     51#
     52export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'
     53
     54## Parse options
     55#
     56if [ $# -gt 1 ]; then
     57        echo "Can use only one option at a time"
     58        exit 1
     59fi
     60
     61skip_tests=0
     62
     63if [ $# -eq 1 ]; then
     64        case $1 in
     65                -s|--skiptests) skip_tests=1;                                   ;;
     66                *) echo "Unknown parameter passed: $1"; exit 1  ;;
     67        esac
     68fi
     69
     70# Clean up from previous packaging
     71echo "Cleaning up existing assets"
     72cd ${ISSM_DIR}
     73rm -rf ${PKG} ${COMPRESSED_PKG}
     74mkdir ${PKG}
     75
     76# Add required binaries and libraries to package and modify them where needed
     77cd ${ISSM_DIR}/bin
     78
     79echo "Modify generic"
     80cat generic_static.py | sed -e "s/generic_static/generic/g" > generic.py
     81
     82echo "Moving MPICH binaries to bin/"
     83if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
     84        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
     85        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
     86elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
     87        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
     88        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
     89else
     90        echo "MPICH not found"
     91        exit 1
     92fi
     93
     94echo "Moving GDAL binaries to bin/"
     95if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
     96        cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
     97        cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
     98else
     99        echo "GDAL not found"
     100        exit 1
     101fi
     102
     103echo "Moving GMT binaries to bin/"
     104if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
     105        cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
     106        cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
     107else
     108        echo "GMT not found"
     109        exit 1
     110fi
     111
     112echo "Moving Gmsh binaries to bin/"
     113if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
     114        cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
     115else
     116        echo "Gmsh not found"
     117        exit 1
     118fi
     119
     120# Run tests
     121if [ ${skip_tests} -eq 0 ]; then
     122        echo "Running tests"
     123        cd ${ISSM_DIR}/test/NightlyRun
     124        rm python.log 2> /dev/null
     125
     126        # Set Python environment
     127        export PYTHONPATH="${ISSM_DIR}/src/m/dev"
     128        export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
     129        export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
     130
     131        # Run tests, redirecting output to logfile and suppressing output to console
     132        ./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
     133
     134        # Check that Python did not exit in error
     135        pythonExitCode=`echo $?`
     136        pythonExitedInError=`grep -E "Error|Standard exception|Traceback|bad interpreter" python.log | wc -l`
     137
     138        if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
     139                echo "----------Python exited in error!----------"
     140                cat python.log
     141                echo "-----------End of python.log-----------"
     142
     143                # Clean up execution directory
     144                rm -rf ${ISSM_DIR}/execution/*
     145
     146                exit 1
     147        fi
     148
     149        # Check that all tests passed
     150        numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
     151
     152        if [ ${numTestsFailed} -ne 0 ]; then
     153                echo "One or more tests FAILED"
     154                exit 1
     155        else
     156                echo "All tests PASSED"
     157        fi
     158else
     159        echo "Skipping tests"
     160fi
     161
     162# Create package
     163cd ${ISSM_DIR}
     164svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
     165echo "Copying assets to package: ${PKG}"
     166cp -rf bin examples lib scripts test ${PKG}
     167mkdir ${PKG}/execution
     168cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist
     169${ISSM_DIR}/scripts/py_to_pyc.sh ${PKG}/bin # Compile Python source files
     170echo "Cleaning up unneeded/unwanted files"
     171rm -f ${PKG}/bin/*.py # Remove all Python scripts
     172rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
     173rm -f ${PKG}/lib/*.a # Remove static libraries from package
     174rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
     175rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
     176
     177# Compress package
     178echo "Compressing package"
     179ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}

  • TabularUnified ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python-3.sh 

    Property changes on: ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python-2.sh
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
     
     1#!/bin/bash
     2
     3################################################################################
     4# Packages and tests ISSM distributable package for macOS with Python 3 API.
     5#
     6# Options:
     7# -s/--skiptests                Skip testing during packaging Use if packaging fails
     8#                                               for some reason but build is valid.
     9#
     10# NOTE:
     11# - Assumes that the following constants are defined,
     12#
     13#               COMPRESSED_PKG
     14#               ISSM_DIR
     15#               PKG
     16#
     17# See also:
     18# - packagers/mac/complete-issm-mac-binaries-python-3.sh
     19# - packagers/mac/sign-issm-mac-binaries-python-3.sh
     20################################################################################
     21
     22# Expand aliases within the context of this script
     23shopt -s expand_aliases
     24
     25# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
     26#
     27#       Command line tool support for Subversion — including svn, git-svn, and
     28#       related commands — is no longer provided by Xcode. (50266910)
     29#
     30# which results in,
     31#
     32#       svn: error: The subversion command line tools are no longer provided by
     33#       Xcode.
     34#
     35# when calling svn, even when subversion is installed via Homebrew and its path
     36# is available in PATH.
     37#
     38# NOTE: May be able to remove this after updating macOS.
     39#
     40#alias svn='/usr/local/bin/svn'
     41
     42## Override certain other aliases
     43#
     44alias grep=$(which grep)
     45
     46## Constants
     47#
     48PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2002 2003 2004 2005 2006 2007 2008 2010 2011 2021 2051 2052 2053 2084 2085 2090 2101 2424 2425 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
     49
     50## Environment
     51#
     52export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'
     53
     54## Parse options
     55#
     56if [ $# -gt 1 ]; then
     57        echo "Can use only one option at a time"
     58        exit 1
     59fi
     60
     61skip_tests=0
     62
     63if [ $# -eq 1 ]; then
     64        case $1 in
     65                -s|--skiptests) skip_tests=1;                                   ;;
     66                *) echo "Unknown parameter passed: $1"; exit 1  ;;
     67        esac
     68fi
     69
     70# Clean up from previous packaging
     71echo "Cleaning up existing assets"
     72cd ${ISSM_DIR}
     73rm -rf ${PKG} ${COMPRESSED_PKG}
     74mkdir ${PKG}
     75
     76# Add required binaries and libraries to package and modify them where needed
     77cd ${ISSM_DIR}/bin
     78
     79echo "Modify generic"
     80cat generic_static.py | sed -e "s/generic_static/generic/g" > generic.py
     81
     82echo "Moving MPICH binaries to bin/"
     83if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
     84        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
     85        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
     86elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
     87        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
     88        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
     89else
     90        echo "MPICH not found"
     91        exit 1
     92fi
     93
     94echo "Moving GDAL binaries to bin/"
     95if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
     96        cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
     97        cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
     98else
     99        echo "GDAL not found"
     100        exit 1
     101fi
     102
     103echo "Moving GMT binaries to bin/"
     104if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
     105        cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
     106        cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
     107else
     108        echo "GMT not found"
     109        exit 1
     110fi
     111
     112echo "Moving Gmsh binaries to bin/"
     113if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
     114        cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
     115else
     116        echo "Gmsh not found"
     117        exit 1
     118fi
     119
     120# Run tests
     121if [ ${skip_tests} -eq 0 ]; then
     122        echo "Running tests"
     123        cd ${ISSM_DIR}/test/NightlyRun
     124        rm python.log 2> /dev/null
     125
     126        # Set Python environment
     127        export PYTHONPATH="${ISSM_DIR}/src/m/dev"
     128        export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
     129        export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
     130
     131        # Ensure that runme.py uses Python 3 interpreter
     132        sed -i "s|/usr/bin/env python|/usr/local/bin/python3|g" ./runme.py
     133
     134        # Run tests, redirecting output to logfile and suppressing output to console
     135        ./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
     136
     137        # Check that Python did not exit in error
     138        pythonExitCode=`echo $?`
     139        pythonExitedInError=`grep -E "Error|Standard exception|Traceback|bad interpreter" python.log | wc -l`
     140
     141        if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
     142                echo "----------Python exited in error!----------"
     143                cat python.log
     144                echo "-----------End of python.log-----------"
     145
     146                # Clean up execution directory
     147                rm -rf ${ISSM_DIR}/execution/*
     148
     149                exit 1
     150        fi
     151
     152        # Check that all tests passed
     153        numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
     154
     155        if [ ${numTestsFailed} -ne 0 ]; then
     156                echo "One or more tests FAILED"
     157                exit 1
     158        else
     159                echo "All tests PASSED"
     160        fi
     161else
     162        echo "Skipping tests"
     163fi
     164
     165# Create package
     166cd ${ISSM_DIR}
     167svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
     168echo "Copying assets to package: ${PKG}"
     169cp -rf bin examples lib scripts test ${PKG}
     170mkdir ${PKG}/execution
     171cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist
     172${ISSM_DIR}/scripts/py_to_pyc.sh ${PKG}/bin # Compile Python source files
     173echo "Cleaning up unneeded/unwanted files"
     174rm -f ${PKG}/bin/*.py # Remove all Python scripts
     175rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
     176rm -f ${PKG}/lib/*.a # Remove static libraries from package
     177rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
     178rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
     179
     180# Compress package
     181echo "Compressing package"
     182ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}

  • TabularUnified ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python-2.sh 

    Property changes on: ../trunk-jpl/packagers/mac/package-issm-mac-binaries-python-3.sh
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
     
     1#!/bin/bash
     2
     3################################################################################
     4# Intended to be run in the context of a Jenkins project on a JPL
     5# Cybersecurity server for signing macOS applications. Polls SCM of the
     6# Subversion repository hosted at
     7# https://issm.ess.uci.edu/svn/issm-binaries/mac/python/2/unsigned to trigger
     8# new builds.
     9#
     10# In order to replicate the requried Jenkins project configuration:
     11# - First, navigate to 'Manage Jenkins' -> 'Manage Plugins' and install the
     12#       'Credentials Bindings Plugin' if it is not already installed.
     13# - Contact one of the members of the ISSM development team for crendentials
     14#       for the ISSM binaries repository (mention that the credentials are stored
     15#       in ISSM-Infrastructure.pdf).
     16# - Navigate to 'Manage Jenkins' -> 'Manage Credentials' -> <domain> ->
     17#       'Add Credentials' and enter the crendentials from above.
     18# - From the 'Dashboard', select 'New Item' -> 'Freestyle project'.
     19# - Under 'Source Code Management', select 'Subversion'.
     20#               - The 'Repository URL' text field should be set to
     21#               "https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned".
     22#               - The 'Credentials' select menu should be set to the new credentials
     23#               created previously.
     24#               - The 'Local module directory' text field should be set to the same
     25#               value as the constant UNSIGNED_REPO_COPY (set below to './unsigned').
     26# - Under 'Build Trigggers', check the box for 'Poll SCM' and set the
     27#       'Schedule' text area to "H/5 * * * *".
     28# - Under 'Build Environment', check the box for 'Use secret text(s) or
     29#       file(s)', then under 'Bindings' click the 'Add...' button and select
     30#       'Username and password (separated)'.
     31#               - Set 'Username Variable' to "ISSM_BINARIES_USER".
     32#               - Set 'Password Variable' to "ISSM_BINARIES_PASS".
     33# - Under 'Credentials', select the same, new credentials that created
     34#       previously.
     35# - The contents of this script can be copied/pasted directly into the ‘Build'
     36#       -> 'Execute Shell' -> ‘Command' textarea of the project configuration (or
     37#       you can simply store the script on disk and call it from there).
     38# - Make sure to click the 'Save' button.
     39#
     40# Current point of contact at JPL Cybersecurity:
     41#       Alex Coward, alexander.g.coward@jpl.nasa.gov
     42#
     43# NOTE:
     44# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
     45#       the 'Bindings' section under a 'Username and password (separated)' binding
     46#       (requires 'Credentials Binding Plugin').
     47# - For local debugging, the aformentioned credentials can be hardcoded into
     48#       the 'USERNAME' and 'PASSWORD' constants below.
     49################################################################################
     50
     51# Expand aliases within the context of this script
     52shopt -s expand_aliases
     53
     54# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
     55#
     56#       Command line tool support for Subversion — including svn, git-svn, and
     57#       related commands — is no longer provided by Xcode. (50266910)
     58#
     59# which results in,
     60#
     61#       svn: error: The subversion command line tools are no longer provided by
     62#       Xcode.
     63#
     64# when calling svn, even when subversion is installed via Homebrew and its path
     65# is available in PATH.
     66#
     67# NOTE: May be able to remove this after updating macOS.
     68#
     69#alias svn='/usr/local/bin/svn'
     70
     71## Override certain other aliases
     72#
     73alias cp=$(which cp)
     74alias grep=$(which grep)
     75
     76## Constants
     77#
     78AD_IDENTITY="**********" # Apple Developer identity
     79AD_USERNAME="**********" # Apple Developer username
     80ALTOOL_PASSWORD="@keychain:**********" # altool password (assumed to be stored in keychain)
     81ASC_PROVIDER="**********"
     82MAX_SVN_ATTEMPTS=10
     83NOTARIZATION_CHECK_ATTEMPTS=20
     84NOTARIZATION_CHECK_PERIOD=60
     85NOTARIZATION_LOGFILE="notarization.log"
     86NOTARIZATION_LOGFILE_PATH="."
     87PASSWORD=${ISSM_BINARIES_PASS}
     88PKG="ISSM-macOS-Python-2"
     89PRIMARY_BUNDLE_ID="gov.nasa.jpl.issm.python"
     90SIGNED_REPO_COPY="./signed"
     91SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/2/signed"
     92SIGNING_LOCK_FILE="signing.lock"
     93SUCCESS_LOGFILE="${SIGNED_REPO_COPY}/success.log"
     94UNSIGNED_REPO_COPY="./unsigned"
     95UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/2/unsigned"
     96USERNAME=${ISSM_BINARIES_USER}
     97
     98COMPRESSED_PKG="${PKG}.zip"
     99EXE_ENTITLEMENTS_PLIST="${PKG}/bin/entitlements.plist"
     100
     101# NOTE: Uncomment the following for local testing (Jenkins checks out copy of
     102#               repository for unsigned packages to working directory)
     103#
     104
     105# # Clean up from previous packaging (not necessary for single builds on Jenkins,
     106# # but useful when testing packaging locally)
     107# echo "Cleaning up existing assets"
     108# rm -rf ${COMPRESSED_PKG} ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${UNSIGNED_REPO_COPY}
     109
     110# # Check out copy of repository for unsigned packages
     111# echo "Checking out copy of respository for unsigned packages"
     112# svn checkout \
     113#       --trust-server-cert \
     114#       --non-interactive \
     115#       --username ${USERNAME} \
     116#       --password ${PASSWORD} \
     117#       ${UNSIGNED_REPO_URL} \
     118#       ${UNSIGNED_REPO_COPY}
     119
     120rm -rf ${PKG} ${SIGNED_REPO_COPY}
     121
     122
     123# Extract package contents
     124echo "Extracting package contents"
     125ditto -xk ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} .
     126
     127# Clear extended attributes on all files
     128xattr -cr ${PKG}
     129
     130# Build list of ISSM executables
     131ISSM_BINS=$(\
     132        find ${PKG}/bin -type f -name *.exe; \
     133        find ${PKG}/bin -type f -name *.pyc; \
     134)
     135
     136# Build list of third party executables
     137THIRD_PARTY_BINS=$(\
     138        echo ${PKG}/bin/mpiexec; \
     139        echo ${PKG}/bin/hydra_pmi_proxy; \
     140        echo ${PKG}/bin/gdalsrsinfo; \
     141        echo ${PKG}/bin/gdaltransform; \
     142        echo ${PKG}/bin/gmt; \
     143        echo ${PKG}/bin/gmtselect; \
     144        echo ${PKG}/bin/gmsh; \
     145)
     146
     147# Sign all executables in package
     148echo "Signing all executables in package"
     149codesign -s ${AD_IDENTITY} --timestamp --options=runtime --entitlements ${EXE_ENTITLEMENTS_PLIST} ${ISSM_BINS}
     150codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${THIRD_PARTY_BINS}
     151
     152# Build list of ISSM libraries
     153ISSM_LIBS=$(\
     154        find ${PKG}/lib -type f -name *.so; \
     155)
     156
     157# Sign all libraries in package
     158echo "Signing all libraries in package"
     159codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${ISSM_LIBS}
     160
     161# NOTE: Skipping signature validation because this is not a true package nor app
     162
     163# Compress signed package
     164echo "Compressing signed package"
     165ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
     166
     167# Submit compressed package for notarization
     168echo "Submitting signed package to Apple for notarization"
     169xcrun altool --notarize-app --primary-bundle-id ${PRIMARY_BUNDLE_ID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} --asc-provider ${ASC_PROVIDER} --file ${COMPRESSED_PKG} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
     170
     171# Sleep until notarization request response is received
     172echo "Waiting for notarizaion request response"
     173while [[ ! -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} || ! -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; do
     174        sleep 30
     175done
     176
     177echo "Notarizaion request response received"
     178
     179# Check if UUID exists in response
     180HAS_UUID=$(grep 'RequestUUID = ' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}) # NOTE: Checking for "RequestUUID = " because "RequestUUID" shows up in some error messages
     181if [ -z "${HAS_UUID}" ]; then
     182        echo "Notarization failed!"
     183        echo "----------------------- Contents of notarization logfile -----------------------"
     184        cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
     185        echo "--------------------------------------------------------------------------------"
     186
     187        # Clean up
     188        rm -rf ${PKG} ${COMPRESSED_PKG}
     189
     190        exit 1
     191fi
     192
     193# Get UUID from notarization request response
     194UUID=$(echo ${HAS_UUID} | sed 's/[[:space:]]*RequestUUID = //')
     195echo "UUID: ${UUID}"
     196
     197# Check notarization status
     198#
     199# NOTE: Currently, this checks if notarization was successful, but we are not
     200#               able to staple notarization as this is not a true package nor app and,
     201#               at the very least, MATLAB Mex files cannot be stapled. As such, clients
     202#               will not be able to clear Gatekeeper if they are offline.
     203#
     204echo "Checking notarization status"
     205SUCCESS=0
     206for ATTEMPT in $(seq 1 ${NOTARIZATION_CHECK_ATTEMPTS}); do
     207        echo "    Attempt #${ATTEMPT}..."
     208        xcrun altool --notarization-info ${UUID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
     209        if [[ -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} && -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; then
     210
     211                # First, check if there is an error
     212                ERROR_CHECK=$(grep 'Error' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE})
     213                if [ ! -z "${ERROR_CHECK}" ]; then
     214                        break
     215                fi
     216
     217                # No error, so check status
     218                STATUS=$(grep 'Status:' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
     219                if [[ "${STATUS}" == "success" ]]; then
     220                        # Staple notarization to all elements of package that were previously signed
     221                        #xcrun stapler staple ${THIRD_PARTY_BINS} # NOTE: Fails with "Stapler is incapable of working with MATLAB Mex files."
     222
     223                        # Validate stapling of notarization
     224                        #xcrun stapler validation ${THIRD_PARTY_BINS} # NOTE: Skipping notarization stapling validation because this is not a true package nor app
     225
     226                        # Compress signed and notarized package
     227                        ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
     228
     229                        # Set flag indicating notarization was successful
     230                        SUCCESS=1
     231
     232                        break
     233                elif [[ "${STATUS}" == "in progress" ]]; then
     234                        echo "    ...in progress still; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds."
     235                        sleep ${NOTARIZATION_CHECK_PERIOD}
     236                elif [[ "${STATUS}" == "invalid" ]]; then
     237                        break
     238                fi
     239        else
     240                if [ ${ATTEMPT} -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then
     241                        echo "    ...not ready yet; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds"
     242                        sleep ${NOTARIZATION_CHECK_PERIOD}
     243                else
     244                        echo "    ...maximum attempts reached, but no response, or something else went wrong"
     245                        echo "    If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again"
     246                        break
     247                fi
     248        fi
     249done
     250
     251if [ ${SUCCESS} -eq 1 ]; then
     252        echo "Notarization successful!"
     253else
     254        echo "Notarization failed!"
     255        echo "----------------------- Contents of notarization logfile -----------------------"
     256        cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
     257        echo "--------------------------------------------------------------------------------"
     258fi
     259
     260# Check out copy of repository for signed packages
     261echo "Checking out copy of respository for signed packages"
     262SVN_ATTEMPT=0
     263SVN_SUCCESS=0
     264while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
     265        rm -rf ${SIGNED_REPO_COPY}
     266        svn checkout \
     267                --trust-server-cert \
     268                --non-interactive \
     269                --username ${USERNAME} \
     270                --password ${PASSWORD} \
     271                ${SIGNED_REPO_URL} \
     272                ${SIGNED_REPO_COPY} > /dev/null 2>&1
     273        if [ $? -eq 0 ]; then
     274                SVN_SUCCESS=1
     275                break
     276        else
     277                ((++SVN_ATTEMPT))
     278                sleep 5
     279        fi
     280done
     281
     282if [ ${SVN_SUCCESS} -eq 0 ]; then
     283        echo "Checkout of respository for signed packages failed"
     284        exit 1
     285fi
     286
     287# Copy notarization file to repository for signed packages
     288cp ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY}
     289svn add ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1
     290
     291# Remove lock file from repository for signed packages
     292svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
     293
     294SVN_ATTEMPT=0
     295SVN_SUCCESS=0
     296if [ ${SUCCESS} -eq 1 ]; then
     297        # Copy signed package to repository for signed packages
     298        cp ${COMPRESSED_PKG} ${SIGNED_REPO_COPY}
     299        svn add ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
     300
     301        # Commit changes
     302        echo "Committing changes to repository for signed packages"
     303        while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
     304                svn commit \
     305                        --trust-server-cert \
     306                        --non-interactive \
     307                        --username ${USERNAME} \
     308                        --password ${PASSWORD} \
     309                        --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
     310                if [ $? -eq 0 ]; then
     311                        SVN_SUCCESS=1
     312                        break
     313                else
     314                        ((++SVN_ATTEMPT))
     315                        sleep 5
     316                fi
     317        done
     318
     319        if [ ${SVN_SUCCESS} -eq 0 ]; then
     320                echo "Commit to respository for signed packages failed"
     321                exit 1
     322        fi
     323else
     324        # Commit changes
     325        echo "Committing changes to repository for signed packages"
     326        while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
     327                svn commit \
     328                        --trust-server-cert \
     329                        --non-interactive \
     330                        --username ${USERNAME} \
     331                        --password ${PASSWORD} \
     332                        --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
     333                if [ $? -eq 0 ]; then
     334                        SVN_SUCCESS=1
     335                        break
     336                else
     337                        ((++SVN_ATTEMPT))
     338                        sleep 5
     339                fi
     340        done
     341
     342        if [ ${SVN_SUCCESS} -eq 0 ]; then
     343                echo "Commit to respository for signed packages failed"
     344                exit 1
     345        fi
     346
     347        exit 1
     348fi

  • TabularUnified ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python-3.sh 

    Property changes on: ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python-2.sh
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
     
     1#!/bin/bash
     2
     3################################################################################
     4# Intended to be run in the context of a Jenkins project on a JPL
     5# Cybersecurity server for signing macOS applications. Polls SCM of the
     6# Subversion repository hosted at
     7# https://issm.ess.uci.edu/svn/issm-binaries/mac/python/3/unsigned to trigger
     8# new builds.
     9#
     10# In order to replicate the requried Jenkins project configuration:
     11# - First, navigate to 'Manage Jenkins' -> 'Manage Plugins' and install the
     12#       'Credentials Bindings Plugin' if it is not already installed.
     13# - Contact one of the members of the ISSM development team for crendentials
     14#       for the ISSM binaries repository (mention that the credentials are stored
     15#       in ISSM-Infrastructure.pdf).
     16# - Navigate to 'Manage Jenkins' -> 'Manage Credentials' -> <domain> ->
     17#       'Add Credentials' and enter the crendentials from above.
     18# - From the 'Dashboard', select 'New Item' -> 'Freestyle project'.
     19# - Under 'Source Code Management', select 'Subversion'.
     20#               - The 'Repository URL' text field should be set to
     21#               "https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned".
     22#               - The 'Credentials' select menu should be set to the new credentials
     23#               created previously.
     24#               - The 'Local module directory' text field should be set to the same
     25#               value as the constant UNSIGNED_REPO_COPY (set below to './unsigned').
     26# - Under 'Build Trigggers', check the box for 'Poll SCM' and set the
     27#       'Schedule' text area to "H/5 * * * *".
     28# - Under 'Build Environment', check the box for 'Use secret text(s) or
     29#       file(s)', then under 'Bindings' click the 'Add...' button and select
     30#       'Username and password (separated)'.
     31#               - Set 'Username Variable' to "ISSM_BINARIES_USER".
     32#               - Set 'Password Variable' to "ISSM_BINARIES_PASS".
     33# - Under 'Credentials', select the same, new credentials that created
     34#       previously.
     35# - The contents of this script can be copied/pasted directly into the ‘Build'
     36#       -> 'Execute Shell' -> ‘Command' textarea of the project configuration (or
     37#       you can simply store the script on disk and call it from there).
     38# - Make sure to click the 'Save' button.
     39#
     40# Current point of contact at JPL Cybersecurity:
     41#       Alex Coward, alexander.g.coward@jpl.nasa.gov
     42#
     43# NOTE:
     44# - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
     45#       the 'Bindings' section under a 'Username and password (separated)' binding
     46#       (requires 'Credentials Binding Plugin').
     47# - For local debugging, the aformentioned credentials can be hardcoded into
     48#       the 'USERNAME' and 'PASSWORD' constants below.
     49################################################################################
     50
     51# Expand aliases within the context of this script
     52shopt -s expand_aliases
     53
     54# From https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes,
     55#
     56#       Command line tool support for Subversion — including svn, git-svn, and
     57#       related commands — is no longer provided by Xcode. (50266910)
     58#
     59# which results in,
     60#
     61#       svn: error: The subversion command line tools are no longer provided by
     62#       Xcode.
     63#
     64# when calling svn, even when subversion is installed via Homebrew and its path
     65# is available in PATH.
     66#
     67# NOTE: May be able to remove this after updating macOS.
     68#
     69#alias svn='/usr/local/bin/svn'
     70
     71## Override certain other aliases
     72#
     73alias cp=$(which cp)
     74alias grep=$(which grep)
     75
     76## Constants
     77#
     78AD_IDENTITY="**********" # Apple Developer identity
     79AD_USERNAME="**********" # Apple Developer username
     80ALTOOL_PASSWORD="@keychain:**********" # altool password (assumed to be stored in keychain)
     81ASC_PROVIDER="**********"
     82MAX_SVN_ATTEMPTS=10
     83NOTARIZATION_CHECK_ATTEMPTS=20
     84NOTARIZATION_CHECK_PERIOD=60
     85NOTARIZATION_LOGFILE="notarization.log"
     86NOTARIZATION_LOGFILE_PATH="."
     87PASSWORD=${ISSM_BINARIES_PASS}
     88PKG="ISSM-macOS-Python-3"
     89PRIMARY_BUNDLE_ID="gov.nasa.jpl.issm.python"
     90SIGNED_REPO_COPY="./signed"
     91SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/3/signed"
     92SIGNING_LOCK_FILE="signing.lock"
     93SUCCESS_LOGFILE="${SIGNED_REPO_COPY}/success.log"
     94UNSIGNED_REPO_COPY="./unsigned"
     95UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/python/3/unsigned"
     96USERNAME=${ISSM_BINARIES_USER}
     97
     98COMPRESSED_PKG="${PKG}.zip"
     99EXE_ENTITLEMENTS_PLIST="${PKG}/bin/entitlements.plist"
     100
     101# NOTE: Uncomment the following for local testing (Jenkins checks out copy of
     102#               repository for unsigned packages to working directory)
     103#
     104
     105# # Clean up from previous packaging (not necessary for single builds on Jenkins,
     106# # but useful when testing packaging locally)
     107# echo "Cleaning up existing assets"
     108# rm -rf ${COMPRESSED_PKG} ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${UNSIGNED_REPO_COPY}
     109
     110# # Check out copy of repository for unsigned packages
     111# echo "Checking out copy of respository for unsigned packages"
     112# svn checkout \
     113#       --trust-server-cert \
     114#       --non-interactive \
     115#       --username ${USERNAME} \
     116#       --password ${PASSWORD} \
     117#       ${UNSIGNED_REPO_URL} \
     118#       ${UNSIGNED_REPO_COPY}
     119
     120rm -rf ${PKG} ${SIGNED_REPO_COPY}
     121
     122
     123# Extract package contents
     124echo "Extracting package contents"
     125ditto -xk ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} .
     126
     127# Clear extended attributes on all files
     128xattr -cr ${PKG}
     129
     130# Build list of ISSM executables
     131ISSM_BINS=$(\
     132        find ${PKG}/bin -type f -name *.exe; \
     133        find ${PKG}/bin -type f -name *.pyc; \
     134)
     135
     136# Build list of third party executables
     137THIRD_PARTY_BINS=$(\
     138        echo ${PKG}/bin/mpiexec; \
     139        echo ${PKG}/bin/hydra_pmi_proxy; \
     140        echo ${PKG}/bin/gdalsrsinfo; \
     141        echo ${PKG}/bin/gdaltransform; \
     142        echo ${PKG}/bin/gmt; \
     143        echo ${PKG}/bin/gmtselect; \
     144        echo ${PKG}/bin/gmsh; \
     145)
     146
     147# Sign all executables in package
     148echo "Signing all executables in package"
     149codesign -s ${AD_IDENTITY} --timestamp --options=runtime --entitlements ${EXE_ENTITLEMENTS_PLIST} ${ISSM_BINS}
     150codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${THIRD_PARTY_BINS}
     151
     152# Build list of ISSM libraries
     153ISSM_LIBS=$(\
     154        find ${PKG}/lib -type f -name *.so; \
     155)
     156
     157# Sign all libraries in package
     158echo "Signing all libraries in package"
     159codesign -s ${AD_IDENTITY} --timestamp --options=runtime ${ISSM_LIBS}
     160
     161# NOTE: Skipping signature validation because this is not a true package nor app
     162
     163# Compress signed package
     164echo "Compressing signed package"
     165ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
     166
     167# Submit compressed package for notarization
     168echo "Submitting signed package to Apple for notarization"
     169xcrun altool --notarize-app --primary-bundle-id ${PRIMARY_BUNDLE_ID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} --asc-provider ${ASC_PROVIDER} --file ${COMPRESSED_PKG} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
     170
     171# Sleep until notarization request response is received
     172echo "Waiting for notarizaion request response"
     173while [[ ! -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} || ! -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; do
     174        sleep 30
     175done
     176
     177echo "Notarizaion request response received"
     178
     179# Check if UUID exists in response
     180HAS_UUID=$(grep 'RequestUUID = ' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}) # NOTE: Checking for "RequestUUID = " because "RequestUUID" shows up in some error messages
     181if [ -z "${HAS_UUID}" ]; then
     182        echo "Notarization failed!"
     183        echo "----------------------- Contents of notarization logfile -----------------------"
     184        cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
     185        echo "--------------------------------------------------------------------------------"
     186
     187        # Clean up
     188        rm -rf ${PKG} ${COMPRESSED_PKG}
     189
     190        exit 1
     191fi
     192
     193# Get UUID from notarization request response
     194UUID=$(echo ${HAS_UUID} | sed 's/[[:space:]]*RequestUUID = //')
     195echo "UUID: ${UUID}"
     196
     197# Check notarization status
     198#
     199# NOTE: Currently, this checks if notarization was successful, but we are not
     200#               able to staple notarization as this is not a true package nor app and,
     201#               at the very least, MATLAB Mex files cannot be stapled. As such, clients
     202#               will not be able to clear Gatekeeper if they are offline.
     203#
     204echo "Checking notarization status"
     205SUCCESS=0
     206for ATTEMPT in $(seq 1 ${NOTARIZATION_CHECK_ATTEMPTS}); do
     207        echo "    Attempt #${ATTEMPT}..."
     208        xcrun altool --notarization-info ${UUID} --username ${AD_USERNAME} --password ${ALTOOL_PASSWORD} &> ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
     209        if [[ -f ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} && -z $(find ${NOTARIZATION_LOGFILE_PATH} -empty -name ${NOTARIZATION_LOGFILE}) ]]; then
     210
     211                # First, check if there is an error
     212                ERROR_CHECK=$(grep 'Error' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE})
     213                if [ ! -z "${ERROR_CHECK}" ]; then
     214                        break
     215                fi
     216
     217                # No error, so check status
     218                STATUS=$(grep 'Status:' ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
     219                if [[ "${STATUS}" == "success" ]]; then
     220                        # Staple notarization to all elements of package that were previously signed
     221                        #xcrun stapler staple ${THIRD_PARTY_BINS} # NOTE: Fails with "Stapler is incapable of working with MATLAB Mex files."
     222
     223                        # Validate stapling of notarization
     224                        #xcrun stapler validation ${THIRD_PARTY_BINS} # NOTE: Skipping notarization stapling validation because this is not a true package nor app
     225
     226                        # Compress signed and notarized package
     227                        ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
     228
     229                        # Set flag indicating notarization was successful
     230                        SUCCESS=1
     231
     232                        break
     233                elif [[ "${STATUS}" == "in progress" ]]; then
     234                        echo "    ...in progress still; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds."
     235                        sleep ${NOTARIZATION_CHECK_PERIOD}
     236                elif [[ "${STATUS}" == "invalid" ]]; then
     237                        break
     238                fi
     239        else
     240                if [ ${ATTEMPT} -lt ${NOTARIZATION_CHECK_ATTEMPTS} ]; then
     241                        echo "    ...not ready yet; checking again in ${NOTARIZATION_CHECK_PERIOD} seconds"
     242                        sleep ${NOTARIZATION_CHECK_PERIOD}
     243                else
     244                        echo "    ...maximum attempts reached, but no response, or something else went wrong"
     245                        echo "    If contents of notarization status check logfile appear to be valid, increase NOTARIZATION_CHECK_ATTEMPTS and run again"
     246                        break
     247                fi
     248        fi
     249done
     250
     251if [ ${SUCCESS} -eq 1 ]; then
     252        echo "Notarization successful!"
     253else
     254        echo "Notarization failed!"
     255        echo "----------------------- Contents of notarization logfile -----------------------"
     256        cat ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE}
     257        echo "--------------------------------------------------------------------------------"
     258fi
     259
     260# Check out copy of repository for signed packages
     261echo "Checking out copy of respository for signed packages"
     262SVN_ATTEMPT=0
     263SVN_SUCCESS=0
     264while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
     265        rm -rf ${SIGNED_REPO_COPY}
     266        svn checkout \
     267                --trust-server-cert \
     268                --non-interactive \
     269                --username ${USERNAME} \
     270                --password ${PASSWORD} \
     271                ${SIGNED_REPO_URL} \
     272                ${SIGNED_REPO_COPY} > /dev/null 2>&1
     273        if [ $? -eq 0 ]; then
     274                SVN_SUCCESS=1
     275                break
     276        else
     277                ((++SVN_ATTEMPT))
     278                sleep 5
     279        fi
     280done
     281
     282if [ ${SVN_SUCCESS} -eq 0 ]; then
     283        echo "Checkout of respository for signed packages failed"
     284        exit 1
     285fi
     286
     287# Copy notarization file to repository for signed packages
     288cp ${NOTARIZATION_LOGFILE_PATH}/${NOTARIZATION_LOGFILE} ${SIGNED_REPO_COPY}
     289svn add ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} > /dev/null 2>&1
     290
     291# Remove lock file from repository for signed packages
     292svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
     293
     294SVN_ATTEMPT=0
     295SVN_SUCCESS=0
     296if [ ${SUCCESS} -eq 1 ]; then
     297        # Copy signed package to repository for signed packages
     298        cp ${COMPRESSED_PKG} ${SIGNED_REPO_COPY}
     299        svn add ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
     300
     301        # Commit changes
     302        echo "Committing changes to repository for signed packages"
     303        while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
     304                svn commit \
     305                        --trust-server-cert \
     306                        --non-interactive \
     307                        --username ${USERNAME} \
     308                        --password ${PASSWORD} \
     309                        --message "CHG: New signed package (success)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
     310                if [ $? -eq 0 ]; then
     311                        SVN_SUCCESS=1
     312                        break
     313                else
     314                        ((++SVN_ATTEMPT))
     315                        sleep 5
     316                fi
     317        done
     318
     319        if [ ${SVN_SUCCESS} -eq 0 ]; then
     320                echo "Commit to respository for signed packages failed"
     321                exit 1
     322        fi
     323else
     324        # Commit changes
     325        echo "Committing changes to repository for signed packages"
     326        while [[ ${SVN_ATTEMPT} -lt ${MAX_SVN_ATTEMPTS} && ${SVN_SUCCESS} -eq 0 ]]; do
     327                svn commit \
     328                        --trust-server-cert \
     329                        --non-interactive \
     330                        --username ${USERNAME} \
     331                        --password ${PASSWORD} \
     332                        --message "CHG: New signed package (failure)" ${SIGNED_REPO_COPY} > /dev/null 2>&1
     333                if [ $? -eq 0 ]; then
     334                        SVN_SUCCESS=1
     335                        break
     336                else
     337                        ((++SVN_ATTEMPT))
     338                        sleep 5
     339                fi
     340        done
     341
     342        if [ ${SVN_SUCCESS} -eq 0 ]; then
     343                echo "Commit to respository for signed packages failed"
     344                exit 1
     345        fi
     346
     347        exit 1
     348fi

  • TabularUnified ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python-2.sh 

    Property changes on: ../trunk-jpl/packagers/mac/sign-issm-mac-binaries-python-3.sh
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
     
     1#!/bin/bash
     2
     3################################################################################
     4# This script is intended to test ISSM macOS Python 2 binaries on an end-user
     5# machine after successful packaging and signing.
     6#
     7# NOTE: Tarball must already exist in INSTALL_DIR
     8################################################################################
     9
     10## Constants
     11#
     12INSTALL_DIR=.
     13PKG="ISSM-macOS-Python-2"
     14PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2002 2003 2004 2005 2006 2007 2008 2010 2011 2021 2051 2052 2053 2084 2085 2090 2101 2424 2425 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
     15
     16COMPRESSED_PKG="${PKG}.zip"
     17
     18export ISSM_DIR="${INSTALL_DIR}/${PKG}"
     19export PATH="${PATH}:${ISSM_DIR}/bin:${ISSM_DIR}/scripts"
     20export PYTHONPATH="${ISSM_DIR}/scripts"
     21export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
     22export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
     23
     24cd ${INSTALL_DIR}
     25rm -rf ${PKG}
     26ditto -xk ${COMPRESSED_PKG} .
     27cd ${PKG}/test/NightlyRun
     28
     29# Run tests, redirecting output to logfile and suppressing output to console
     30echo "Running tests"
     31rm python.log 2> /dev/null
     32./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
     33
     34# Check that Python did not exit in error
     35pythonExitCode=`echo $?`
     36pythonExitedInError=`grep -E "runme.py: error" python.log | wc -l`
     37
     38if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
     39        echo "----------Python exited in error!----------"
     40        cat python.log
     41        echo "-----------End of python.log-----------"
     42        exit 1
     43fi
     44
     45# Check that all tests passed
     46numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
     47
     48if [[ ${numTestsFailed} -ne 0 ]]; then
     49        echo "One or more tests FAILED"
     50        exit 1
     51else
     52        echo "All tests PASSED"
     53fi

  • TabularUnified ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python-3.sh 

    Property changes on: ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python-2.sh
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
     
     1#!/bin/bash
     2
     3################################################################################
     4# This script is intended to test ISSM macOS Python 3 binaries on an end-user
     5# machine after successful packaging and signing.
     6#
     7# NOTE: Tarball must already exist in INSTALL_DIR
     8################################################################################
     9
     10## Constants
     11#
     12INSTALL_DIR=.
     13PKG="ISSM-macOS-Python-3"
     14PYTHON_NROPTIONS="--benchmark all --exclude 125 126 234 235 418 420 435 444 445 701 702 703 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1201 1202 1203 1204 1205 1206 1207 1208 1301 1302 1303 1304 1401 1402 1601 1602 2002 2003 2004 2005 2006 2007 2008 2010 2011 2021 2051 2052 2053 2084 2085 2090 2101 2424 2425 3001:3200 3201 3202 3300 3480 3481 4001 4002 4003" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
     15
     16COMPRESSED_PKG="${PKG}.zip"
     17
     18export ISSM_DIR="${INSTALL_DIR}/${PKG}"
     19export PATH="${PATH}:${ISSM_DIR}/bin:${ISSM_DIR}/scripts"
     20export PYTHONPATH="${ISSM_DIR}/scripts"
     21export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
     22export PYTHONUNBUFFERED=1 # We don't want Python to buffer output, otherwise issm.exe output is not captured
     23
     24cd ${INSTALL_DIR}
     25rm -rf ${PKG}
     26ditto -xk ${COMPRESSED_PKG} .
     27cd ${PKG}/test/NightlyRun
     28
     29# Run tests, redirecting output to logfile and suppressing output to console
     30echo "Running tests"
     31rm python.log 2> /dev/null
     32./runme.py ${PYTHON_NROPTIONS} &> python.log 2>&1
     33
     34# Check that Python did not exit in error
     35pythonExitCode=`echo $?`
     36pythonExitedInError=`grep -E "runme.py: error" python.log | wc -l`
     37
     38if [[ ${pythonExitCode} -ne 0 || ${pythonExitedInError} -ne 0 ]]; then
     39        echo "----------Python exited in error!----------"
     40        cat python.log
     41        echo "-----------End of python.log-----------"
     42        exit 1
     43fi
     44
     45# Check that all tests passed
     46numTestsFailed=`cat python.log | grep -c -e "FAILED|ERROR"`
     47
     48if [[ ${numTestsFailed} -ne 0 ]]; then
     49        echo "One or more tests FAILED"
     50        exit 1
     51else
     52        echo "All tests PASSED"
     53fi

  • TabularUnified ../trunk-jpl/packagers/mac/transfer-issm-mac-binaries.sh 

    Property changes on: ../trunk-jpl/packagers/mac/test-issm-mac-binaries-python-3.sh
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
     
    11#!/bin/bash
    22
    33################################################################################
    4 # Transfers ISSM distributable package for macOS with MATLAB API to ISSM Web
    5 # site.
     4# Transfers ISSM distributable package for macOS to ISSM website.
    65#
    76# NOTE:
    87# - Assumes that the following constants are defined,
    … …  
    1514#
    1615# See also:
    1716# - packagers/mac/complete-issm-mac-binaries-matlab.sh
    18 # - packagers/mac/complete-issm-mac-binaries-python.sh
     17# - packagers/mac/complete-issm-mac-binaries-python-2.sh
     18# - packagers/mac/complete-issm-mac-binaries-python-3.sh
    1919################################################################################
    2020
    2121# Expand aliases within the context of this script
Note: See TracBrowser for help on using the repository browser.