Ignore:
Timestamp:
12/18/20 16:06:19 (4 years ago)
Author:
jdquinn
Message:

CHG: Similar packaging script modifications for Linux binaries; clean up

File:
1 edited

Legend:

Unmodified
Added
Removed

  • issm/trunk-jpl/packagers/linux/package-issm-linux-binaries-matlab.sh

    r25809 r25896  
    22
    33################################################################################
    4 # To be used after running,
    5 #
    6 #       ${ISSM_DIR}/jenkins/jenkins.sh ${ISSM_DIR}/jenkins/ross-debian_linux-binaries-matlab
    7 #
    8 # in the context of a Jenkins project.
    9 #
    10 # When no runtime errors occur, performs the following:
    11 # - Checks resulting executables and libraries against test suite.
    12 # - Packages and compresses executables and libraries.
    13 # - Transmits it to ISSM Web site for distribution.
     4# Packages and tests ISSM distributable package for Linux with MATLAB API.
    145#
    156# Options:
    16 # -s/--skiptests                Skip tests (use if this script fails for some reason
    17 #                                               after tests have successfully passed to save time)
    18 # -t/--transferonly             Transfer package to ISSM Web site only (use if
    19 #                                               transfer fails for some reason to skip testing and
    20 #                                               signing)
     7# -s/--skiptests                Skip testing during packaging Use if packaging fails
     8#                                               for some reason but build is valid.
     9#
     10# NOTE:
     11# - Assumes that the following constants are defined,
     12#
     13#               COMPRESSED_PKG
     14#               ISSM_DIR
     15#               PKG
     16#
     17# See also:
     18# - packagers/linux/complete-issm-linux-binaries-matlab.sh
    2119################################################################################
    2220
    … …  
    3432MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2002,2003,2005,2006,2010,2020,2021,2051,2052,2053,2101,2102,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
    3533MATLAB_PATH="/usr/local/MATLAB/R2019b"
    36 PKG="ISSM-Linux-MATLAB" # Name of directory to copy distributable files to
    37 
    38 COMPRESSED_PKG="${PKG}.tar.gz"
    3934
    4035## Environment
    … …  
    5045
    5146skip_tests=0
    52 transfer_only=0
    5347while [ $# -gt 0 ]; do
    5448    case $1 in
    55         -s|--skiptests) skip_tests=1; shift ;;
    56         -t|--transferonly) transfer_only=1; shift ;;
    57         *) echo "Unknown parameter passed: $1"; exit 1 ;;
     49        -s|--skiptests) skip_tests=1;                                   ;;
     50        *) echo "Unknown parameter passed: $1"; exit 1  ;;
    5851    esac
    5952    shift
    6053done
    6154
    62 if [ ${transfer_only} -eq 0 ]; then
    63         # Check if MATLAB exists
    64         if ! [ -d ${MATLAB_PATH} ]; then
    65                 echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again."
     55# Check if MATLAB exists
     56if ! [ -d ${MATLAB_PATH} ]; then
     57        echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again."
     58        exit 1
     59fi
     60
     61# Clean up from previous packaging
     62echo "Cleaning up existing assets"
     63cd ${ISSM_DIR}
     64rm -rf ${PKG} ${COMPRESSED_PKG}
     65mkdir ${PKG}
     66
     67# Add/modify required binaries and libraries
     68cd ${ISSM_DIR}/bin
     69
     70echo "Modify generic"
     71cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m
     72
     73echo "Moving MPICH binaries to bin/"
     74if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
     75        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
     76        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
     77elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
     78        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
     79        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
     80else
     81        echo "MPICH not found"
     82        exit 1
     83fi
     84
     85echo "Moving GDAL binaries to bin/"
     86if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
     87        cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
     88        cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
     89else
     90        echo "GDAL not found"
     91        exit 1
     92fi
     93
     94echo "Moving GMT binaries to bin/"
     95if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
     96        cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
     97        cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
     98else
     99        echo "GMT not found"
     100        exit 1
     101fi
     102
     103echo "Moving Gmsh binaries to bin/"
     104if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
     105        cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
     106else
     107        echo "Gmsh not found"
     108        exit 1
     109fi
     110
     111echo "Moving libgfortran to lib/"
     112cp ${LIBGFORTRAN} ${LIBGFORTRAN_DIST} 2> /dev/null
     113
     114# Run tests
     115if [ ${skip_tests} -eq 0 ]; then
     116        echo "Running tests"
     117        cd ${ISSM_DIR}/test/NightlyRun
     118        rm matlab.log 2> /dev/null
     119
     120        # Run tests, redirecting output to logfile and suppressing output to console
     121        ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
     122
     123        # Check that MATLAB did not exit in error
     124        matlabExitCode=`echo $?`
     125        matlabExitedInError=`grep -E "Activation cannot proceed|Error in matlab_run" matlab.log | wc -l`
     126
     127        if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then
     128                echo "----------MATLAB exited in error!----------"
     129                cat matlab.log
     130                echo "-----------End of matlab.log-----------"
     131
     132                # Clean up execution directory
     133                rm -rf ${ISSM_DIR}/execution/*
     134
    66135                exit 1
    67136        fi
    68137
    69         # Clean up from previous packaging
    70         echo "Cleaning up existing assets"
    71         cd ${ISSM_DIR}
    72         rm -rf ${PKG} ${COMPRESSED_PKG}
    73         mkdir ${PKG}
     138        # Check that all tests passed
     139        numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
    74140
    75         # Add/modify required binaries and libraries
    76         cd ${ISSM_DIR}/bin
    77 
    78         echo "Modify generic"
    79         cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m
    80 
    81         echo "Moving MPICH binaries to bin/"
    82         if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
    83                 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
    84                 cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
    85         elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
    86                 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
    87                 cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
     141        if [[ ${numTestsFailed} -ne 0 ]]; then
     142                echo "One or more tests FAILED"
     143                exit 1
    88144        else
    89                 echo "MPICH not found"
    90                 exit 1
     145                echo "All tests PASSED"
    91146        fi
    92 
    93         echo "Moving GDAL binaries to bin/"
    94         if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
    95                 cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
    96                 cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
    97         else
    98                 echo "GDAL not found"
    99                 exit 1
    100         fi
    101 
    102         echo "Moving GMT binaries to bin/"
    103         if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
    104                 cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
    105                 cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
    106         else
    107                 echo "GMT not found"
    108                 exit 1
    109         fi
    110 
    111         echo "Moving Gmsh binaries to bin/"
    112         if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
    113                 cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
    114         else
    115                 echo "Gmsh not found"
    116                 exit 1
    117         fi
    118 
    119         echo "Moving libgfortran to lib/"
    120         cp ${LIBGFORTRAN} ${LIBGFORTRAN_DIST} 2> /dev/null
    121 
    122 
    123         # Run tests
    124         if [ ${skip_tests} -eq 0 ]; then
    125                 echo "Running tests"
    126                 cd ${ISSM_DIR}/test/NightlyRun
    127                 rm matlab.log 2> /dev/null
    128 
    129                 # Run tests, redirecting output to logfile and suppressing output to console
    130                 ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
    131 
    132                 # Check that MATLAB did not exit in error
    133                 matlabExitCode=`echo $?`
    134                 matlabExitedInError=`grep -E "Activation cannot proceed|license|Error" matlab.log | wc -l`
    135 
    136                 if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then
    137                         echo "----------MATLAB exited in error!----------"
    138                         cat matlab.log
    139                         echo "-----------End of matlab.log-----------"
    140 
    141                         # Clean up execution directory
    142                         rm -rf ${ISSM_DIR}/execution/*
    143 
    144                         exit 1
    145                 fi
    146 
    147                 # Check that all tests passed
    148                 numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
    149 
    150                 if [[ ${numTestsFailed} -ne 0 ]]; then
    151                         echo "One or more tests FAILED"
    152                         exit 1
    153                 else
    154                         echo "All tests PASSED"
    155                 fi
    156         else
    157                 echo "Skipping tests"
    158         fi
    159 
    160         # Create package
    161         cd ${ISSM_DIR}
    162         svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
    163         echo "Copying assets to package: ${PKG}"
    164         cp -rf bin examples lib scripts test ${PKG}/
    165         mkdir ${PKG}/execution
    166         echo "Cleaning up unneeded/unwanted files"
    167         rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
    168         rm -f ${PKG}/lib/*.a # Remove static libraries from package
    169         rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
    170         rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
    171 
    172         # Compress package
    173         echo "Compressing package"
    174         tar -czf ${COMPRESSED_PKG} ${PKG}
    175         ls -lah ${ISSM_DIR}/${COMPRESSED_PKG}
     147else
     148        echo "Skipping tests"
    176149fi
    177150
    178 # Transfer package to ISSM Web site
    179 echo "Transferring package to ISSM Web site"
    180 scp -i ~/.ssh/debian_linux-vm_to_ross ${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG}
     151# Create package
     152cd ${ISSM_DIR}
     153svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
     154echo "Copying assets to package: ${PKG}"
     155cp -rf bin examples lib scripts test ${PKG}/
     156mkdir ${PKG}/execution
     157echo "Cleaning up unneeded/unwanted files"
     158rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
     159rm -f ${PKG}/lib/*.a # Remove static libraries from package
     160rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
     161rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
    181162
    182 if [ $? -ne 0 ]; then
    183         echo "Transfer failed! Verify connection then build this project again."
    184         exit 1
    185 fi
     163# Compress package
     164echo "Compressing package"
     165tar -czf ${COMPRESSED_PKG} ${PKG}
Note: See TracChangeset for help on using the changeset viewer.