Changeset 25884


Ignore:
Timestamp:
12/17/20 19:45:30 (4 years ago)
Author:
jdquinn
Message:

CHG: Split macOS packaging into multiple scripts

Location:
issm/trunk-jpl/packagers/mac
Files:
3 added
1 edited

Legend:

Unmodified
Added
Removed
  • issm/trunk-jpl/packagers/mac/package-issm-mac-binaries-matlab.sh

    r25879 r25884  
    22
    33################################################################################
    4 # To be used after running,
    5 #
    6 #       ${ISSM_DIR}/jenkins/jenkins.sh ${ISSM_DIR}/jenkins/pine_island-mac-binaries-matlab
    7 #
    8 # in the context of a Jenkins project.
    9 #
    10 # When no runtime errors occur, performs the following:
    11 # - Checks resulting executables and libraries against test suite.
    12 # - Packages and compresses executables and libraries.
    13 # - Commits compressed package to repository to be signed by JPL Cybersecurity.
    14 # - Retrieves signed package and transmits it to ISSM Web site for
    15 #       distribution.
     4# Packages and tests ISSM distributable package for macOS with MATLAB API.
    165#
    176# Options:
    18 # -n/--notarizeonly             Sign/notarize only (use if signing/notarization fails
    19 #                                               to skip tests/packaging)
    20 # -s/--skiptests                Skip tests (use if this script fails for some reason
    21 #                                               after tests have successfully passed to save time)
    22 # -t/--transferonly             Transfer package to ISSM Web site only (use if
    23 #                                               transfer fails for some reason to skip testing and
    24 #                                               signing)
    25 # -u/--unlock                   Remove lock file from signed package repository (use if
    26 #                                               build is aborted to allow for subsequent fresh build)
    27 #
    28 # Debugging:
    29 # - Relies on a very tight handshake with project on remote JPL Cybersecurity
    30 #       Jenkins server. Debugging may be perfomed locally by running,
    31 #
    32 #               packagers/mac/sign-issm-mac-binaries-matlab.sh
    33 #
    34 #       with Apple Developer credentials.
    35 # - Removing stdout/stderr redirections to null device (> /dev/null 2>&1) can
    36 #       help debug potential SVN issues.
     7# -s/--skiptests                Skip tests and package only.
    378#
    389# NOTE:
    39 # - Assumes that "ISSM_BINARIES_USER" and "ISSM_BINARIES_PASS" are set up in
    40 #       the 'Bindings' section under a 'Username and password (separated)' binding
    41 #       (requires 'Credentials Binding Plugin').
    42 # - For local debugging, the aformentioned credentials can be hardcoded into
    43 #       the 'USERNAME' and 'PASSWORD' constants below.
     10# - Assumes that the following constants are defined,
     11#
     12#               COMPRESSED_PKG
     13#               ISSM_DIR
     14#               PKG
     15#
     16# See also:
     17# - packagers/mac/complete-issm-mac-binaries-matlab.sh
     18# - packagers/mac/sign-issm-mac-binaries-matlab.sh
    4419################################################################################
    4520
     
    6641## Override certain other aliases
    6742#
     43alias cp=$(which cp)
    6844alias grep=$(which grep)
    6945
     
    7248MATLAB_NROPTIONS="'benchmark','all','exclude',[125,126,234,235,418,420,435,444,445,701,702,703,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1201,1202,1203,1204,1205,1206,1207,1208,1301,1302,1303,1304,1401,1402,1601,1602,2006,2020,2021,2051,2052,2053,3001:3200,3201,3202,3300,3480,3481,4001,4002,4003]" # NOTE: Combination of test suites from basic, Dakota, and Solid Earth builds, with tests that require a restart and those that require the JVM excluded
    7349MATLAB_PATH="/Applications/MATLAB_R2018a.app"
    74 NOTARIZATION_LOGFILE="notarization.log"
    75 PASSWORD=${ISSM_BINARIES_PASS}
    76 PKG="ISSM-macOS-MATLAB" # Name of directory to copy distributable files to
    77 RETRIGGER_SIGNING_FILE="retrigger.txt"
    78 SIGNED_REPO_COPY="./signed"
    79 SIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/signed"
    80 SIGNING_CHECK_PERIOD=60 # in seconds
    81 SIGNING_LOCK_FILE="signing.lock"
    82 UNSIGNED_REPO_COPY="./unsigned"
    83 UNSIGNED_REPO_URL="https://issm.ess.uci.edu/svn/issm-binaries/mac/matlab/unsigned"
    84 USERNAME=${ISSM_BINARIES_USER}
    85 
    86 COMPRESSED_PKG="${PKG}.zip"
    8750
    8851## Environment
    8952#
    90 export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'
    91 
    92 ## Functions
    93 #
    94 checkout_signed_repo_copy(){
    95         echo "Checking out copy of repository for signed packages"
    96 
    97         # NOTE: Get empty copy because we do not want to have to check out package
    98         #               from previous signing.
    99         #
    100         svn checkout \
    101                 --trust-server-cert \
    102                 --non-interactive \
    103                 --depth empty \
    104                 --username ${USERNAME} \
    105                 --password ${PASSWORD} \
    106                 ${SIGNED_REPO_URL} \
    107                 ${SIGNED_REPO_COPY} > /dev/null 2>&1
    108 }
    109 checkout_unsigned_repo_copy(){
    110         echo "Checking out copy of repository for unsigned packages"
    111         svn checkout \
    112                 --trust-server-cert \
    113                 --non-interactive \
    114                 --username ${USERNAME} \
    115                 --password ${PASSWORD} \
    116                 ${UNSIGNED_REPO_URL} \
    117                 ${UNSIGNED_REPO_COPY} > /dev/null 2>&1
    118 }
    119 validate_signed_repo_copy(){
    120         # Validate copy of repository for signed binaries (e.g.
    121         # 'Check-out Strategy' was set to 'Use 'svn update' as much as possible';
    122         # initial checkout failed)
    123         if [[ ! -d ${SIGNED_REPO_COPY} || ! -d ${SIGNED_REPO_COPY}/.svn ]]; then
    124                 rm -rf ${SIGNED_REPO_COPY}
    125                 checkout_signed_repo_copy
    126         fi
    127 }
     53export PATH="${ISSM_DIR}/bin:$(getconf PATH)" # Ensure that we pick up binaries from 'bin' directory rather than 'externalpackages'; used when running tests
    12854
    12955## Parse options
     
    13460fi
    13561
    136 notarize_only=0
    13762skip_tests=0
    138 transfer_only=0
    139 unlock=0
    140 while [ $# -gt 0 ]; do
    141     case $1 in
    142         -n|--notarizeonly) notarize_only=1; shift ;;
    143         -s|--skiptests) skip_tests=1; shift ;;
    144         -t|--transferonly) transfer_only=1; shift ;;
    145         -u|--unlock) unlock=1; shift ;;
    146         *) echo "Unknown parameter passed: $1"; exit 1 ;;
    147     esac
    148     shift
    149 done
    15063
    151 if [ ${unlock} -eq 1 ]; then
    152         # Remove signing lock file from signed package repository so that a new
    153         # build can run
    154         echo "Removing lock file from repository for signed packages"
    155         checkout_signed_repo_copy
    156         svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
    157         svn delete ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
    158         svn commit \
    159                 --trust-server-cert \
    160                 --non-interactive \
    161                 --username ${USERNAME} \
    162                 --password ${PASSWORD} \
    163                 --message "DEL: Removing lock file after failed build" ${SIGNED_REPO_COPY}
    164         svn cleanup ${SIGNED_REPO_COPY}
     64if [ $# -eq 1 ]; then
     65        case $1 in
     66                -s|--skiptests) skip_tests=1;                                   ;;
     67                *) echo "Unknown parameter passed: $1"; exit 1  ;;
     68        esac
     69fi
     70
     71# Check if MATLAB exists
     72if ! [ -d ${MATLAB_PATH} ]; then
     73        echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again."
    16574        exit 1
    16675fi
    16776
    168 if [ ${transfer_only} -eq 0 ]; then
    169         rm -rf ${SIGNED_REPO_COPY}
     77# Clean up from previous packaging
     78echo "Cleaning up existing assets"
     79cd ${ISSM_DIR}
     80rm -rf ${PKG} ${COMPRESSED_PKG}
     81mkdir ${PKG}
    17082
    171         checkout_signed_repo_copy
     83# Add required binaries and libraries to package and modify them where needed
     84cd ${ISSM_DIR}/bin
    17285
    173         # If lock file exists, a signing build is still in process by JPL Cybersecurity
    174         svn up ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
    175         if [ -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
    176                 echo "Previous signing job still in process by JPL Cybersecurity. Please try again later."
     86echo "Modify generic"
     87cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m
     88
     89echo "Moving MPICH binaries to bin/"
     90if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
     91        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
     92        cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
     93elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
     94        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
     95        cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
     96else
     97        echo "MPICH not found"
     98        exit 1
     99fi
     100
     101echo "Moving GDAL binaries to bin/"
     102if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
     103        cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
     104        cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
     105else
     106        echo "GDAL not found"
     107        exit 1
     108fi
     109
     110echo "Moving GMT binaries to bin/"
     111if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
     112        cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
     113        cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
     114else
     115        echo "GMT not found"
     116        exit 1
     117fi
     118
     119echo "Moving Gmsh binaries to bin/"
     120if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
     121        cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
     122else
     123        echo "Gmsh not found"
     124        exit 1
     125fi
     126
     127# Run tests
     128if [ ${skip_tests} -eq 0 ]; then
     129        echo "Running tests"
     130        cd ${ISSM_DIR}/test/NightlyRun
     131        rm matlab.log 2> /dev/null
     132
     133        # Run tests, redirecting output to logfile and suppressing output to console
     134        ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
     135
     136        # Check that MATLAB did not exit in error
     137        matlabExitCode=`echo $?`
     138        matlabExitedInError=`grep -E "Activation cannot proceed|Error in matlab_run" matlab.log | wc -l`
     139
     140        if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then
     141                echo "----------MATLAB exited in error!----------"
     142                cat matlab.log
     143                echo "-----------End of matlab.log-----------"
     144
     145                # Clean up execution directory
     146                rm -rf ${ISSM_DIR}/execution/*
     147
    177148                exit 1
    178149        fi
    179150
    180         if [ ${notarize_only} -eq 0 ]; then
    181                 # Check if MATLAB exists
    182                 if ! [ -d ${MATLAB_PATH} ]; then
    183                         echo "${MATLAB_PATH} does not point to a MATLAB installation! Please modify MATLAB_PATH variable in $(basename $0) and try again."
    184                         exit 1
    185                 fi
     151        # Check that all tests passed
     152        numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
    186153
    187                 # Clean up from previous packaging
    188                 echo "Cleaning up existing assets"
    189                 cd ${ISSM_DIR}
    190                 rm -rf ${PKG} ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
    191                 mkdir ${PKG}
    192 
    193                 # Add required binaries and libraries to package and modify them where needed
    194                 cd ${ISSM_DIR}/bin
    195 
    196                 echo "Modify generic"
    197                 cat generic_static.m | sed -e "s/generic_static/generic/g" > generic.m
    198 
    199                 echo "Moving MPICH binaries to bin/"
    200                 if [ -f ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec ]; then
    201                         cp ${ISSM_DIR}/externalpackages/petsc/install/bin/mpiexec .
    202                         cp ${ISSM_DIR}/externalpackages/petsc/install/bin/hydra_pmi_proxy .
    203                 elif [ -f ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec ]; then
    204                         cp ${ISSM_DIR}/externalpackages/mpich/install/bin/mpiexec .
    205                         cp ${ISSM_DIR}/externalpackages/mpich/install/bin/hydra_pmi_proxy .
    206                 else
    207                         echo "MPICH not found"
    208                         exit 1
    209                 fi
    210 
    211                 echo "Moving GDAL binaries to bin/"
    212                 if [ -f ${ISSM_DIR}/externalpackages/gdal/install/bin/gdal-config ]; then
    213                         cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdalsrsinfo .
    214                         cp ${ISSM_DIR}/externalpackages/gdal/install/bin/gdaltransform .
    215                 else
    216                         echo "GDAL not found"
    217                         exit 1
    218                 fi
    219 
    220                 echo "Moving GMT binaries to bin/"
    221                 if [ -f ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt-config ]; then
    222                         cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmt .
    223                         cp ${ISSM_DIR}/externalpackages/gmt/install/bin/gmtselect .
    224                 else
    225                         echo "GMT not found"
    226                         exit 1
    227                 fi
    228 
    229                 echo "Moving Gmsh binaries to bin/"
    230                 if [ -f ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh ]; then
    231                         cp ${ISSM_DIR}/externalpackages/gmsh/install/bin/gmsh .
    232                 else
    233                         echo "Gmsh not found"
    234                         exit 1
    235                 fi
    236 
    237                 # Run tests
    238                 if [ ${skip_tests} -eq 0 ]; then
    239                         echo "Running tests"
    240                         cd ${ISSM_DIR}/test/NightlyRun
    241                         rm matlab.log 2> /dev/null
    242 
    243                         # Run tests, redirecting output to logfile and suppressing output to console
    244                         ${MATLAB_PATH}/bin/matlab -nojvm -nosplash -r "try, addpath ${ISSM_DIR}/bin ${ISSM_DIR}/lib; runme(${MATLAB_NROPTIONS}); exit; catch me,fprintf('%s',getReport(me)); exit; end" -logfile matlab.log &> /dev/null
    245 
    246                         # Check that MATLAB did not exit in error
    247                         matlabExitCode=`echo $?`
    248                         matlabExitedInError=`grep -E "Activation cannot proceed|Error in matlab_run" matlab.log | wc -l`
    249 
    250                         if [[ ${matlabExitCode} -ne 0 || ${matlabExitedInError} -ne 0 ]]; then
    251                                 echo "----------MATLAB exited in error!----------"
    252                                 cat matlab.log
    253                                 echo "-----------End of matlab.log-----------"
    254 
    255                                 # Clean up execution directory
    256                                 rm -rf ${ISSM_DIR}/execution/*
    257 
    258                                 exit 1
    259                         fi
    260 
    261                         # Check that all tests passed
    262                         numTestsFailed=`cat matlab.log | grep -c -e "FAILED|ERROR"`
    263 
    264                         if [ ${numTestsFailed} -ne 0 ]; then
    265                                 echo "One or more tests FAILED"
    266                                 exit 1
    267                         else
    268                                 echo "All tests PASSED"
    269                         fi
    270                 else
    271                         echo "Skipping tests"
    272                 fi
    273 
    274                 # Create package
    275                 cd ${ISSM_DIR}
    276                 svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
    277                 echo "Copying assets to package: ${PKG}"
    278                 cp -rf bin examples lib scripts test ${PKG}/
    279                 mkdir ${PKG}/execution
    280                 cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist
    281                 echo "Cleaning up unneeded/unwanted files"
    282                 rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
    283                 rm -f ${PKG}/lib/*.a # Remove static libraries from package
    284                 rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
    285                 rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
    286 
    287                 # Compress package
    288                 echo "Compressing package"
    289                 ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
     154        if [ ${numTestsFailed} -ne 0 ]; then
     155                echo "One or more tests FAILED"
     156                exit 1
    290157        else
    291                 # Assume that previous build was successful, but signing/notarization
    292                 # failed.
    293                 #
    294                 echo "Notarizing only"
     158                echo "All tests PASSED"
    295159        fi
    296 
    297         # Commit lock file to repository for signed packages
    298         echo "Committing lock file to repository for signed packages"
    299         touch ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE}
    300         svn add ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} > /dev/null 2>&1
    301         svn commit \
    302                 --trust-server-cert \
    303                 --non-interactive \
    304                 --username ${USERNAME} \
    305                 --password ${PASSWORD} \
    306                 --message "ADD: New lock file" ${SIGNED_REPO_COPY}
    307 
    308         # Check out copy of repository for unsigned packages
    309         checkout_unsigned_repo_copy
    310 
    311         if [ ${notarize_only} -eq 0 ]; then
    312                 # Commit new compressed package to repository for unsigned binaries
    313                 #
    314                 # NOTE: This will not work if, for any reason, the checksum on the compressed
    315                 #               package is unchanged.
    316                 #
    317                 echo "Committing package to repository for unsigned packages"
    318                 cp ${COMPRESSED_PKG} ${UNSIGNED_REPO_COPY}
    319                 svn add ${UNSIGNED_REPO_COPY}/${COMPRESSED_PKG} > /dev/null 2>&1
    320                 svn commit \
    321                         --trust-server-cert \
    322                         --non-interactive \
    323                         --username ${USERNAME} \
    324                         --password ${PASSWORD} \
    325                         --message "CHG: New unsigned package" ${UNSIGNED_REPO_COPY}
    326         else
    327                 # NOTE: If notarize_only == 1, we commit a dummy file as the signing
    328                 #               build on the remote JPL Cybersecurity Jenkins server is
    329                 #               triggered by polling SCM.
    330                 #
    331                 echo "Attempting to sign existing package again"
    332                 echo $(date +'%Y-%m-%d-%H-%M-%S') > ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} # Write datetime stamp to file to ensure modification is made
    333                 svcheckout_unsigned_repo_copyn add ${UNSIGNED_REPO_COPY}/${RETRIGGER_SIGNING_FILE} > /dev/null 2>&1
    334                 svn commit \
    335                         --trust-server-cert \
    336                         --non-interactive \
    337                         --username ${USERNAME} \
    338                         --password ${PASSWORD} \
    339                         --message "ADD: Retriggering signing with same package (previous attempt failed)" ${UNSIGNED_REPO_COPY}
    340         fi
    341 
    342         # Check status of signing
    343         echo "Checking progress of signing..."
    344         IN_PROCESS=1
    345         SUCCESS=0
    346 
    347         while [ ${IN_PROCESS} -eq 1 ]; do
    348                 echo "...in progress still; checking again in ${SIGNING_CHECK_PERIOD} seconds"
    349                 sleep ${SIGNING_CHECK_PERIOD}
    350                 svn up ${SIGNED_REPO_COPY}
    351 
    352                 if [ ! -f ${SIGNED_REPO_COPY}/${SIGNING_LOCK_FILE} ]; then
    353                         IN_PROCESS=0
    354 
    355                         # Retrieve notarization lock file
    356                         svn up ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
    357 
    358                         # Check status
    359                         STATUS=$(grep 'Status:' ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE} | sed -e 's/[[:space:]]*Status: //')
    360                         if [[ "${STATUS}" == "success" ]]; then
    361                                 echo "Notarization successful!"
    362 
    363                                 # Set flag indicating notarization was successful
    364                                 SUCCESS=1
    365                         else
    366                                 echo "Notarization failed!"
    367                         fi
    368                 fi
    369         done
    370160else
    371         # Assume that previous build resulted in successful signing of package but
    372         # that transfer to ISSM Web site failed and user built this project again
    373         # with -t/--transferonly option.
    374         #
    375 
    376         # Make sure copy of repository for signed packages exists
    377         validate_signed_repo_copy
    378 
    379         SUCCESS=1
     161        echo "Skipping tests"
    380162fi
    381163
    382 # Handle result of signing
    383 if [ ${SUCCESS} -eq 1 ]; then
    384         # Retrieve signed and notarized package
    385         svn up ${SIGNED_REPO_COPY}/${COMPRESSED_PKG}
     164# Create package
     165cd ${ISSM_DIR}
     166svn cleanup --remove-ignored --remove-unversioned test # Clean up test directory (before copying to package)
     167echo "Copying assets to package: ${PKG}"
     168cp -rf bin examples lib scripts test ${PKG}/
     169mkdir ${PKG}/execution
     170cp packagers/mac/issm-executable_entitlements.plist ${PKG}/bin/entitlements.plist
     171echo "Cleaning up unneeded/unwanted files"
     172rm -f ${PKG}/bin/generic_static.* # Remove static versions of generic cluster classes
     173rm -f ${PKG}/lib/*.a # Remove static libraries from package
     174rm -f ${PKG}/lib/*.la # Remove libtool libraries from package
     175rm -rf ${PKG}/test/SandBox # Remove testing sandbox from package
    386176
    387         # Transfer signed package to ISSM Web site
    388         echo "Transferring signed package to ISSM Web site"
    389         scp -i ~/.ssh/pine_island_to_ross ${SIGNED_REPO_COPY}/${COMPRESSED_PKG} jenkins@ross.ics.uci.edu:/var/www/html/${COMPRESSED_PKG}
    390 
    391         if [ $? -ne 0 ]; then
    392                 echo "Transfer failed! Verify connection then build this project again (with -t/--transferonly option to skip testing and signing)."
    393                 exit 1
    394         fi
    395 else
    396         echo "----------------------- Contents of notarization logfile -----------------------"
    397         cat ${SIGNED_REPO_COPY}/${NOTARIZATION_LOGFILE}
    398         echo "--------------------------------------------------------------------------------"
    399 
    400         exit 1
    401 fi
     177# Compress package
     178echo "Compressing package"
     179ditto -ck --sequesterRsrc --keepParent ${PKG} ${COMPRESSED_PKG}
Note: See TracChangeset for help on using the changeset viewer.