Index: /issm/trunk/cron/configs/linux64_morlighem2 =================================================================== --- /issm/trunk/cron/configs/linux64_morlighem2 (revision 4999) +++ /issm/trunk/cron/configs/linux64_morlighem2 (revision 4999) @@ -0,0 +1,94 @@ +# +########### Configuration file for Mathieu Morlighem's nightly run on Linux ############ + +#-------------------------------# +# 1: ISSM general configuration # +#-------------------------------# + +#ISSM main directory (full pathto ISSM trunk) +ISSM_DIR="/u/wilkes-r1b/morlighe/svn/issm/trunk/cron/trunk" +#ISSM_DIR="/u/wilkes-r1b/morlighe/svn/issm/trunk/cron/issm2.2" + +#ISSM Architecture +ISSM_ARCH="linux-gnu-amd64" + +#Operating System +OS="linux64" + +#MATLAB path +MATLAB_PATH="/usr/local/pkgs/matlab-7.6/" + +#----------------------# +# 2: ISSM Installation # +#----------------------# + +#ISSM_INSTALLATION can have 3 values: +# - "checkout" the existing version of the code will be erased and +# the latest version will be checked out +# - "update" the directory won't be erased but ISSM will be updated +# ->skip to section 3 +# - "none" leave ISSM as is in its directory +# ->skip to section 3 +ISSM_INSTALLATION="update" + +#SVN repository +REPOSITORY="http://s383-rhat/issm/svn/issm/trunk" +#REPOSITORY="http://s383-rhat/issm/svn/issm/branches/issm2.2" + +#execution path used for parallel runs +EXECUTION_PATH="/u/wilkes-r1b/morlighe/ExecutionNightlyRun" + +#-----------------------------------# +# 3: External packages installation # +#-----------------------------------# + +#ISSM_EXTERNALPACKAGES can have 3 values: +# - "install" install all external packages listed below +# - "copy" copy existing directories (EXTERNALPACKAGESDIR +# and DEVPACKAGESDIR) +# - "none" leave external packages as is +# ->skip to section 4 +ISSM_EXTERNALPACKAGES="none" +EXTERNALPACKAGESDIR="/u/wilkes-r1b/morlighe/svn/issm/trunk/externalpackages" +DEVPACKAGESDIR="/u/wilkes-r1b/morlighe/svn/issm/trunk/devpackages" + +#List of external pakages to be installed +EXTERNALPACKAGES="matlab mpich2 petsc metis triangle dakota chaco" + +#---------------------# +# 4: ISSM Compilation # +#---------------------# + +#ISSM_COMPILATION can have 2 values: +# - "yes" compile ISSM +# - "no" do not compile ISSM +ISSM_COMPILATION="no" + +#----------------------# +# 5: Mail notification # +#----------------------# + +#Mail delivery. If SKIPMAIL="no", the html nightly run report will be +#sent to the adresses present in $ISSM_DIR/cron/mailinglist. +SKIPMAIL="no" + +#Sender email address +EMAIL_ADRESS="mathieu.morlighem@jpl.nasa.gov" + +#------------------------# +# 6: Nightly run options # +#------------------------# + +#number of cpus used in ISSM installation and compilation (one is usually +#safer as some packages are very sensitive to parallel compilation) +NUMCPUS_INSTALL=8 + +#number of cpus used in the nightly runs. +NUMCPUS_RUN=7 + +#Nightly run options. The matlab routine nightlyrun.m will be called +#as follows: nightlyrun($NROPTIONS). The options must be understandable +#by Matlab and nightlyrun.m +#ex: "'id',[101 102 103]" + +NROPTIONS="" Index: /issm/trunk/cron/nightlyrun2.sh =================================================================== --- /issm/trunk/cron/nightlyrun2.sh (revision 4999) +++ /issm/trunk/cron/nightlyrun2.sh (revision 4999) @@ -0,0 +1,296 @@ +#!/bin/bash +#This bash script calls the nightlyrun.m matlab file to run our nightly test decks. +#It then processes the results and sends an email to the Ice developpers. + +#some functions +function timer() #{{{1 +{ + if [[ $# -eq 0 ]]; then + echo $(date '+%s') + else + local stime=$1 + etime=$(date '+%s') + + if [[ -z "$stime" ]]; then stime=$etime; fi + + dt=$((etime - stime)) + ds=$((dt % 60)) + dm=$(((dt / 60) % 60)) + dh=$((dt / 3600)) + printf '%d:%02d:%02d' $dh $dm $ds + fi +} #}}} +function todaydate() #{{{1 +{ + suffix=`date | awk '{printf("%s-%s-%s %s",$2,$3,$6,$4);}'` + echo $suffix; +} #}}} +function host_name() #{{{1 +{ + #return host name depending on the OS + if [ "$1" = "winxp32" ] + then + HOST_NAME=`hostname`; + else + HOST_NAME=`hostname -s`; + fi + echo $HOST_NAME; +} #}}} + +#Get configuration +#Check that no other nightly run is running {{{1 +if [ "$(ps aux | grep "nighltyrun\.sh" | grep -v "grep")" ] +then + echo "WARNING: A nightly run is already running. Killing..." + + for JOBID in $(ps aux | grep "nighltyrun\.sh" | grep -v "grep" | awk '{printf("%s\n",$2);}'); + do + kill -9 $JOBID; + done; +fi +#}}} +#Source config file{{{1 +if [ $# -ne 1 ]; +then + #no config file specified: exit + echo "no config file specified. Exiting..." >&2 # Error message to stderr. + exit 1 +fi +if [ ! -f "$1" ] +then + echo "File $1 not found!" >&2 # Error message to stderr. + exit 1 +fi +source $1; +#}}} +#Export ISSM_* variables{{{1 +export ISSM_DIR +export ISSM_ARCH +#}}} +#Initialize variables {{{1 +TODAY=$(todaydate); +HOST_NAME=$(host_name $OS); +START_TIME=$(timer); +ISSM_RELEASE=$(basename $(echo $REPOSITORY)); +USER=$(whoami); +INIT_PATH=$(pwd); +#}}} + + +#Lauch installation +#Checkout/update/none ISSM if requested (ISSM_INSTALLATION){{{1 +if [ "$ISSM_INSTALLATION" == "checkout" ] +then + + #Erase previous code and Fetch the new one + rm -rf $ISSM_RELEASE + svn checkout $REPOSITORY + + + #create simpler cluster.rc file, with only the cluster we are interested in. + cat << END > cluster.rc +begin +cluster_name=$HOST_NAME +cluster_codepath=$ISSM_DIR/bin +cluster_executionpath=$EXECUTION_PATH +cluster_login=$USER +cluster_port=0 + +end +END + +elif [ "$ISSM_INSTALLATION" == "update" ] +then + + #only update ISSM + cd $ISSM_DIR + svn update + +elif [ "$ISSM_INSTALLATION" == "none" ] +then + + #do nothing + echo "Skipping ISSM installation" + +else + + #Error message + echo "ISSM_INSTALLATION supported values are: checkout, update and none. Exiting..." >&2 # Error message to stderr. + exit 1 + +fi +#}}} +#Source environment variables with new matlab path {{{1 +cd $ISSM_DIR/etc +source environment.sh MATLAB_DIR=$MATLAB_PATH +#}}} +#install/copy/none external packages (ISSM_EXTERNALPACKAGES){{{1 +if [ "$ISSM_EXTERNALPACKAGES" == "install" ] +then + cd $ISSM_DIR/externalpackages + for ep in $EXTERNALPACKAGES + do + cd $ep + if [ "$ep" == "petsc" ] + then + #For Petsc (2.3 for now) + cp configs/2.3.2-p3/$OS/* . + #cp configs/3.0.0-p11/$OS/* . + #cat install.sh | sed -e "s/version='2.3.2-p3'/version='3.0.0-p11'/g" > temp.sh + #mv temp.sh install.sh + else + cp configs/$OS/* . + fi + ./install.sh $NUMCPUS_INSTALL + cd .. + done + + #3: install automake and autoconf + cd $ISSM_DIR/devpackages + make + cd .. + +elif [ "$ISSM_EXTERNALPACKAGES" == "copy" ] +then + + #erase externapackages, and link with externalpackages_dir + cd $ISSM_DIR + rm -rf externalpackages + cp -Rf $EXTERNALPACKAGESDIR ./ + + rm -rf devpackages + cp -Rf $DEVPACKAGESDIR ./ + +elif [ "$ISSM_EXTERNALPACKAGES" == "none" ] +then + + #Do nothing + echo "Skipping external packages installation" + +else + + #Error message + echo "ISSM_EXTERNALPACKAGES supported values are: install, copy and none. Exiting..." >&2 # Error message to stderr. + exit 1 + +fi +#}}} +#ISSM compilation yes/no (ISSM_COMPILATION) {{{1 +if [ "$ISSM_COMPILATION" == "yes" ] +then + + cd $ISSM_DIR + make distclean + sh scripts/automakererun.sh + sh configs/$OS/$OS.sh.petsc2 #use petsc2 for now + #sh configs/$OS/$OS.sh.petsc3 #switch to petsc3 to debug + + #4: compile and install ISSM + if [ "$OS" = "winxp32" ] + then + cd $ISSM_DIR/src/c + ./intel-compile.sh + cd $ISSM_DIR/src/mex + make install + cd $ISSM_DIR + else + make -j $NUMCPUS_INSTALL + make -j $NUMCPUS_INSTALL install + fi + +elif [ "$ISSM_COMPILATION" == "no" ] +then + + #do nothing + echo "Skipping ISSM compilation" + +else + + #Error message + echo "ISSM_COMPILATION supported values are: yes and no. Exiting..." >&2 # Error message to stderr. + exit 1 + +fi +#}}} + +#Prepare run +#Windows hack for startup.m {{{1 +#windows environments: ISSM_DIR_WIN variable not correctly picked up when using +#the cron job. just get startup to take the ISSM_DIR variable as the pwd: +if [ "$OS" = "winxp32" ] +then + cat startup.m | sed 's/clear status/clear status; ISSM_DIR=pwd;/g' > startup.m.bak + mv startup.m.bak startup.m +fi +#}}} +#Create TEMP directory and nightly.log {{{1 +#put installation elapsed time in nightly.log +INSTALL_TIME=$(timer) +ELAPSED_INSTALL=$(timer $START_TIME) +rm -rf $ISSM_DIR/TEMP +mkdir $ISSM_DIR/TEMP +cat << END > $ISSM_DIR/TEMP/nightly.log +today: $(echo $TODAY) +user: $(echo $USER) +host: $(echo $HOST_NAME) +OS: $(echo $OS) +release: $(echo $ISSM_RELEASE) +init_path: $(echo $INIT_PATH) +elapsed_install: $(echo $ELAPSED_INSTALL) +END +#}}} + +#Run tests +#Call run2.sh script{{{1 +cd $ISSM_DIR/cron/ +./run2.sh $1 +#}}} +#Complete nightly.log {{{1 +ELAPSED_RUN=$(timer $INSTALL_TIME) +ELAPSED_TOTAL=$(timer $START_TIME) +cat << END >> $ISSM_DIR/TEMP/nightly.log +elapsed_run: $(echo $ELAPSED_RUN) +elapsed_total: $(echo $ELAPSED_TOTAL) +END +#}}} + +#Send Report +#Build html report {{{1 +cd $ISSM_DIR/cron/ +./report2.sh +echo "html report located in $ISSM_DIR/TEMP/report.html" +#}}} +exit 1 +#send mail if requested (SKIPMAIL) {{{1 +if [ "$SKIPMAIL" != "yes" ] +then + echo "sending report..." + source $ISSM_DIR/cron/mailinglist + for i in `echo $MAILINGLIST`; do + + if [ "$OS" = "winxp32" ] + then + email -html -f "ISSM Nightly run $EMAIL_ADRESS" -s "Nightly runs of $ISSM_RELEASE , operating system: $OS, host: $HOST_NAME, user: $USER. " $i < $ISSM_DIR/test/Verification/NightlyRun/report.html + else + if [ "$OS" = "linux64" ] + then +cat - $ISSM_DIR/TEMP/report.html < +To: $i +Subject: Nightly runs of $ISSM_RELEASE, operating system: $OS, host: $HOST_NAME, user: $USER. +Mime-Version: 1.0 +Content-Type: text/html +HERE + else +cat - $ISSM_DIR/TEMP/report.html < +To: $i +Subject: Nightly runs of $ISSM_RELEASE, operating system: $OS, host: $HOST_NAME, user: $USER. +Mime-Version: 1.0 +Content-Type: text/html +HERE + fi + fi + done +fi +#}}} Index: /issm/trunk/cron/run2.sh =================================================================== --- /issm/trunk/cron/run2.sh (revision 4999) +++ /issm/trunk/cron/run2.sh (revision 4999) @@ -0,0 +1,77 @@ +#!/bin/bash + +#prepare runs +#get configs {{{1 +if [ $# -ne 1 ]; +then + #no config file specified: exit + echo "no config file specified. Exiting..." + exit 1 +fi + +#Go to init_Path and source config +INIT_PATH=$(cat $ISSM_DIR/TEMP/nightly.log | grep "init_path" | awk '{print $2}') +cd $INIT_PATH +source $1; +#}}} +#check NUMCPUS_RUN options {{{1 +if [ "$NUMCPUS_RUN" = "" ] +then + echo "NUMCPUS_RUN option not found, defaulting to NUMCPUS_RUN = 1" + NUMCPUS_RUN=1 +fi +#}}} +#create softlink to startup {{{1 +cd $ISSM_DIR/test/NightlyRun/ +ln -s $ISSM_DIR/startup.m . +#}}} + +#Launch all tests on different cpus +for (( i=1;i<=$NUMCPUS_RUN;i++ )) +do + #Launch matlab and the nightly run script + cat > $ISSM_DIR/TEMP/matlab_run$i.m << EOF + warning off %necessary to avoid a nightly.log of several Go for parallel runs + try, + cd $ISSM_DIR/test/NightlyRun + startup; + $(if [ "$NROPTIONS" = "" ] + then + echo "runme('rank',$i,'numprocs',$NUMCPUS_RUN);" + else + echo "runme($NROPTIONS,'rank',$i,'numprocs',$NUMCPUS_RUN);" + fi + ) + catch me, + %An error occured, get report and exit + directory=strsplit(pwd,'/'); + message=getReport(me) + fid=fopen([ISSM_DIR '/TEMP/matlaberror.log'], 'at'); + fprintf(fid,'\nMatlab error occured in: %s\n\n',directory{end}); + fprintf(fid,'%s',message); + fclose(fid); + end + exit +EOF + + #Start run from TEMP directory + cd $ISSM_DIR/TEMP/ + + #Start test + MATLAB_VERSION="7.6" #7.2,7.4,7.6 and 7.8 + /usr/local/pkgs/matlab-$MATLAB_VERSION/bin/matlab -nojvm -nosplash -r matlab_run$i -logfile matlab_log$i.log & + +done + +#wait until matlab closes +wait + +#concatenate all reports +mv matlab_log1.log matlab_log.log +for (( i=2;i<=$NUMCPUS_RUN;i++ )) +do + cat matlab_log.log matlab_log$i.log > matlab_log.log.bak + mv matlab_log.log.bak matlab_log.log + rm matlab_log$i.log + +done Index: /issm/trunk/test/NightlyRun/runme.m =================================================================== --- /issm/trunk/test/NightlyRun/runme.m (revision 4998) +++ /issm/trunk/test/NightlyRun/runme.m (revision 4999) @@ -54,4 +54,9 @@ end +%Get RANK and NUMPROCS for mutlithreaded runs +rank=getfieldvalue(options,'rank',1); +numprocs=getfieldvalue(options,'numprocs',1); +if (numprocstolerance); - disp(sprintf(['ERROR difference: %-7.2g > %7.2g test id: %i test name: %s field: %s'],... - error_diff,tolerance,id,Id2Name(id),fieldname)); - else - disp(sprintf(['SUCCESS difference: %-7.2g < %7.2g test id: %i test name: %s field: %s'],... - error_diff,tolerance,id,Id2Name(id),fieldname)); + %disp test result + if (error_diff>tolerance); + disp(sprintf(['ERROR difference: %-7.2g > %7.2g test id: %i test name: %s field: %s'],... + error_diff,tolerance,id,Id2Name(id),fieldname)); + else + disp(sprintf(['SUCCESS difference: %-7.2g < %7.2g test id: %i test name: %s field: %s'],... + error_diff,tolerance,id,Id2Name(id),fieldname)); + end + + catch me2 + + %something went wrong, print failure message: + directory=strsplit(pwd,'/'); + message=getReport(me2) + fid=fopen([ISSM_DIR '/TEMP/matlaberror.log'], 'at'); + fprintf(fid,'%s',message); + fprintf(fid,'\n--------------------------------------------\n'); + fclose(fid); + disp(sprintf(['FAILURE difference: N/A test id: %i test name: %s field: %s'],id,Id2Name(id),fieldname)); + end end @@ -119,10 +141,9 @@ directory=strsplit(pwd,'/'); message=getReport(me) - fid=fopen([ISSM_DIR '/test/Verification/NightlyRun/matlaberror.log'], 'at'); - fprintf(fid,'\n\nMatlab error occured in: %s\n\n',directory{end}); + fid=fopen([ISSM_DIR '/TEMP/matlaberror.log'], 'at'); fprintf(fid,'%s',message); + fprintf(fid,'\n--------------------------------------------\n'); fclose(fid); - %disp(sprintf(['\nFAILURE test: %s analysis_type: %s sub_analysis_type: %s qmu: %i control: %i control_fit: %s parallel: %i\n'],... - %testname,EnumAsString(analysis_type),EnumAsString(sub_analysis_type),qmu,control,control_fit,parallel)); + disp(sprintf(['FAILURE difference: N/A test id: %i test name: %s field: N/A'],id,Id2Name(id))); end end Index: /issm/trunk/test/NightlyRun/test101.m =================================================================== --- /issm/trunk/test/NightlyRun/test101.m (revision 4998) +++ /issm/trunk/test/NightlyRun/test101.m (revision 4999) @@ -2,4 +2,5 @@ md=geography(md,'all',''); md=parameterize(md,'../Par/SquareShelfConstrained.par'); +return md=setelementstype(md,'macayeal','all'); md.cluster='none'; Index: /issm/trunk/test/NightlyRun/test111.m =================================================================== --- /issm/trunk/test/NightlyRun/test111.m (revision 4998) +++ /issm/trunk/test/NightlyRun/test111.m (revision 4999) @@ -5,4 +5,4 @@ md=setelementstype(md,'macayeal','all'); md.cluster='none'; -md.prognostic_DG=1 +md.prognostic_DG=1; md=solve(md,'analysis_type',PrognosticSolutionEnum); Index: /issm/trunk/test/NightlyRun/test112.m =================================================================== --- /issm/trunk/test/NightlyRun/test112.m (revision 4998) +++ /issm/trunk/test/NightlyRun/test112.m (revision 4999) @@ -5,4 +5,4 @@ md=setelementstype(md,'macayeal','all'); md.cluster=oshostname; -md.prognostic_DG=1 +md.prognostic_DG=1; md=solve(md,'analysis_type',PrognosticSolutionEnum); Index: /issm/trunk/test/Par/79North.par =================================================================== --- /issm/trunk/test/Par/79North.par (revision 4998) +++ /issm/trunk/test/Par/79North.par (revision 4999) @@ -46,2 +46,6 @@ md.spcthickness(pos,1)=1; md.spcthickness(pos,2)=md.thickness(pos); + +%Change name so that no test have the same name +A=dbstack; +if (length(A)>2), md.name=A(3).file(1:end-2); end Index: /issm/trunk/test/Par/Pig.par =================================================================== --- /issm/trunk/test/Par/Pig.par (revision 4998) +++ /issm/trunk/test/Par/Pig.par (revision 4999) @@ -42,2 +42,6 @@ %Boundary conditions: md=SetMarineIceSheetBC(md); + +%Change name so that no test have the same name +A=dbstack; +if (length(A)==3), md.name=A(3).file(1:end-2); end Index: /issm/trunk/test/Par/SquareSheetConstrained.par =================================================================== --- /issm/trunk/test/Par/SquareSheetConstrained.par (revision 4998) +++ /issm/trunk/test/Par/SquareSheetConstrained.par (revision 4999) @@ -46,2 +46,6 @@ %Boundary conditions: md=SetIceSheetBC(md); + +%Change name so that no test have the same name +A=dbstack; +if (length(A)>2), md.name=A(3).file(1:end-2); end Index: /issm/trunk/test/Par/SquareSheetShelf.par =================================================================== --- /issm/trunk/test/Par/SquareSheetShelf.par (revision 4998) +++ /issm/trunk/test/Par/SquareSheetShelf.par (revision 4999) @@ -53,2 +53,6 @@ %Deal with boundary conditions: md=SetMarineIceSheetBC(md,'./../Exp/SquareFront.exp'); + +%Change name so that no test have the same name +A=dbstack; +if (length(A)>2), md.name=A(3).file(1:end-2); end Index: /issm/trunk/test/Par/SquareShelf.par =================================================================== --- /issm/trunk/test/Par/SquareShelf.par (revision 4998) +++ /issm/trunk/test/Par/SquareShelf.par (revision 4999) @@ -46,2 +46,6 @@ %Boundary conditions: md=SetIceShelfBC(md,'./../Exp/SquareFront.exp'); + +%Change name so that no test have the same name +A=dbstack; +if (length(A)>2), md.name=A(3).file(1:end-2); end Index: /issm/trunk/test/Par/SquareShelfConstrained.par =================================================================== --- /issm/trunk/test/Par/SquareShelfConstrained.par (revision 4998) +++ /issm/trunk/test/Par/SquareShelfConstrained.par (revision 4999) @@ -50,2 +50,6 @@ %Deal with boundary conditions: md=SetIceShelfBC(md); + +%Change name so that no test have the same name +A=dbstack; +if (length(A)>2), md.name=A(3).file(1:end-2); end