Index: /issm/trunk/cron/nightlyrun.sh =================================================================== --- /issm/trunk/cron/nightlyrun.sh (revision 2184) +++ /issm/trunk/cron/nightlyrun.sh (revision 2185) @@ -23,6 +23,5 @@ function todaydate() { - export today=`date | awk '{printf("%s-%s-%s",$2,$3,$6);}'`; - suffix=`date | awk '{printf("%s-%s-%s",$2,$3,$6);}'` + suffix=`date | awk '{printf("%s-%s-%s %s",$2,$3,$6,$4);}'` echo $suffix; } @@ -30,5 +29,5 @@ { #return host name depending on the OS - if [[ "$1" == "winxp32" ]] + if [ "$1" = "winxp32" ] then HOST_NAME=`hostname`; @@ -45,15 +44,22 @@ echo "no config file specified. Exiting..." exit 1 -else - #launch nightly run (get start time) - source $1; - START_TIME=$(timer) -fi - -#create nightly run variables -export TODAY=$(todaydate); -export HOST_NAME=$(host_name $OS); -export ISSM_RELEASE="issm" -export OS +fi + +#launch nightly run (get start time) +source $1; +TODAY=$(todaydate); +HOST_NAME=$(host_name $OS); +START_TIME=$(timer) + +#Initialize nightly run output +rm -rf $ISSM_DIR/test/Verification/NightlyRun +mkdir $ISSM_DIR/test/Verification/NightlyRun +cat << END > $ISSM_DIR/test/Verification/NightlyRun/nightly.log +today: $(echo $TODAY) +user: $(whoami) +host: $(echo $HOST_NAME) +OS: $(echo $OS) +release: $(echo "issm") +END #create ISSM variables @@ -61,18 +67,18 @@ export ISSM_ARCH -#Erase previous code and Fetch the new one -rm -rf trunk -svn checkout $REPOSITORY - -#install ISSM - -#1: modify some variables -cd $ISSM_DIR/etc - -#source environment variables with new matlab path -source environment.sh MATLAB_DIR=$MATLAB_PATH - -#create simpler cluster.rc file, with only the cluster we are interested in. -cat << END > cluster.rc +#Checkout and install ISSM if required +if [ "$SKIPINSTALLATION" != "yes" ] +then + + #Erase previous code and Fetch the new one + rm -rf trunk + svn checkout $REPOSITORY + + #source environment variables with new matlab path + cd $ISSM_DIR/etc + source environment.sh MATLAB_DIR=$MATLAB_PATH + + #create simpler cluster.rc file, with only the cluster we are interested in. + cat << END > cluster.rc begin cluster_name=$HOST_NAME @@ -83,54 +89,63 @@ END -if [[ $SKIPPACKAGESCOMPILATION == "yes" ]] -then - #erase externapackages, and link with externalpackages_dir + #Install external packages + if [ "$SKIPPACKAGESCOMPILATION" = "yes" ] + then + + #erase externapackages, and link with externalpackages_dir + cd $ISSM_DIR + rm -rf externalpackages + cp -Rf $EXTERNALPACKAGESDIR ./ + + rm -rf devpackages + cp -Rf $DEVPACKAGESDIR ./ + else + + cd $ISSM_DIR/externalpackages + for ep in $EXTERNALPACKAGES + do + cd $ep + cp configs/$OS/* . + ./install.sh $NUMCPUS + cd .. + done + + #3: install automake and autoconf + cd $ISSM_DIR/devpackages + make + cd .. + fi +else + + #source environment variables with new matlab path + cd $ISSM_DIR/etc + source environment.sh MATLAB_DIR=$MATLAB_PATH + +fi + +#Generate Makefiles and compile ISSM if requested +if [ "$SKIPCOMPILATION" != "yes" ] +then cd $ISSM_DIR - rm -rf externalpackages - cp -Rf $EXTERNALPACKAGESDIR ./ - - rm -rf devpackages - cp -Rf $DEVPACKAGESDIR ./ -else - #2: install some required external packages - cd $ISSM_DIR/externalpackages - for ep in $EXTERNALPACKAGES - do - cd $ep - cp configs/$OS/* . - ./install.sh $NUMCPUS - cd .. - done - - #3: install automake and autoconf - cd $ISSM_DIR/devpackages - make - cd .. -fi - -#3: generate MakeFiles -cd $ISSM_DIR -sh scripts/automakererun.sh -sh configs/$OS/$OS.sh - -#4: compile and install ISSM -if [[ "$OS" == "winxp32" ]] -then - cd $ISSM_DIR/src/c - ./intel-compile.sh >> ../../c-compilation.log 2>&1 - cd $ISSM_DIR/src/mex - make install >> ../../mex-compilation.log 2>&1 - cd $ISSM_DIR -else - make -j $NUMCPUS >> ../../cmex-compilation.log 2>&1 - make -j $NUMCPUS install >> ../../cmex-compilation.log 2>&1 -fi - -INSTAL_TIME=$(timer) -export ELAPSED_INSTAL=$(timer $START_TIME) + sh scripts/automakererun.sh + sh configs/$OS/$OS.sh + + #4: compile and install ISSM + if [ "$OS" = "winxp32" ] + then + cd $ISSM_DIR/src/c + ./intel-compile.sh >> ../../c-compilation.log 2>&1 + cd $ISSM_DIR/src/mex + make install >> ../../mex-compilation.log 2>&1 + cd $ISSM_DIR + else + make -j $NUMCPUS >> ../../cmex-compilation.log 2>&1 + make -j $NUMCPUS install >> ../../cmex-compilation.log 2>&1 + fi +fi #windows environments: ISSM_DIR_WIN variable not correctly picked up when using #the cron job. just get startup to take the ISSM_DIR variable as the pwd: -if [[ "$OS" == "winxp32" ]] +if [ "$OS" = "winxp32" ] then cat startup.m | sed 's/clear status/clear status; ISSM_DIR=pwd;/g' > startup.m.bak @@ -138,33 +153,26 @@ fi +#put installation elapsed time in nightly.log +INSTALL_TIME=$(timer) +ELAPSED_INSTALL=$(timer $START_TIME) +cat << END >> $ISSM_DIR/test/Verification/NightlyRun/nightly.log +elapsed_install: $(echo $ELAPSED_INSTALL) +END + #Launch matlab and the nightly run script -mktemp matlaberror.log -cat > matlab_run.m << EOF -startup -warning off %necessary to avoid a nightly.log of several Go for cielo_parallel -try, - nightlyrun; -catch me, - %An error occured, get report and exit - directory=strsplit(pwd,'/'); - message=getReport(me) - fid=fopen([ISSM_DIR '/matlaberror.log'], 'wt'); - fprintf(fid,'Matlab error occured in: %s\n',directory{end}); - fprintf(fid,'%s',message); - fclose(fid); -end +cd $ISSM_DIR/cron/ +./run.sh $NROPTIONS +ELAPSED_RUN=$(timer $INSTALL_TIME) +ELAPSED_TOTAL=$(timer $START_TIME) +cat << END >> $ISSM_DIR/test/Verification/NightlyRun/nightly.log +elapsed_run: $(echo $ELAPSED_RUN) +elapsed_total: $(echo $ELAPSED_TOTAL) +END + +#Build html report +cd $ISSM_DIR/cron/ +./report.sh +echo "html report located in $ISSM_DIR/test/Verification/NightlyRun/report.html" exit -EOF - -matlab -nojvm -nosplash -r matlab_run -logfile nightlylong.log - -#removed unused lines to deal with a smaller file -cat nightlylong.log | egrep 'ERROR|SUCCESS|FAILURE' > nightly.log -#rm nightlylong.log - -#Build html report -export ELAPSED_RUN=$(timer $INSTAL_TIME) -export ELAPSED_TOTAL=$(timer $START_TIME) -./cron/report.sh #send mail @@ -172,9 +180,9 @@ for i in `echo $MAILINGLIST`; do - if [[ "$OS" == "winxp32" ]] + if [ "$OS" = "winxp32" ] then email -html -f "ISSM Nightly run $EMAIL_ADRESS" -s "Nightly runs of $ISSM_RELEASE , operating system: $OS. " $i < report.html else - if [[ "$OS" == "linux64" ]] + if [ "$OS" = "linux64" ] then cat - report.html < errors.log -cat nightly.log | grep 'SUCCESS' > success.log -cat nightly.log | grep 'FAILURE' > failures.log +cd $ISSM_DIR/test/Verification/NightlyRun/ + +#process nightly.log +TODAY=` cat nightly.log | grep "today" | awk '{printf("%s %s",$2,$3);}'` +USER=` cat nightly.log | grep "user" | awk '{print $2}'` +HOST_NAME=` cat nightly.log | grep "host" | awk '{print $2}'` +OS=` cat nightly.log | grep "OS" | awk '{print $2}'` +RELEASE=` cat nightly.log | grep "release" | awk '{print $2}'` +EL_INSTALL=`cat nightly.log | grep "elapsed_install" | awk '{print $2}'` +EL_RUN=` cat nightly.log | grep "elapsed_run" | awk '{print $2}'` +EL_TOTAL=` cat nightly.log | grep "elapsed_total" | awk '{print $2}'` + +#Process matlab_log.log +cat matlab_log.log | egrep 'ERROR|SUCCESS|FAILURE' > matlab.log +NUM_TOT=`wc -l matlab.log | awk '{print $1}'` +NUM_ERR=`cat matlab.log | grep 'ERROR' | grep -v "PETSC ERROR" | wc -l` +NUM_SUC=`cat matlab.log | grep 'SUCCESS' | wc -l` +NUM_FAI=`cat matlab.log | grep 'FAILURE' | wc -l` + +#style +H1_STYLE='width="1000px" cellpadding="20"' +H1_FONT='style="color:#6495ed; font-family:Arial, Verdana, Tahoma; font-weight: bold; font-size:35px;" align="center"' + +H2_STYLE='width="900px" cellpadding="20"' +H2_FONT='style="color:#6495ed; font-family:Arial, Verdana, Tahoma; font-size:28px; font-weight: bold;" align="left"' + +TABLE_STYLE='width="800px" rules=none bgcolor="#ffffdd" border=1 bordercolor="#000000" cellpadding="3"' +TABLE_FONT='style="color:#404040; font-family:Arial, Verdana, Tahoma; font-size:14; font-weight: normal;" align="left"' + +MATLAB_STYLE='width="1000px" rules=none' +MATLAB_FONT='style="color:#404040; font-family:Arial, Verdana, Tahoma; font-size:14px; font-weight: normal;" align="left"' + +BODY_STYLE='width="1000px"' +BODY_FONT="style=\"color:#404040; font-family:Arial, Verdana, Tahoma; font-size:14px;\"" + +FOOTER_STYLE='width="800px" cellpadding="10"' +FOOTER_FONT='style="color:#404040; font-family:Arial, Verdana, Tahoma; font-size:12px; font-weight: normal;" align="center"' + +#style 2 +BODY_FONTC=`echo $BODY_FONT | sed -e "s/style=\"/style=\"text-align:center; /g"` +BODY_FONTL=`echo $BODY_FONT | sed -e "s/style=\"/style=\"text-align:left; /g"` #create some variables @@ -14,5 +51,5 @@ IS_INSTALL=1 fi -if [ $(wc -l nightly.log | awk '{printf("%s",$1);}') -eq 0 ] +if [ $(wc -l matlab.log | awk '{printf("%s",$1);}') -eq 0 ] then IS_RUN=0 @@ -20,26 +57,4 @@ IS_RUN=1 fi - -#style -H1_STYLE='width="800px" cellpadding="20"' -H1_FONT='style="color:#6495ed; font-family:Arial, Verdana, Tahoma; font-weight: bold; font-size:35px;" align="center"' - -H2_STYLE='width="840px" cellpadding="20"' -H2_FONT='style="color:#6495ed; font-family:Arial, Verdana, Tahoma; font-size:28px; font-weight: bold;" align="left"' - -TABLE_STYLE='width="820px" rules=none bgcolor="#ffffdd" border=1 bordercolor="#000000" cellpadding="3"' -TABLE_FONT='style="color:#404040; font-family:Arial, Verdana, Tahoma; font-size:14; font-weight: normal;" align="left"' - -MATLAB_STYLE='width="820px" rules=none' -MATLAB_FONT='style="color:#404040; font-family:Arial, Verdana, Tahoma; font-size:14px; font-weight: normal;" align="left"' - -BODY_STYLE='width="820px"' -BODY_FONT="style=\"color:#404040; font-family:Arial, Verdana, Tahoma; font-size:14px;\"" - -NOTEST_STYLE='width="650px"' -NOTEST_FONT='style="color:#FF0000; font-family:Arial, Verdana, Tahoma; font-size:14px;" align="left"' - -FOOTER_STYLE='width="800px" cellpadding="10"' -FOOTER_FONT='style="color:#404040; font-family:Arial, Verdana, Tahoma; font-size:12px; font-weight: normal;" align="center"' #build report.html @@ -56,55 +71,47 @@ -user: $(whoami) -total elapsed time: $ELAPSED_TOTAL - - OS: $OS -installation elapsed time: $ELAPSED_INSTAL - - -$(#print status -if [ $(echo $IS_INSTALL) -eq 0 ]; -then - #installation failed, end of report - echo "status: installation failed" - echo "execution elapsed time: $ELAPSED_RUN" - echo "" - echo "" - rm errors.log success.log -else - #installation successful. Did we go to the end? - - if [ $(echo $IS_RUN) -eq 0 ]; +user: $USER + + +status: STATUS +release: $RELEASE + + +number of success: $NUM_SUC/$NUM_TOT +total elapsed time: $EL_TOTAL + + +number of errors: $NUM_ERR/$NUM_TOT + +installation elapsed time: $EL_INSTALL + + +number of failures: $NUM_FAI/$NUM_TOT +installation elapsed time: $EL_INSTALL + + +
+END + +#update status +if [ $IS_RUN -eq 1 ] +then + cat summary.html | sed -e "s/STATUS/all test desks have been run<\/span>/g" > summary2.html + mv summary2.html summary.html +else + if [ $IS_INSTALL -eq 1 ] then - #run failed, end of report - echo "status: installation successful but tests runs failed" - echo "execution elapsed time: $ELAPSED_RUN" - echo "" - echo "" - rm errors.log success.log failures.log - + cat summary.html | sed -e "s/STATUS/installation successful but tests runs failed<\/span>/g" > summary2.html + mv summary2.html summary.html else - - echo "status: all test desks have been run" - echo "execution elapsed time: $ELAPSED_RUN" - echo "" - echo "" - echo "number of errors: $(wc -l errors.log | awk '{printf("%s",$1);}')/$(wc -l nightly.log | awk '{printf("%s",$1);}'; rm errors.log)" - echo " " - echo "" - echo "" - echo "number of success: $(wc -l success.log | awk '{printf("%s",$1);}')/$(wc -l nightly.log | awk '{printf("%s",$1);}'; rm success.log)" - echo " " - echo "" - echo "" - - #draw a line and clean up - echo "
" + cat summary.html | sed -e "s/STATUS/installation failed<\/span>/g" > summary2.html + mv summary2.html summary.html fi -fi) -END +fi #Matlab error report +if [ -e matlaberror.log ] +then cat << END > matlaberror.html
Matlab error
@@ -113,7 +120,10 @@ END +else + mktemp matlaberror.html +fi #report content -if [ $(echo $IS_INSTALL) = 1 ]; +if [ $IS_INSTALL -eq 1 ]; then cat << END > content.html @@ -130,35 +140,46 @@ Control fit Parallel - -$( COUNTER=0 -MAX=`wc -l nightly.log | awk '{printf("%s",$1);}'` -while [ $COUNTER -lt $MAX ]; do - let COUNTER=COUNTER+1 - - echo "" - - #see wether it is success or error (get color: red or green) - RESULT="$(cat nightly.log | awk '{ printf "line%so %s\n", NR, $0 }' | awk '/line'$COUNTER'o/ {printf("%s\n",$2);}';)" - if [ "$RESULT" = "SUCCESS" ]; - then - color="bgcolor=#ddffdd"; - else - if [ "$RESULT" = "ERROR" ]; - then - color="bgcolor=#ffdddd"; - else - color="bgcolor=#ddddff"; - fi - fi - FONT=$(echo "$BODY_FONT $color") - - #build html corresponding line - cat nightly.log | awk '{ printf "line%so %s\n", NR, $0 }' | awk -v FONT="$FONT" ' - /line'$COUNTER'o/ { printf("%s%s%s%s%s%s%s%s%s%s%s",FONT,$2,FONT,$4,$5,$6,FONT,$8,FONT,$10,FONT,$12,FONT,$14,FONT,$16,FONT,$18,FONT,$20);} - '; - - echo "" - -done) +Field checked + +$(cat matlab.log | while read line + do + echo "" + + #get status + STATUS=`echo $line | awk '{print $1}'` + + #FAILURE + if [ "$STATUS" = "FAILURE" ] + then + + FONTC=$(echo "$BODY_FONTC bgcolor=#ffff00"); + FONTL=$(echo "$BODY_FONTL bgcolor=#ffff00"); + echo $line | awk -v FONTC="$FONTC" -v FONTL="$FONTL" ' + { printf("%s%s%s%s%s%s%s%s%s%s",FONTL,$1,FONTC,"N/A",FONTL,$3,FONTL,$5,FONTL,$7,FONTC,$9,FONTC,$11,FONTC,$13,FONTC,$15,FONTC,"N/A");} + '; + + else + + #SUCCESS + if [ "$STATUS" = "SUCCESS" ] + then + color="bgcolor=#ddffdd"; + + #ERROR + else + color="bgcolor=#ffdddd"; + fi + + FONTC=$(echo "$BODY_FONTC $color") + FONTL=$(echo "$BODY_FONTL $color") + echo $line | awk -v FONTC="$FONTC" -v FONTL="$FONTL" ' + { printf("%s%s%s%s%s%s%s%s%s%s%s%s",FONTL,$1,FONTL,$3,$4,$5,FONTL,$7,FONTL,$9,FONTL,$11,FONTC,$13,FONTC,$15,FONTC,$17,FONTC,$19,FONTL,$21);} + '; + fi + + echo "" + + done + )
@@ -176,9 +197,4 @@ #concatenate files -if [ $IS_RUN -eq 0 ]; -then - cat summary.html matlaberror.html content.html footer.html > report.html -else - cat summary.html content.html footer.html > report.html -fi -rm summary.html content.html footer.html matlaberror.html +cat summary.html matlaberror.html content.html footer.html > report.html +rm summary.html content.html footer.html matlaberror.html matlab.log Index: /issm/trunk/cron/run.sh =================================================================== --- /issm/trunk/cron/run.sh (revision 2184) +++ /issm/trunk/cron/run.sh (revision 2185) @@ -3,29 +3,29 @@ numprocs=8; -#figure out if startup.m is here, otherwise, symlink it. -if [ -e startup.m ] -then - echo "" - #startup.m exists, do nothing -else - echo "creating startup.m symlink file" - ln -s $ISSM_DIR/startup.m -fi +#create softlink to startup +rm startup.m +cd $ISSM_DIR/test/Verification/NightlyRun/ +ln -s $ISSM_DIR/startup.m . - +#Launch all tests on different cpus for (( i=1;i<=$numprocs;i++ )) do #Launch matlab and the nightly run script - cat > matlab_run$i.m << EOF - startup + cat > $ISSM_DIR/test/Verification/NightlyRun/matlab_run$i.m << EOF warning off %necessary to avoid a nightly.log of several Go for cielo_parallel try, - nightlyrun($i,$numprocs); + $(if [ "$1" = "" ] + then + echo "nightlyrun('rank',$i,'numprocs',$numprocs);" + else + echo "nightlyrun($1,'rank',$i,'numprocs',$numprocs);" + fi + ) catch me, %An error occured, get report and exit directory=strsplit(pwd,'/'); message=getReport(me) - fid=fopen([ISSM_DIR '/matlaberror.log'], 'wt'); - fprintf(fid,'Matlab error occured in: %s\n',directory{end}); + fid=fopen([ISSM_DIR '/test/Verification/NightlyRun/matlaberror.log'], 'wt'); + fprintf(fid,'\nMatlab error occured in: %s\n\n',directory{end}); fprintf(fid,'%s',message); fclose(fid); @@ -33,23 +33,19 @@ exit EOF - matlab -nojvm -nosplash -r matlab_run$i -logfile nightlylong$i.log & + matlab -nojvm -nosplash -r matlab_run$i -logfile matlab_log$i.log & done wait +#remove matlab_run scripts. +rm -rf matlab_run*.m + #concatenate all reports -mv nightlylong1.log nightlylong.log +mv matlab_log1.log matlab_log.log for (( i=2;i<=$numprocs;i++ )) do - cat nightlylong.log nightlylong$i.log > nightlylong.log.bak - mv nightlylong.log.bak nightlylong.log - rm nightlylong$i.log + cat matlab_log.log matlab_log$i.log > matlab_log.log.bak + mv matlab_log.log.bak matlab_log.log + rm matlab_log$i.log done - -#remove unused lines to deal with a smaller file -cat nightlylong.log | egrep 'difference: |NIGHTLYRUNTERMINATEDCORRECTLY' > nightly.log -rm nightlylong.log - -#remove matlab_run scripts. -rm -rf matlab_run*.m