Changeset 26034 for issm/trunk-jpl/jenkins/jenkins.sh

Timestamp:

02/26/21 10:56:08 (4 years ago)

Author:

jdquinn

Message:

CHG: Modifications to build system and additional configurations in support of Windows 10 MSYS2 MinGW build

File:

• issm/trunk-jpl/jenkins/jenkins.sh (modified) (27 diffs)

issm/trunk-jpl/jenkins/jenkins.sh

@@ -7 +7 @@
 # members of the ISSM development team.
 #
+# NOTE:
+# - Variable OS is set in environment by running 
+#       `source $ISSM_DIR/etc/environment.sh`.
+#
 # TODO:
 # - Rename TEMP to something more descriptive (ensure first that other parts of
@@ -14 +18 @@
 
 echo "Cleaning up execution directory"
-rm -rf $ISSM_DIR/execution/*
-rm -rf $ISSM_DIR/nightlylog
-mkdir $ISSM_DIR/nightlylog
+rm -rf ${ISSM_DIR}/execution/*
+rm -rf ${ISSM_DIR}/nightlylog
+mkdir ${ISSM_DIR}/nightlylog
 
 #Server URI
@@ -53 +57 @@
 fi
 
-#Get Operating system (OS) name{{{
-OS=$(uname -s)
-if [[ $OS == CYGWIN* ]]; then
-        OS="win";
-fi
-#}}}
-
 #Install ISSM
 #Determining installation type depending on svn changes{{{
@@ -65 +62 @@
 echo "             Determining Installation type            "
 echo "======================================================";
-if [ -a $ISSM_DIR/svn_revision_old ]; then
-        SVN_PREVIOUS=$(cat $ISSM_DIR/svn_revision_old)
+if [ -a ${ISSM_DIR}/svn_revision_old ]; then
+        SVN_PREVIOUS=$(cat ${ISSM_DIR}/svn_revision_old)
         SVN_CURRENT=$SVN_REVISION_1
         echo "Previous revision number: $SVN_PREVIOUS"
@@ -74 +71 @@
         #
 
-        #svn --non-interactive --no-auth-cache --trust-server-cert diff -r $SVN_PREVIOUS:$SVN_CURRENT --summarize $ISSM_DIR | awk '{print $NF}' > $ISSM_DIR/TEMP
+        #svn --non-interactive --no-auth-cache --trust-server-cert diff -r $SVN_PREVIOUS:$SVN_CURRENT --summarize ${ISSM_DIR} | awk '{print $NF}' > ${ISSM_DIR}/TEMP
 
         # Get list of changes from Jenkins itself as svn requires credentials
@@ -81 +78 @@
 
         # Process html page and get the list of files that has changed (tricky...)
-        #cat changes | grep '="The file was modified"' | sed -e 's/.*<\/td><td><a>\(.*\)<\/a><\/td><td>.*/\1/' > $ISSM_DIR/TEMP
+        #cat changes | grep '="The file was modified"' | sed -e 's/.*<\/td><td><a>\(.*\)<\/a><\/td><td>.*/\1/' > ${ISSM_DIR}/TEMP
         #cat changes | grep 'document_edit' |sed -e 's/document_edit.png/document_edit.png\
-                #/g' | sed -e 's/.*<\/a><\/td><td>\(.*\)<\/td><\/tr>.*/\1/' | grep -v 'document_edit.png' > $ISSM_DIR/TEMP
-        cat changes | tr " " "\n" | grep trunk | sed -e 's/.*<a>\(.*\)<\/a>.*/\1/' > $ISSM_DIR/TEMP
+                #/g' | sed -e 's/.*<\/a><\/td><td>\(.*\)<\/td><\/tr>.*/\1/' | grep -v 'document_edit.png' > ${ISSM_DIR}/TEMP
+        cat changes | tr " " "\n" | grep trunk | sed -e 's/.*<a>\(.*\)<\/a>.*/\1/' > ${ISSM_DIR}/TEMP
 
         # Print list of changed files
         echo "   "
         echo "List of updated files"
-        cat $ISSM_DIR/TEMP
+        cat ${ISSM_DIR}/TEMP
         echo "   "
 
@@ -99 +96 @@
         # If the contents of the externalpackages directory were modified in any
         # way, check for changed external packages
-        if [ ! -z "$(cat $ISSM_DIR/TEMP | grep externalpackages)" ]; then
+        if [ ! -z "$(cat ${ISSM_DIR}/TEMP | grep externalpackages)" ]; then
                 echo "  -- checking for changed externalpackages... yes"
                 ISSM_EXTERNALPACKAGES="yes"
@@ -109 +106 @@
         # If the Makefile or m4 diirectory were changed in any way or if certain
         # binary files from a previous compilation do not exist, reconfigure
-        if [ ! -z "$(cat $ISSM_DIR/TEMP | grep -e "Makefile.am" -e "m4" )" ] ||
-                [ ! -f "$ISSM_DIR/bin/issm.exe" ] && [ ! -f "$ISSM_DIR/bin/issm-bin.js" ] ||
+        if [ ! -z "$(cat ${ISSM_DIR}/TEMP | grep -e "Makefile.am" -e "m4" )" ] ||
+                [ ! -f "${ISSM_DIR}/bin/issm.exe" ] && [ ! -f "${ISSM_DIR}/bin/issm-bin.js" ] ||
                 [ "$ISSM_EXTERNALPACKAGES" == "yes" ]; then
                 echo "  -- checking for reconfiguration... yes"
@@ -120 +117 @@
 
         # If source files were changed in any way, recompile
-        if [ ! -z "$(cat $ISSM_DIR/TEMP | grep -e "\.cpp" -e "\.h" )" ] ||
+        if [ ! -z "$(cat ${ISSM_DIR}/TEMP | grep -e "\.cpp" -e "\.h" )" ] ||
                 [ "$ISSM_RECONFIGURE" == "yes" ]; then
                 echo "  -- checking for recompilation... yes"
@@ -131 +128 @@
         # Cleanup
         rm changes
-        rm $ISSM_DIR/TEMP
+        rm ${ISSM_DIR}/TEMP
 else
         echo "Previous revision not found, this must be a fresh install"
@@ -144 +141 @@
 echo " "
 echo "Recording current svn version: $SVN_REVISION_1"
-echo $SVN_REVISION_1 > $ISSM_DIR/svn_revision_old
+echo $SVN_REVISION_1 > ${ISSM_DIR}/svn_revision_old
 #}}}
 
@@ -154 +151 @@
 
 #Jenkins XML files for individual packages
-EXTERNAL_TEST_FILE="$ISSM_DIR/nightlylog/results/external.xml"
-mkdir -p $ISSM_DIR/nightlylog/results
+EXTERNAL_TEST_FILE="${ISSM_DIR}/nightlylog/results/external.xml"
+mkdir -p ${ISSM_DIR}/nightlylog/results
 echo "<testsuite tests=\"$NUMPACKAGES\">" > $EXTERNAL_TEST_FILE
 
 # Need a source here for when builds start midway through installation of externalpackages.
-source $ISSM_DIR/etc/environment.sh
-
-if [ "$OS" == "win" ]; then
+source ${ISSM_DIR}/etc/environment.sh
+
+if [ "${OS}" == CYGWIN* ]; then
         echo " == WINDOWS ENVIRONMENT DETECTED =="
-        source $ISSM_DIR/externalpackages/windows/windows_environment.sh
+        source ${ISSM_DIR}/externalpackages/windows/windows_environment.sh
 fi
 
@@ -175 +172 @@
 
         #install if requested or if previous install has not been successful
-        if [ "$ISSM_EXTERNALPACKAGES" == "yes" ]; then # NOTE: Removed check on if 'install' directory exist
-                cd $ISSM_DIR/externalpackages/$PACKAGENAME
+        if [ "${ISSM_EXTERNALPACKAGES}" == "yes" ]; then # NOTE: Removed check on if 'install' directory exist
+                cd ${ISSM_DIR}/externalpackages/$PACKAGENAME
 
                 echo "======================================================";
@@ -183 +180 @@
 
                 ./$PACKAGEINST $NUMCPUS_INSTALL &> compil.log
-                if [ $? -ne 0 ] && [ "$PACKAGENAME" != "boost" ]; then
+                if [ $? -ne 0 ] && [ "${PACKAGENAME}" != "boost" ]; then
                         cat compil.log
                         echo "======================================================";
@@ -197 +194 @@
                         echo "<testcase classname=\"externalpackages\" name=\"$PACKAGENAME\"/>" >> $EXTERNAL_TEST_FILE
                 fi
-                source $ISSM_DIR/etc/environment.sh
+                source ${ISSM_DIR}/etc/environment.sh
 
                 #If external package is rebuilt, we also need to recompile
@@ -220 +217 @@
 
 # Source here to include any newly installed externalpackages on the path.
-source $ISSM_DIR/etc/environment.sh
-
-if [ "$OS" == "win" ]; then
+source ${ISSM_DIR}/etc/environment.sh
+
+if [ "${OS}" == CYGWIN* ]; then
         echo " == WINDOWS ENVIRONMENT DETECTED =="
-        source $ISSM_DIR/externalpackages/windows/windows_environment.sh
+        source ${ISSM_DIR}/externalpackages/windows/windows_environment.sh
 fi
 
@@ -244 +241 @@
         #export EMCC_DEBUG=1 # Uncomment to enable debugging
         export EMCC_CFLAGS="-s ERROR_ON_UNDEFINED_SYMBOLS=0" # Required after v1.38.14 to avoid undefined symbol warnings from our Fortran object files being treated as errors
-        source $ISSM_DIR/externalpackages/emscripten/install/emsdk_env.sh
+        source ${ISSM_DIR}/externalpackages/emscripten/install/emsdk_env.sh
 fi
 
 #}}}
 #ISSM compilation yes/no                (ISSM_COMPILATION) {{{
-if [ "$ISSM_COMPILATION" == "yes" ]; then
+if [ "${ISSM_COMPILATION}" == "yes" ]; then
         cd $ISSM_DIR
-        if [ "$ISSM_RECONFIGURE" == "yes" ]; then
+        if [ "${ISSM_RECONFIGURE}" == "yes" ]; then
                 echo "======================================================";
                 echo "             Cleaning up and reconfiguring            "
@@ -262 +259 @@
                         exit 1
                 fi
-                eval "./configure $ISSM_CONFIG"
+                eval "./configure ${ISSM_CONFIG}"
                 if [ $? -ne 0 ]; then
                         echo "ISSM configuration failed (see options below)"
@@ -276 +273 @@
         echo "======================================================"
         if [ $NUMCPUS_INSTALL -gt 1 ]; then
-                echo "Making with " $NUMCPUS_INSTALL " cpus"
+                echo "Making with ${NUMCPUS_INSTALL} cpus"
+
+                # To debug compilation/linking, add 'V=1' option to the call to make
+                #make -j $NUMCPUS_INSTALL V=1
                 make -j $NUMCPUS_INSTALL
         else
+                #make V=1
                 make
         fi
@@ -292 +293 @@
         fi
         make install
-elif [ "$ISSM_COMPILATION" == "no" ]
-then
+elif [ "${ISSM_COMPILATION}" == "no" ]; then
         echo "Skipping ISSM compilation"
 else
@@ -311 +311 @@
         for (( i=1;i<=$NUMCPUS_RUN;i++ )); do
                 #Launch matlab and the nightly run script
-                cat > $ISSM_DIR/nightlylog/matlab_run$i.m << EOF
+                cat > ${ISSM_DIR}/nightlylog/matlab_run$i.m << EOF
                 warning off %necessary to avoid a log of several Go for parallel runs
                 try,
-                        $(if [ "$MATLAB_NROPTIONS" = "" ]; then
-                                echo "runme('output','nightly','rank',$i,'numprocs',$NUMCPUS_RUN);"
+                        $(if [ "${MATLAB_NROPTIONS}" = "" ]; then
+                                echo "runme('output','nightly','rank',${i},'numprocs',${NUMCPUS_RUN});"
                         else
-                                echo "runme($MATLAB_NROPTIONS,'output','nightly','rank',$i,'numprocs',$NUMCPUS_RUN);"
+                                echo "runme(${MATLAB_NROPTIONS},'output','nightly','rank',${i},'numprocs',${NUMCPUS_RUN});"
                         fi)
                 catch me,
@@ -332 +332 @@
 EOF
                 cd $ISSM_DIR/test/NightlyRun
-                if [ "$OS" = "win" ]; then
-                        $MATLAB_PATH/bin/matlab -nodisplay -nosplash -r "addpath $ISSM_DIR_WIN/src/m/dev; devpath; addpath $ISSM_DIR_WIN/nightlylog/; matlab_run$i" -logfile $ISSM_DIR_WIN/nightlylog/matlab_log$i.log &
+                if [[ "${OS}" == CYGWIN* || "${OS}" == MINGW* ]]; then
+                        $MATLAB_PATH/bin/matlab -nodesktop -nosplash -nojvm -r "addpath ${ISSM_DIR_WIN}/src/m/dev; devpath; addpath ${ISSM_DIR_WIN}/nightlylog; matlab_run$i" -logfile ${ISSM_DIR_WIN}/nightlylog/matlab_log$i.log &
                 else
-                        $MATLAB_PATH/bin/matlab -nodisplay -nosplash -r "addpath $ISSM_DIR/src/m/dev; devpath; addpath $ISSM_DIR/nightlylog/; matlab_run$i" -logfile $ISSM_DIR/nightlylog/matlab_log$i.log &
+                        $MATLAB_PATH/bin/matlab -nodisplay -nosplash -r "addpath ${ISSM_DIR}/src/m/dev; devpath; addpath ${ISSM_DIR}/nightlylog; matlab_run$i" -logfile ${ISSM_DIR}/nightlylog/matlab_log$i.log &
                 fi
         done
 
-        #wait until matlab closes
-        if [ "$OS" = "win" ]; then
+        # Wait for MATLAB to exit
+        if [[ "${OS}" == CYGWIN* ]]; then
                 sleep 5;
-                echo "Waiting for matlab on windows"
+                echo "Waiting for MATLAB to exit"
                 pid=$(ps aux -W | grep MATLAB | awk '{printf("%s\n","MATLAB");}')
                 echo '-----------------------------'
-                echo "pid: $pid"
+                echo "pid: ${pid}"
                 echo '-----------------------------'
-                while [ -n "$pid" ]; do
+                while [ -n "${pid}" ]; do
                         pid=$(ps aux -W | grep MATLAB | awk '{printf("%s\n","MATLAB");}')
                         sleep 1;
                 done
                 echo "DONE!"
+        elif [[ "${OS}" == MINGW* ]]; then
+                # NOTE: The PID query used here may work as well on Cygwin. If so, combine the two conditional blocks.
+                sleep 5;
+                echo "Waiting for MATLAB to exit"
+                pid=$(ps -W | grep MATLAB | awk '{print $1}')
+                echo '-----------------------------'
+                echo "pid: ${pid}"
+                echo '-----------------------------'
+                while [ -n "${pid}" ]; do
+                        pid=$(ps -W | grep MATLAB | awk '{print $1}')
+                        sleep 1;
+                done
         else
                 wait
         fi
 
-        #concatenate reports
-        cd $ISSM_DIR/nightlylog/
-        #echo 'CHECKING NIGHTLYLOG DIRECTORY'
-        #echo '-----------------------------'
-        #ls -la
-        #echo '-----------------------------'
+        # Concatenate logs
+        cd $ISSM_DIR/nightlylog
 
         if [ -f matlab_log.log ]; then
@@ -368 +376 @@
 
         for job in `jobs -p`; do
-                echo "Waiting on: $job"
+                #echo "Waiting on: ${job}" # Commented out because it really has nothing to do with MATLAB processes
                 wait $job
         done
@@ -387 +395 @@
 if [ $PYTHON_TEST -eq 1 ]; then
         #Launch all tests on different cpus
-        export PYTHONPATH="$ISSM_DIR/src/m/dev"
+        export PYTHONPATH="${ISSM_DIR}/src/m/dev"
         export PYTHONSTARTUP="${PYTHONPATH}/devpath.py"
         export PYTHONUNBUFFERED=1 #we don't want python to buffer otherwise issm.exe output is not captured
@@ -398 +406 @@
         done
 
-        # concatenate reports
-        cd $ISSM_DIR/nightlylog/
+        # Concatenate logs
+        cd $ISSM_DIR/nightlylog
         if [ -f python_log.log ]; then
                 rm python_log.log
@@ -406 +414 @@
         for job in `jobs -p`
         do
-                echo "Waiting on: $job"
+                echo "Waiting on: ${job}"
                 wait $job
         done
@@ -429 +437 @@
 # }}}
 
-#process logs to be junit compatible
+# Process logs to be JUnit compatible
 #{{{
 cd $ISSM_DIR/nightlylog