Changeset 25685

Timestamp:

10/14/20 00:19:52 (4 years ago)

Author:

jdquinn

Message:

CHG: MATLAB -> Python; calling external package install scripts with value of NUMCPUS_INSTALL in jenkins.sh; clean up

Location:

issm/trunk-jpl

Files:

• jenkins/jenkins.sh (modified) (3 diffs)
• src/m/classes/issmsettings.py (modified) (2 diffs)
• src/m/classes/qmu.py (modified) (1 diff)
• src/m/classes/qmustatistics.py (modified) (2 diffs)
• src/m/exp/contourlevelzero.m (modified) (1 diff)
• src/m/geometry/polyarea.py (modified) (1 diff)
• src/m/mesh/meshintersect3d.m (modified) (1 diff)
• src/m/plot/plot_coastlines.m (modified) (1 diff)
• src/m/plot/plot_coastlines.py (added)
• src/m/plot/processdata.py (modified) (2 diffs)
• src/m/qmu/postqmu.py (modified) (2 diffs)
• src/m/solve/loadresultsfromcluster.py (modified) (2 diffs)
• src/m/solve/loadresultsfromdisk.py (modified) (3 diffs)

issm/trunk-jpl/jenkins/jenkins.sh ¶

@@ -58 +58 @@
 
 #Install ISSM
-#Determinig Installation type depending on svn changes{{{
+#Determining installation type depending on svn changes{{{
 echo "======================================================";
 echo "             Determining Installation type            "
@@ -153 +153 @@
 NUMPACKAGES=$(($(echo ${EXTERNALPACKAGES} | wc -w ) / 2))
 
-#Jenkins xml files for individual packages
+#Jenkins XML files for individual packages
 EXTERNAL_TEST_FILE="$ISSM_DIR/nightlylog/results/external.xml"
 mkdir -p $ISSM_DIR/nightlylog/results
@@ -183 +183 @@
                 echo "======================================================";
 
-                ./$PACKAGEINST &> compil.log
+                ./$PACKAGEINST $NUMCPUS_INSTALL &> compil.log
                 if [ $? -ne 0 ] && [ "$PACKAGENAME" != "boost" ]; then
                         cat compil.log

issm/trunk-jpl/src/m/classes/issmsettings.py ¶

@@ +1 @@
+from checkfield import checkfield
 from fielddisplay import fielddisplay
-from checkfield import checkfield
 from WriteData import WriteData
 
 
 class issmsettings(object):
-    """
-    ISSMSETTINGS class definition
+    """ISSMSETTINGS class definition
 
-       Usage:
-          issmsettings = issmsettings()
+    Usage:
+        issmsettings = issmsettings()
     """
 
@@ -88 +87 @@
         else:
             WriteData(fid, prefix, 'name', 'md.settings.waitonlock', 'data', False, 'format', 'Boolean')
+
         WriteData(fid, prefix, 'object', self, 'fieldname', 'solver_residue_threshold', 'format', 'Double')
     # }}}

issm/trunk-jpl/src/m/classes/qmu.py ¶

@@ -206 +206 @@
             WriteData(fid, prefix, 'data', False, 'name', 'md.qmu.mass_flux_segments_present', 'format', 'Boolean')
             return
+        WriteData(fid, prefix, 'data', self.method.params.samples, 'name', 'md.qmu.method.params.samples', 'format', 'Integer')
         WriteData(fid, prefix, 'object', self, 'fieldname', 'numberofresponses', 'format', 'Integer')
         WriteData(fid, prefix, 'object', self, 'fieldname', 'variabledescriptors', 'format', 'StringArray')

issm/trunk-jpl/src/m/classes/qmustatistics.py ¶

@@ -1 @@
-# NOTE: Uncomment 
-
 import numpy as np
 
@@ -53 +51 @@
 
         if self.method[0]['name'] == 'None':
-            return
+            return s
 
         s += '{}\n'.format(fielddisplay(self, 'nfiles_per_directory', 'number of files per output directory'))

issm/trunk-jpl/src/m/exp/contourlevelzero.m ¶

@@ -1 +1 @@
 function contours=contourlevelzero(md,mask,level)
-%CONTOURLEVELZERO - figure out the zero level (or offset thereof, specified by the level value) 
-%                   of a vectorial mask, and vectorialize it into an exp or shp compatible 
-%structure.
+%CONTOURLEVELZERO - figure out the zero level (or offset thereof, specified by 
+%the level value) of a vectorial mask, and vectorialize it into an exp or shp 
+%compatible structure.
 %
 %   Usage:

issm/trunk-jpl/src/m/geometry/polyarea.py ¶

@@ -5 +5 @@
 
 def polyarea(x, y): #{{{
-    """
-    POLYAREA - returns the area of the 2-D polygon defined by the vertices in 
+    """POLYAREA - returns the area of the 2-D polygon defined by the vertices in 
     lists x and y
     

issm/trunk-jpl/src/m/mesh/meshintersect3d.m ¶

@@ -1 +1 @@
 function indices=meshintersect3d(x,y,z,xs,ys,zs,varargin)
-%MESHINTERSECT - return indices (into x,y and z) of common values between (x,y,z) and (xs,ys,zs). 
-%  i.e: x(index)=xs; y(index)=ys;
+%MESHINTERSECT - return indices (into x,y and z) of common values between 
+%(x,y,z) and (xs,ys,zs).
+%   i.e: x(index)=xs; y(index)=ys;
 %
 

issm/trunk-jpl/src/m/plot/plot_coastlines.m ¶

@@ -3 +3 @@
 %define coastline: {{{
 coast=[-90 -180; ...
-        -83.83 -180 ;...
+-83.83 -180 ;...
 -84.33 -178 ;...
 -84.5 -174 ;...

issm/trunk-jpl/src/m/plot/processdata.py ¶

@@ -3 +3 @@
 
 def processdata(md, data, options):
-    """
-    PROCESSDATA - process data to be plotted
+    """PROCESSDATA - process data to be plotted
 
     datatype = 1 -> elements
@@ -16 +15 @@
     See also: PLOTMODEL, PROCESSMESH
     """
-    # {{{ Initialisation and grabbing auxiliaries
+    # {{{ Initialization and grabbing auxiliaries
     # check format
     if (len(data) == 0 or (len(data) == 1 and not isinstance(data, dict) and np.isnan(data).all())):

issm/trunk-jpl/src/m/qmu/postqmu.py ¶

@@ -45 +45 @@
         dakotaresults.dresp_dat = dresp_dat
 
-    if md.qmu.output:
-        if md.qmu.method.method == 'nond_sampling' and md.qmu.statistics.method[0].name == 'None':
+    if md.qmu.output and md.qmu.statistics.method[0]['name'] == 'None':
+        if md.qmu.method.method == 'nond_sampling':
             dakotaresults.modelresults = []
             md2 = copy.deepcopy(md)
@@ -55 +55 @@
                 dakotaresults.modelresults.append(md2.results)
 
-    # TODO: Uncomment the following once qmustatistics class is translated
-    # if md.qmu.statistics.method[0].name != 'None':
-    #     md.qmu.isdakota = 0
-    #     md = loadresultsfromdisk(md, [md.miscellaneous.name,'.stats'])
-    #     md.qmu.isdakota = 1
+    if md.qmu.statistics.method[0]['name'] != 'None':
+        md.qmu.isdakota = 0
+        md = loadresultsfromdisk(md, [md.miscellaneous.name,'.stats'])
+        md.qmu.isdakota = 1
 
     # put dakotaresults in their right location.

issm/trunk-jpl/src/m/solve/loadresultsfromcluster.py ¶

@@ -5 +5 @@
 from helpers import *
 from loadresultsfromdisk import loadresultsfromdisk
+from MatlabFuncs import ispc
+from pairoptions import pairoptions
 
 
-def loadresultsfromcluster(md, runtimename=False):
-    '''
-    LOADRESULTSFROMCLUSTER - load results of solution sequence from cluster
+def remove(filename):
+    try:
+        os.remove(filename)
+    except OSError:
+        print(('WARNING: ' + filename + ' does not exist'))
 
-       Usage:
-          md = loadresultsfromcluster(md, runtimename)
-    '''
+def loadresultsfromcluster(md, *args):
+    """LOADRESULTSFROMCLUSTER - load results of solution sequence from cluster
 
-    #retrieve cluster, to be able to call its methods
+    Usage:
+        md = loadresultsfromcluster(md, runtimename)
+    """
+
+    # Process options
+    options = pairoptions(*args)
+    nolog = options.getfieldvalue('nolog', 0)
+    md.private.runtimename = options.getfieldvalue('runtimename', md.private.runtimename)
+
+    # Retrieve cluster, to be able to call its methods
     cluster = md.cluster
 
-    if runtimename:
-        md.private.runtimename = runtimename
-
-    #Download outputs from the cluster
-    filelist = [md.miscellaneous.name + '.outlog', md.miscellaneous.name + '.errlog']
+    # Download outputs from the cluster
+    if not nolog:
+        filelist = [md.miscellaneous.name + '.outlog', md.miscellaneous.name + '.errlog']
+    else:
+        filelist = []
     if md.qmu.isdakota:
         filelist.append(md.miscellaneous.name + '.qmu.err')
@@ -29 +41 @@
             if md.qmu.params.tabular_graphics_data:
                 filelist.append('dakota_tabular.dat')
+        if md.qmu.output and md.qmu.statistics.method[0]['name'] == 'None':
+            if md.qmu.method.method == 'nond_sampling':
+                for i in range(len(md.qmu.method.params.samples)):
+                    filelist.append(md.miscellaneous.name + '.outbin' + str(i))
+        if md.qmu.statistics.method[0]['name'] != 'None':
+            filelist.append(md.miscellaneous.name + '.stats')
     else:
         filelist.append(md.miscellaneous.name + '.outbin')
     cluster.Download(md.private.runtimename, filelist)
 
-    #If we are here, no errors in the solution sequence, call loadresultsfromdisk.
-    if os.path.exists(md.miscellaneous.name + '.outbin'):
-        if os.path.getsize(md.miscellaneous.name + '.outbin') > 0:
-            md = loadresultsfromdisk(md, md.miscellaneous.name + '.outbin')
-        else:
-            print(('WARNING, outbin file is empty for run ' + md.miscellaneous.name))
-    elif not md.qmu.isdakota:
-        print(('WARNING, outbin file does not exist ' + md.miscellaneous.name))
+    # If we are here, no errors in the solution sequence, call loadresultsfromdisk
+    md = loadresultsfromdisk(md, md.miscellaneous.name + '.outbin')
 
-    #erase the log and output files
-    if md.qmu.isdakota:
-        #filename = os.path.join('qmu' + str(os.getpid()), md.miscellaneous.name)
-        filename = md.miscellaneous.name
+    # Erase the log and output files
+    for i in range(len(filelist)):
+        filename = filelist[i]
+        remove(filename)
+    if not ispc():
+        remove(md.private.runtimename + '.tar.gz')
 
-        # this will not work like normal as dakota doesn't generate outbin files,
-        #   instead calls postqmu to store results directly in the model
-        #   at md.results.dakota via dakota_out_parse
-        md = loadresultsfromdisk(md, md.miscellaneous.name + '.outbin')
-    else:
-        filename = md.miscellaneous.name
-        TryRem('.outbin', filename)
-        if not platform.system() == 'Windows':
-            TryRem('.tar.gz', md.private.runtimename)
-
-    TryRem('.errlog', filename)
-    TryRem('.outlog', filename)
-
-    #erase input file if run was carried out on same platform.
+    # Erase input file if run was carried out on same platform
     hostname = socket.gethostname()
     if hostname == cluster.name:
+        remove(md.miscellaneous.name + '.bin')
+        remove(md.miscellaneous.name + '.toolkits')
         if md.qmu.isdakota:
-            #filename = os.path.join('qmu' + str(os.getpid()), md.miscellaneous.name)
-            filename = md.miscellaneous.name
-            TryRem('.queue', filename)
+            remove(md.miscellaneous.name + '.qmu.in')
+        if not ispc():
+            remove(md.miscellaneous.name + '.queue')
         else:
-            filename = md.miscellaneous.name
-            TryRem('.toolkits', filename)
-            if not platform.system() == 'Windows':
-                TryRem('.queue', filename)
-            else:
-                TryRem('.bat', filename)
-
-        # remove this for bin file debugging
-        TryRem('.bin', filename)
+            remove(md.miscellaneous.name + '.bat')
 
     return md
-
-def TryRem(extension, filename):
-    try:
-        os.remove(filename + extension)
-    except OSError:
-        print(('WARNING, no ' + extension + '  is present for run ' + filename))

issm/trunk-jpl/src/m/solve/loadresultsfromdisk.py ¶

@@ -1 +1 @@
 import os
-from results import results
+
+from helpers import fieldnames
 from parseresultsfromdisk import parseresultsfromdisk
 from postqmu import postqmu
+from results import results
 
 
 def loadresultsfromdisk(md, filename):
-    '''
-    LOADRESULTSFROMDISK - load results of solution sequence from disk file "filename"
+    """LOADRESULTSFROMDISK - load results of solution sequence from disk file 
+    "filename"
 
-       Usage:
-          md = loadresultsfromdisk(md=False,filename=False);
-    '''
-    #check number of inputs/outputs
+    Usage:
+        md = loadresultsfromdisk(md=False, filename=False)
+    """
+
+    # Check number of inputs/outputs
     if not md or not filename:
         raise ValueError("loadresultsfromdisk: error message.")
@@ -18 +21 @@
     if not md.qmu.isdakota:
 
-        #Check that file exists
+        # Check that file exists
         if not os.path.exists(filename):
-            raise OSError("binary file '{}' not found.".format(filename))
+            err_msg  = '============================================================\n'
+            err_msg += '   Binary file {} not found                                 \n'.format(filename)
+            err_msg += '                                                            \n'
+            err_msg += '   This typically happens when the run crashed.             \n'
+            err_msg += '   Please check for error messages above or in the outlog   \n'
+            err_msg += '============================================================\n'
 
-        #initialize md.results if not a structure yet
+            raise OSError(err_msg)
+
+        # Initialize md.results if not a structure yet
         if not isinstance(md.results, results):
             md.results = results()
 
-        #load results onto model
+        # Load results onto model
         structure = parseresultsfromdisk(md, filename, not md.settings.io_gather)
-        if not len(structure):
+        if not structure:
             raise RuntimeError("No result found in binary file '{}'. Check for solution crash.".format(filename))
-
+        if not structure[0].SolutionType:
+            if structure[-1].SolutionType:
+                structure[0].SolutionType = structure[-1].SolutionType
+            else:
+                print('Cannot find a solution type in the results! Ascribing one: \'NoneSolution\'.')
+                structure[0].SolutionType = 'NoneSolution'
         setattr(md.results, structure[0].SolutionType, structure)
 
-        #recover solution_type from results
+        # Recover solution_type from results
         md.private.solution = structure[0].SolutionType
 
-        #read log files onto fields
+        # Read log files onto fields
         if os.path.exists(md.miscellaneous.name + '.errlog'):
             with open(md.miscellaneous.name + '.errlog', 'r') as f:
                 setattr(getattr(md.results, structure[0].SolutionType)[0], 'errlog', [line[:-1] for line in f])
         else:
-            setattr(getattr(md.results, structure[0].SolutionType)[0], 'errlog', [])
+            setattr(getattr(md.results, structure[0].SolutionType)[0], 'errlog', '')
 
         if os.path.exists(md.miscellaneous.name + '.outlog'):
@@ -47 +62 @@
                 setattr(getattr(md.results, structure[0].SolutionType)[0], 'outlog', [line[:-1] for line in f])
         else:
-            setattr(getattr(md.results, structure[0].SolutionType)[0], 'outlog', [])
+            setattr(getattr(md.results, structure[0].SolutionType)[0], 'outlog', '')
 
-        if len(getattr(md.results, structure[0].SolutionType)[0].errlog):
+        if getattr(md.results, structure[0].SolutionType)[0].errlog:
             print("loadresultsfromcluster info message: error during solution. Check your errlog and outlog model fields.")
 
-        #if only one solution, extract it from list for user friendliness
+        # If only one solution, extract it from list for user friendliness
         if len(structure) == 1 and structure[0].SolutionType != 'TransientSolution':
             setattr(md.results, structure[0].SolutionType, structure[0])
 
-    #post processes qmu results if necessary
+    # Post processes QMU results if necessary
     else:
         md = postqmu(md)