Changeset 26353

Timestamp:

07/18/21 16:19:07 (4 years ago)

Author:

jdquinn

Message:

CHG: Checking on cluster type rather than name in waitonlock.py; silencing cluster settings warnings by default; formatting; cleanup

Location:

issm/trunk-jpl

Files:

• src/m/classes/clusters/generic.py (modified) (2 diffs)
• src/m/classes/clusters/pfe.py (modified) (3 diffs)
• src/m/classes/issmsettings.py (modified) (1 diff)
• src/m/classes/model.py (modified) (1 diff)
• src/m/classes/solidearthsettings.py (modified) (1 diff)
• src/m/coordsystems/gdaltransform.m (modified) (1 diff)
• src/m/dev/devpath.py (modified) (2 diffs)
• src/m/miscellaneous/MatlabFuncs.py (modified) (1 diff)
• src/m/plot/googlemaps.m (modified) (2 diffs)
• src/m/plot/plot_manager.py (modified) (4 diffs)
• src/m/plot/plot_overlay.py (modified) (3 diffs)
• src/m/plot/radarpower.m (modified) (1 diff)
• src/m/solve/solve.m (modified) (1 diff)
• src/m/solve/solve.py (modified) (2 diffs)
• src/m/solve/waitonlock.m (modified) (1 diff)
• src/m/solve/waitonlock.py (modified) (5 diffs)
• test/NightlyRun/runme.py (modified) (3 diffs)

issm/trunk-jpl/src/m/classes/clusters/generic.py

@@ -6 +6 @@
     from generic_settings import generic_settings
 except ImportError:
-    print('Warning generic_settings.py not found, default will be used')
+    #print('Warning: generic.py: generic_settings.py not found, default will be used')
+    pass
 from MatlabFuncs import *
 from IssmConfig import IssmConfig
@@ -48 +49 @@
             self = generic_settings(self)
         except NameError:
-            print('generic_settings.py not found, using default settings')
+            # print('generic_settings.py not found, using default settings')
+            pass
 
         # OK get other fields

issm/trunk-jpl/src/m/classes/clusters/pfe.py

@@ -12 +12 @@
     from pfe_settings import pfe_settings
 except ImportError:
-    print('You need pfe_settings.py to proceed, check presence and sys.path')
+    # print('You need pfe_settings.py to proceed, check presence and sys.path')
+    pass
 from QueueRequirements import QueueRequirements
 
@@ -25 +26 @@
 
     def __init__(self, *args):  # {{{
-        self.name = 'pfe'
+        self.name = ''
         self.login = ''
         self.modules = ['comp-intel/2016.2.181', 'mpi-sgi/mpt']
@@ -50 +51 @@
             self = pfe_settings(self)
         except NameError:
-            print('pfe_settings.py not found, using default settings')
+            #print('pfe_settings.py not found, using default settings')
+            pass
 
         # OK get other fields

issm/trunk-jpl/src/m/classes/issmsettings.py

@@ -21 +21 @@
         self.solver_residue_threshold = 0
 
-    #set defaults
+        # Set defaults
         self.setdefaultparameters()
-
     #}}}
 
     def __repr__(self):  # {{{
-        string = "   general issmsettings parameters:"
-
-        string = "%s\n%s" % (string, fielddisplay(self, "results_on_nodes", "list of output for which results will be output for all the nodes of each element, Use 'all' for all output on nodes."))
-        string = "%s\n%s" % (string, fielddisplay(self, "io_gather", "I / O gathering strategy for result outputs (default 1)"))
-        string = "%s\n%s" % (string, fielddisplay(self, "lowmem", "is the memory limited ? (0 or 1)"))
-        string = "%s\n%s" % (string, fielddisplay(self, "output_frequency", "frequency at which results are saved in all solutions with multiple time_steps"))
-        string = "%s\n%s" % (string, fielddisplay(self, "sb_coupling_frequency", "frequency at which StressBalance solver is coupled (default 1)"))
-        string = "%s\n%s" % (string, fielddisplay(self, "checkpoint_frequency", "frequency at which the runs are being recorded, allowing for a restart"))
-        string = "%s\n%s" % (string, fielddisplay(self, "waitonlock", "maximum number of minutes to wait for batch results, or return 0"))
-        string = "%s\n%s" % (string, fielddisplay(self, "solver_residue_threshold", "throw an error if solver residue exceeds this value (NaN to deactivate)"))
-        return string
+        s = "   general issmsettings parameters:\n"
+        s += '{}\n'.format(fielddisplay(self, "results_on_nodes", "list of output for which results will be output for all the nodes of each element, Use 'all' for all output on nodes."))
+        s += '{}\n'.format(fielddisplay(self, "io_gather", "I / O gathering strategy for result outputs (default 1)"))
+        s += '{}\n'.format(fielddisplay(self, "lowmem", "is the memory limited ? (0 or 1)"))
+        s += '{}\n'.format(fielddisplay(self, "output_frequency", "frequency at which results are saved in all solutions with multiple time_steps"))
+        s += '{}\n'.format(fielddisplay(self, "sb_coupling_frequency", "frequency at which StressBalance solver is coupled (default 1)"))
+        s += '{}\n'.format(fielddisplay(self, "checkpoint_frequency", "frequency at which the runs are being recorded, allowing for a restart"))
+        s += '{}\n'.format(fielddisplay(self, "waitonlock", "maximum number of minutes to wait for batch results, or return 0"))
+        s += '{}\n'.format(fielddisplay(self, "solver_residue_threshold", "throw an error if solver residue exceeds this value (NaN to deactivate)"))
+        return s
     #}}}
 

issm/trunk-jpl/src/m/classes/model.py

@@ -37 +37 @@
 from toolkits import toolkits
 from generic import generic
-from pfe import pfe
-from cyclone import cyclone
-from saga import saga
 from balancethickness import balancethickness
 from stressbalance import stressbalance

issm/trunk-jpl/src/m/classes/solidearthsettings.py

@@ -118 +118 @@
         # A GRD computation has been requested, make some checks on the nature of the meshes provided
         if self.isgrd:
-            if md.mesh.__class__.__name__ is 'mesh3dsurface':
+            if md.mesh.__class__.__name__ == 'mesh3dsurface':
                 if self.grdmodel == 2:
                     raise RuntimeException('model requires a 2D mesh to run gia Ivins computations (change mesh from mesh3dsurface to mesh2d)')

issm/trunk-jpl/src/m/coordsystems/gdaltransform.m

@@ -20 +20 @@
 %                       +proj=stere +lat_0=90 +lat_ts=71 +lon_0=-39 +k=1 +x_0=0 +y_0=0 +a=6378273 +b=6356889.448564109 +units=m +no_defs
 %
-%       To get proj.4 string from EPSG, use gdalsrsinfo. Example:
+%       To get PROJ.4 string from EPSG, use gdalsrsinfo. Example:
 %               gdalsrsinfo epsg:4326 | grep "PROJ.4" | sed "s/PROJ.4 : //"
 

issm/trunk-jpl/src/m/dev/devpath.py

@@ -15 +15 @@
         dirs.remove('.svn')
     for file in files:
-        if file.find(".py") != -1:
-            if file.find(".pyc") == -1:
+        if file.find('.py') != -1:
+            if file.find('.pyc') == -1:
                 if root not in sys.path:
                     sys.path.append(root)
@@ -46 +46 @@
 #c.InteractiveShellApp.exec_lines.append('print "Warning: disable autoreload in startup.py to improve performance." ')
 
-print("\n  ISSM development path correctly loaded")
-print("Current path is {}\n\n".format(ISSM_DIR))
+# print("\n  ISSM development path correctly loaded")
+# print("Current path is {}\n\n".format(ISSM_DIR))

issm/trunk-jpl/src/m/miscellaneous/MatlabFuncs.py

@@ -104 +104 @@
 #}}}
 
+def isa(A, dataType):  #{{{
+    """FUNCTION ISA
+
+    NOTE:
+    - Takes a type as its second argument (in contrast to the MATLAB function 
+    that it replicates, which takes a string representing the name of a type)
+    """
+    return type(A) == dataType
+#}}}
+
 def isfile(fileName):  #{{{
     import os

issm/trunk-jpl/src/m/plot/googlemaps.m

@@ -175 +175 @@
 delete('temp.png');
 
-%If not gdal, exit
+%If not GDAL, exit
 if status~=0,
         disp(result);
-        disp('googlemaps info: gdal not found or not working properly, the Google image will not be transformed');
+        disp('googlemaps info: GDAL not found or not working properly, the Google image will not be transformed');
         [gX gY]=meshgrid(ulx:ulx+size(final,2)-1,uly:-1:uly-size(final,1)+1);
         [LAT LON]=pixelstolatlon(gX,gY, zoom);
@@ -204 +204 @@
 if ~isempty(strfind(result,'ERROR')),
         disp(result);
-        disp(' ');disp('googlemaps info: gdal not working properly (missing proj.4 library?), Google image will not be transformed');
+        disp(' ');disp('googlemaps info: GDAL not working properly (missing PROJ.4 library?), Google image will not be transformed');
         disp(result);
         [gX gY]=meshgrid(ulx:ulx+size(final,2)-1,uly:-1:uly-size(final,1)+1);

issm/trunk-jpl/src/m/plot/plot_manager.py

@@ -3 +3 @@
     overlaysupport = True
 except ImportError:
-    print('osgeo/gdal for python not installed, overlay plots are not enabled')
+    print('OSGeo/GDAL for Python not installed, overlay plots are not enabled')
     overlaysupport = False
 
@@ -20 +20 @@
 
 def plot_manager(md, options, fig, axgrid, gridindex):
-    '''
-    PLOT_MANAGER - distribute the plots called by plotmodel
+    """PLOT_MANAGER - distribute the plots called by plotmodel
 
     'fig' is a handle to the figure instance created by plotmodel.
@@ -35 +34 @@
 
     See also: PLOTMODEL, PLOT_UNIT
-    '''
+    """
     #parse options and get a structure of options
     options = checkplotoptions(md, options)
@@ -46 +45 @@
     # TODO: Check why we are doing this and if it is absolutely necessary (see
     #       also src/m/plot/plot_mesh.py, src/m/plot/applyoptions.py)
+    #
     options.addfielddefault('ticklabels', 'on')
 

issm/trunk-jpl/src/m/plot/plot_overlay.py

@@ -11 +11 @@
     from osgeo import gdal
 except ImportError:
-    print('osgeo/gdal for python not installed, plot_overlay is disabled')
+    print('OSGeo/GDAL for python not installed, plot_overlay is disabled')
 
 from processdata import processdata
@@ -19 +19 @@
 
 def plot_overlay(md, data, options, ax):
-    '''
-    Function for plotting a georeferenced image. Called from plot_manager by 
+    """Function for plotting a georeferenced image. Called from plot_manager by 
     call to plotmodel.
 
@@ -27 +26 @@
 
     See also: PLOTMODEL
-    '''
+    """
 
     x, y, z, elements, is2d, isplanet = processmesh(md, [], options)

issm/trunk-jpl/src/m/plot/radarpower.m

@@ -29 +29 @@
 d = getfieldvalue(options,'overlay_adjust_d',1);
 
-%find gdal coordinates
+%find GDAL coordinates
 x0=min(xlim); x1=max(xlim);
 y0=min(ylim); y1=max(ylim);

issm/trunk-jpl/src/m/solve/solve.m

@@ -173 +173 @@
         end
 elseif md.settings.waitonlock>0,
-        %we wait for the done file
+        %wait for done file
         done=waitonlock(md);
         if md.verbose.solution,

issm/trunk-jpl/src/m/solve/solve.py

@@ -141 +141 @@
     if md.qmu.isdakota:
         filelist.append(modelname + '.qmu.in')
-        
+
     if not restart:
         cluster.UploadQueueJob(md.miscellaneous.name, md.private.runtimename, filelist)
@@ -150 +150 @@
     # Wait on lock
     if md.settings.waitonlock > 0:
-        islock = waitonlock(md)
-        if islock == 0: # no results to be loaded
-            print('The results must be loaded manually with md = loadresultsfromcluster(md).')
-        else: # load results
-            if md.verbose.solution:
-                print('loading results from cluster')
-            md = loadresultsfromcluster(md)
+        # Wait for done file
+        done = waitonlock(md)
+        if md.verbose.solution:
+            print('loading results from cluster')
+        md = loadresultsfromcluster(md)
+    elif md.settings.waitonlock == 0:
+        print('Model results must be loaded manually with md = loadresultsfromcluster(md).')
 
     return md

issm/trunk-jpl/src/m/solve/waitonlock.m

@@ -5 +5 @@
 %   disk. Also check for outlog file be cause it might be written several 
 %   seconds after the lock file.
+%
 %   If the time limit given in input is exceeded, return 0
 %

issm/trunk-jpl/src/m/solve/waitonlock.py

@@ -2 +2 @@
 import sys
 import time
+from generic import *
+from generic_static import *
+# from localpfe import *
 from MatlabFuncs import *
+from pfe import *
 
 def waitonlock(md):
@@ -15 +19 @@
     Usage:
         flag = waitonlock(md)
+
+    TODO:
+    - Uncomment import of localpfe and check on cluster type once localpfe.py 
+    has been translated from localpfe.m.
     """
 
@@ -22 +30 @@
     cluster = md.cluster
 
-    """
-    NOTE: We check cluster.name against string as cluster classes are not 
-          defined globally and we do not have to import them all
-    """
-    if cluster.name == 'pfe' and cluster.interactive > 1:
+    if isa(cluster, pfe) and cluster.interactive > 1:
         lockfilename = '{}/Interactive{}/{}.lock'.format(executionpath, cluster.interactive, md.miscellaneous.name)
         logfilename = '{}/Interactive{}/{}.outlog'.format(executionpath, cluster.interactive, md.miscellaneous.name)
-    elif cluster.name == 'localpfe':
-        lockfilename = '{}/{}.lock'.format(executionpath, md.miscellaneous.name)
-        logfilename = '{}/{}.outlog'.format(executionpath, md.miscellaneous.name)
+    # elif isa(cluster, localpfe):
+    #     lockfilename = '{}/{}.lock'.format(executionpath, md.miscellaneous.name)
+    #     logfilename = '{}/{}.outlog'.format(executionpath, md.miscellaneous.name)
     else:
         lockfilename = '{}/{}/{}.lock'.format(executionpath, md.private.runtimename, md.miscellaneous.name)
@@ -37 +41 @@
 
     # If we are using the generic cluster in interactive mode, job is already complete
-    if (cluster.name == 'generic' and cluster.interactive) or (cluster.name == 'generic_static'):
+    if (isa(cluster, generic) and cluster.interactive) or (isa(cluster, generic_static)):
         # We are in interactive mode, no need to check for job completion
         return 1
@@ -73 +77 @@
             subproc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             outs, errs = subproc.communicate() # NOTE: Need to consume output before checking return code
+
             # TODO: Debug the following check under Linux (exits after first iteration with errs = "b")
             # UPDATE: Works in testing under Debian Linux system. Leaving comment for now so that it is easier to backtrace this issue if someone else encounters it.
             #
             if errs != '':
-                raise Exception("waitonlock: check for existence of files failed: {}".format(errs))
+                raise Exception('waitonlock: check for existence of files failed: {}'.format(errs))
             ispresent = not subproc.returncode
             if ispresent:

issm/trunk-jpl/test/NightlyRun/runme.py

@@ -10 +10 @@
 
 try:
-    from parallelrange import parallelrange
-except ImportError:  #we don't have issm code in path, just get it
+    from arch import archread
+except: # ISSM_DIR is not on path
     import devpath
-    from parallelrange import parallelrange
+
 from arch import archread
 from arch import archwrite
 from GetIds import *
 from IdToName import IdToName
+from parallelrange import parallelrange
 
 
@@ -33 +34 @@
 
     Options:
-        -i/--id             followed by the list of ids or (parts of) test names requested
-                            NOTE: runs all tests by default
-        -e/--exclude        ids or (parts of) test names to be excluded (same format as id)
-                            NOTE: exclude does nothing if 'id' is specified with different values
-        -b/--benchmark      'all'           : (all of the tests)
-                            'nightly'       : (nightly run/daily run)
-                            'validation'    : (validation)
-                            'adolc'         : validation of adolc tests
-                            'eismint'       : validation of eismint tests
-                            'ismip'         : validation of ismip-hom tests
-                            'mesh'          : validation of mesh tests
-                            'qmu'           : validation of qmu tests
-                            'referential'   : validation of referential tests
-                            'slc'           : validation of slc tests
-                            'thermal'       : validation of thermal tests
-                            'tranforcing'   : validation of transient forcing tests
-        -p/--procedure      'check'         : run the test (default)
-                            'update'        : update the archive
+        -i/--id         Followed by the list of ids or (parts of) test names 
+                        requested
+        -e/--exclude    Ids or (parts of) test names to be excluded (same 
+                        format as id). Does nothing if 'id' is specified with 
+                        different values.
+        -b/--benchmark  'all'           : (all of the tests)
+                        'nightly'       : (nightly run/daily run)
+                        'validation'    : (validation)
+                        'adolc'         : validation of adolc tests
+                        'eismint'       : validation of eismint tests
+                        'ismip'         : validation of ismip-hom tests
+                        'mesh'          : validation of mesh tests
+                        'qmu'           : validation of qmu tests
+                        'referential'   : validation of referential tests
+                        'slc'           : validation of slc tests
+                        'thermal'       : validation of thermal tests
+                        'tranforcing'   : validation of transient forcing tests
+        -p/--procedure  'check'         : run the test (default)
+                        'update'        : update the archive
 
     Usage:
@@ -103 +105 @@
     #GET ids  {{{
     flist = glob('test*.py')  #File name must start with 'test' and must end by '.py' and must be different than 'test.py'
-    list_ids = [int(re.search(r'\d+',file.split('.')[0]).group()) for file in flist if not file == 'test.py'] #Keep test id only (skip 'test' and '.py')
+    list_ids = [int(re.search(r'\d+',file.split('.')[0]).group()) for file in flist if not file == 'test.py'] # Keep test id only (skip 'test' and '.py')
 
     i1, i2 = parallelrange(rank, numprocs, len(list_ids))  #Get tests for this cpu only