Changeset 19157

Timestamp:

02/26/15 09:07:32 (10 years ago)

Author:

bdef

Message:

NEW: adding cluster (pfe) launch ability in python

Location:

issm/trunk-jpl/src/m

Files:

• classes/clusters/pfe.py (added)
• classes/model.py (modified) (1 diff)
• consistency/QueueRequirements.py (added)
• contrib/netCDF/export_netCDF.py (modified) (8 diffs)
• contrib/netCDF/read_netCDF.m (modified) (6 diffs)
• contrib/paraview/exportVTK.m (modified) (1 diff)
• dev/devpath.py (modified) (2 diffs)
• io/structtonc.m (modified) (1 diff)
• os/issmscpin.py (modified) (1 diff)
• solve/loadresultsfromcluster.py (modified) (1 diff)

issm/trunk-jpl/src/m/classes/model.py

@@ -29 +29 @@
 from toolkits import toolkits
 from generic import generic
+from pfe import pfe
+from greenplanet import greenplanet
 from balancethickness import balancethickness
 from stressbalance import stressbalance

issm/trunk-jpl/src/m/contrib/netCDF/export_netCDF.py

@@ -1 @@
-from netCDF4 import Dataset
+from netCDF4 import Dataset, stringtochar
 import numpy
 import time
@@ -34 +34 @@
         Dimension3=NCData.createDimension('Dimension3',numpy.shape(md.mesh.elements)[1])
         Dimension4=NCData.createDimension('Dimension4',StepNum)
-        Dimension5=NCData.createDimension('Dimension5',2) 
+        Dimension5=NCData.createDimension('Dimension5',40)
+        Dimension6=NCData.createDimension('Dimension6',2) 
 
         DimDict = {len(Dimension1):'Dimension1',
@@ -40 +41 @@
                                                  len(Dimension3):'Dimension3',
                                                  len(Dimension4):'Dimension4',
-                                                 len(Dimension5):'Dimension5'}
+                                                 len(Dimension5):'Dimension5',
+                                                 len(Dimension6):'Dimension6'}
 
         #get all model classes and create respective groups
@@ -114 +116 @@
         elif val_type==list:
                 dimensions,DimDict=GetDim(NCData,var,val_shape,DimDict,val_dim,istime)
-                ncvar = Group.createVariable(str(field),str,dimensions,zlib=True)
+                ncvar = Group.createVariable(str(field),'S1',dimensions,zlib=True)
+                charvar=stringtochar(numpy.array(var))
+                print charvar
+                print charvar.shape
                 for elt in range(0,val_dim):
                         try:
-                                ncvar[elt] = var[elt]
+                                ncvar[elt] = charvar[elt]
                         except IndexError:
                                 ncvar[0]= " "
@@ -123 +128 @@
         elif val_type=='bool':
                 dimensions,DimDict=GetDim(NCData,var,val_shape,DimDict,val_dim,istime)
-                ncvar = Group.createVariable(str(field),str,dimensions,zlib=True)
+                ncvar = Group.createVariable(str(field),'S1',dimensions,zlib=True)
                 for elt in range(0,val_shape[0]):
                         ncvar[elt] = str(var[elt])
@@ -129 +134 @@
         elif val_type==collections.OrderedDict:
                 dimensions,DimDict=GetDim(NCData,var,val_shape,DimDict,val_dim,istime)
-                ncvar = Group.createVariable(str(field),str,dimensions,zlib=True)
+                ncvar = Group.createVariable(str(field),'S1',dimensions,zlib=True)
                 for elt in range(0,val_dim):
                         ncvar[elt,0]=dict.keys(var)[elt]
@@ -169 +174 @@
 def GetDim(NCData,var,shape,DimDict,i,istime):
         output=[]
+        #grab type
+        try:
+                val_type=str(var.dtype)
+        except AttributeError:
+                val_type=type(var)
+        #grab dimension
         for dim in range(0,i): #loop on the dimensions
                 if type(shape[0])==int: 
@@ -187 +198 @@
                                 NewDim=NCData.createDimension('Dimension'+str(index),numpy.shape(shape)[0])
                                 DimDict[len(NewDim)]='Dimension'+str(index)
-                                output=[str(DimDict[numpy.shape(dict.keys(var))[0]])]+['Dimension5']
+                                output=[str(DimDict[numpy.shape(dict.keys(var))[0]])]+['Dimension6']
                                 print 'Defining dimension ' +'Dimension'+str(index)
                         break
         if istime:
                 output=output+['Dimension4']
+        #dealing with char and not string as we should so we need to had a string length
+        if val_type=='bool' or val_type==collections.OrderedDict or val_type==list:
+                charvar=stringtochar(numpy.array(var))
+                stringlength=charvar.shape[charvar.ndim-1]
+                try:
+                        output=output+[str(DimDict[stringlength])] #test if the dimension allready exist
+                except KeyError: #if not create it
+                        if (shape[dim])>1:
+                                index=len(DimDict)+1
+                                NewDim=NCData.createDimension('Dimension'+str(index),(stringlength))
+                                DimDict[len(NewDim)]='Dimension'+str(index)
+                                output=output+[str(DimDict[stringlength])]
+                                print 'Defining dimension ' +'Dimension'+str(index)
         return tuple(output), DimDict

issm/trunk-jpl/src/m/contrib/netCDF/read_netCDF.m

@@ -21 +21 @@
                                 subclass = netcdf.getAtt(subgroupIDs(j),netcdf.getConstant('NC_GLOBAL'),'classtype');
                                 self.results=setfield(self.results,subclass,struct);
-                                [ndims nvar natts]=netcdf.inq(groupIDs(j));
+                                [ndims nvar natts]=netcdf.inq(subgroupIDs(j));
                                 varIDs=netcdf.inqVarIDs(subgroupIDs(j));
                                 %first loop on group atributes
@@ -27 +27 @@
                                         attname = netcdf.inqAttName(subgroupIDs(j),netcdf.getConstant('NC_GLOBAL'),k-1);
                                         [xtype,attlen] = netcdf.inqAtt(subgroupIDs(j),netcdf.getConstant('NC_GLOBAL'),attname);
-                                        %disp(sprintf('In %s, Treating attribute %s of type %i',subclass,attname,xtype));
+                                        disp(sprintf('In %s, Treating attribute %s of type %i',subclass,attname,xtype));
                                         %classtype have done is job, no need to keep it any more
                                         if ~strcmp(attname,'classtype'),
@@ -43 +43 @@
                                 for k=1:length(varIDs),
                                         [varname, xtype, varDimIDs, varAtts] =netcdf.inqVar(subgroupIDs(j),varIDs(k));
-                                        %disp(sprintf('In %s, Treating variable %s of type %i',whichclass,varname,xtype));
+                                        disp(sprintf('In %s, Treating variable %s of type %i',whichclass,varname,xtype));
                                         %time dimension seems to be last in our construction
                                         for l=1:length(varDimIDs),
@@ -68 +68 @@
                         varID=netcdf.inqVarIDs(groupIDs(i));
                         [varname, xtype, varDimIDs, varAtts] =netcdf.inqVar(groupIDs(i),varID);
-                        %disp(sprintf('In %s, Treating variable %s of type %i',whichclass,varname,xtype));
+                        disp(sprintf('In %s, Treating variable %s of type %i',whichclass,varname,xtype));
                         [dimname,numoffields] = netcdf.inqDim(ncid,varDimIDs(end));
                         self.(groupName)=eval(whichclass);
@@ -97 +97 @@
                                 attname = netcdf.inqAttName(groupIDs(i),netcdf.getConstant('NC_GLOBAL'),j-1);
                                 [xtype,attlen] = netcdf.inqAtt(groupIDs(i),netcdf.getConstant('NC_GLOBAL'),attname);
-                                %disp(sprintf('In %s, Treating attribute %s of type %i',whichclass,attname,xtype));
+                                disp(sprintf('In %s, Treating attribute %s of type %i',whichclass,attname,xtype));
                                 %classtype have done is job, no need to keep it any more
                                 if ~strcmp(attname,'classtype'),
@@ -113 +113 @@
                         for j=1:length(varIDs),
                                 [varname, xtype, varDimIDs, varAtts] =netcdf.inqVar(groupIDs(i),varIDs(j));
-                                %disp(sprintf('In %s, Treating variable %s of type %i',whichclass,varname,xtype));                      
+                                disp(sprintf('In %s, Treating variable %s of type %i',whichclass,varname,xtype));                       
                                 %if the value is a single string, we need to transpose it (cross check with python file is necessary)
                                 if xtype==2

issm/trunk-jpl/src/m/contrib/paraview/exportVTK.m

@@ -112 +112 @@
                         %check which field is a real result and print
                         for k=1:num_of_fields
-                                if ((numel(sol_struct{j}(timestep).(fieldnames{k})))== ...
-                                                num_of_points);
+                                if ((numel(sol_struct{j}(timestep).(fieldnames{k})))==num_of_points);
                                         %paraview does not like NaN, replacing
                                         nanval=find(isnan(sol_struct{j}(timestep).(fieldnames{k})));

issm/trunk-jpl/src/m/dev/devpath.py

@@ -1 +1 @@
 #!/usr/bin/env python
-
 import os,sys
+import warnings
 
 #Recover ISSM_DIR and USERNAME
-ISSM_DIR=os.getenv('ISSM_DIR')
-USERNAME =os.getenv('USER')
+ISSM_DIR = os.getenv('ISSM_DIR')
+USERNAME = os.getenv('USER')
+JPL_SVN  = os.getenv('JPL_SVN')
 if(ISSM_DIR==None):
         raise NameError('"ISSM_DIR" environment variable is empty! You should define ISSM_DIR in your .cshrc or .bashrc!')
+if(JPL_SVN==None):
+        warnings.warn('"JPL_SVN" environment variable is empty! add it to your .cshrc or .bashrc if you want to do distant computing')
 
 #Go through src/m and append any directory that contains a *.py file to PATH 
@@ -21 +24 @@
 sys.path.append(ISSM_DIR + '/lib')
 sys.path.append(ISSM_DIR + '/src/wrappers/python/.libs')
+# If using clusters, we need to have the path to the cluster settings directory
+if(JPL_SVN!=None):
+        if os.path.exists(JPL_SVN + '/usr/' + USERNAME):
+                sys.path.append(JPL_SVN + '/usr/' + USERNAME)
+        else:
+                raise NameError ('cluster settings should be in, '+ JPL_SVN +'/usr/' + USERNAME)
 
 #Manual imports for commonly used functions

issm/trunk-jpl/src/m/io/structtonc.m

@@ -28 +28 @@
 %Double vector
 elseif isa(field,'double') & size(field,2)==1,
-        if step==1,
+
+if step==1,
                 dim_id          = netcdf.defDim(ncid,[fieldname '_size1'],size(field,1));
                 var_id(counter) = netcdf.defVar(ncid,fieldname,'NC_DOUBLE',dim_id);

issm/trunk-jpl/src/m/os/issmscpin.py

@@ -55 +55 @@
                         #just use standard unix scp
                         #string to copy multiple files using scp: 
-                        if len(packages)==1:
-                                string=packages[0]
+                        string='\{'+','.join([str(x) for x in packages])+'\}'
+                        
+                        if port:
+                                subprocess.call('scp -P %d %s@localhost:%s %s/. ' % (port,login,os.path.join(path,string),os.getcwd()),shell=True)
                         else:
-                                string='{'
-                                for package in packages:
-                                        string+=packages[i]+','
-                                string=string[:-1]+'}'
-
-                        if port:
-                                subprocess.call('scp -P %d %s@localhost:%s %s' % (port,login,os.path.join(path,string),os.getcwd),shell=True)
-                        else:
-                                subprocess.call('scp %s@%s:%s %s' % (login,host,os.path.join(path,string),os.getcwd),shell=True)
+                                subprocess.call('scp %s@%s:%s %s/.' % (login,host,os.path.join(path,string),os.getcwd()),shell=True)
                 
                         #check scp worked

issm/trunk-jpl/src/m/solve/loadresultsfromcluster.py

@@ -35 +35 @@
         #erase the log and output files
         if md.qmu.isdakota:
-                os.remove(os.path.join('qmu'+str(os.getpid()),md.miscellaneous.name+'.outlog'))
-                os.remove(os.path.join('qmu'+str(os.getpid()),md.miscellaneous.name+'.errlog'))
+                filename=os.path.join('qmu'+str(os.getpid()),md.miscellaneous.name)
         else:
-                os.remove(md.miscellaneous.name+'.outlog')
-                os.remove(md.miscellaneous.name+'.errlog')
-                os.remove(md.miscellaneous.name+'.outbin')
-                if not m.ispc():
-                        os.remove(md.private.runtimename+'.tar.gz')
+                filename=md.miscellaneous.name
 
-        #erase input file if run was carried out on same platform.
+        TryRem('.errlog',filename)
+        TryRem('.outlog',filename)
+        
+        if not m.ispc():
+                TryRem('.tar.gz',md.private.runtimename)
+                if not md.qmu.isdakota:
+                        TryRem('.outbin',filename)
+
+                        #erase input file if run was carried out on same platform.
         hostname=socket.gethostname()
         if m.strcmpi(hostname,cluster.name):
                 if md.qmu.isdakota:
-                        os.remove(os.path.join('qmu'+str(os.getpid()),md.miscellaneous.name+'.bin'))
-                        os.remove(os.path.join('qmu'+str(os.getpid()),md.miscellaneous.name+'.queue'))
+                        filename=os.path.join('qmu'+str(os.getpid()),md.miscellaneous.name)
+                        TryRem('.queue',filename)
                 else:
-                        os.remove(md.miscellaneous.name+'.bin')
-                        os.remove(md.miscellaneous.name+'.toolkits')
+                        filename=md.miscellaneous.name
+                        TryRem('.toolkits',filename)
                         if not m.ispc():
-                                os.remove(md.miscellaneous.name+'.queue')
+                                TryRem('.queue',filename)
                         else:
-                                os.remove(md.miscellaneous.name+'.bat')
+                                TryRem('.bat',filename)
+
+                TryRem('.bin',filename)
 
         return md
 
+def TryRem(extension,filename):
+        try:    
+                os.remove(filename+extension)
+        except OSError:
+                print 'WARNING, no '+extension+'  is present for run '+filename