![(please configure the [header_logo] section in trac.ini)](/trac/issm/chrome/common/trac_banner.png)
Changeset 24313 for issm/trunk/src/m/io/loadvars.py
- Timestamp:
- 11/01/19 12:01:57 (5 years ago)
- Location:
- issm/trunk
- Files:
-
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
issm/trunk
-
issm/trunk/src
- Property svn:mergeinfo changed
-
issm/trunk/src/m/io/loadvars.py
r23189 r24313 1 1 import shelve 2 import os.path 3 import numpy as np 2 import numpy as np 4 3 from netCDF4 import Dataset 5 from netCDF4 import chartostring6 4 from re import findall 7 from os import path8 5 from collections import OrderedDict 9 from whichdb import whichdb10 6 from model import * 7 #hack to keep python 2 compatibility 8 try: 9 #py3 import 10 from dbm import whichdb 11 except ImportError: 12 #py2 import 13 from whichdb import whichdb 14 11 15 12 16 def loadvars(*args): 13 """ 14 LOADVARS - function to load variables to a file. 15 16 This function loads one or more variables from a file. The names of the variables 17 must be supplied. If more than one variable is specified, it may be done with 18 a list of names or a dictionary of name as keys. The output type will correspond 19 to the input type. All the variables in the file may be loaded by specifying only 20 the file name. 21 22 Usage: 23 a=loadvars('shelve.dat','a') 24 [a,b]=loadvars('shelve.dat',['a','b']) 25 nvdict=loadvars('shelve.dat',{'a':None,'b':None}) 26 nvdict=loadvars('shelve.dat') 27 28 """ 29 30 filename='' 31 nvdict={} 32 33 if len(args) >= 1 and isinstance(args[0],(str,unicode)): 34 filename=args[0] 35 if not filename: 36 filename='/tmp/shelve.dat' 37 38 else: 39 raise TypeError("Missing file name.") 40 41 if len(args) >= 2 and isinstance(args[1],(str,unicode)): # (filename,name) 42 for name in args[1:]: 43 nvdict[name]=None 44 45 elif len(args) == 2 and isinstance(args[1],list): # (filename,[names]) 46 for name in args[1]: 47 nvdict[name]=None 48 49 elif len(args) == 2 and isinstance(args[1],dict): # (filename,{names:values}) 50 nvdict=args[1] 51 52 elif len(args) == 1: # (filename) 53 pass 54 55 else: 56 raise TypeError("Unrecognized input arguments.") 57 58 if whichdb(filename): 59 print "Loading variables from file '%s'." % filename 60 61 my_shelf = shelve.open(filename,'r') # 'r' for read-only 62 if nvdict: 63 for name in nvdict.iterkeys(): 64 try: 65 nvdict[name] = my_shelf[name] 66 print "Variable '%s' loaded." % name 67 except KeyError: 68 value = None 69 print "Variable '%s' not found." % name 70 71 else: 72 for name in my_shelf.iterkeys(): 73 nvdict[name] = my_shelf[name] 74 print "Variable '%s' loaded." % name 75 76 my_shelf.close() 77 78 else: 79 try: 80 NCFile=Dataset(filename,mode='r') 81 NCFile.close() 82 except RuntimeError: 83 raise IOError("File '%s' not found." % filename) 84 85 classtype,classtree=netCDFread(filename) 86 nvdict['md']=model() 87 NCFile=Dataset(filename,mode='r') 88 for mod in dict.keys(classtype): 89 if np.size(classtree[mod])>1: 90 curclass=NCFile.groups[classtree[mod][0]].groups[classtree[mod][1]] 91 if classtype[mod][0]=='list': 92 keylist=[key for key in curclass.groups] 93 try: 94 steplist=[int(key) for key in curclass.groups] 95 except ValueError: 96 steplist=[int(findall(r'\d+',key)[0]) for key in keylist] 97 indexlist=[index*(len(curclass.groups)-1)/max(1,max(steplist)) for index in steplist] 98 listtype=curclass.groups[keylist[0]].classtype 99 if listtype=='dict': 100 nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [OrderedDict() for i in range(max(1,len(curclass.groups)))] 101 else: 102 nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(__import__(listtype),listtype)() for i in range(max(1,len(curclass.groups)))] 103 Tree=nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]][:] 104 else: 105 nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = getattr(classtype[mod][1],classtype[mod][0])() 106 Tree=nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] 107 else: 108 curclass=NCFile.groups[classtree[mod][0]] 109 nvdict['md'].__dict__[mod] = getattr(classtype[mod][1],classtype[mod][0])() 110 Tree=nvdict['md'].__dict__[classtree[mod][0]] 111 #treating groups that are lists 112 for i in range(0,max(1,len(curclass.groups))): 113 if len(curclass.groups)>0: 114 listclass=curclass.groups[keylist[i]] 115 else: 116 listclass=curclass 117 for var in listclass.variables: 118 if var not in ['errlog','outlog']: 119 varval=listclass.variables[str(var)] 120 vardim=varval.ndim 121 try: 122 val_type=str(varval.dtype) 123 except AttributeError: 124 val_type=type(varval) 125 if vardim==0: 126 if type(Tree)==list: 127 t=indexlist[i] 128 if listtype=='dict': 129 Tree[t][str(var)]=varval[0] 130 else: 131 Tree[t].__dict__[str(var)]=varval[0] 132 else: 133 if str(varval[0])=='': 134 Tree.__dict__[str(var)]=[] 135 elif varval[0]=='True': 136 Tree.__dict__[str(var)]=True 137 elif varval[0]=='False': 138 Tree.__dict__[str(var)]=False 139 else: 140 Tree.__dict__[str(var)]=varval[0] 141 142 elif vardim==1: 143 if varval.dtype==str: 144 if varval.shape[0]==1: 145 Tree.__dict__[str(var)]=[str(varval[0]),] 146 elif 'True' in varval[:] or 'False' in varval[:]: 147 Tree.__dict__[str(var)]=np.asarray([V=='True' for V in varval[:]],dtype=bool) 148 else: 149 Tree.__dict__[str(var)]=[str(vallue) for vallue in varval[:]] 150 else: 151 if type(Tree)==list: 152 t=indexlist[i] 153 if listtype=='dict': 154 Tree[t][str(var)]=varval[:] 155 else: 156 Tree[t].__dict__[str(var)]=varval[:] 157 else: 158 try: 159 #some thing specifically require a list 160 mdtype=type(Tree.__dict__[str(var)]) 161 except KeyError: 162 mdtype=float 163 if mdtype==list: 164 Tree.__dict__[str(var)]=[mdval for mdval in varval[:]] 165 else: 166 Tree.__dict__[str(var)]=varval[:] 167 elif vardim==2: 168 #dealling with dict 169 if varval.dtype==str: #that is for toolkits wich needs to be ordered 170 if any(varval[:,0]=='toolkit'): #toolkit definition have to be first 171 Tree.__dict__[str(var)]=OrderedDict([('toolkit', str(varval[np.where(varval[:,0]=='toolkit')[0][0],1]))]) 172 173 strings1=[str(arg[0]) for arg in varval if arg[0]!='toolkits'] 174 strings2=[str(arg[1]) for arg in varval if arg[0]!='toolkits'] 175 Tree.__dict__[str(var)].update(zip(strings1, strings2)) 176 else: 177 if type(Tree)==list: 178 #t=int(keylist[i][-1])-1 179 t=indexlist[i] 180 if listtype=='dict': 181 Tree[t][str(var)]=varval[:,:] 182 else: 183 Tree[t].__dict__[str(var)]=varval[:,:] 184 else: 185 Tree.__dict__[str(var)]=varval[:,:] 186 elif vardim==3: 187 if type(Tree)==list: 188 t=indexlist[i] 189 if listtype=='dict': 190 Tree[t][str(var)]=varval[:,:,:] 191 else: 192 Tree[t].__dict__[str(var)]=varval[:,:,:] 193 else: 194 Tree.__dict__[str(var)]=varval[:,:,:] 195 else: 196 print 'table dimension greater than 3 not implemented yet' 197 for attr in listclass.ncattrs(): 198 if attr!='classtype': #classtype is for treatment, don't get it back 199 if type(Tree)==list: 200 t=indexlist[i] 201 if listtype=='dict': 202 Tree[t][str(attr).swapcase()]=str(listclass.getncattr(attr)) 203 else: 204 Tree[t].__dict__[str(attr).swapcase()]=str(listclass.getncattr(attr)) 205 else: 206 Tree.__dict__[str(attr).swapcase()]=str(listclass.getncattr(attr)) 207 if listclass.getncattr(attr)=='True': 208 Tree.__dict__[str(attr).swapcase()]=True 209 elif listclass.getncattr(attr)=='False': 210 Tree.__dict__[str(attr).swapcase()]=False 211 NCFile.close() 212 if len(args) >= 2 and isinstance(args[1],(str,unicode)): # (value) 213 value=[nvdict[name] for name in args[1:]] 214 return value 215 216 elif len(args) == 2 and isinstance(args[1],list): # ([values]) 217 value=[nvdict[name] for name in args[1]] 218 return value 219 220 elif (len(args) == 2 and isinstance(args[1],dict)) or (len(args) == 1): # ({names:values}) 221 return nvdict 17 """ 18 LOADVARS - function to load variables to a file. 19 20 This function loads one or more variables from a file. The names of the variables 21 must be supplied. If more than one variable is specified, it may be done with 22 a list of names or a dictionary of name as keys. The output type will correspond 23 to the input type. All the variables in the file may be loaded by specifying only 24 the file name. 25 26 Usage: 27 a = loadvars('shelve.dat', 'a') 28 [a, b] = loadvars('shelve.dat', ['a', 'b']) 29 nvdict = loadvars('shelve.dat', {'a':None, 'b':None}) 30 nvdict = loadvars('shelve.dat') 31 32 """ 33 34 filename = '' 35 nvdict = {} 36 37 if len(args) >= 1 and isinstance(args[0], str): 38 filename = args[0] 39 if not filename: 40 filename = '/tmp/shelve.dat' 41 42 else: 43 raise TypeError("Missing file name.") 44 45 if len(args) >= 2 and isinstance(args[1], str): # (filename, name) 46 for name in args[1:]: 47 nvdict[name] = None 48 49 elif len(args) == 2 and isinstance(args[1], list): # (filename, [names]) 50 for name in args[1]: 51 nvdict[name] = None 52 53 elif len(args) == 2 and isinstance(args[1], dict): # (filename, {names:values}) 54 nvdict = args[1] 55 56 elif len(args) == 1: # (filename) 57 pass 58 59 else: 60 raise TypeError("Unrecognized input arguments.") 61 62 if whichdb(filename): 63 print("Loading variables from file {}.".format(filename)) 64 my_shelf = shelve.open(filename, 'r') # 'r' for read - only 65 if nvdict: 66 for name in list(nvdict.keys()): 67 try: 68 nvdict[name] = my_shelf[name] 69 print(("Variable '%s' loaded." % name)) 70 except KeyError: 71 value = None 72 print("Variable '{}' not found.".format(name)) 73 74 else: 75 for name in list(my_shelf.keys()): 76 nvdict[name] = my_shelf[name] 77 print(("Variable '%s' loaded." % name)) 78 my_shelf.close() 79 80 else: 81 try: 82 NCFile = Dataset(filename, mode='r') 83 NCFile.close() 84 except RuntimeError: 85 raise IOError("File '{}' not found.".format(filename)) 86 87 classtype, classtree = netCDFread(filename) 88 nvdict['md'] = model() 89 NCFile = Dataset(filename, mode='r') 90 for mod in dict.keys(classtype): 91 #print(' - Now treating classtype {}'.format(mod)) 92 if np.size(classtree[mod]) > 1: 93 curclass = NCFile.groups[classtree[mod][0]].groups[classtree[mod][1]] 94 if classtype[mod][0] == 'list': 95 keylist = [key for key in curclass.groups] 96 try: 97 steplist = [int(key) for key in curclass.groups] 98 except ValueError: 99 steplist = [int(findall(r'\d + ', key)[0]) for key in keylist] 100 indexlist = [index * (len(curclass.groups) - 1) / max(1, max(steplist)) for index in steplist] 101 listtype = curclass.groups[keylist[0]].classtype 102 if listtype == 'dict': 103 nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [OrderedDict() for i in range(max(1, len(curclass.groups)))] 104 else: 105 nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(__import__(listtype), listtype)() for i in range(max(1, len(curclass.groups)))] 106 Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]][:] 107 else: 108 nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = getattr(classtype[mod][1], classtype[mod][0])() 109 Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] 110 else: 111 curclass = NCFile.groups[classtree[mod][0]] 112 nvdict['md'].__dict__[mod] = getattr(classtype[mod][1], classtype[mod][0])() 113 Tree = nvdict['md'].__dict__[classtree[mod][0]] 114 #treating groups that are lists 115 for i in range(0, max(1, len(curclass.groups))): 116 if len(curclass.groups) > 0: 117 listclass = curclass.groups[keylist[i]] 118 else: 119 listclass = curclass 120 for var in listclass.variables: 121 #print("treating var {}".format(var)) 122 if var not in ['errlog', 'outlog']: 123 varval = listclass.variables[str(var)] 124 vardim = varval.ndim 125 if vardim == 0: 126 if type(Tree) == list: 127 t = int(indexlist[i]) 128 if listtype == 'dict': 129 Tree[t][str(var)] = varval[0] 130 131 else: 132 Tree[t].__dict__[str(var)] = varval[0] 133 134 else: 135 if str(varval[0]) == '': #no value 136 Tree.__dict__[str(var)] = [] 137 138 elif varval[0] == 'True': #treatin bool 139 Tree.__dict__[str(var)] = True 140 141 elif varval[0] == 'False': #treatin bool 142 Tree.__dict__[str(var)] = False 143 144 else: 145 Tree.__dict__[str(var)] = varval[0].item() 146 147 elif vardim == 1: 148 if varval.dtype == str: 149 if varval.shape[0] == 1: 150 Tree.__dict__[str(var)] = [str(varval[0]), ] 151 152 elif 'True' in varval[:] or 'False' in varval[:]: 153 Tree.__dict__[str(var)] = np.asarray([V == 'True' for V in varval[:]], dtype=bool) 154 155 else: 156 Tree.__dict__[str(var)] = [str(vallue) for vallue in varval[:]] 157 158 else: 159 if type(Tree) == list: 160 t = int(indexlist[i]) 161 if listtype == 'dict': 162 Tree[t][str(var)] = varval[:] 163 164 else: 165 Tree[t].__dict__[str(var)] = varval[:] 166 167 else: 168 try: 169 #some thing specifically require a list 170 mdtype = type(Tree.__dict__[str(var)]) 171 except KeyError: 172 mdtype = float 173 if mdtype == list: 174 Tree.__dict__[str(var)] = [mdval for mdval in varval[:]] 175 176 else: 177 Tree.__dict__[str(var)] = varval[:].data 178 179 elif vardim == 2: 180 #dealling with dict 181 if varval.dtype == str: #that is for toolkits wich needs to be ordered 182 if any(varval[:, 0] == 'toolkit'): #toolkit definition have to be first 183 Tree.__dict__[str(var)] = OrderedDict([('toolkit', str(varval[np.where(varval[:, 0] == 'toolkit')[0][0], 1]))]) 184 strings1 = [str(arg[0]) for arg in varval if arg[0] != 'toolkits'] 185 strings2 = [str(arg[1]) for arg in varval if arg[0] != 'toolkits'] 186 Tree.__dict__[str(var)].update(list(zip(strings1, strings2))) 187 else: 188 if type(Tree) == list: 189 t = int(indexlist[i]) 190 if listtype == 'dict': 191 Tree[t][str(var)] = varval[:, :] 192 else: 193 Tree[t].__dict__[str(var)] = varval[:, :] 194 else: 195 Tree.__dict__[str(var)] = varval[:, :].data 196 elif vardim == 3: 197 if type(Tree) == list: 198 t = int(indexlist[i]) 199 if listtype == 'dict': 200 Tree[t][str(var)] = varval[:, :, :] 201 else: 202 Tree[t].__dict__[str(var)] = varval[:, :, :] 203 else: 204 Tree.__dict__[str(var)] = varval[:, :, :].data 205 else: 206 print('table dimension greater than 3 not implemented yet') 207 for attr in listclass.ncattrs(): 208 if attr != 'classtype': #classtype is for treatment, don't get it back 209 if type(Tree) == list: 210 t = int(indexlist[i]) 211 if listtype == 'dict': 212 Tree[t][str(attr).swapcase()] = str(listclass.getncattr(attr)) 213 else: 214 Tree[t].__dict__[str(attr).swapcase()] = str(listclass.getncattr(attr)) 215 else: 216 Tree.__dict__[str(attr).swapcase()] = str(listclass.getncattr(attr)) 217 if listclass.getncattr(attr) == 'True': 218 Tree.__dict__[str(attr).swapcase()] = True 219 elif listclass.getncattr(attr) == 'False': 220 Tree.__dict__[str(attr).swapcase()] = False 221 NCFile.close() 222 if len(args) >= 2 and isinstance(args[1], str): # (value) 223 value = [nvdict[name] for name in args[1:]] 224 return value 225 226 elif len(args) == 2 and isinstance(args[1], list): # ([values]) 227 value = [nvdict[name] for name in args[1]] 228 return value 229 230 elif (len(args) == 2 and isinstance(args[1], dict)) or (len(args) == 1): # ({names:values}) 231 return nvdict 222 232 223 233 224 234 def netCDFread(filename): 225 print ('Opening {} for reading '.format(filename))226 NCData=Dataset(filename, 'r')227 class_dict={}228 class_tree={}229 230 231 if len(NCData.groups[group].groups)>0:232 233 classe=str(group)+'.'+str(subgroup)234 class_dict[classe]=[str(getattr(NCData.groups[group].groups[subgroup],'classtype')),]235 if class_dict[classe][0] not in ['dict','list','cell']:236 237 class_tree[classe]=[group,subgroup]238 239 classe=str(group)240 241 class_dict[classe]=[str(getattr(NCData.groups[group],'classtype')),]242 if class_dict[classe][0] not in ['dict','list','cell']:243 244 class_tree[classe]=[group,]245 246 print('group {} is empty'.format(group))247 248 return class_dict,class_tree235 print(('Opening {} for reading '.format(filename))) 236 NCData = Dataset(filename, 'r') 237 class_dict = {} 238 class_tree = {} 239 240 for group in NCData.groups: 241 if len(NCData.groups[group].groups) > 0: 242 for subgroup in NCData.groups[group].groups: 243 classe = str(group) + '.' + str(subgroup) 244 class_dict[classe] = [str(getattr(NCData.groups[group].groups[subgroup], 'classtype')), ] 245 if class_dict[classe][0] not in ['dict', 'list', 'cell']: 246 class_dict[classe].append(__import__(class_dict[classe][0])) 247 class_tree[classe] = [group, subgroup] 248 else: 249 classe = str(group) 250 try: 251 class_dict[classe] = [str(getattr(NCData.groups[group], 'classtype')), ] 252 if class_dict[classe][0] not in ['dict', 'list', 'cell']: 253 class_dict[classe].append(__import__(class_dict[classe][0])) 254 class_tree[classe] = [group, ] 255 except AttributeError: 256 print(('group {} is empty'.format(group))) 257 NCData.close() 258 return class_dict, class_tree
Note:
See TracChangeset
for help on using the changeset viewer.
