Index: /issm/trunk-jpl/src/m/contrib/defleurian/netCDF/export_netCDF.py =================================================================== --- /issm/trunk-jpl/src/m/contrib/defleurian/netCDF/export_netCDF.py (revision 21246) +++ /issm/trunk-jpl/src/m/contrib/defleurian/netCDF/export_netCDF.py (revision 21247) @@ -24,23 +24,21 @@ #gather geometry and timestepping as dimensions - Duration=md.timestepping.final_time-md.timestepping.start_time - if Duration>0 and md.timestepping.time_step*md.settings.output_frequency>0: - StepNum=int(Duration/(md.timestepping.time_step*md.settings.output_frequency)) - else: + try: + StepNum=np.shape(dict.values(md.results.__dict__))[1] + except IndexError: StepNum=1 - - Dimension1=NCData.createDimension('Dimension1',md.mesh.numberofelements) - Dimension2=NCData.createDimension('Dimension2',md.mesh.numberofvertices) - Dimension3=NCData.createDimension('Dimension3',np.shape(md.mesh.elements)[1]) - Dimension4=NCData.createDimension('Dimension4',StepNum) - Dimension5=NCData.createDimension('Dimension5',2) - - DimDict = {len(Dimension1):'Dimension1', - len(Dimension2):'Dimension2', - len(Dimension3):'Dimension3', - len(Dimension4):'Dimension4', - len(Dimension5):'Dimension5'} + Dimension1=NCData.createDimension('DimNum1',StepNum)#time is first + DimDict={len(Dimension1):'DimNum1'} + dimindex=1 + + dimlist=[2,md.mesh.numberofelements,md.mesh.numberofvertices,np.shape(md.mesh.elements)[1]] + for i in range(0,4): + if dimlist[i] not in DimDict.keys(): + dimindex+=1 + NewDim=NCData.createDimension('DimNum'+str(dimindex),dimlist[i]) + DimDict[len(NewDim)]='DimNum'+str(dimindex) #get all model classes and create respective groups + typelist=[bool,str,unicode,int,float,complex,numpy.ndarray,collections.OrderedDict] groups=dict.keys(md.__dict__) for group in groups: @@ -49,36 +47,57 @@ fields=dict.keys(md.__dict__[group].__dict__) - #Special treatment for the results - if str(group)=='results': - for supfield in fields:#looping on the different solutions - NCgroup.__setattr__('classtype', "results") - Subgroup=NCgroup.createGroup(str(supfield)) - #Subgroup.__setattr__('classtype',md.results.__dict__[supfield].__class__.__name__) - Subgroup.__setattr__('classtype',md.results.__class__.__name__) - if type(md.results.__dict__[supfield])==list:#the solution have several timestep - #get last timesteps and output frequency - last_step = np.size(md.results.__dict__[supfield]) - step_freq = md.settings.output_frequency - #grab first time step - subfields=dict.keys(md.results.__dict__[supfield].__getitem__(0).__dict__) - for field in subfields: - if str(field)!='errlog' and str(field)!='outlog' and str(field)!='SolutionType': - Var=md.results.__dict__[supfield].__getitem__(0).__dict__[field] - DimDict=CreateVar(NCData,Var,field,Subgroup,DimDict,True,last_step,step_freq,md,supfield) - - elif type(md.results.__dict__[supfield])==results:#only one timestep - subfields=dict.keys(md.results.__dict__[supfield].__dict__) - for field in subfields: - if str(field)!='errlog' and str(field)!='outlog' and str(field)!='SolutionType': - Var=md.results.__dict__[supfield].__dict__[field] - DimDict=CreateVar(NCData,Var,field,NCgroup,DimDict,False) + #looping on fields + for field in fields: + #Special treatment for list fields + if type(md.__dict__[group].__dict__[field])==list: + StdList=False + if len(md.__dict__[group].__dict__[field])==0: + StdList=True else: - print ('Result format not suported') - else: - - for field in fields: + StdList=type(md.__dict__[group].__dict__[field][0]) in typelist + + if StdList: #this is a standard or empty list just proceed + NCgroup.__setattr__('classtype', md.__dict__[group].__class__.__name__) + Var=md.__dict__[group].__dict__[field] + DimDict=CreateVar(NCData,Var,field,NCgroup,DimDict,False) + else: #this is a list of fields, specific treatment needed + Listsize=len(md.__dict__[group].__dict__[field]) + NCgroup.__setattr__('classtype', md.__dict__[group].__class__.__name__) + Subgroup=NCgroup.createGroup(str(field)) + Subgroup.__setattr__('classtype',md.__dict__[group].__dict__[field].__class__.__name__) + for listindex in range(0,Listsize): + try: + Listgroup=Subgroup.createGroup(str(md.__dict__[group].__dict__[field].__getitem__(listindex).__dict__['name'])) + except KeyError: + for naming in ['step']: + Listgroup=Subgroup.createGroup(str(md.__dict__[group].__dict__[field].__getitem__(listindex).__dict__[naming])) + except AttributeError: + Listgroup=Subgroup.createGroup(str(md.__dict__[group].__dict__[field].__class__.__name__)+str(listindex)) + Listgroup.__setattr__('classtype',md.__dict__[group].__dict__[field].__getitem__(listindex).__class__.__name__) + try: + subfields=dict.keys(md.__dict__[group].__dict__[field].__getitem__(listindex).__dict__) + except AttributeError: + subfields=dict.keys(md.__dict__[group].__dict__[field].__getitem__(listindex)) + for subfield in subfields: + try: + Var=md.__dict__[group].__dict__[field].__getitem__(listindex).__dict__[subfield] + except AttributeError: + Var=md.__dict__[group].__dict__[field].__getitem__(listindex)[subfield] + DimDict=CreateVar(NCData,Var,subfield,Listgroup,DimDict,False,md.__dict__[group],field,listindex) + #No subgroup, we directly treat the variable + elif type(md.__dict__[group].__dict__[field]) in typelist: NCgroup.__setattr__('classtype', md.__dict__[group].__class__.__name__) Var=md.__dict__[group].__dict__[field] DimDict=CreateVar(NCData,Var,field,NCgroup,DimDict,False) + else: + NCgroup.__setattr__('classtype', str(group)) + Subgroup=NCgroup.createGroup(str(field)) + Subgroup.__setattr__('classtype',md.__dict__[group].__class__.__name__) + subfields=dict.keys(md.__dict__[group].__dict__[field].__dict__) + for subfield in subfields: + if str(subfield)!='errlog' and str(subfield)!='outlog' and str(subfield)!='SolutionType': + Var=md.__dict__[group].__dict__[field].__dict__[subfield] + DimDict=CreateVar(NCData,Var,subfield,Subgroup,DimDict,False) + NCData.close() @@ -97,9 +116,10 @@ val_shape=np.shape(var) - TypeDict = {float:'f8', 'float64':'f8', int:'i8', - 'int64':'i8'} + 'int64':'i8', + str:'str', + dict:'str'} val_dim=np.shape(val_shape)[0] @@ -111,10 +131,15 @@ elif val_type==list: dimensions,DimDict=GetDim(NCData,var,val_shape,DimDict,val_dim,istime) - ncvar = Group.createVariable(str(field),'str',dimensions,zlib=True) - for elt in range(0,val_dim): - try: + #try to get the type from the first element + try: + nctype=TypeDict[type(var[0])] + except IndexError: + nctype='str' #most probably an empty list take str for that + ncvar = Group.createVariable(str(field),nctype,dimensions,zlib=True) + if val_shape==0: + ncvar= [] + else: + for elt in range(0,val_shape[0]): ncvar[elt] = var[elt] - except IndexError: - ncvar= [] #treating bool tables as string tables elif val_type=='bool': @@ -135,16 +160,13 @@ ncvar = Group.createVariable(str(field),TypeDict[val_type],dimensions,zlib=True) if istime: - last=step_args[0] - md=step_args[2] - supfield=step_args[3] + md=step_args[0] + supfield=step_args[1] + index=step_args[2] vartab=var - for time in range(0,last,1): - if time!=0: - timevar=md.results.__dict__[supfield].__getitem__(time).__dict__[field] - vartab=np.column_stack((vartab,timevar)) - try: - ncvar[:,:]=vartab[:,:] + timevar=md.__dict__[supfield].__getitem__(index).__dict__[field] + try: + ncvar[:,:]=timevar[:,:] except ValueError: - ncvar[:]=vartab.T[:] + ncvar[:]=timevar.T[:] else: try: @@ -170,18 +192,18 @@ if (shape[dim])>0: index=len(DimDict)+1 - NewDim=NCData.createDimension('Dimension'+str(index),(shape[dim])) - DimDict[len(NewDim)]='Dimension'+str(index) + NewDim=NCData.createDimension('DimNum'+str(index),(shape[dim])) + DimDict[len(NewDim)]='DimNum'+str(index) output=output+[str(DimDict[shape[dim]])] elif type(shape[0])==str:#dealling with a dictionnary try: #dimension5 is 2 to treat with dict - output=[str(DimDict[np.shape(shape)[0]])]+['Dimension5'] + output=[str(DimDict[np.shape(shape)[0]])]+[DimDict[2]] except KeyError: index=len(DimDict)+1 - NewDim=NCData.createDimension('Dimension'+str(index),np.shape(shape)[0]) - DimDict[len(NewDim)]='Dimension'+str(index) - output=[str(DimDict[np.shape(dict.keys(var))[0]])]+['Dimension5'] + NewDim=NCData.createDimension('DimNum'+str(index),np.shape(shape)[0]) + DimDict[len(NewDim)]='DimNum'+str(index) + output=[str(DimDict[np.shape(dict.keys(var))[0]])]+[DimDict[2]] break if istime: - output=output+['Dimension4'] + output=output+['DimNum1'] return tuple(output), DimDict Index: /issm/trunk-jpl/src/m/io/loadvars.py =================================================================== --- /issm/trunk-jpl/src/m/io/loadvars.py (revision 21246) +++ /issm/trunk-jpl/src/m/io/loadvars.py (revision 21247) @@ -78,86 +78,123 @@ NCFile=Dataset(filename,mode='r') NCFile.close() - classtype,classtree=netCDFread(filename) - nvdict['md']=model() - module=map(__import__,dict.values(classtype)) - for i,mod in enumerate(dict.keys(classtype)): -# print('treating md.{}'.format(mod)) - if np.size(classtree[mod])>1: - if classtree[mod][0]=='results': - #treating results (Dimension4 is time) - resdim=len(NCFile.dimensions['Dimension4']) - curclass=NCFile.groups[classtree[mod][0]].groups[classtree[mod][1]] - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(module[i],classtype[mod])()] - if resdim>1: - for t in range(1,resdim): - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]].append(getattr(module[i],classtype[mod])()) - Tree=nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]][:] - else: - curclass=NCFile.groups[classtree[mod][0]].groups[classtree[mod][1]] - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = getattr(module[i],classtype[mod])() - Tree=nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] - else: - curclass=NCFile.groups[classtree[mod][0]] - nvdict['md'].__dict__[mod] = getattr(module[i],classtype[mod])() - Tree=nvdict['md'].__dict__[classtree[mod][0]] - for var in curclass.variables: - #print(' treating {}'.format(var)) - varval=curclass.variables[str(var)] - vardim=varval.ndim - try: - val_type=str(varval.dtype) - except AttributeError: - val_type=type(varval) - if vardim==0: - try: - Tree.__dict__[str(var)]=varval.getValue() - if varval.getValue()=='True': - Tree.__dict__[str(var)]=True - elif varval.getValue()=='False': - Tree.__dict__[str(var)]=False - except IndexError: - Tree.__dict__[str(var)]=[] - elif vardim==1: - if varval.dtype==str: - if varval.shape==1: - Tree.__dict__[str(var)]=str(varval[0]) - if 'True' in varval[:] or 'False' in varval[:]: - Tree.__dict__[str(var)]=np.asarray(varval[:],dtype=bool) - else: - if classtree[mod][0]=='results' and resdim>1: - for t in range(0,resdim): - Tree[t].__dict__[str(var)]=varval[t] - else: - Tree.__dict__[str(var)]=varval[:] - elif vardim==2: - #dealling with dict - if varval.dtype==str: - Tree.__dict__[str(var)]=dict(zip(varval[:,0], varval[:,1])) - else: - if classtree[mod][0]=='results' and resdim>1: - for t in range(0,resdim): - Tree[t].__dict__[str(var)]=varval[:,t] - else: - Tree.__dict__[str(var)]=varval[:,:] - elif vardim==3: - if classtree[mod][0]=='results' and resdim>1: - for t in range(0,resdim): - Tree[t].__dict__[str(var)]=varval[:,:,t] - else: - Tree.__dict__[str(var)]=varval[:,:,:] - else: - print 'table dimension greater than 3 not implemented yet' - for attr in curclass.ncattrs(): - #print(' treating {}'.format(attr)) - if classtree[mod][0]!='results' and attr!='classtype': #no attributes in results - Tree.__dict__[str(attr)]=str(curclass.getncattr(attr)) - if curclass.getncattr(attr)=='True': - Tree.__dict__[str(attr)]=True - elif curclass.getncattr(attr)=='False': - Tree.__dict__[str(attr)]=False - except RuntimeError: raise IOError("File '%s' not found." % filename) + classtype,classtree=netCDFread(filename) + nvdict['md']=model() + NCFile=Dataset(filename,mode='r') + for mod in dict.keys(classtype): + if np.size(classtree[mod])>1: + curclass=NCFile.groups[classtree[mod][0]].groups[classtree[mod][1]] + if classtype[mod][0]=='list': + keylist=[key for key in curclass.groups] + listtype=curclass.groups[keylist[0]].classtype + if listtype=='dict': + nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [{} for i in range(max(1,len(curclass.groups)))] + else: + nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(__import__(listtype),listtype)() for i in range(max(1,len(curclass.groups)))] + Tree=nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]][:] + else: + nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = getattr(classtype[mod][1],classtype[mod][0])() + Tree=nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] + else: + curclass=NCFile.groups[classtree[mod][0]] + nvdict['md'].__dict__[mod] = getattr(classtype[mod][1],classtype[mod][0])() + Tree=nvdict['md'].__dict__[classtree[mod][0]] + #treating groups that are lists + for i in range(0,max(1,len(curclass.groups))): + if len(curclass.groups)>0: + listclass=curclass.groups[keylist[i]] + else: + listclass=curclass + for var in listclass.variables: + if var not in ['errlog','outlog']: + varval=listclass.variables[str(var)] + vardim=varval.ndim + try: + val_type=str(varval.dtype) + except AttributeError: + val_type=type(varval) + if vardim==0: + if type(Tree)==list: + t=int(keylist[i][-1])-1 + if listtype=='dict': + Tree[t][str(var)]=varval.getValue() + else: + Tree[t].__dict__[str(var)]=varval.getValue() + else: + try: + Tree.__dict__[str(var)]=varval.getValue() + if varval.getValue()=='True': + Tree.__dict__[str(var)]=True + elif varval.getValue()=='False': + Tree.__dict__[str(var)]=False + except IndexError: + Tree.__dict__[str(var)]=[] + elif vardim==1: + if varval.dtype==str: + if varval.shape[0]==1: + Tree.__dict__[str(var)]=[str(varval[0]),] + elif 'True' in varval[:] or 'False' in varval[:]: + Tree.__dict__[str(var)]=np.asarray([V=='True' for V in varval[:]],dtype=bool) + else: + Tree.__dict__[str(var)]=[str(vallue) for vallue in varval[:]] + else: + if type(Tree)==list: + t=int(keylist[i][-1])-1 + if listtype=='dict': + Tree[t][str(var)]=varval[:] + else: + Tree[t].__dict__[str(var)]=varval[:] + else: + try: + #some thing specifically require a list + mdtype=type(Tree.__dict__[str(var)]) + except KeyError: + print 'not defined yet' + mdtype=float + if mdtype==list: + Tree.__dict__[str(var)]=[mdval for mdval in varval[:]] + else: + Tree.__dict__[str(var)]=varval[:] + elif vardim==2: + #dealling with dict + if varval.dtype==str: + Tree.__dict__[str(var)]=dict(zip(varval[:,0], varval[:,1])) + else: + if type(Tree)==list: + t=int(keylist[i][-1])-1 + if listtype=='dict': + Tree[t][str(var)]=varval[:,:] + else: + Tree[t].__dict__[str(var)]=varval[:,:] + else: + Tree.__dict__[str(var)]=varval[:,:] + elif vardim==3: + if type(Tree)==list: + t=int(keylist[i][-1])-1 + if listtype=='dict': + Tree[t][str(var)]=varval[:,:,:] + else: + Tree[t].__dict__[str(var)]=varval[:,:,:] + else: + Tree.__dict__[str(var)]=varval[:,:,:] + else: + print 'table dimension greater than 3 not implemented yet' + for attr in listclass.ncattrs(): + if attr!='classtype': #classtype is for treatment, don't get it back + if type(Tree)==list: + t=int(keylist[i][-1])-1 + if listtype=='dict': + Tree[t][str(attr)]=str(listclass.getncattr(attr)) + else: + Tree[t].__dict__[str(attr)]=str(listclass.getncattr(attr)) + else: + Tree.__dict__[str(attr)]=str(listclass.getncattr(attr)) + if listclass.getncattr(attr)=='True': + Tree.__dict__[str(attr)]=True + elif listclass.getncattr(attr)=='False': + Tree.__dict__[str(attr)]=False + NCFile.close() if len(args) >= 2 and isinstance(args[1],(str,unicode)): # (value) value=[nvdict[name] for name in args[1:]] @@ -173,26 +210,24 @@ def netCDFread(filename): - def walktree(data): - keys = data.groups.keys() - for key in keys: - yield [str(key)] - for children in walktree(data.groups[str(key)]): - child=[str(key)] - child.append(str(children[0])) - yield child print ('Opening {} for reading '.format(filename)) NCData=Dataset(filename, 'r') class_dict={} class_tree={} + + for group in NCData.groups: + if len(NCData.groups[group].groups)>0: + for subgroup in NCData.groups[group].groups: + classe=str(group)+'.'+str(subgroup) + class_dict[classe]=[str(getattr(NCData.groups[group].groups[subgroup],'classtype')),] + if class_dict[classe][0] not in ['dict','list']: + class_dict[classe].append(__import__(class_dict[classe][0])) + class_tree[classe]=[group,subgroup] + else: + classe=str(group) + class_dict[classe]=[str(getattr(NCData.groups[group],'classtype')),] + if class_dict[classe][0] not in ['dict','list']: + class_dict[classe].append(__import__(class_dict[classe][0])) + class_tree[classe]=[group,] - for children in walktree(NCData): - classe=str(children[0]) - if np.size(children)>1: - for name in children[1:]: - classe=classe+'.'+name - class_dict[classe]=str(getattr(NCData.groups[children[0]].groups[children[1]],'classtype')) - else: - class_dict[classe]=str(getattr(NCData.groups[classe],'classtype')) - class_tree[classe]=children - + NCData.close() return class_dict,class_tree