Index: /issm/trunk-jpl/src/m/contrib/defleurian/netCDF/export_netCDF.py =================================================================== --- /issm/trunk-jpl/src/m/contrib/defleurian/netCDF/export_netCDF.py (revision 25061) +++ /issm/trunk-jpl/src/m/contrib/defleurian/netCDF/export_netCDF.py (revision 25062) @@ -6,5 +6,7 @@ -def export_netCDF(md, filename): +def export_netCDF(md, filename): # {{{ + res_time = time.time() + debug = False if path.exists(filename): print('File {} allready exist'.format(filename)) @@ -15,33 +17,42 @@ print(('New file name is {}'.format(newname))) filename = newname + #create file and define it NCData = Dataset(filename, 'w', format='NETCDF4') NCData.description = 'Results for run' + md.miscellaneous.name NCData.history = 'Created ' + time.ctime(time.time()) # define netCDF dimensions + #grab time from Transient if it exists try: - StepNum = np.shape(dict.values(md.results.__dict__))[1] - except IndexError: + StepNum = len(md.results.TransientSolution) + except AttributeError: #no transient so just one timestep StepNum = 1 - Dimension1 = NCData.createDimension('DimNum1', StepNum) # time is first - DimDict = {len(Dimension1): 'DimNum1'} + TimeDim = NCData.createDimension('Time', StepNum) # time is first + DimDict = {len(TimeDim): 'Time'} dimindex = 1 + UnlimDim = NCData.createDimension('Unlim', None) # unlimited dimension if needed + DimDict[len(UnlimDim)] = {'Inf'} + dimindex = 2 + #add mesh related dimension that we know are needed dimlist = [2, md.mesh.numberofelements, md.mesh.numberofvertices, np.shape(md.mesh.elements)[1]] + dimnames = ['DictDummy', 'EltNum', 'VertNum', 'VertPerElt'] print('===Creating dimensions ===') for i, newdim in enumerate(dimlist): if newdim not in list(DimDict.keys()): dimindex += 1 - NewDim = NCData.createDimension('DimNum' + str(dimindex), newdim) - DimDict[len(NewDim)] = 'DimNum' + str(dimindex) + NewDim = NCData.createDimension(dimnames[i], newdim) + DimDict[len(NewDim)] = dimnames[i] + typelist = [bool, str, str, int, float, complex, collections.OrderedDict, np.int64, np.ndarray, np.float64] + + # get all model classes and create respective groups groups = dict.keys(md.__dict__) - # get all model classes and create respective groups print('===Creating and populating groups===') for group in groups: NCgroup = NCData.createGroup(str(group)) - # In each group gather the fields of the class + # In each group gather the fields of the class fields = dict.keys(md.__dict__[group].__dict__) - # looping on fields + # looping on fields in each group for field in fields: # Special treatment for list fields @@ -49,40 +60,90 @@ StdList = False if len(md.__dict__[group].__dict__[field]) == 0: - StdList = True + StdList = True #this is an empty list else: + #returns False for exotic types (typicaly results) StdList = type(md.__dict__[group].__dict__[field][0]) in typelist NCgroup.__setattr__('classtype', md.__dict__[group].__class__.__name__) if StdList: # this is a standard or empty list just proceed Var = md.__dict__[group].__dict__[field] - DimDict = CreateVar(NCData, Var, field, NCgroup, DimDict) + Var = SqueezeVar(Var) + if debug: + print("=££=creating var for {}.{}".format(group, field)) + DimDict, ncvar = CreateVar(NCData, Var, field, NCgroup, DimDict) + if ncvar is not None: + FillVar(ncvar, Var) else: # this is a list of fields, specific treatment needed + if debug: + print("list of fields happens for {}.{}".format(group, field)) Listsize = len(md.__dict__[group].__dict__[field]) Subgroup = NCgroup.createGroup(str(field)) - Subgroup.__setattr__('classtype', md.__dict__[group].__dict__[field].__class__.__name__) - for listindex in range(0, Listsize): - try: - Listgroup = Subgroup.createGroup(str(md.__dict__[group].__dict__[field].__getitem__(listindex).__dict__['name'])) - except KeyError: - for naming in ['step']: - Listgroup = Subgroup.createGroup(str(md.__dict__[group].__dict__[field].__getitem__(listindex).__dict__[naming])) - except AttributeError: - Listgroup = Subgroup.createGroup(str(md.__dict__[group].__dict__[field].__class__.__name__) + str(listindex)) - Listgroup.__setattr__('classtype', md.__dict__[group].__dict__[field].__getitem__(listindex).__class__.__name__) - try: - subfields = dict.keys(md.__dict__[group].__dict__[field].__getitem__(listindex).__dict__) - except AttributeError: - subfields = dict.keys(md.__dict__[group].__dict__[field].__getitem__(listindex)) - for subfield in subfields: - if subfield != 'outlog': + try: + #take the class of the first element to define nc class and get the list of variables + Subgroup.__setattr__('classtype', md.__dict__[group].__dict__[field][0].__class__.__name__) + subfields = dict.keys(md.__dict__[group].__dict__[field][0].__dict__) + except IndexError: + Subgroup.__setattr__('classtype', md.__dict__[group].__dict__[field].__class__.__name__) + except AttributeError: + subfields = dict.keys(md.__dict__[group].__dict__[field].__getitem__(0)) + + # for subfield in subfields: + # if subfield not in['errlog', 'outlog']: + # #first create the variable + # try: + # Var = md.__dict__[group].__dict__[field].__getitem__(0).__dict__[subfield] + # except AttributeError: + # Var = md.__dict__[group].__dict__[field].__getitem__(0)[subfield] + # Var = SqueezeVar(Var) + # #If first time we create a nc variable + # if Listsize == len(TimeDim): + # ExtraDim = 'Time' + # else: + # ExtraDim = 'Unlim' + # if debug: + # print("=$$=creating var for {}.{}.{}".format(group, field, subfield)) + # DimDict, ncvar = CreateVar(NCData, Var, subfield, Subgroup, DimDict, ExtraDim) + + # for listindex in range(0, Listsize): + # try: + # Var = md.__dict__[group].__dict__[field].__getitem__(listindex).__dict__[subfield] + # except AttributeError: + # Var = md.__dict__[group].__dict__[field].__getitem__(listindex)[subfield] + # Var = SqueezeVar(Var) + # #and fill it up + # FillVar(ncvar, Var, listindex) + + for subfield in subfields: + if subfield not in['errlog', 'outlog']: + try: + Var = md.__dict__[group].__dict__[field].__getitem__(0).__dict__[subfield] + except AttributeError: + Var = md.__dict__[group].__dict__[field].__getitem__(0)[subfield] + StackedVar = SqueezeVar(Var) + + for listindex in range(1, Listsize): try: Var = md.__dict__[group].__dict__[field].__getitem__(listindex).__dict__[subfield] except AttributeError: Var = md.__dict__[group].__dict__[field].__getitem__(listindex)[subfield] - DimDict = CreateVar(NCData, Var, subfield, Listgroup, DimDict, md.__dict__[group], field, listindex) + Var = SqueezeVar(Var) + StackedVar = np.vstack((StackedVar, Var)) + if debug: + print("=$$=creating var for {}.{}.{}".format(group, field, subfield)) + StackedVar = SqueezeVar(StackedVar) + DimDict, ncvar = CreateVar(NCData, StackedVar, subfield, Subgroup, DimDict) + #and fill it up + if ncvar is not None: + FillVar(ncvar, StackedVar) + # No subgroup, we directly treat the variable elif type(md.__dict__[group].__dict__[field]) in typelist or field == 'bamg': NCgroup.__setattr__('classtype', md.__dict__[group].__class__.__name__) Var = md.__dict__[group].__dict__[field] - DimDict = CreateVar(NCData, Var, field, NCgroup, DimDict) + Var = SqueezeVar(Var) + if debug: + print("====creating var for {}.{}".format(group, field)) + DimDict, ncvar = CreateVar(NCData, Var, field, NCgroup, DimDict) + if ncvar is not None: + FillVar(ncvar, Var) elif md.__dict__[group].__dict__[field] is None: print('field md.{}.{} is None'.format(group, field)) @@ -92,5 +153,10 @@ NCgroup.__setattr__('classtype', md.__dict__[group].__class__.__name__) Var = md.__dict__[group].__dict__[field].data - DimDict = CreateVar(NCData, Var, field, NCgroup, DimDict) + Var = SqueezeVar(Var) + if debug: + print("=++=creating var for {}.{}".format(group, field)) + DimDict, ncvar = CreateVar(NCData, Var, field, NCgroup, DimDict) + if ncvar is not None: + FillVar(ncvar, Var) else: NCgroup.__setattr__('classtype', str(group)) @@ -99,17 +165,27 @@ subfields = dict.keys(md.__dict__[group].__dict__[field].__dict__) for subfield in subfields: - if str(subfield) != 'outlog': + if str(subfield) not in ['errlog', 'outlog']: Var = md.__dict__[group].__dict__[field].__dict__[subfield] - DimDict = CreateVar(NCData, Var, subfield, Subgroup, DimDict) + Var = SqueezeVar(Var) + if debug: + print("+==+creating var for {}.{}.{}".format(group, field, subfield)) + DimDict, ncvar = CreateVar(NCData, Var, subfield, Subgroup, DimDict) + if ncvar is not None: + FillVar(ncvar, Var) NCData.close() - - #============================================================================ + print("export time is {}".format(time.time() - res_time)) + +# }}} + + +def CreateVar(NCData, var, field, Group, DimDict, *SupDim): # {{{ + #================================================================= # Define the variables - - -def CreateVar(NCData, var, field, Group, DimDict, * step_args): + #================================================================= # grab type try: val_type = str(var.dtype) + if val_type.startswith(' 1: + Var = np.squeeze(Var) + + return Var +# }}} Index: /issm/trunk-jpl/src/m/io/loadvars.py =================================================================== --- /issm/trunk-jpl/src/m/io/loadvars.py (revision 25061) +++ /issm/trunk-jpl/src/m/io/loadvars.py (revision 25062) @@ -34,4 +34,5 @@ filename = '' nvdict = {} + debug = False #print messages if true if len(args) >= 1 and isinstance(args[0], str): @@ -60,5 +61,5 @@ raise TypeError("Unrecognized input arguments.") - if whichdb(filename): + if whichdb(filename): #We used python pickle for the save print("Loading variables from file {}.".format(filename)) my_shelf = shelve.open(filename, 'r') # 'r' for read - only @@ -78,5 +79,5 @@ my_shelf.close() - else: + else: #We used netcdf for the save try: NCFile = Dataset(filename, mode='r') @@ -89,20 +90,46 @@ NCFile = Dataset(filename, mode='r') for mod in dict.keys(classtype): - #print(' - Now treating classtype {}'.format(mod)) + #==== First we create the model structure {{{ + if debug: + print(' - Now treating classtype {}'.format(mod)) if np.size(classtree[mod]) > 1: + # this points to a subclass (results.TransientSolution for example) curclass = NCFile.groups[classtree[mod][0]].groups[classtree[mod][1]] - if classtype[mod][0] == 'list': + if debug: + print("now checking {} for list : {}".format(mod, classtype[mod][0] == 'list' or (classtype[mod][0] == 'results' and 'Time' in NCFile.dimensions))) + if classtype[mod][0] == 'list' or (classtype[mod][0] == 'results' and 'Time' in NCFile.dimensions): #We have a list of variables keylist = [key for key in curclass.groups] - try: - steplist = [int(key) for key in curclass.groups] - except ValueError: - steplist = [int(findall(r'\d + ', key)[0]) for key in keylist] - indexlist = [index * (len(curclass.groups) - 1) / max(1, max(steplist)) for index in steplist] - listtype = curclass.groups[keylist[0]].classtype - if listtype == 'dict': - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [OrderedDict() for i in range(max(1, len(curclass.groups)))] + # this is related to the old structure of NC files where every steps of results had its own group + if len(keylist) > 0: + #this is kept for compatibility + #and treatment of list of dicts?? + try: + #group are named after their step + steplist = [int(key) for key in curclass.groups] + except ValueError: + #or a number is appended at the end of the name + steplist = [int(findall(r'\d + ', key)[0]) for key in keylist] + indexlist = [int(index * (len(curclass.groups) - 1) / max(1, max(steplist))) for index in steplist] + listtype = curclass.groups[keylist[0]].classtype + #discriminate between dict and results + if listtype == 'dict': + nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [OrderedDict() for i in range(max(1, len(curclass.groups)))] + else: + nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(__import__(listtype), listtype)() for i in range(max(1, len(curclass.groups)))] + Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]][:] else: - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(__import__(listtype), listtype)() for i in range(max(1, len(curclass.groups)))] - Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]][:] + #that is the current treatment + #here we have a more NC approach with time being a dimension + keylist = [key for key in curclass.variables] + dimlist = [curclass.variables[key].dimensions for key in keylist] + indexlist = np.arange(0, len(NCFile.dimensions['Time'])) + if 'Time' in np.all(dimlist): + #Time dimension is in all the variables so we take that as stepnumber for the results + listtype = curclass.classtype + nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(__import__(listtype), listtype)() for i in range(max(1, len(NCFile.dimensions['Time'])))] + Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]][:] + else: + print("ERROR: Time dimension is not in all results. That has been overlooked for now but your resulat are not saved.") + else: nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = getattr(classtype[mod][1], classtype[mod][0])() @@ -112,5 +139,8 @@ nvdict['md'].__dict__[mod] = getattr(classtype[mod][1], classtype[mod][0])() Tree = nvdict['md'].__dict__[classtree[mod][0]] - #treating groups that are lists + if debug: + print("for {} Tree is a {}".format(mod, Tree.__class__.__name__)) + # }}} + #==== Then we populate it {{{ for i in range(0, max(1, len(curclass.groups))): if len(curclass.groups) > 0: @@ -118,89 +148,98 @@ else: listclass = curclass + #==== We deal with Variables {{{ for var in listclass.variables: - #print("treating var {}".format(var)) + if debug: + print("treating var {}".format(var)) if var not in ['errlog', 'outlog']: varval = listclass.variables[str(var)] vardim = varval.ndim - if vardim == 0: - if type(Tree) == list: - t = int(indexlist[i]) - if listtype == 'dict': - Tree[t][str(var)] = varval[0].data - - else: - Tree[t].__dict__[str(var)] = varval[0].data - - else: - if str(varval[0]) == '': #no value - Tree.__dict__[str(var)] = [] - - elif varval[0] == 'True': #treatin bool - Tree.__dict__[str(var)] = True - - elif varval[0] == 'False': #treatin bool - Tree.__dict__[str(var)] = False - - else: - Tree.__dict__[str(var)] = varval[0].item() - elif vardim == 1: - if varval.dtype == str: - if varval.shape[0] == 1: - Tree.__dict__[str(var)] = [str(varval[0]), ] - - elif 'True' in varval[:] or 'False' in varval[:]: - Tree.__dict__[str(var)] = np.asarray([V == 'True' for V in varval[:]], dtype=bool) - - else: - Tree.__dict__[str(var)] = [str(vallue) for vallue in varval[:]] - - else: + #There is a special treatment for results to account for its specific structure + #that is the new export version where time is a named dimension + NewFormat = 'Time' in NCFile.dimensions + if type(Tree) == list and NewFormat: + for t in indexlist: + if vardim == 1: + Tree[t].__dict__[str(var)] = varval[t].data + elif vardim == 2: + Tree[t].__dict__[str(var)] = varval[t, :].data + elif vardim == 3: + Tree[t].__dict__[str(var)] = varval[t, :, :].data + else: + print('table dimension greater than 3 not implemented yet') + else: + if vardim == 0: #that is a scalar + if type(Tree) == list: + t = indexlist[i] + if listtype == 'dict': + Tree[t][str(var)] = varval[0].data + else: + Tree[t].__dict__[str(var)] = varval[0].data + else: + if str(varval[0]) == '': #no value + Tree.__dict__[str(var)] = [] + elif varval[0] == 'True': #treatin bool + Tree.__dict__[str(var)] = True + elif varval[0] == 'False': #treatin bool + Tree.__dict__[str(var)] = False + else: + Tree.__dict__[str(var)] = varval[0].item() + + elif vardim == 1: #that is a vector + if varval.dtype == str: + if varval.shape[0] == 1: + Tree.__dict__[str(var)] = [str(varval[0]), ] + elif 'True' in varval[:] or 'False' in varval[:]: + Tree.__dict__[str(var)] = np.asarray([V == 'True' for V in varval[:]], dtype=bool) + else: + Tree.__dict__[str(var)] = [str(vallue) for vallue in varval[:]] + else: + if type(Tree) == list: + t = indexlist[i] + if listtype == 'dict': + Tree[t][str(var)] = varval[:].data + else: + Tree[t].__dict__[str(var)] = varval[:].data + else: + try: + #some thing specifically require a list + mdtype = type(Tree.__dict__[str(var)]) + except KeyError: + mdtype = float + if mdtype == list: + Tree.__dict__[str(var)] = [mdval for mdval in varval[:]] + else: + Tree.__dict__[str(var)] = varval[:].data + + elif vardim == 2: + #dealling with dict + if varval.dtype == str: #that is for toolkits wich needs to be ordered + if any(varval[:, 0] == 'toolkit'): #toolkit definition have to be first + Tree.__dict__[str(var)] = OrderedDict([('toolkit', str(varval[np.where(varval[:, 0] == 'toolkit')[0][0], 1]))]) + strings1 = [str(arg[0]) for arg in varval if arg[0] != 'toolkits'] + strings2 = [str(arg[1]) for arg in varval if arg[0] != 'toolkits'] + Tree.__dict__[str(var)].update(list(zip(strings1, strings2))) + else: + if type(Tree) == list: + t = indexlist[i] + if listtype == 'dict': + Tree[t][str(var)] = varval[:, :].data + else: + Tree[t].__dict__[str(var)] = varval[:, :].data + else: + Tree.__dict__[str(var)] = varval[:, :].data + elif vardim == 3: if type(Tree) == list: t = int(indexlist[i]) if listtype == 'dict': - Tree[t][str(var)] = varval[:].data - - else: - Tree[t].__dict__[str(var)] = varval[:].data - else: - try: - #some thing specifically require a list - mdtype = type(Tree.__dict__[str(var)]) - except KeyError: - mdtype = float - if mdtype == list: - Tree.__dict__[str(var)] = [mdval for mdval in varval[:]] - - else: - Tree.__dict__[str(var)] = varval[:].data - - elif vardim == 2: - #dealling with dict - if varval.dtype == str: #that is for toolkits wich needs to be ordered - if any(varval[:, 0] == 'toolkit'): #toolkit definition have to be first - Tree.__dict__[str(var)] = OrderedDict([('toolkit', str(varval[np.where(varval[:, 0] == 'toolkit')[0][0], 1]))]) - strings1 = [str(arg[0]) for arg in varval if arg[0] != 'toolkits'] - strings2 = [str(arg[1]) for arg in varval if arg[0] != 'toolkits'] - Tree.__dict__[str(var)].update(list(zip(strings1, strings2))) + Tree[t][str(var)] = varval[:, :, :].data + else: + Tree[t].__dict__[str(var)] = varval[:, :, :] + else: + Tree.__dict__[str(var)] = varval[:, :, :].data else: - if type(Tree) == list: - t = int(indexlist[i]) - if listtype == 'dict': - Tree[t][str(var)] = varval[:, :].data - else: - Tree[t].__dict__[str(var)] = varval[:, :].data - else: - Tree.__dict__[str(var)] = varval[:, :].data - elif vardim == 3: - if type(Tree) == list: - t = int(indexlist[i]) - if listtype == 'dict': - Tree[t][str(var)] = varval[:, :, :].data - else: - Tree[t].__dict__[str(var)] = varval[:, :, :] - else: - Tree.__dict__[str(var)] = varval[:, :, :].data - else: - print('table dimension greater than 3 not implemented yet') + print('table dimension greater than 3 not implemented yet') + # }}} + #==== And with atribute {{{ for attr in listclass.ncattrs(): if attr != 'classtype': #classtype is for treatment, don't get it back @@ -217,4 +256,6 @@ elif listclass.getncattr(attr) == 'False': Tree.__dict__[str(attr).swapcase()] = False + # }}} + # }}} NCFile.close() if len(args) >= 2 and isinstance(args[1], str): # (value)