Index: /issm/trunk-jpl/src/m/contrib/defleurian/netCDF/export_netCDF.py =================================================================== --- /issm/trunk-jpl/src/m/contrib/defleurian/netCDF/export_netCDF.py (revision 25818) +++ /issm/trunk-jpl/src/m/contrib/defleurian/netCDF/export_netCDF.py (revision 25819) @@ -75,4 +75,7 @@ if verbose > 1: print('===Now treating {}==='.format(group)) + if group == 'qmu': + print("qmu is skipped until it is more stable") + continue NCgroup = NCData.createGroup(str(group)) # In each group gather the fields of the class @@ -88,8 +91,9 @@ #returns False for exotic types (typicaly results) StdList = type(md.__dict__[group].__dict__[field][0]) in typelist - NCgroup.__setattr__('classtype', md.__dict__[group].__class__.__name__) + klass = type(md.__dict__[group]).__module__ + '.' + type(md.__dict__[group]).__name__ + NCgroup.__setattr__('classtype', klass) if StdList: # this is a standard or empty list just proceed if verbose > 4: - print("=££=creating var for {}.{}".format(group, field)) + print("=££=creating var for {}.{} with classtype : {}".format(group, field, klass)) Var = md.__dict__[group].__dict__[field] Var = SqueezeVar(Var) @@ -99,5 +103,5 @@ else: # this is a list of fields, specific treatment needed (usually results or outputdefinitions) if verbose > 4: - print("list of fields happens for {}.{}".format(group, field)) + print("list of fields happens for {}.{} with classtype : {}".format(group, field, klass)) Listsize = len(md.__dict__[group].__dict__[field]) if group == 'results': #for results we reshape the datas following time rather than subgrouping @@ -105,8 +109,10 @@ try: #take the class of the first element to define nc class and get the list of variables - Subgroup.__setattr__('classtype', md.__dict__[group].__dict__[field][0].__class__.__name__) + klass = type(md.__dict__[group].__dict__[field][0]).__module__ + '.' + type(md.__dict__[group].__dict__[field][0]).__name__ + Subgroup.__setattr__('classtype', klass) subfields = dict.keys(md.__dict__[group].__dict__[field][0].__dict__) except (IndexError, AttributeError): - Subgroup.__setattr__('classtype', md.__dict__[group].__dict__[field].__class__.__name__) + klass = type(md.__dict__[group].__dict__[field]).__module__ + '.' + type(md.__dict__[group].__dict__[field]).__name__ + Subgroup.__setattr__('classtype', klass) subfields = dict.keys(md.__dict__[group].__dict__[field].__getitem__(0)) for subfield in subfields: @@ -137,5 +143,6 @@ Subgroupname = str(md.__dict__[group].__dict__[field][listindex].definitionstring) Subgroup = NCgroup.createGroup(Subgroupname) - Subgroup.__setattr__('classtype', md.__dict__[group].__dict__[field][listindex].__class__.__name__) + klass = type(md.__dict__[group].__dict__[field][listindex]).__module__ + '.' + type(md.__dict__[group].__dict__[field][listindex]).__name__ + Subgroup.__setattr__('classtype', klass) subfields = dict.keys(md.__dict__[group].__dict__[field][listindex].__dict__) for subfield in subfields: @@ -150,8 +157,8 @@ else: print("WARNING: unknown treatment for md.{}".format(group)) - # No subgroup, we directly treat the variable elif type(md.__dict__[group].__dict__[field]) in typelist or field == 'bamg': - NCgroup.__setattr__('classtype', md.__dict__[group].__class__.__name__) + klass = type(md.__dict__[group]).__module__ + '.' + type(md.__dict__[group]).__name__ + NCgroup.__setattr__('classtype', klass) Var = md.__dict__[group].__dict__[field] Var = SqueezeVar(Var) @@ -161,10 +168,11 @@ if ncvar is not None: FillVar(ncvar, Var) - # empty fiel, do nothing + # empty field, do nothing elif md.__dict__[group].__dict__[field] is None: print('field md.{}.{} is None'.format(group, field)) # if it is a masked array elif type(md.__dict__[group].__dict__[field]) is np.ma.core.MaskedArray: - NCgroup.__setattr__('classtype', md.__dict__[group].__class__.__name__) + klass = type(md.__dict__[group]).__module__ + '.' + type(md.__dict__[group]).__name__ + NCgroup.__setattr__('classtype', klass) Var = md.__dict__[group].__dict__[field].data Var = SqueezeVar(Var) @@ -176,25 +184,56 @@ # this is an issm class elif isclass(type(md.__dict__[group].__dict__[field])): - classtype = getmodule(type(md.__dict__[group])).__name__ - NCgroup.__setattr__('classtype', classtype) - Subgroup = NCgroup.createGroup(str(field)) - classname = type(md.__dict__[group].__dict__[field]).__name__ - modulename = getmodule(type(md.__dict__[group].__dict__[field])).__name__ - if classname == modulename: - #when module and class name have the same name - classtype = classname + if type(md.__dict__[group].__dict__[field]).__name__ == 'solution': + #for results we reshape the datas following time rather than subgrouping + Listsize = len(md.__dict__[group].__dict__[field]) + Subgroup = NCgroup.createGroup(str(field)) + try: + #take the class of the first element to define nc class and get the list of variables + klass = type(md.__dict__[group].__dict__[field][0]).__module__ + '.' + type(md.__dict__[group].__dict__[field][0]).__name__ + Subgroup.__setattr__('classtype', klass) + subfields = dict.keys(md.__dict__[group].__dict__[field][0].__dict__) + except (IndexError, AttributeError): + klass = type(md.__dict__[group].__dict__[field]).__module__ + '.' + type(md.__dict__[group].__dict__[field]).__name__ + Subgroup.__setattr__('classtype', klass) + subfields = dict.keys(md.__dict__[group].__dict__[field].__getitem__(0)) + for subfield in subfields: + if subfield not in['errlog', 'outlog']: + StackedVar = ResTable() + for listindex in range(0, Listsize): + try: + Var = md.__dict__[group].__dict__[field].__getitem__(listindex).__dict__[subfield] + lastindex = listindex + 1 + except AttributeError: + Var = md.__dict__[group].__dict__[field].__getitem__(listindex)[subfield] + except KeyError: + #Some fields only exist for the first step + lastindex = listindex + continue + Var = SqueezeVar(Var) + StackedVar.update(Var) + if verbose > 4: + print("=$$=creating var for {}.{}.{}".format(group, field, subfield)) + print("last index of the list is {}".format(lastindex)) + StackedVar = SqueezeVar(StackedVar.finalize(int(lastindex))) + DimDict, ncvar = CreateVar(NCData, StackedVar, subfield, Subgroup, DimDict) + #and fill it up + if ncvar is not None: + FillVar(ncvar, StackedVar) else: - classtype = modulename + '.' + classname - Subgroup.__setattr__('classtype', classtype) - subfields = dict.keys(md.__dict__[group].__dict__[field].__dict__) - for subfield in subfields: - if str(subfield) not in ['errlog', 'outlog']: - Var = md.__dict__[group].__dict__[field].__dict__[subfield] - Var = SqueezeVar(Var) - if verbose > 4: - print("+==+creating var for {}.{}.{}".format(group, field, subfield)) - DimDict, ncvar = CreateVar(NCData, Var, subfield, Subgroup, DimDict) - if ncvar is not None: - FillVar(ncvar, Var) + klass = type(md.__dict__[group]).__module__ + '.' + type(md.__dict__[group]).__name__ + NCgroup.__setattr__('classtype', klass) + Subgroup = NCgroup.createGroup(str(field)) + klass = type(md.__dict__[group].__dict__[field]).__module__ + '.' + type(md.__dict__[group].__dict__[field]).__name__ + Subgroup.__setattr__('classtype', klass) + subfields = dict.keys(md.__dict__[group].__dict__[field].__dict__) + for subfield in subfields: + if str(subfield) not in ['errlog', 'outlog']: + Var = md.__dict__[group].__dict__[field].__dict__[subfield] + Var = SqueezeVar(Var) + if verbose > 4: + print("+==+creating var for {}.{}.{}".format(group, field, subfield)) + DimDict, ncvar = CreateVar(NCData, Var, subfield, Subgroup, DimDict) + if ncvar is not None: + FillVar(ncvar, Var) else: print("WARNING, md.{}.{} is not treated as it does not fall in one of the existing cases.".format(str(group), str(field))) Index: /issm/trunk-jpl/src/m/io/loadvars.py =================================================================== --- /issm/trunk-jpl/src/m/io/loadvars.py (revision 25818) +++ /issm/trunk-jpl/src/m/io/loadvars.py (revision 25819) @@ -93,71 +93,28 @@ curclass = NCFile.groups[classtree[mod][0]].groups[classtree[mod][1]] if debug: - print("===> {} is of class {}".format(mod, classtype[mod][0])) - if classtype[mod][0] == 'results' and 'Time' in NCFile.dimensions: #Treating results {{{ + print("===> {} is of class {}".format(mod, classtype[mod])) + if classtype[mod][0] == 'results.solutionstep': #Treating results {{{ keylist = [key for key in curclass.groups] # this is related to the old structure of NC files where every steps of results had its own group - if len(keylist) > 0: - #this is kept for compatibility - #and treatment of list of dicts?? - try: - #group are named after their step - steplist = [int(key) for key in curclass.groups] - except ValueError: - #or a number is appended at the end of the name - steplist = [int(findall(r'\d + ', key)[0]) for key in keylist] - indexlist = [int(index * (len(curclass.groups) - 1) / max(1, max(steplist))) for index in steplist] - listtype = curclass.groups[keylist[0]].classtype - #discriminate between dict and results - if listtype == 'dict': - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [OrderedDict() for i in range(max(1, len(curclass.groups)))] - else: - if onlylast: #we load only the last result to save on time and memory - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(__import__(listtype), listtype)()] - Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] - else: - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(__import__(listtype), listtype)() for i in range(max(1, len(curclass.groups)))] - Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]][:] + #that is the current treatment + #here we have a more NC approach with time being a dimension + listtype = split(r'\.', classtype[mod][0])[0] + print("listtype is {}".format(listtype)) + if len(NCFile.dimensions['Time']) == 1: + nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = getattr(classtype[mod][1], listtype)() + Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] else: - #that is the current treatment - #here we have a more NC approach with time being a dimension - listtype = curclass.classtype - if len(NCFile.dimensions['Time']) == 1: - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = getattr(__import__(listtype), listtype)() + #Time dimension is in all the variables so we take that as stepnumber for the results + if onlylast: #we load only the last result to save on time and memory + nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(classtype[mod][1], listtype)()] Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] else: - #Time dimension is in all the variables so we take that as stepnumber for the results - if onlylast: #we load only the last result to save on time and memory - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(__import__(listtype), listtype)()] - Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] - else: - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(__import__(listtype), listtype)() for i in range(max(1, len(NCFile.dimensions['Time'])))] - Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]][:] - #}}} - elif classtype[mod][0] == 'list': #Treating the old style of results {{{ - keylist = [key for key in curclass.groups] - # this is related to the old structure of NC files where every steps of results had its own group - if len(keylist) > 0: - #this is kept for compatibility - #and treatment of list of dicts?? - try: - #group are named after their step - steplist = [int(key) for key in curclass.groups] - except ValueError: - #or a number is appended at the end of the name - steplist = [int(findall(r'\d + ', key)[0]) for key in keylist] - indexlist = [int(index * (len(curclass.groups) - 1) / max(1, max(steplist))) for index in steplist] - listtype = curclass.groups[keylist[0]].classtype - #discriminate between dict and results - if listtype == 'dict': - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [OrderedDict() for i in range(max(1, len(curclass.groups)))] - else: - if onlylast: #we load only the last result to save on time and memory - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(__import__(listtype), listtype)()] - Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] - else: - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(__import__(listtype), listtype)() for i in range(max(1, len(curclass.groups)))] - Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]][:] - else: - print("WARNING: we don't get the expected format, there might be an issue somewhere") + #nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = [getattr(classtype[mod][1], listtype)() for i in range(max(1, len(NCFile.dimensions['Time'])))] + #Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]][:] + setattr(nvdict['md'].__dict__[classtree[mod][0]], classtree[mod][1], getattr(classtype[mod][1], 'solution')([])) + for i in range(max(1, len(NCFile.dimensions['Time']))): + nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]].steps.append(getattr(classtype[mod][1], 'solutionstep')()) + Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]][:] + print(nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]].steps[0]) # }}} elif classtype[mod][0] == 'massfluxatgate': #this is for output definitions {{{ @@ -170,9 +127,16 @@ if debug: print("Using the default for md.{}.{}, is that right??".format(classtree[mod][0], classtree[mod][1])) - nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = getattr(classtype[mod][1], classtype[mod][0])() + try: + modulename = split(r'\.', classtype[mod][0])[0] + if debug: + print("trying to import {} from {}".format(classtype[mod][0], modulename)) + nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] = getattr(classtype[mod][1], modulename)() + except AttributeError: + print("WARNING: md.{}.{} is not initialized, hopefully that was done in the main group:".format(classtree[mod][0], classtree[mod][1])) Tree = nvdict['md'].__dict__[classtree[mod][0]].__dict__[classtree[mod][1]] else: curclass = NCFile.groups[classtree[mod][0]] - nvdict['md'].__dict__[mod] = getattr(classtype[mod][1], classtype[mod][0])() + modulename = split(r'\.', classtype[mod][0])[0] + nvdict['md'].__dict__[mod] = getattr(classtype[mod][1], modulename)() Tree = nvdict['md'].__dict__[classtree[mod][0]] if debug: @@ -351,12 +315,15 @@ for subgroup in NCData.groups[group].groups: classe = str(group) + '.' + str(subgroup) - submodule = str(getattr(NCData.groups[group].groups[subgroup], 'classtype')).split('.')[-1] - class_dict[classe] = [submodule, ] + grpclass = str(getattr(NCData.groups[group].groups[subgroup], 'classtype')) + class_dict[classe] = [grpclass, ] if class_dict[classe][0] not in ['dict', 'list', 'cell']: try: - class_dict[classe].append(__import__(class_dict[classe][0])) + modulename = split(r'\.', class_dict[classe][0])[0] + #class_dict[classe].append(__import__(class_dict[classe][0])) + class_dict[classe].append(__import__(modulename)) except ModuleNotFoundError: #submodule probably has a different name - modulename = str(getattr(NCData.groups[group].groups[subgroup], 'classtype')).split('.')[0] + modulename = str(getattr(NCData.groups[group].groups[subgroup], 'classtype')) + print("WARNING importing {} rather than {}".format(modulename, class_dict[classe][0])) class_dict[classe].append(__import__(modulename)) class_tree[classe] = [group, subgroup] @@ -366,5 +333,6 @@ class_dict[classe] = [str(getattr(NCData.groups[group], 'classtype')), ] if class_dict[classe][0] not in ['dict', 'list', 'cell']: - class_dict[classe].append(__import__(class_dict[classe][0])) + modulename = split(r'\.', class_dict[classe][0])[0] + class_dict[classe].append(__import__(modulename)) class_tree[classe] = [group, ] except AttributeError: