Index: /issm/trunk/src/m/contrib/musselman/read_netCDF.py =================================================================== --- /issm/trunk/src/m/contrib/musselman/read_netCDF.py (revision 27895) +++ /issm/trunk/src/m/contrib/musselman/read_netCDF.py (revision 27896) @@ -10,4 +10,7 @@ from taoinversion import taoinversion from collections import OrderedDict +import sys +from massfluxatgate import massfluxatgate + @@ -117,34 +120,38 @@ for variable in eval(group_location_in_file + '.variables.keys()'): - if variable == 'this_is_a_nested' and 'results' in group_location_in_file: - # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']' - pattern = r"\['(.*?)'\]" - matches = re.findall(pattern, group_location_in_file) - name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") - copy_multidimensional_results_struct(group_location_in_file, name_of_struct, NCData) - istruct = True - - elif variable == 'this_is_a_nested' and 'qmu' in group_location_in_file: - print('encountered qmu structure that is not yet supported.') - # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']' - #pattern = r"\['(.*?)'\]" - #matches = re.findall(pattern, group_location_in_file) - #name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") - #name_of_struct = eval(group_location_in_file + ".variables['']") - #copy_multidimensional_qmu_struct(group_location_in_file, name_of_struct) - isstruct = True - - else: - location_of_variable_in_file = group_location_in_file + ".variables['" + str(variable) + "']" - # group_location_in_file is like filename.groups['group1'].groups['group1.1'].groups['group1.1.1'] - # Define the regex pattern to match the groups within brackets - pattern = r"\['(.*?)'\]" - # Use regex to find all matches and return something like 'group1.group1.1.group1.1.1 ...' where the last value is the name of the variable - matches = re.findall(pattern, location_of_variable_in_file) - variable_name = matches[-1] - location_of_variable_in_model = '.'.join(matches[:-1]) - copy_variable_data_to_new_model(location_of_variable_in_file, location_of_variable_in_model, variable_name, NCData, verbose=verbose) - - if 'istruct' in locals(): + if 'is_object' not in locals(): + if variable == 'this_is_a_nested' and 'results' in group_location_in_file and 'qmu' not in group_location_in_file: + # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']' + pattern = r"\['(.*?)'\]" + matches = re.findall(pattern, group_location_in_file) + name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") + copy_multidimensional_results_struct(group_location_in_file, name_of_struct, NCData) + is_object = True + + elif variable == 'name_of_cell_array': + # reconstruct an array of elements + copy_cell_array_of_objects(group_location_in_file, model_copy, NCData, verbose) + is_object = True + + elif variable == 'this_is_a_nested' and 'qmu' in group_location_in_file: + if verbose: + print('encountered qmu structure that is not yet supported.') + else: pass + + is_object = True + + else: + location_of_variable_in_file = group_location_in_file + ".variables['" + str(variable) + "']" + # group_location_in_file is like filename.groups['group1'].groups['group1.1'].groups['group1.1.1'] + # Define the regex pattern to match the groups within brackets + pattern = r"\['(.*?)'\]" + # Use regex to find all matches and return something like 'group1.group1.1.group1.1.1 ...' where the last value is the name of the variable + matches = re.findall(pattern, location_of_variable_in_file) + variable_name = matches[-1] + location_of_variable_in_model = '.'.join(matches[:-1]) + copy_variable_data_to_new_model(location_of_variable_in_file, location_of_variable_in_model, variable_name, NCData, verbose=verbose) + + # if one of the variables above was an object, further subclasses will be taken care of when reconstructing it + if 'is_object' in locals(): pass else: @@ -152,4 +159,171 @@ new_nested_group = group_location_in_file + ".groups['" + str(nested_group) + "']" walk_nested_groups(new_nested_group, NCData, verbose=verbose) + + +def copy_cell_array_of_objects(group_location_in_file, model_copy, NCData, verbose): + ''' + The structure in netcdf for groups with the name_of_cell_array variable is like: + + group: 2x6_cell_array_of_objects { + name_of_cell_array = + + group: Row_1_of_2 { + group: Col_1_of_6 { + ... other groups can be here that refer to objects + } // group Col_6_of_6 + } // group Row_1_of_2 + + group: Row_2_of_2 { + group: Col_1_of_6 { + ... other groups can be here that refer to objects + } // group Col_6_of_6 + } // group Row_2_of_2 + } // group 2x6_cell_array_of_objects + + We have to navigate this structure to extract all the data and recreate the + original structure when the model was saved + ''' + + # get the name_of_cell_array, rows and cols vars + name_of_cell_array_varID = eval(group_location_in_file + ".variables['name_of_cell_array']") + rows_varID = eval(group_location_in_file + ".variables['rows']") + cols_varID = eval(group_location_in_file + ".variables['cols']") + + name_of_cell_array = name_of_cell_array_varID[:][...].tobytes().decode() + rows = rows_varID[:] + cols = cols_varID[:] + + # now we work backwards: make the array, fill it in, and assign it to the model + + # make the array + array = list() + + subgroups = eval(group_location_in_file + ".groups") #.keys()") + + # enter each subgroup, get the data, assign it to the corresponding index of cell array + if rows > 1: + # we go over rows + # set index for rows + row_idx = 0 + for row in list(subgroups): + # make list for each row + current_row = list() + columns = subgroups[str(row)].groups.keys() + + # set index for columns + col_idx = 0 + + # iterate over columns + for col in list(columns): + # now get the variables + current_col_vars = columns.groups[str(col)].variables + + # check for special datastructures + if "class_is_a" in current_col_vars: + class_name = subgroups[str(col)].variables['class_is_a'][:][...].tobytes().decode() + col_data = deserialize_class(class_name, columns.groups[str(col)], NCData, verbose) + is_object = True + elif "this_is_a_nested" in current_col_vars: + # functionality not yet supported + print('Error: Cell Arrays of structs not yet supported!') + is_object = True + else: + if 'is_object_' in locals(): + pass + # already taken care of + else: + # store the variables as normal -- to be added later + print('Error: Arrays of mixed objects not yet supported!') + for var in current_col_vars: + # this is where that functionality would be handled + pass + col_idx += 1 + # add the entry to our row list + current_row.append(col_data) + + # add the list of columns to the array + array.append(current_row) + row_idx += 1 + + else: + # set index for columns + col_idx = 0 + + # iterate over columns + for col in list(subgroups): + # now get the variables + current_col_vars = subgroups[str(col)].variables + + # check for special datastructures + if "class_is_a" in current_col_vars: + class_name = subgroups[str(col)].variables['class_is_a'][:][...].tobytes().decode() + col_data = deserialize_class(class_name, subgroups[str(col)], NCData, verbose) + is_object = True + elif "this_is_a_nested" in current_col_vars: + # functionality not yet supported + print('Error: Cell Arrays of structs not yet supported!') + is_object = True + else: + if 'is_object_' in locals(): + pass + # already taken care of + else: + # store the variables as normal -- to be added later + print('Error: Arrays of mixed objects not yet supported!') + for var in current_col_vars: + # this is where that functionality would be handled + pass + col_idx += 1 + # add the list of columns to the array + array.append(col_data) + + # finally, add the attribute to the model + # borrow some code from above: + pattern = r"\['(.*?)'\]" + matches = re.findall(pattern, group_location_in_file) + variable_name = matches[0] + setattr(model_copy.__dict__[variable_name], name_of_cell_array, array) + + + +def deserialize_class(class_name, group, NCData, verbose=False): + + # most ISSM classes are imported by from import + class ModuleError(Exception): + pass + + if class_name not in sys.modules: + raise ModuleError(str('Model requires the following class to be imported from a module: ' + class_name + ". Please add the import to read_netCDF.py in order to continue.")) + + # Instantiate the class + class_instance = eval(class_name + "()") + + # Get and assign properties + subgroups = list(group.groups.keys()) + + if len(subgroups) == 1: + # Get properties + subgroup = group[subgroups[0]] + varIDs = subgroup.variables.keys() + for varname in varIDs: + # Variable metadata + var = subgroup[varname] + + # Data + if 'char' in var.dimensions[0]: + data = var[:][...].tobytes().decode() + else: + data = var[:] + + # Some classes may have permissions, so we skip those + try: + setattr(class_instance, varname, data) + except: + pass + else: + # Not supported + pass + + return class_instance @@ -208,5 +382,4 @@ try: # results band-aid... - #print(str(location_of_variable_in_model + '.' + variable_name)) if str(location_of_variable_in_model + '.' + variable_name) in ['results.solutionstep', 'results.solution', 'results.resultsdakota']: pass @@ -240,7 +413,7 @@ else: # we have to convert numpy datatypes to native python types with .item() - setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, var_to_save.item()) + setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, var_to_save.item()) except: - setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]')) + setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]')) except AttributeError: copy_variable_data_to_new_model_dict(location_of_variable_in_file, location_of_variable_in_model, NCData, verbose=verbose) @@ -248,4 +421,5 @@ if verbose: print('Successfully loaded ' + location_of_variable_in_model + '.' + variable_name + ' into model.') +