Index: /issm/trunk/src/m/contrib/musselman/read_netCDF_commit.py =================================================================== --- /issm/trunk/src/m/contrib/musselman/read_netCDF_commit.py (revision 27877) +++ /issm/trunk/src/m/contrib/musselman/read_netCDF_commit.py (revision 27878) @@ -9,4 +9,6 @@ from m1qn3inversion import m1qn3inversion from taoinversion import taoinversion +from collections import OrderedDict + @@ -66,8 +68,23 @@ def make_results_subclasses(): + ''' + There are 3 possible subclasses: solution, solutionstep, resultsdakota. + In the NetCDF file these are saved as a list of strings. Ie, say there are 2 + instances of solution under results, StressbalanceSolution and TransientSolution. + In the NetCDF file we would see solution = "StressbalanceSolution", "TransientSolution" + To deconstruct this, we need to iteratively assign md.results.StressbalanceSolution = solution() + and md.results.TransientSolution = solution() and whatever else. + ''' + # start with the subclasses for subclass in NCData.groups['results'].variables.keys(): class_instance = subclass + '()' - class_instance_name = NCData.groups['results'].variables[subclass][:][...].tobytes().decode() - setattr(model_copy.results, class_instance_name, eval(class_instance)) + + # now handle the instances + for instance in NCData.groups['results'].variables[subclass][:]: + # this is an ndarray of numpy bytes_ that we have to convert to strings + class_instance_name = instance.tobytes().decode('utf-8').strip() + # from here we can make new subclasses named as they were in the model saved + setattr(model_copy.results, class_instance_name, eval(class_instance)) + print(f'Successfully created results subclass instance {class_instance} named {class_instance_name}.') @@ -90,4 +107,5 @@ # first, we enter the group by: filename.groups['group_name'] # second we search the current level for variables: filename.groups['group_name'].variables.keys() + # at this step we check for multidimensional structure arrays and filter them out # third we get nested group keys by: filename.groups['group_name'].groups.keys() # if a variables exists, copy the data to the model framework by calling copy function @@ -95,17 +113,84 @@ for variable in eval(group_location_in_file + '.variables.keys()'): - location_of_variable_in_file = group_location_in_file + ".variables['" + str(variable) + "']" - # group_location_in_file is like filename.groups['group1'].groups['group1.1'].groups['group1.1.1'] - # Define the regex pattern to match the groups within brackets - pattern = r"\['(.*?)'\]" - # Use regex to find all matches and return something like 'group1.group1.1.group1.1.1 ...' where the last value is the name of the variable - matches = re.findall(pattern, location_of_variable_in_file) - variable_name = matches[-1] - location_of_variable_in_model = '.'.join(matches[:-1]) - copy_variable_data_to_new_model(location_of_variable_in_file, location_of_variable_in_model, variable_name) - - for nested_group in eval(group_location_in_file + '.groups.keys()'): - new_nested_group = group_location_in_file + ".groups['" + str(nested_group) + "']" - walk_nested_groups(new_nested_group) + if variable == 'this_is_a_nested' and 'results' in group_location_in_file: + # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']' + pattern = r"\['(.*?)'\]" + matches = re.findall(pattern, group_location_in_file) + name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") + copy_multidimensional_results_struct(group_location_in_file, name_of_struct) + istruct = True + + elif variable == 'this_is_a_nested' and 'qmu' in group_location_in_file: + print('encountered qmu structure that is not yet supported.') + # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']' + #pattern = r"\['(.*?)'\]" + #matches = re.findall(pattern, group_location_in_file) + #name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") + #name_of_struct = eval(group_location_in_file + ".variables['']") + #copy_multidimensional_qmu_struct(group_location_in_file, name_of_struct) + isstruct = True + + else: + location_of_variable_in_file = group_location_in_file + ".variables['" + str(variable) + "']" + # group_location_in_file is like filename.groups['group1'].groups['group1.1'].groups['group1.1.1'] + # Define the regex pattern to match the groups within brackets + pattern = r"\['(.*?)'\]" + # Use regex to find all matches and return something like 'group1.group1.1.group1.1.1 ...' where the last value is the name of the variable + matches = re.findall(pattern, location_of_variable_in_file) + variable_name = matches[-1] + location_of_variable_in_model = '.'.join(matches[:-1]) + copy_variable_data_to_new_model(location_of_variable_in_file, location_of_variable_in_model, variable_name) + + if 'istruct' in locals(): + pass + else: + for nested_group in eval(group_location_in_file + '.groups.keys()'): + new_nested_group = group_location_in_file + ".groups['" + str(nested_group) + "']" + walk_nested_groups(new_nested_group) + + + +''' + MATLAB has Multidimensional Structure Arrays in 2 known classes: results and qmu. + The python classes results.py and qmu.py emulate this MATLAB object in their own + unique ways. The functions in this script will assign data to either of these + classes such that the final structure is compatible with its parent class. +''' + +def copy_multidimensional_results_struct(group_location_in_file, name_of_struct): + ''' + A common multidimensional array is the 1xn md.results.TransientSolution object. + + The way that this object emulates the MATLAB mutli-dim. struct. array is with + the solution().steps attr. which is a list of solutionstep() instances + The process to recreate is as follows: + 1. Get instance of solution() with solution variable (the instance is made in make_results_subclasses) + 2. For each subgroup, create a solutionstep() class instance + 2a. Populate the instance with the key:value pairs + 2b. Append the instance to the solution().steps list + ''' + # step 1 + class_instance_name = name_of_struct + + # for some reason steps is not already a list + setattr(model_copy.results.__dict__[class_instance_name], 'steps', list()) + + steps = model_copy.results.__dict__[class_instance_name].steps + + # step 2 + layer = 1 + for subgroup in eval(group_location_in_file + ".groups.keys()"): + solutionstep_instance = solutionstep() + # step 2a + subgroup_location_in_file = group_location_in_file + ".groups['" + subgroup + "']" + for key in eval(subgroup_location_in_file + ".variables.keys()"): + value = eval(subgroup_location_in_file + ".variables['" + str(key) + "'][:]") + setattr(solutionstep_instance, key, value) + # step 2b + steps.append(solutionstep_instance) + print('Succesfully saved layer ' + str(layer) + ' to results.' + str(class_instance_name) + ' struct.') + layer += 1 + + print('Successfully recreated results structure ' + str(class_instance_name)) @@ -113,67 +198,119 @@ def copy_variable_data_to_new_model(location_of_variable_in_file, location_of_variable_in_model, variable_name): # as simple as navigating to the location_of_variable_in_model and setting it equal to the location_of_variable_in_file - # NetCDF4 has a property called "_FillValue" that sometimes saves empty lists, so we have to catch those FillValue = -9223372036854775806 - - # results band-aid... - if str(location_of_variable_in_model + '.' + variable_name) =='results.solutionstep': - pass - # qmu band-aid - elif 'qmu.statistics.method' in str(location_of_variable_in_model + '.' + variable_name): - pass - # handle any strings: - elif 'char' in eval(location_of_variable_in_file + '.dimensions[0]'): - setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:][...].tobytes().decode()')) - # handle ndarrays + lists - elif len(eval(location_of_variable_in_file + '[:]'))>1: - # check for bool - try: # there is only one datatype assigned the attribute 'units' and that is bool, so anything else will go right to except - if eval(location_of_variable_in_file + '.units') == 'bool': - setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, np.array(eval(location_of_variable_in_file + '[:]'), dtype = bool)) - else: + try: + # results band-aid... + #print(str(location_of_variable_in_model + '.' + variable_name)) + if str(location_of_variable_in_model + '.' + variable_name) in ['results.solutionstep', 'results.solution', 'results.resultsdakota']: + pass + # qmu band-aid + elif 'qmu.statistics.method' in str(location_of_variable_in_model + '.' + variable_name): + pass + # handle any strings: + elif 'char' in eval(location_of_variable_in_file + '.dimensions[0]'): + setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:][...].tobytes().decode()')) + # handle ndarrays + lists + elif len(eval(location_of_variable_in_file + '[:]'))>1: + # check for bool + try: # there is only one datatype assigned the attribute 'units' and that is bool, so anything else will go right to except + if eval(location_of_variable_in_file + '.units') == 'bool': + setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, np.array(eval(location_of_variable_in_file + '[:]'), dtype = bool)) + else: + setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]')) + except: setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]')) - except: - setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]')) - # catch everything else + # catch everything else + else: + # check for FillValue. use try/except because try block will only work on datatypes like int64, float, single element lists/arrays ect and not nd-arrays/n-lists etc + try: + # this try block will only work on single ints/floats/doubles and will skip to the except block for all other cases + if FillValue == eval(location_of_variable_in_file + '[:][0]'): + setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, []) + else: + # we have to convert numpy datatypes to native python types with .item() + var_to_save = eval(location_of_variable_in_file + '[:][0]') # note the [0] on the end + setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, var_to_save.item()) + except: + setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]')) + except AttributeError: + copy_variable_data_to_new_model_dict(location_of_variable_in_file, location_of_variable_in_model) + + print('Successfully saved ' + location_of_variable_in_model + '.' + variable_name + ' to model.') + + + + + + +def copy_variable_data_to_new_model_dict(location_of_variable_in_file, location_of_variable_in_model): + # as simple as navigating to the location_of_variable_in_model and setting it equal to the location_of_variable_in_file + + # NetCDF4 has a property called "_FillValue" that sometimes saves empty lists, so we have to catch those + FillValue = -9223372036854775806 + + # the key will be the last item in the location + key = ''.join(location_of_variable_in_model.split('.')[-1]) + + # update the location to point to the dict instead of the dict key + location_of_variable_in_model = '.'.join(location_of_variable_in_model.split('.')[:-1]) + + # verify we're working with a dict: + if isinstance(eval('model_copy.' + location_of_variable_in_model), OrderedDict): + dict_object = eval('model_copy.' + location_of_variable_in_model) + + # handle any strings: + if 'char' in eval(location_of_variable_in_file + '.dimensions[0]'): + data = eval(location_of_variable_in_file + '[:][...].tobytes().decode()') + dict_object.update({key: data}) + + # handle ndarrays + lists + elif len(eval(location_of_variable_in_file + '[:]'))>1: + # check for bool + try: # there is only one datatype assigned the attribute 'units' and that is bool, so anything else will go right to except + if eval(location_of_variable_in_file + '.units') == 'bool': + data = np.array(eval(location_of_variable_in_file + '[:]'), dtype = bool) + dict_object.update({key: data}) + else: + data = eval(location_of_variable_in_file + '[:]') + dict_object.update({key: data}) + except: + data = eval(location_of_variable_in_file + '[:]') + dict_object.update({key: data}) + # catch everything else + else: + # check for FillValue. use try/except because try block will only work on datatypes like int64, float, single element lists/arrays ect and not nd-arrays/n-lists etc + try: + # this try block will only work on single ints/floats/doubles and will skip to the except block for all other cases + if FillValue == eval(location_of_variable_in_file + '[:][0]'): + dict_object.update({key: []}) + else: + # we have to convert numpy datatypes to native python types with .item() + var_to_save = eval(location_of_variable_in_file + '[:][0]') # note the [0] on the end + dict_object.update({key: var_to_save.item()}) + except: + data = eval(location_of_variable_in_file + '[:]') + dict_object.update({key: data}) else: - # check for FillValue. use try/except because try block will only work on datatypes like int64, float, single element lists/arrays ect and not nd-arrays/n-lists etc - try: - # this try block will only work on single ints/floats/doubles and will skip to the except block for all other cases - if FillValue == eval(location_of_variable_in_file + '[:][0]'): - setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, []) - else: - # we have to convert numpy datatypes to native python types with .item() - var_to_save = eval(location_of_variable_in_file + '[:][0]') # note the [0] on the end - setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, var_to_save.item()) - except: - setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]')) - print('Successfully saved ' + location_of_variable_in_model + '.' + variable_name + ' to model.') - - - - - - - - - - - - - - - - - - - - - - - - - - - - + print(f"Unrecognized object was saved and cannot be reconstructed: {location_of_variable_in_model}") + + + + + + + + + + + + + + + + + + + + +