Changeset 27878

Timestamp:

08/16/23 21:26:04 (19 months ago)

Author:

musselman

Message:

Added functionality to read_netCDF_commit.py to handle mutlidimensional structure arrays generated in matlab under the results subclass.

File:

• issm/trunk/src/m/contrib/musselman/read_netCDF_commit.py (modified) (5 diffs)

issm/trunk/src/m/contrib/musselman/read_netCDF_commit.py

@@ -9 +9 @@
 from m1qn3inversion import m1qn3inversion
 from taoinversion import taoinversion
+from collections import OrderedDict
+
 
 
@@ -66 +68 @@
 
 def make_results_subclasses():
+    '''
+        There are 3 possible subclasses: solution, solutionstep, resultsdakota.
+        In the NetCDF file these are saved as a list of strings. Ie, say there are 2
+        instances of solution under results, StressbalanceSolution and TransientSolution. 
+        In the NetCDF file we would see solution = "StressbalanceSolution", "TransientSolution"
+        To deconstruct this, we need to iteratively assign md.results.StressbalanceSolution = solution()
+        and md.results.TransientSolution = solution() and whatever else.
+    '''
+    # start with the subclasses
     for subclass in NCData.groups['results'].variables.keys():
         class_instance = subclass + '()'
-        class_instance_name = NCData.groups['results'].variables[subclass][:][...].tobytes().decode()
-        setattr(model_copy.results, class_instance_name, eval(class_instance))
+
+        # now handle the instances
+        for instance in NCData.groups['results'].variables[subclass][:]:
+            # this is an ndarray of numpy bytes_ that we have to convert to strings
+            class_instance_name = instance.tobytes().decode('utf-8').strip()
+            # from here we can make new subclasses named as they were in the model saved
+            setattr(model_copy.results, class_instance_name, eval(class_instance))
+            print(f'Successfully created results subclass instance {class_instance} named {class_instance_name}.')
 
 
@@ -90 +107 @@
     # first, we enter the group by: filename.groups['group_name']
     # second we search the current level for variables: filename.groups['group_name'].variables.keys()
+    # at this step we check for multidimensional structure arrays and filter them out
     # third we get nested group keys by: filename.groups['group_name'].groups.keys()
     # if a variables exists, copy the data to the model framework by calling copy function
@@ -95 +113 @@
 
     for variable in eval(group_location_in_file + '.variables.keys()'):
-        location_of_variable_in_file = group_location_in_file + ".variables['" + str(variable) + "']"
-        # group_location_in_file is like filename.groups['group1'].groups['group1.1'].groups['group1.1.1']
-        # Define the regex pattern to match the groups within brackets
-        pattern = r"\['(.*?)'\]"
-        # Use regex to find all matches and return something like 'group1.group1.1.group1.1.1 ...' where the last value is the name of the variable
-        matches = re.findall(pattern, location_of_variable_in_file)
-        variable_name = matches[-1]
-        location_of_variable_in_model = '.'.join(matches[:-1])
-        copy_variable_data_to_new_model(location_of_variable_in_file, location_of_variable_in_model, variable_name)
-
-    for nested_group in eval(group_location_in_file + '.groups.keys()'):
-        new_nested_group = group_location_in_file + ".groups['" + str(nested_group) + "']"
-        walk_nested_groups(new_nested_group)
+        if variable == 'this_is_a_nested' and 'results' in group_location_in_file:
+            # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']'
+            pattern = r"\['(.*?)'\]"
+            matches = re.findall(pattern, group_location_in_file)
+            name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") 
+            copy_multidimensional_results_struct(group_location_in_file, name_of_struct)
+            istruct = True
+
+        elif variable == 'this_is_a_nested' and 'qmu' in group_location_in_file:
+            print('encountered qmu structure that is not yet supported.')
+            # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']'
+            #pattern = r"\['(.*?)'\]"
+            #matches = re.findall(pattern, group_location_in_file)
+            #name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") 
+            #name_of_struct = eval(group_location_in_file + ".variables['']")
+            #copy_multidimensional_qmu_struct(group_location_in_file, name_of_struct)
+            isstruct = True
+    
+        else:
+            location_of_variable_in_file = group_location_in_file + ".variables['" + str(variable) + "']"
+            # group_location_in_file is like filename.groups['group1'].groups['group1.1'].groups['group1.1.1']
+            # Define the regex pattern to match the groups within brackets
+            pattern = r"\['(.*?)'\]"
+            # Use regex to find all matches and return something like 'group1.group1.1.group1.1.1 ...' where the last value is the name of the variable
+            matches = re.findall(pattern, location_of_variable_in_file)
+            variable_name = matches[-1]
+            location_of_variable_in_model = '.'.join(matches[:-1])
+            copy_variable_data_to_new_model(location_of_variable_in_file, location_of_variable_in_model, variable_name)
+            
+    if 'istruct' in locals():
+        pass
+    else:
+        for nested_group in eval(group_location_in_file + '.groups.keys()'):
+            new_nested_group = group_location_in_file + ".groups['" + str(nested_group) + "']"
+            walk_nested_groups(new_nested_group)
+
+
+
+'''
+    MATLAB has Multidimensional Structure Arrays in 2 known classes: results and qmu.
+    The python classes results.py and qmu.py emulate this MATLAB object in their own
+    unique ways. The functions in this script will assign data to either of these 
+    classes such that the final structure is compatible with its parent class.
+'''
+
+def copy_multidimensional_results_struct(group_location_in_file, name_of_struct):
+    '''
+    A common multidimensional array is the 1xn md.results.TransientSolution object.
+
+    The way that this object emulates the MATLAB mutli-dim. struct. array is with 
+    the solution().steps attr. which is a list of solutionstep() instances
+        The process to recreate is as follows:
+            1. Get instance of solution() with solution variable (the instance is made in make_results_subclasses)
+            2. For each subgroup, create a solutionstep() class instance
+             2a. Populate the instance with the key:value pairs
+             2b. Append the instance to the solution().steps list
+    '''
+    # step 1
+    class_instance_name = name_of_struct
+    
+    # for some reason steps is not already a list
+    setattr(model_copy.results.__dict__[class_instance_name], 'steps', list())
+
+    steps = model_copy.results.__dict__[class_instance_name].steps
+    
+    # step 2
+    layer = 1
+    for subgroup in eval(group_location_in_file + ".groups.keys()"):
+        solutionstep_instance = solutionstep()
+        # step 2a
+        subgroup_location_in_file = group_location_in_file + ".groups['" + subgroup + "']"
+        for key in eval(subgroup_location_in_file + ".variables.keys()"):
+            value = eval(subgroup_location_in_file + ".variables['" + str(key) + "'][:]")
+            setattr(solutionstep_instance, key, value)
+        # step 2b
+        steps.append(solutionstep_instance)
+        print('Succesfully saved layer ' + str(layer) + ' to results.' + str(class_instance_name) + ' struct.')
+        layer += 1
+
+    print('Successfully recreated results structure ' + str(class_instance_name))
 
 
@@ -113 +198 @@
 def copy_variable_data_to_new_model(location_of_variable_in_file, location_of_variable_in_model, variable_name):
     # as simple as navigating to the location_of_variable_in_model and setting it equal to the location_of_variable_in_file
-
     # NetCDF4 has a property called "_FillValue" that sometimes saves empty lists, so we have to catch those
     FillValue = -9223372036854775806
-
-    # results band-aid...
-    if str(location_of_variable_in_model + '.' + variable_name) =='results.solutionstep':
-        pass
-    # qmu band-aid
-    elif 'qmu.statistics.method' in str(location_of_variable_in_model + '.' + variable_name):
-        pass
-    # handle any strings:
-    elif 'char' in eval(location_of_variable_in_file + '.dimensions[0]'):
-        setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:][...].tobytes().decode()'))
-    # handle ndarrays + lists
-    elif len(eval(location_of_variable_in_file + '[:]'))>1:
-        # check for bool
-        try: # there is only one datatype assigned the attribute 'units' and that is bool, so anything else will go right to except
-            if eval(location_of_variable_in_file + '.units') == 'bool':
-                setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, np.array(eval(location_of_variable_in_file + '[:]'), dtype = bool))
-            else:
+    try:
+        # results band-aid...
+        #print(str(location_of_variable_in_model + '.' + variable_name))
+        if str(location_of_variable_in_model + '.' + variable_name) in ['results.solutionstep', 'results.solution', 'results.resultsdakota']:
+            pass
+        # qmu band-aid
+        elif 'qmu.statistics.method' in str(location_of_variable_in_model + '.' + variable_name):
+            pass
+        # handle any strings:
+        elif 'char' in eval(location_of_variable_in_file + '.dimensions[0]'):
+            setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:][...].tobytes().decode()'))
+        # handle ndarrays + lists
+        elif len(eval(location_of_variable_in_file + '[:]'))>1:
+            # check for bool
+            try: # there is only one datatype assigned the attribute 'units' and that is bool, so anything else will go right to except
+                if eval(location_of_variable_in_file + '.units') == 'bool':
+                    setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, np.array(eval(location_of_variable_in_file + '[:]'), dtype = bool))
+                else:
+                    setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]'))
+            except:
                 setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]'))
-        except:
-            setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]'))
-    # catch everything else
+        # catch everything else
+        else:
+            # check for FillValue. use try/except because try block will only work on datatypes like int64, float, single element lists/arrays ect and not nd-arrays/n-lists etc
+            try:
+                # this try block will only work on single ints/floats/doubles and will skip to the except block for all other cases
+                if FillValue == eval(location_of_variable_in_file + '[:][0]'):
+                    setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, [])
+                else:
+                    # we have to convert numpy datatypes to native python types with .item()
+                    var_to_save = eval(location_of_variable_in_file + '[:][0]')  # note the [0] on the end
+                    setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, var_to_save.item())
+            except:
+                    setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]'))
+    except AttributeError:
+        copy_variable_data_to_new_model_dict(location_of_variable_in_file, location_of_variable_in_model)
+
+    print('Successfully saved ' + location_of_variable_in_model + '.' + variable_name + ' to model.')
+
+
+
+
+
+
+def copy_variable_data_to_new_model_dict(location_of_variable_in_file, location_of_variable_in_model):
+    # as simple as navigating to the location_of_variable_in_model and setting it equal to the location_of_variable_in_file
+
+    # NetCDF4 has a property called "_FillValue" that sometimes saves empty lists, so we have to catch those
+    FillValue = -9223372036854775806
+
+    # the key will be the last item in the location
+    key = ''.join(location_of_variable_in_model.split('.')[-1])
+
+    # update the location to point to the dict instead of the dict key
+    location_of_variable_in_model = '.'.join(location_of_variable_in_model.split('.')[:-1])
+
+    # verify we're working with a dict:
+    if isinstance(eval('model_copy.' + location_of_variable_in_model), OrderedDict):
+        dict_object = eval('model_copy.' + location_of_variable_in_model)
+        
+        # handle any strings:
+        if 'char' in eval(location_of_variable_in_file + '.dimensions[0]'):
+            data = eval(location_of_variable_in_file + '[:][...].tobytes().decode()')
+            dict_object.update({key: data})
+            
+        # handle ndarrays + lists
+        elif len(eval(location_of_variable_in_file + '[:]'))>1:
+            # check for bool
+            try: # there is only one datatype assigned the attribute 'units' and that is bool, so anything else will go right to except
+                if eval(location_of_variable_in_file + '.units') == 'bool':
+                    data = np.array(eval(location_of_variable_in_file + '[:]'), dtype = bool)
+                    dict_object.update({key: data})
+                else:
+                    data = eval(location_of_variable_in_file + '[:]')
+                    dict_object.update({key: data})
+            except:
+                data = eval(location_of_variable_in_file + '[:]')
+                dict_object.update({key: data})
+        # catch everything else
+        else:
+            # check for FillValue. use try/except because try block will only work on datatypes like int64, float, single element lists/arrays ect and not nd-arrays/n-lists etc
+            try:
+                # this try block will only work on single ints/floats/doubles and will skip to the except block for all other cases
+                if FillValue == eval(location_of_variable_in_file + '[:][0]'):
+                    dict_object.update({key: []})
+                else:
+                    # we have to convert numpy datatypes to native python types with .item()
+                    var_to_save = eval(location_of_variable_in_file + '[:][0]')  # note the [0] on the end
+                    dict_object.update({key:  var_to_save.item()})
+            except:
+                data = eval(location_of_variable_in_file + '[:]')
+                dict_object.update({key: data})
     else:
-        # check for FillValue. use try/except because try block will only work on datatypes like int64, float, single element lists/arrays ect and not nd-arrays/n-lists etc
-        try:
-            # this try block will only work on single ints/floats/doubles and will skip to the except block for all other cases
-            if FillValue == eval(location_of_variable_in_file + '[:][0]'):
-                setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, [])
-            else:
-                # we have to convert numpy datatypes to native python types with .item()
-                var_to_save = eval(location_of_variable_in_file + '[:][0]')  # note the [0] on the end
-                setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, var_to_save.item())
-        except:
-            setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]'))
-    print('Successfully saved ' + location_of_variable_in_model + '.' + variable_name + ' to model.')
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+        print(f"Unrecognized object was saved and cannot be reconstructed: {location_of_variable_in_model}")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+