Changeset 27896 for issm/trunk

Timestamp:

09/05/23 15:48:58 (19 months ago)

Author:

musselman

Message:

Updated read_netCDF.py to include functionality for arrays of objects saved from MATLAB.

File:

• issm/trunk/src/m/contrib/musselman/read_netCDF.py (modified) (6 diffs)

issm/trunk/src/m/contrib/musselman/read_netCDF.py

@@ -10 +10 @@
 from taoinversion import taoinversion
 from collections import OrderedDict
+import sys
+from massfluxatgate import massfluxatgate
+
 
 
@@ -117 +120 @@
 
     for variable in eval(group_location_in_file + '.variables.keys()'):
-        if variable == 'this_is_a_nested' and 'results' in group_location_in_file:
-            # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']'
-            pattern = r"\['(.*?)'\]"
-            matches = re.findall(pattern, group_location_in_file)
-            name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") 
-            copy_multidimensional_results_struct(group_location_in_file, name_of_struct, NCData)
-            istruct = True
-
-        elif variable == 'this_is_a_nested' and 'qmu' in group_location_in_file:
-            print('encountered qmu structure that is not yet supported.')
-            # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']'
-            #pattern = r"\['(.*?)'\]"
-            #matches = re.findall(pattern, group_location_in_file)
-            #name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") 
-            #name_of_struct = eval(group_location_in_file + ".variables['']")
-            #copy_multidimensional_qmu_struct(group_location_in_file, name_of_struct)
-            isstruct = True
-    
-        else:
-            location_of_variable_in_file = group_location_in_file + ".variables['" + str(variable) + "']"
-            # group_location_in_file is like filename.groups['group1'].groups['group1.1'].groups['group1.1.1']
-            # Define the regex pattern to match the groups within brackets
-            pattern = r"\['(.*?)'\]"
-            # Use regex to find all matches and return something like 'group1.group1.1.group1.1.1 ...' where the last value is the name of the variable
-            matches = re.findall(pattern, location_of_variable_in_file)
-            variable_name = matches[-1]
-            location_of_variable_in_model = '.'.join(matches[:-1])
-            copy_variable_data_to_new_model(location_of_variable_in_file, location_of_variable_in_model, variable_name, NCData, verbose=verbose)
-            
-    if 'istruct' in locals():
+        if 'is_object' not in locals():
+            if variable == 'this_is_a_nested' and 'results' in group_location_in_file and 'qmu' not in group_location_in_file:
+                # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']'
+                pattern = r"\['(.*?)'\]"
+                matches = re.findall(pattern, group_location_in_file)
+                name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") 
+                copy_multidimensional_results_struct(group_location_in_file, name_of_struct, NCData)
+                is_object = True
+    
+            elif variable == 'name_of_cell_array':
+                # reconstruct an array of elements
+                copy_cell_array_of_objects(group_location_in_file, model_copy, NCData, verbose)
+                is_object = True
+    
+            elif variable == 'this_is_a_nested' and 'qmu' in group_location_in_file:
+                if verbose:
+                    print('encountered qmu structure that is not yet supported.')
+                else: pass
+                    
+                is_object = True
+        
+            else:
+                location_of_variable_in_file = group_location_in_file + ".variables['" + str(variable) + "']"
+                # group_location_in_file is like filename.groups['group1'].groups['group1.1'].groups['group1.1.1']
+                # Define the regex pattern to match the groups within brackets
+                pattern = r"\['(.*?)'\]"
+                # Use regex to find all matches and return something like 'group1.group1.1.group1.1.1 ...' where the last value is the name of the variable
+                matches = re.findall(pattern, location_of_variable_in_file)
+                variable_name = matches[-1]
+                location_of_variable_in_model = '.'.join(matches[:-1])
+                copy_variable_data_to_new_model(location_of_variable_in_file, location_of_variable_in_model, variable_name, NCData, verbose=verbose)
+
+    # if one of the variables above was an object, further subclasses will be taken care of when reconstructing it
+    if 'is_object' in locals():
         pass
     else:
@@ -152 +159 @@
             new_nested_group = group_location_in_file + ".groups['" + str(nested_group) + "']"
             walk_nested_groups(new_nested_group, NCData, verbose=verbose)
+
+
+def copy_cell_array_of_objects(group_location_in_file, model_copy, NCData, verbose):
+    '''
+        The structure in netcdf for groups with the name_of_cell_array variable is like:
+
+        group: 2x6_cell_array_of_objects {
+            name_of_cell_array = <name_of_cell_array>
+
+            group: Row_1_of_2 {
+                group: Col_1_of_6 {
+                    ... other groups can be here that refer to objects
+                } // group Col_6_of_6
+            } // group Row_1_of_2
+
+            group: Row_2_of_2 {
+                group: Col_1_of_6 {
+                    ... other groups can be here that refer to objects
+                } // group Col_6_of_6
+            } // group Row_2_of_2
+        } // group 2x6_cell_array_of_objects
+
+        We have to navigate this structure to extract all the data and recreate the 
+        original structure when the model was saved
+    '''
+
+    # get the name_of_cell_array, rows and cols vars
+    name_of_cell_array_varID = eval(group_location_in_file + ".variables['name_of_cell_array']")
+    rows_varID = eval(group_location_in_file + ".variables['rows']")
+    cols_varID = eval(group_location_in_file + ".variables['cols']")
+
+    name_of_cell_array = name_of_cell_array_varID[:][...].tobytes().decode()
+    rows = rows_varID[:]
+    cols = cols_varID[:]
+
+    # now we work backwards: make the array, fill it in, and assign it to the model
+
+    # make the array
+    array = list()
+
+    subgroups = eval(group_location_in_file + ".groups") #.keys()")
+
+    # enter each subgroup, get the data, assign it to the corresponding index of cell array
+    if rows > 1:
+        # we go over rows
+        # set index for rows
+        row_idx = 0
+        for row in list(subgroups):
+            # make list for each row
+            current_row = list()
+            columns = subgroups[str(row)].groups.keys()
+
+            # set index for columns
+            col_idx = 0
+
+            # iterate over columns
+            for col in list(columns):
+                # now get the variables 
+                current_col_vars = columns.groups[str(col)].variables
+
+                # check for special datastructures                
+                if "class_is_a" in current_col_vars:
+                    class_name = subgroups[str(col)].variables['class_is_a'][:][...].tobytes().decode()
+                    col_data = deserialize_class(class_name, columns.groups[str(col)], NCData, verbose)
+                    is_object = True
+                elif "this_is_a_nested" in current_col_vars:
+                    # functionality not yet supported
+                    print('Error: Cell Arrays of structs not yet supported!')
+                    is_object = True
+                else:
+                    if 'is_object_' in locals():
+                        pass
+                        # already taken care of
+                    else:
+                        # store the variables as normal -- to be added later
+                        print('Error: Arrays of mixed objects not yet supported!')
+                        for var in current_col_vars:
+                            # this is where that functionality would be handled
+                            pass
+                col_idx += 1
+                # add the entry to our row list
+                current_row.append(col_data)
+
+            # add the list of columns to the array
+            array.append(current_row)
+            row_idx += 1
+
+    else:
+        # set index for columns
+        col_idx = 0
+
+        # iterate over columns
+        for col in list(subgroups):
+            # now get the variables 
+            current_col_vars = subgroups[str(col)].variables
+            
+            # check for special datastructures
+            if "class_is_a" in current_col_vars:
+                class_name = subgroups[str(col)].variables['class_is_a'][:][...].tobytes().decode()
+                col_data = deserialize_class(class_name, subgroups[str(col)], NCData, verbose)
+                is_object = True
+            elif "this_is_a_nested" in current_col_vars:
+                # functionality not yet supported
+                print('Error: Cell Arrays of structs not yet supported!')
+                is_object = True
+            else:
+                if 'is_object_' in locals():
+                    pass
+                    # already taken care of
+                else:
+                    # store the variables as normal -- to be added later
+                    print('Error: Arrays of mixed objects not yet supported!')
+                    for var in current_col_vars:
+                        # this is where that functionality would be handled
+                        pass
+            col_idx += 1
+            # add the list of columns to the array
+            array.append(col_data)
+
+    # finally, add the attribute to the model
+    # borrow some code from above:
+    pattern = r"\['(.*?)'\]"
+    matches = re.findall(pattern, group_location_in_file)
+    variable_name = matches[0]
+    setattr(model_copy.__dict__[variable_name], name_of_cell_array, array)
+
+
+
+def deserialize_class(class_name, group, NCData, verbose=False):
+
+    # most ISSM classes are imported by from <name> import <name>
+    class ModuleError(Exception):
+        pass
+    
+    if class_name not in sys.modules:
+        raise ModuleError(str('Model requires the following class to be imported from a module: ' + class_name + ". Please add the import to read_netCDF.py in order to continue."))
+
+    # Instantiate the class
+    class_instance = eval(class_name + "()")
+
+    # Get and assign properties
+    subgroups = list(group.groups.keys())
+
+    if len(subgroups) == 1:
+        # Get properties
+        subgroup = group[subgroups[0]]
+        varIDs = subgroup.variables.keys()
+        for varname in varIDs:
+            # Variable metadata
+            var = subgroup[varname]
+
+            # Data
+            if 'char' in var.dimensions[0]:
+                data = var[:][...].tobytes().decode()
+            else:
+                data = var[:]
+
+            # Some classes may have permissions, so we skip those
+            try:
+                setattr(class_instance, varname, data)
+            except:
+                pass
+    else:
+        # Not supported
+        pass
+        
+    return class_instance
 
 
@@ -208 +382 @@
     try:
         # results band-aid...
-        #print(str(location_of_variable_in_model + '.' + variable_name))
         if str(location_of_variable_in_model + '.' + variable_name) in ['results.solutionstep', 'results.solution', 'results.resultsdakota']:
             pass
@@ -240 +413 @@
                     else:
                         # we have to convert numpy datatypes to native python types with .item()
-                        setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, var_to_save.item())                        
+                        setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, var_to_save.item())
             except:
-                    setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]'))
+                setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]'))
     except AttributeError:
         copy_variable_data_to_new_model_dict(location_of_variable_in_file, location_of_variable_in_model, NCData, verbose=verbose)
@@ -248 +421 @@
     if verbose:
         print('Successfully loaded ' + location_of_variable_in_model + '.' + variable_name + ' into model.')
+