Index: /issm/trunk-jpl/src/m/netcdf/read_netCDF.m =================================================================== --- /issm/trunk-jpl/src/m/netcdf/read_netCDF.m (revision 27899) +++ /issm/trunk-jpl/src/m/netcdf/read_netCDF.m (revision 27900) @@ -130,10 +130,17 @@ % keep an eye out for nested structs: if strcmp(varname, 'this_is_a_nested') - is_nested = true; + is_object = true; model_copy = copy_nested_struct(group_location_in_file, model_copy, NCData, verbose); + elseif strcmp(varname, 'name_of_cell_array') + is_object = true; + model_copy = copy_cell_array_of_objects(variables, group_location_in_file, model_copy, NCData, verbose); elseif strcmp(varname, 'solution') % band-aid pass.. else - model_copy = copy_variable_data_to_new_model(group_location_in_file, varname, xtype, model_copy, NCData, verbose); + if logical(exist('is_object', 'var')) + % already handled + else + model_copy = copy_variable_data_to_new_model(group_location_in_file, varname, xtype, model_copy, NCData, verbose); + end end end @@ -142,5 +149,5 @@ %try % if it's a nested struct the function copy_nested_struct has already been called - if logical(exist('is_nested', 'var')) + if logical(exist('is_object', 'var')) % do nothing else @@ -159,4 +166,197 @@ end + +% to read cell arrays with objects: +function model_copy = copy_cell_array_of_objects(variables, group_location_in_file, model_copy, NCData, verbose); + %{ + The structure in netcdf for groups with the name_of_cell_array variable is like: + + group: 2x6_cell_array_of_objects { + name_of_cell_array = + + group: Row_1_of_2 { + group: Col_1_of_6 { + ... other groups can be here that refer to objects + } // group Col_6_of_6 + } // group Row_1_of_2 + + group: Row_2_of_2 { + group: Col_1_of_6 { + ... other groups can be here that refer to objects + } // group Col_6_of_6 + } // group Row_2_of_2 + } // group 2x6_cell_array_of_objects + + We have to navigate this structure to extract all the data and recreate the + original structure when the model was saved + %} + + % get the name_of_cell_array, rows and cols vars + name_of_cell_array_varID = netcdf.inqVarID(group_location_in_file, 'name_of_cell_array'); + rows_varID = netcdf.inqVarID(group_location_in_file, 'rows'); + cols_varID = netcdf.inqVarID(group_location_in_file, 'cols'); + + name_of_cell_array = netcdf.getVar(group_location_in_file, name_of_cell_array_varID).'; % transpose + rows = netcdf.getVar(group_location_in_file, rows_varID); + cols = netcdf.getVar(group_location_in_file, cols_varID); + + % now we work backwards: make the cell array, fill it in, and assign it to the model + + % make the cell array + cell_array_placeholder = cell(rows, cols); + + % get subgroups which are elements of the cell array + subgroups = netcdf.inqGrps(group_location_in_file); % numerical cell array with ID's of subgroups + + % enter each subgroup, get the data, assign it to the corresponding index of cell array + if rows > 1 + % we go over rows + % set index for cell array rows + row_idx = 1; + for row = subgroups + % now columns + columns = netcdf.inqGrps(group_location_in_file); + + % set index for cell array cols + col_idx = 1; + for column = columns + % now variables + current_column_varids = netcdf.inqVarIDs(column); + + % if 'class_is_a' or 'this_is_a_nested' variables is present at this level we have to handle them accordingly + try + class_is_aID = netcdf.inqVarID(column, 'class_is_a'); + col_data = deserialize_class(column, NCData, verbose); + is_object = true; + catch + end + + try + this_is_a_nestedID = netcdf.inqVarID(column, 'this_is_a_nested'); + % functionality not supported + disp('Error: Cell Arrays of structs not yet supported!') + % copy_nested_struct(column, model_copy, NCData, verbose) + is_object = true; + catch + end + + if logical(exist('is_object', 'var')) + % already taken care of + else + % store the variables as normal -- to be added later + disp('Error: Cell Arrays of mixed objects not yet supported!') + for var = current_column_varids + % not supported + end + end + + cell_array_placeholder{row_idx, col_idx} = col_data; + col_idx = col_idx + 1; + end + row_idx = row_idx + 1; + end + else + % set index for cell array + col_idx = 1; + for column = subgroups + % now variables + current_column_varids = netcdf.inqVarIDs(column); + + % if 'class_is_a' or 'this_is_a_nested' variables is present at this level we have to handle them accordingly + try + classID = netcdf.inqVarID(column, 'class_is_a'); + col_data = deserialize_class(classID, column, NCData, verbose); + is_object = true; + catch ME + rethrow(ME) + end + + try + this_is_a_nestedID = netcdf.inqVarID(column, 'this_is_a_nested'); + % functionality not supported + disp('Error: Cell Arrays of structs not yet supported!') + % col_data = copy_nested_struct(column, model_copy, NCData, verbose); + is_object = true; + catch + end + if logical(exist('is_object', 'var')) + % already taken care of + else + % store the variables as normal -- to be added later + disp('Error: Cell Arrays of mixed objects not yet supported!') + for var = current_column_varids + % col_data = not supported + end + end + + cell_array_placeholder{col_idx} = col_data; + col_idx = col_idx + 1; + + end + end + + + % Like in copy_nested_struct, we can only handle things 1 layer deep. + % assign cell array to model + address_to_attr_list = split(netcdf.inqGrpNameFull(group_location_in_file), '/'); + address_to_attr = address_to_attr_list{2}; + if isprop(model_copy.(address_to_attr), name_of_cell_array); + model_copy.(address_to_attr).(name_of_cell_array) = cell_array_placeholder; + else + model_copy = addprop(model_copy.(address_to_attr), name_of_cell_array, cell_array_placeholder); + end + + if verbose + fprintf("Successfully loaded cell array %s to %s\n", name_of_cell_array,address_to_attr_list{2}) + end +end + + + + +function output = deserialize_class(classID, group, NCData, verbose) + %{ + This function will recreate a class + %} + + % get the name of the class + name = netcdf.getVar(group, classID).'; + + % instantiate it + class_instance = eval([name, '()']); + + % get and assign properties + subgroups = netcdf.inqGrps(group); % numerical cell array with ID's of subgroups + + if numel(subgroups) == 1 + % get properties + varIDs = netcdf.inqVarIDs(subgroups); + for varID = varIDs + % var metadata + [varname, xtype, dimids, numatts] = netcdf.inqVar(subgroups, varID); + % data + data = netcdf.getVar(subgroups, varID); + + % netcdf uses Row Major Order but MATLAB uses Column Major Order so we need to transpose all arrays w/ more than 1 dim + if all(size(data)~=1) || xtype == 2 + data = data.'; + end + + % some classes have permissions... so we skip those + try + % if property already exists, assign new value + if isprop(class_instance, varname) + class_instance.(varname) = data; + else + addprop(class_instance, varname, data); + end + catch + end + end + else + % not supported + end + output = class_instance; +end @@ -192,5 +392,5 @@ % and not pointers to the same memory address % this means that if address_in_model has more than 1 layer, we need to modify the code. For now, - % we just hope this will do. An example of a no-solution would be model().abc.def.ghi.field + % we just hope this will do. An example of a no-solution would be model().abc.def.ghi.field whereas we're only assuming model().abc.field now model_copy.(address_in_model).(name_of_struct) = struct(); Index: /issm/trunk-jpl/src/m/netcdf/read_netCDF.py =================================================================== --- /issm/trunk-jpl/src/m/netcdf/read_netCDF.py (revision 27899) +++ /issm/trunk-jpl/src/m/netcdf/read_netCDF.py (revision 27900) @@ -10,4 +10,7 @@ from taoinversion import taoinversion from collections import OrderedDict +import sys +from massfluxatgate import massfluxatgate + @@ -25,47 +28,63 @@ def read_netCDF(filename, verbose = False): if verbose: - print('NetCDF42C v1.1.13') - - # check if path exists - if path.exists(filename): + print('NetCDF42C v1.2.0') + + ''' + filename = path and name to save file under + verbose = T/F = show or muted log statements. Naturally muted + ''' + + # this is a precaution so that data is not lost + try: + # check if path exists + if path.exists(filename): + if verbose: + print('Opening {} for reading'.format(filename)) + else: pass + + # open the given netCDF4 file + NCData = Dataset(filename, 'r') + # remove masks from numpy arrays for easy conversion + NCData.set_auto_mask(False) + else: + return 'The file you entered does not exist or cannot be found in the current directory' + + # in order to handle some subclasses in the results class, we have to utilize this band-aid + # there will likely be more band-aids added unless a class name library is created with all class names that might be added to a md + try: + # if results has meaningful data, save the name of the subclass and class instance + NCData.groups['results'] + make_results_subclasses(NCData, verbose) + except: + pass + + # similarly, we need to check and see if we have an m1qn3inversion class instance + try: + NCData.groups['inversion'] + check_inversion_class(NCData, verbose) + except: + pass + + # walk through each group looking for subgroups and variables + for group in NCData.groups.keys(): + if 'debris' in group: + pass + else: + # have to send a custom name to this function: filename.groups['group'] + name = "NCData.groups['" + str(group) + "']" + walk_nested_groups(name, NCData, verbose) + if verbose: - print('Opening {} for reading'.format(filename)) - else: pass - - # open the given netCDF4 file - NCData = Dataset(filename, 'r') - # remove masks from numpy arrays for easy conversion - NCData.set_auto_mask(False) - else: - print('The file you entered does not exist or cannot be found in the current directory') - return print() - - # continuation of band-aid for results class - try: - NCData.groups['results'] - make_results_subclasses(NCData, verbose) - except: - pass - - # similarly, we need to check and see if we have an m1qn3inversion class instance - try: - NCData.groups['inversion'] - check_inversion_class(NCData, verbose) - except: - pass - - # walk through each group looking for subgroups and variables - for group in NCData.groups.keys(): - if 'debris' in group: - pass - else: - # have to send a custom name to this function: filename.groups['group'] - name = "NCData.groups['" + str(group) + "']" - walk_nested_groups(name, NCData, verbose) - - if verbose: - print("Model Successfully Loaded.") - return model_copy - + print("Model Successfully Loaded.") + + NCData.close() + + return model_copy + + # just in case something unexpected happens + except Exception as e: + if 'NCData' in locals(): + NCData.close() + raise e def make_results_subclasses(NCData, verbose = False): @@ -111,40 +130,43 @@ # first, we enter the group by: filename.groups['group_name'] # second we search the current level for variables: filename.groups['group_name'].variables.keys() - # at this step we check for multidimensional structure arrays and filter them out + # at this step we check for multidimensional structure arrays/ arrays of objects and filter them out # third we get nested group keys by: filename.groups['group_name'].groups.keys() - # if a variables exists, copy the data to the model framework by calling copy function # if a nested groups exist, repeat all for variable in eval(group_location_in_file + '.variables.keys()'): - if variable == 'this_is_a_nested' and 'results' in group_location_in_file: - # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']' - pattern = r"\['(.*?)'\]" - matches = re.findall(pattern, group_location_in_file) - name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") - copy_multidimensional_results_struct(group_location_in_file, name_of_struct, NCData) - istruct = True - - elif variable == 'this_is_a_nested' and 'qmu' in group_location_in_file: - print('encountered qmu structure that is not yet supported.') - # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']' - #pattern = r"\['(.*?)'\]" - #matches = re.findall(pattern, group_location_in_file) - #name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") - #name_of_struct = eval(group_location_in_file + ".variables['']") - #copy_multidimensional_qmu_struct(group_location_in_file, name_of_struct) - isstruct = True - - else: - location_of_variable_in_file = group_location_in_file + ".variables['" + str(variable) + "']" - # group_location_in_file is like filename.groups['group1'].groups['group1.1'].groups['group1.1.1'] - # Define the regex pattern to match the groups within brackets - pattern = r"\['(.*?)'\]" - # Use regex to find all matches and return something like 'group1.group1.1.group1.1.1 ...' where the last value is the name of the variable - matches = re.findall(pattern, location_of_variable_in_file) - variable_name = matches[-1] - location_of_variable_in_model = '.'.join(matches[:-1]) - copy_variable_data_to_new_model(location_of_variable_in_file, location_of_variable_in_model, variable_name, NCData, verbose=verbose) - - if 'istruct' in locals(): + if 'is_object' not in locals(): + if variable == 'this_is_a_nested' and 'results' in group_location_in_file and 'qmu' not in group_location_in_file: + # have to do some string deconstruction to get the name of the class instance/last group from 'NetCDF.groups['group1'].groups['group1.1']' + pattern = r"\['(.*?)'\]" + matches = re.findall(pattern, group_location_in_file) + name_of_struct = matches[-1] #eval(group_location_in_file + ".variables['solution']") + deserialize_nested_results_struct(group_location_in_file, name_of_struct, NCData) + is_object = True + + elif variable == 'name_of_cell_array': + # reconstruct an array of elements + deserialize_array_of_objects(group_location_in_file, model_copy, NCData, verbose) + is_object = True + + elif variable == 'this_is_a_nested' and 'qmu' in group_location_in_file: + if verbose: + print('encountered qmu structure that is not yet supported.') + else: pass + + is_object = True + + else: + location_of_variable_in_file = group_location_in_file + ".variables['" + str(variable) + "']" + # group_location_in_file is like filename.groups['group1'].groups['group1.1'].groups['group1.1.1'] + # Define the regex pattern to match the groups within brackets + pattern = r"\['(.*?)'\]" + # Use regex to find all matches and return something like 'group1.group1.1.group1.1.1 ...' where the last value is the name of the variable + matches = re.findall(pattern, location_of_variable_in_file) + variable_name = matches[-1] + location_of_variable_in_model = '.'.join(matches[:-1]) + deserialize_data(location_of_variable_in_file, location_of_variable_in_model, variable_name, NCData, verbose=verbose) + + # if one of the variables above was an object, further subclasses will be taken care of when reconstructing it + if 'is_object' in locals(): pass else: @@ -162,5 +184,5 @@ ''' -def copy_multidimensional_results_struct(group_location_in_file, name_of_struct, NCData, verbose = False): +def deserialize_nested_results_struct(group_location_in_file, name_of_struct, NCData, verbose = False): ''' A common multidimensional array is the 1xn md.results.TransientSolution object. @@ -202,5 +224,186 @@ -def copy_variable_data_to_new_model(location_of_variable_in_file, location_of_variable_in_model, variable_name, NCData, verbose = False): + +def deserialize_array_of_objects(group_location_in_file, model_copy, NCData, verbose): + ''' + The structure in netcdf for groups with the name_of_cell_array variable is like: + + group: 2x6_cell_array_of_objects { + name_of_cell_array = + + group: Row_1_of_2 { + group: Col_1_of_6 { + ... other groups can be here that refer to objects + } // group Col_6_of_6 + } // group Row_1_of_2 + + group: Row_2_of_2 { + group: Col_1_of_6 { + ... other groups can be here that refer to objects + } // group Col_6_of_6 + } // group Row_2_of_2 + } // group 2x6_cell_array_of_objects + + We have to navigate this structure to extract all the data and recreate the + original structure when the model was saved + ''' + + if verbose: + print(f"Loading array of objects.") + + # get the name_of_cell_array, rows and cols vars + name_of_cell_array_varID = eval(group_location_in_file + ".variables['name_of_cell_array']") + rows_varID = eval(group_location_in_file + ".variables['rows']") + cols_varID = eval(group_location_in_file + ".variables['cols']") + + name_of_cell_array = name_of_cell_array_varID[:][...].tobytes().decode() + rows = rows_varID[:] + cols = cols_varID[:] + + # now we work backwards: make the array, fill it in, and assign it to the model + + # make the array + array = list() + + subgroups = eval(group_location_in_file + ".groups") #.keys()") + + # enter each subgroup, get the data, assign it to the corresponding index of cell array + if rows > 1: + # we go over rows + # set index for rows + row_idx = 0 + for row in list(subgroups): + # make list for each row + current_row = list() + columns = subgroups[str(row)].groups.keys() + + # set index for columns + col_idx = 0 + + # iterate over columns + for col in list(columns): + # now get the variables + current_col_vars = columns.groups[str(col)].variables + + # check for special datastructures + if "class_is_a" in current_col_vars: + class_name = subgroups[str(col)].variables['class_is_a'][:][...].tobytes().decode() + col_data = deserialize_class_instance(class_name, columns.groups[str(col)], NCData, verbose) + is_object = True + elif "this_is_a_nested" in current_col_vars: + # functionality not yet supported + print('Error: Cell Arrays of structs not yet supported!') + is_object = True + else: + if 'is_object_' in locals(): + pass + # already taken care of + else: + # store the variables as normal -- to be added later + print('Error: Arrays of mixed objects not yet supported!') + for var in current_col_vars: + # this is where that functionality would be handled + pass + col_idx += 1 + # add the entry to our row list + current_row.append(col_data) + + # add the list of columns to the array + array.append(current_row) + row_idx += 1 + + else: + # set index for columns + col_idx = 0 + + # iterate over columns + for col in list(subgroups): + # now get the variables + current_col_vars = subgroups[str(col)].variables + + # check for special datastructures + if "class_is_a" in current_col_vars: + class_name = subgroups[str(col)].variables['class_is_a'][:][...].tobytes().decode() + col_data = deserialize_class_instance(class_name, subgroups[str(col)], NCData, verbose) + is_object = True + elif "this_is_a_nested" in current_col_vars: + # functionality not yet supported + print('Error: Cell Arrays of structs not yet supported!') + is_object = True + else: + if 'is_object_' in locals(): + pass + # already taken care of + else: + # store the variables as normal -- to be added later + print('Error: Arrays of mixed objects not yet supported!') + for var in current_col_vars: + # this is where that functionality would be handled + pass + col_idx += 1 + # add the list of columns to the array + array.append(col_data) + + # finally, add the attribute to the model + pattern = r"\['(.*?)'\]" + matches = re.findall(pattern, group_location_in_file) + variable_name = matches[0] + setattr(model_copy.__dict__[variable_name], name_of_cell_array, array) + + if verbose: + print(f"Successfully loaded array of objects: {name_of_cell_array} to {variable_name}") + + + +def deserialize_class_instance(class_name, group, NCData, verbose=False): + + if verbose: + print(f"Loading class: {class_name}") + + # this function requires the class module to be imported into the namespace of this file. + # we make a custom error in case the class module is not in the list of imported classes. + # most ISSM classes are imported by from import + class ModuleError(Exception): + pass + + if class_name not in sys.modules: + raise ModuleError(str('Model requires the following class to be imported from a module: ' + class_name + ". Please add the import to read_netCDF.py in order to continue.")) + + # Instantiate the class + class_instance = eval(class_name + "()") + + # Get and assign properties + subgroups = list(group.groups.keys()) + + if len(subgroups) == 1: + # Get properties + subgroup = group[subgroups[0]] + varIDs = subgroup.variables.keys() + for varname in varIDs: + # Variable metadata + var = subgroup[varname] + + # Data + if 'char' in var.dimensions[0]: + data = var[:][...].tobytes().decode() + else: + data = var[:] + + # Some classes may have permissions, so we skip those + try: + setattr(class_instance, varname, data) + except: + pass + else: + # Not supported + pass + + if verbose: + print(f"Successfully loaded class instance {class_name} to model") + return class_instance + + + +def deserialize_data(location_of_variable_in_file, location_of_variable_in_model, variable_name, NCData, verbose = False): # as simple as navigating to the location_of_variable_in_model and setting it equal to the location_of_variable_in_file # NetCDF4 has a property called "_FillValue" that sometimes saves empty lists, so we have to catch those @@ -208,5 +411,4 @@ try: # results band-aid... - #print(str(location_of_variable_in_model + '.' + variable_name)) if str(location_of_variable_in_model + '.' + variable_name) in ['results.solutionstep', 'results.solution', 'results.resultsdakota']: pass @@ -240,9 +442,9 @@ else: # we have to convert numpy datatypes to native python types with .item() - setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, var_to_save.item()) + setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, var_to_save.item()) except: - setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]')) + setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]')) except AttributeError: - copy_variable_data_to_new_model_dict(location_of_variable_in_file, location_of_variable_in_model, NCData, verbose=verbose) + deserialize_dict(location_of_variable_in_file, location_of_variable_in_model, NCData, verbose=verbose) if verbose: @@ -250,8 +452,6 @@ -def copy_variable_data_to_new_model_dict(location_of_variable_in_file, location_of_variable_in_model, NCData, verbose = False): - # as simple as navigating to the location_of_variable_in_model and setting it equal to the location_of_variable_in_file - - # NetCDF4 has a property called "_FillValue" that sometimes saves empty lists, so we have to catch those + +def deserialize_dict(location_of_variable_in_file, location_of_variable_in_model, NCData, verbose = False): FillValue = -9223372036854775806 Index: /issm/trunk-jpl/src/m/netcdf/write_netCDF.m =================================================================== --- /issm/trunk-jpl/src/m/netcdf/write_netCDF.m (revision 27899) +++ /issm/trunk-jpl/src/m/netcdf/write_netCDF.m (revision 27900) @@ -159,5 +159,4 @@ if numel(quality_control) ~= numel(class_instance_names) disp('Error: The class instance within your model.results class is not currently supported by this application'); - disp(class(results_var.(class_instance_name))); else if verbose @@ -299,4 +298,5 @@ function create_group(location_of_child, list_of_layers, NetCDF, verbose) + %disp(list_of_layers) % location_of_child is an object % list_of_layers is a list like {'inversion', 'StressbalanceSolution','cost_functions_coefficients'} @@ -324,20 +324,103 @@ end end - % we may be dealing with an object + % sometimes objects are passed through twice so we account for that with this try/catch try - % if this line works, we're still dealing with a struct and need lower levels - if isempty(fieldnames(location_of_child)) - % do nothing - elseif isstruct(location_of_child) && any(size(location_of_child) > 1) - % we have a nested struct - copy_nested_struct(variable_name, location_of_child, group, NetCDF, verbose) + % we may be dealing with an object + % first we screen for structs + if isstruct(location_of_child) % && any(size(location_of_child) > 1) -- this is being tested + % we have a struct + copy_nested_struct(variable_name, location_of_child, group, NetCDF, verbose); + + % now for cell arrays of datastructures: + elseif logical(~isstruct(location_of_child) && iscell(location_of_child) && isobject(location_of_child{1})) + copy_cell_array_of_objects(variable_name, location_of_child, group, NetCDF, verbose); + else + if ~isobject(location_of_child) && ~isstruct(location_of_child) + % we're dealing with raw data + create_var(variable_name, location_of_child, group, NetCDF, verbose); + end end catch - % if that line doesn't work, it means we're dealing with raw data - % not a nested struct, so we can pass - create_var(variable_name, location_of_child, group, NetCDF, verbose); - end -end - + % do nothing + end +end + + + +function copy_cell_array_of_objects(variable_name, address_of_child, group, NetCDF, verbose) + % make subgroup to represent the array + [rows, cols] = size(address_of_child); + name_of_subgroup = [num2str(rows), 'x', num2str(cols), '_cell_array_of_objects']; + subgroup = netcdf.defGrp(group, name_of_subgroup); + + % save the name of the cell array + write_string_to_netcdf('name_of_cell_array', variable_name, subgroup, NetCDF, verbose); + + % save the dimensions of the cell array + create_var('rows', rows, subgroup, NetCDF, verbose); + create_var('cols', cols, subgroup, NetCDF, verbose); + + % if this is a multidimensional cell array, iterate over rows here and cols in copy_objects + if rows>1 + for row = 1:rows + % make a subgroup for each row + name_of_subgroup = ['Row_', num2str(row), '_of_', num2str(rows)]; + subgroup = netcdf.defGrp(group, name_of_subgroup); + copy_objects(address_of_child, subgroup, NetCDF, cols, verbose); + end + else + copy_objects(address_of_child, subgroup, NetCDF, cols, verbose); + end +end + + + +function copy_objects(address_of_child, group, NetCDF, cols, verbose) + for col = 1:cols + % make subgroup to contain each col of array + name_of_subgroup = ['Col_', num2str(col), '_of_', num2str(cols)]; + subgroup = netcdf.defGrp(group, name_of_subgroup); + + % get the kind of object we're working with: + if isstruct(address_of_child{col}) + % handle structs + name_raw = fields(address_of_child{col}); + variable_name = name_raw{1}; + copy_nested_struct(variable_name, address_of_child, subgroup, NetCDF, verbose); + + elseif numel(properties(address_of_child{col})) > 0 + % handle class instances + copy_class_instance(address_of_child{col}, subgroup, NetCDF, verbose); + else + disp('ERROR: Cell arrays of mixed types are not yet supported in read_netCDF!\n Deserialization will not be able to complete!') + % handle regular datastructures that are already supported + name_raw = fields(address_of_child); + variable_name = name_raw{col}; + create_var(variable_name, address_of_child, subgroup, NetCDF, verbose); + end + end +end + + +function copy_class_instance(address_of_child, subgroup, NetCDF, verbose) + % get parent class name + name = class(address_of_child); + + % save the name of the class + write_string_to_netcdf('class_is_a', name, subgroup, NetCDF, verbose); + + % make subgroup to contain properties + name_of_subgroup = ['Properties_of_', name]; + subgroup = netcdf.defGrp(subgroup, name_of_subgroup); + + % get properties + props = properties(address_of_child); + + for property = 1:length(props) + variable_name = props{property}; + create_var(variable_name, address_of_child.(variable_name), subgroup, NetCDF, verbose); + end + +end @@ -345,4 +428,6 @@ %{ This function takes a struct of structs and saves them to netcdf. + + It also works with single structs. To do this, we get the number of dimensions (substructs) of the parent struct. @@ -352,7 +437,4 @@ %} - if verbose - disp("Beginning transfer of nested MATLAB struct to the NetCDF") - end % make a new subgroup to contain all the others: group = netcdf.defGrp(group, parent_struct_name); @@ -386,5 +468,5 @@ end if verbose - fprintf('Succesfully transferred nested MATLAB struct %s to the NetCDF\n', parent_struct_name) + fprintf(["Succesfully transferred nested MATLAB struct ", parent_struct_name, " to the NetCDF\n"]) end end @@ -583,4 +665,5 @@ function write_numeric_array_to_netcdf(variable_name, address_of_child, group, NetCDF, verbose) + % get the dimensions we'll need intdim = netcdf.inqDimID(NetCDF,'int'); Index: /issm/trunk-jpl/src/m/netcdf/write_netCDF.py =================================================================== --- /issm/trunk-jpl/src/m/netcdf/write_netCDF.py (revision 27899) +++ /issm/trunk-jpl/src/m/netcdf/write_netCDF.py (revision 27900) @@ -15,10 +15,9 @@ ''' Given a md, this set of functions will perform the following: - 1. Enter each nested class of the md. - 2. View each attribute of each nested class. - 3. Compare state of attribute in the model to an empty model class. - 4. If states are identical, pass. - 5. Otherwise, create nested groups named after class structure. - 6. Create variable named after class attribute and assign value to it. + 1. View each attribute of each nested class. + 2. Compare state of attribute in the model to an empty model. + 3. If states are identical, pass. (except for np arrays which will always be saved) + 4. Otherwise, create nested groups named after class structure. + 5. Create variable named after class attribute and assign value to it. ''' @@ -26,43 +25,50 @@ def write_netCDF(md, filename: str, verbose = False): if verbose: - print('Python C2NetCDF4 v1.1.14') + print('Python C2NetCDF4 v1.2.0') else: pass ''' - md = model() class instance to be saved + md = model class instance to be saved filename = path and name to save file under - verbose = T/F muted or show log statements. Naturally muted + verbose = T/F = show or muted log statements. Naturally muted ''' - - # Create a NetCDF file to write to - NetCDF = make_NetCDF(filename, verbose) - - # Create an instance of an empty md class to compare md_var against - empty_model = model() - - # Walk through the md class and compare subclass states to empty_model - walk_through_model(md, empty_model, NetCDF, verbose) - - # in order to handle some subclasses in the results class, we have to utilize this band-aid - # there will likely be more band-aids added unless a class name library is created with all class names that might be added to a md + # this is a precaution so that data is not lost try: - # if results has meaningful data, save the name of the subclass and class instance - NetCDF.groups['results'] - results_subclasses_bandaid(md, NetCDF, verbose) - # otherwise, ignore - except KeyError: - pass + # Create a NCData file to write to + NCData = create_NetCDF(filename, verbose) - NetCDF.close() - if verbose: - print('Model successfully saved as NetCDF4') - else: pass - - -def results_subclasses_bandaid(md, NetCDF, verbose = False): + # Create an instance of an empty md class to compare md_var against + empty_model = model() + + # Walk through the md class and compare subclass states to empty_model + walk_through_model(md, empty_model, NCData, verbose) + + # in order to handle some subclasses in the results class, we have to utilize this band-aid + # there will likely be more band-aids added unless a class name library is created with all class names that might be added to a md + try: + # if results has meaningful data, save the name of the subclass and class instance + NCData.groups['results'] + results_subclasses_bandaid(md, NCData, verbose) + # otherwise, ignore + except KeyError: + pass + + NCData.close() + if verbose: + print('Model successfully saved as NetCDF4') + else: pass + + # just in case something unexpected happens + except Exception as e: + if 'NCData' in locals(): + NCData.close() + raise e + + +def results_subclasses_bandaid(md, NCData, verbose = False): # since the results class may have nested classes within it, we need to record the name of the # nested class instance variable as it appears in the md that we're trying to save quality_control = [] - # we save lists of instances to the netcdf + # we save lists of instances to the NCData solutions = [] solutionsteps = [] @@ -72,5 +78,5 @@ if verbose: print(class_instance_name) - # for each class instance in results, see which class its from and record that info in the netcdf to recreate structure later + # for each class instance in results, see which class its from and record that info in the NCData to recreate structure later # check to see if there is a solutionstep class instance if isinstance(md.results.__dict__[class_instance_name],solutionstep): @@ -89,11 +95,11 @@ if solutionsteps != []: - write_string_to_netcdf(variable_name=str('solutionstep'), address_of_child=solutionsteps, group=NetCDF.groups['results'], list=True, NetCDF=NetCDF, verbose=verbose) + serialize_string(variable_name=str('solutionstep'), address_of_child=solutionsteps, group=NCData.groups['results'], list=True, NCData=NCData, verbose=verbose) if solutions != []: - write_string_to_netcdf(variable_name=str('solution'), address_of_child=solutions, group=NetCDF.groups['results'], list=True, NetCDF=NetCDF, verbose=verbose) + serialize_string(variable_name=str('solution'), address_of_child=solutions, group=NCData.groups['results'], list=True, NCData=NCData, verbose=verbose) if resultsdakotas != []: - write_string_to_netcdf(variable_name=str('resultsdakota'), address_of_child=resultsdakotas, group=NetCDF.groups['results'], list=True, NetCDF=NetCDF, verbose=verbose) + serialize_string(variable_name=str('resultsdakota'), address_of_child=resultsdakotas, group=NCData.groups['results'], list=True, NCData=NCData, verbose=verbose) @@ -107,5 +113,5 @@ -def make_NetCDF(filename: str, verbose = False): +def create_NetCDF(filename: str, verbose = False): # If file already exists delete / rename it if os.path.exists(filename): @@ -122,29 +128,29 @@ else: # Otherwise create the file and define it globally so other functions can call it - NetCDF = Dataset(filename, 'w', format='NETCDF4') - NetCDF.history = 'Created ' + time.ctime(time.time()) - NetCDF.createDimension('Unlim', None) # unlimited dimension - NetCDF.createDimension('float', 1) # single integer dimension - NetCDF.createDimension('int', 1) # single float dimension + NCData = Dataset(filename, 'w', format='NETCDF4') + NCData.history = 'Created ' + time.ctime(time.time()) + NCData.createDimension('Unlim', None) # unlimited dimension + NCData.createDimension('float', 1) # single integer dimension + NCData.createDimension('int', 1) # single float dimension if verbose: print('Successfully created ' + filename) - return NetCDF - - -def walk_through_model(md, empty_model, NetCDF, verbose= False): - # Iterate over first layer of md_var attributes and assume this first layer is only classes + return NCData + + +def walk_through_model(md, empty_model, NCData, verbose= False): + # Iterate over first layer of md attributes and assume this first layer is only classes for group in md.__dict__.keys(): address = md.__dict__[group] empty_address = empty_model.__dict__[group] - # we need to record the layers of the md so we can save them to the netcdf file + # we need to record the layers of the md so we can save them to the NCData file layers = [group] # Recursively walk through subclasses - walk_through_subclasses(address, empty_address, layers, NetCDF, empty_model, verbose) - - -def walk_through_subclasses(address, empty_address, layers: list, NetCDF, empty_model, verbose = False): + walk_through_subclasses(address, empty_address, layers, NCData, empty_model, verbose) + + +def walk_through_subclasses(address, empty_address, layers: list, NCData, empty_model, verbose = False): # See if we have an object with keys or a not try: @@ -155,5 +161,5 @@ if is_object: # enter the subclass, see if it has nested classes and/or attributes - # then compare attributes between mds and write to netCDF if they differ + # then compare attributes between mds and write to NCData if they differ # if subclass found, walk through it and repeat for child in address.__dict__.keys(): @@ -165,13 +171,17 @@ address_of_child = address.__dict__[child] - # if the current object is a results. object and has the steps attr it needs special treatment + # if the current object is a results. object and has nonzero steps attr it needs special treatment if isinstance(address_of_child, solution) and len(address_of_child.steps) != 0: - create_group(address_of_child, current_layer, is_struct = True, NetCDF=NetCDF, verbose = verbose) + create_group(address_of_child, current_layer, is_struct = True, is_special_list = False, NCData=NCData, verbose = verbose) + + # if the current object is a list of objects (currently only filters for lists/arrays of classes) + elif isinstance(address_of_child, list) and len(address_of_child) > 0 and hasattr(address_of_child[0], '__dict__'): + create_group(address_of_child, current_layer, is_struct = False, is_special_list = True, NCData=NCData, verbose = verbose) # if the variable is an array, assume it has relevant data (this is because the next line cannot evaluate "==" with an array) elif isinstance(address_of_child, np.ndarray): - create_group(address_of_child, current_layer, is_struct = False, NetCDF=NetCDF, verbose = verbose) + create_group(address_of_child, current_layer, is_struct = False, is_special_list = False, NCData=NCData, verbose = verbose) - # see if the child exists in the empty md. If not, record it in the netcdf + # see if the child exists in the empty md. If not, record it in the NCData else: try: @@ -181,32 +191,34 @@ # if the attributes are identical we don't need to save anything if address_of_child == address_of_child_in_empty_class: - walk_through_subclasses(address_of_child, address_of_child_in_empty_class, current_layer, NetCDF, empty_model, verbose) - - # If it has been modified, record it in the NetCDF file + walk_through_subclasses(address_of_child, address_of_child_in_empty_class, current_layer, NCData, empty_model, verbose) + + # If it has been modified, record it in the NCData file else: - create_group(address_of_child, current_layer, is_struct = False, NetCDF=NetCDF, verbose = verbose) - walk_through_subclasses(address_of_child, address_of_child_in_empty_class, current_layer, NetCDF, empty_model, verbose) - - except KeyError: # record in netcdf and continue to walk thru md - walk_through_subclasses(address_of_child, empty_address, current_layer, NetCDF, empty_model, verbose) - create_group(address_of_child, current_layer, is_struct = False, NetCDF=NetCDF, verbose = verbose) + create_group(address_of_child, current_layer, is_struct = False, is_special_list = False, NCData=NCData, verbose = verbose) + walk_through_subclasses(address_of_child, address_of_child_in_empty_class, current_layer, NCData, empty_model, verbose) + + except KeyError: # record in NCData and continue to walk thru md + walk_through_subclasses(address_of_child, empty_address, current_layer, NCData, empty_model, verbose) + create_group(address_of_child, current_layer, is_struct = False, is_special_list = False, NCData=NCData, verbose = verbose) else: pass -def create_group(address_of_child, layers, is_struct = False, NetCDF=None, verbose = False): +def create_group(address_of_child, layers, is_struct = False, is_special_list = False, NCData=None, verbose = False): # Handle the first layer of the group(s) group_name = layers[0] + + # try to make a group unless the group is already made try: - group = NetCDF.createGroup(str(group_name)) + group = NCData.createGroup(str(group_name)) except: - group = NetCDF.groups[str(group_name)] + group = NCData.groups[str(group_name)] # need to check if inversion or m1qn3inversion class if group_name == 'inversion': - check_inversion_class(address_of_child, NetCDF, verbose) + check_inversion_class(address_of_child, NCData, verbose) else: pass - # if the data is nested, create nested groups to match class structure + # if the data is nested in md, create nested groups to match class structure if len(layers) > 2: for name in layers[1:-1]: @@ -214,5 +226,5 @@ group = group.createGroup(str(name)) except: - group = NetCDF.groups[str(name)] + group = NCData.groups[str(name)] else: pass @@ -220,9 +232,13 @@ if is_struct: parent_struct_name = layers[-1] - copy_nested_results_struct(parent_struct_name, address_of_child, group, NetCDF, verbose) + serialize_nested_results_struct(parent_struct_name, address_of_child, group, NCData, verbose) + + elif is_special_list: + list_name = layers[-1] + serialize_array_of_objects(list_name, address_of_child, group, NCData, verbose) else: variable_name = layers[-1] - create_var(variable_name, address_of_child, group, NetCDF, verbose) + serialize_var(variable_name, address_of_child, group, NCData, verbose) @@ -242,25 +258,25 @@ @singleton -def check_inversion_class(address_of_child, NetCDF, verbose = False): +def check_inversion_class(address_of_child, NCData, verbose = False): # need to make sure that we have the right inversion class: inversion, m1qn3inversion, taoinversion if isinstance(address_of_child, m1qn3inversion): - write_string_to_netcdf(variable_name=str('inversion_class_name'), address_of_child=str('m1qn3inversion'), group=NetCDF.groups['inversion'], NetCDF=NetCDF, verbose = verbose) + serialize_string(variable_name=str('inversion_class_name'), address_of_child=str('m1qn3inversion'), group=NCData.groups['inversion'], NCData=NCData, verbose = verbose) if verbose: print('Successfully saved inversion class instance ' + 'm1qn3inversion') elif isinstance(address_of_child, taoinversion): - write_string_to_netcdf(variable_name=str('inversion_class_name'), address_of_child=str('taoinversion'), group=NetCDF.groups['inversion'], NetCDF=NetCDF, verbose = verbose) + serialize_string(variable_name=str('inversion_class_name'), address_of_child=str('taoinversion'), group=NCData.groups['inversion'], NCData=NCData, verbose = verbose) if verbose: print('Successfully saved inversion class instance ' + 'taoinversion') else: - write_string_to_netcdf(variable_name=str('inversion_class_name'), address_of_child=str('inversion'), group=NetCDF.groups['inversion'], NetCDF=NetCDF, verbose = verbose) + serialize_string(variable_name=str('inversion_class_name'), address_of_child=str('inversion'), group=NCData.groups['inversion'], NCData=NCData, verbose = verbose) if verbose: print('Successfully saved inversion class instance ' + 'inversion') -def copy_nested_results_struct(parent_struct_name, address_of_struct, group, NetCDF, verbose = False): +def serialize_nested_results_struct(parent_struct_name, address_of_struct, group, NCData, verbose = False): ''' - This function takes a solution class instance and saves the solutionstep instances from .steps to the netcdf. - - To do this, we get the number of dimensions (substructs) of the parent struct. + This function takes a results.solution class instance and saves the solutionstep instances from .steps to the NCData. + + To do this, we get the number of dimensions (substructs) of the parent struct (list). Next, we iterate through each substruct and record the data. For each substruct, we create a subgroup of the main struct. @@ -268,5 +284,5 @@ ''' if verbose: - print("Beginning transfer of nested MATLAB struct to the NetCDF") + print("Beginning transfer of nested MATLAB struct to the NCData") # make a new subgroup to contain all the others: @@ -274,5 +290,5 @@ # make sure other systems can flag the nested struct type - write_string_to_netcdf('this_is_a_nested', 'struct', group, list=False, NetCDF=NetCDF, verbose = verbose) + serialize_string('this_is_a_nested', 'struct', group, list=False, NCData=NCData, verbose = verbose) # other systems know the name of the parent struct because it's covered by the results/qmu functions above @@ -290,16 +306,101 @@ for variable in substruct_fields: address_of_child = current_substruct.__dict__[variable] - create_var(variable, address_of_child, subgroup, NetCDF, verbose = verbose) + serialize_var(variable, address_of_child, subgroup, NCData, verbose = verbose) if verbose: - print(f'Successfully transferred struct {parent_struct_name} to the NetCDF\n') - + print(f'Successfully transferred struct {parent_struct_name} to the NCData\n') + + + + +def serialize_array_of_objects(list_name, address_of_child, group, NCData, verbose): + if verbose: + print(f"Serializing array of objects.") + + # Get the dimensions of the cell array + if len(np.shape(address_of_child)) > 1: + rows, cols = np.shape(address_of_child) + else: rows, cols = 1, np.shape(address_of_child)[0] + + # Make subgroup to represent the array + name_of_subgroup = f"{str(rows)}x{str(cols)}_cell_array_of_objects" + subgroup = group.createGroup(name_of_subgroup) + + # Save the name of the cell array + serialize_string('name_of_cell_array', list_name, subgroup, NCData, verbose) + + # Save the dimensions of the cell array + rowsID = subgroup.createVariable('rows', int, ('int',)) + colsID = subgroup.createVariable('cols', int, ('int',)) + rowsID[:] = rows + colsID[:] = cols + + + # If this is a multidimensional cell array, iterate over rows here and cols in serialize_objects + if rows > 1: + for row in range(rows): + # Make a subgroup for each row + name_of_subgroup = f"Row_{row+1}_of_{rows}" + subgroup = group.createGroup(name_of_subgroup) + serialize_objects(address_of_child, subgroup, NCData, cols, verbose) + else: + serialize_objects(address_of_child, subgroup, NCData, cols, verbose) -def create_var(variable_name, address_of_child, group, NetCDF, verbose = False): + if verbose: + print(f"Successfully serialized array of objects: {list_name}") + + +def serialize_objects(address_of_child, group, NCData, cols, verbose): + for col in range(cols): + # Make subgroup to contain each col of array + name_of_subgroup = f'Col_{col+1}_of_{cols}' + subgroup = group.createGroup(name_of_subgroup) + + # index the current item + variable = address_of_child[col] + + # Get the kind of object we're working with: + # see if it's a solution instance + if isinstance(variable, solution) and len(variable.steps) != 0: + pass + # this needs more work... + + # see if it's a general class -- assume ISSM classes all have __dict__ + elif hasattr(variable, '__dict__'): + # Handle class instances + serialize_class_instance(variable, subgroup, NCData, verbose) + else: + print('ERROR: Cell arrays of mixed types are not yet supported in read_NCData!') + print('Deserialization will not be able to complete!') + # Handle regular data structures that are already supported + serialize_var(variable_name, variable, subgroup, NCData, verbose) + + +def serialize_class_instance(instance, group, NCData, verbose): + # get parent class name: + name = instance.__class__.__name__ + + # save the name of the class + serialize_string(variable_name='class_is_a', address_of_child=name, group=group, NCData=NCData, verbose = verbose) + + # make subgroup to contain attributes + name_of_subgroup = 'Properties_of_' + name + subgroup = group.createGroup(name_of_subgroup) + + # get attributes + keys = instance.__dict__.keys() + + for name in keys: + serialize_var(name, instance.__dict__[name], subgroup, NCData, verbose) + + + + +def serialize_var(variable_name, address_of_child, group, NCData, verbose = False): # There are lots of different variable types that we need to handle from the md class # This first conditional statement will catch numpy arrays of any dimension and save them if isinstance(address_of_child, np.ndarray): - write_numpy_array_to_netcdf(variable_name, address_of_child, group, NetCDF, verbose=verbose) + serialize_numpy_array(variable_name, address_of_child, group, NCData, verbose=verbose) # check if it's an int @@ -315,9 +416,9 @@ # or a string elif isinstance(address_of_child, str): - write_string_to_netcdf(variable_name, address_of_child, group, NetCDF, verbose=verbose) + serialize_string(variable_name, address_of_child, group, NCData, verbose=verbose) #or a bool elif isinstance(address_of_child, bool) or isinstance(address_of_child, np.bool_): - # netcdf4 can't handle bool types like True/False so we convert all to int 1/0 and add an attribute named units with value 'bool' + # NetCDF can't handle bool types like True/False so we convert all to int 1/0 and add an attribute named units with value 'bool' variable = group.createVariable(variable_name, int, ('int',)) variable[:] = int(address_of_child) @@ -331,5 +432,5 @@ elif isinstance(address_of_child,list) and isinstance(address_of_child[0],str): for string in address_of_child: - write_string_to_netcdf(variable_name, string, group, list=True, NetCDF=NetCDF, verbose=verbose) + serialize_string(variable_name, string, group, list=True, NCData=NCData, verbose=verbose) # or a regular list @@ -352,16 +453,14 @@ if verbose: - print(f'Successfully transferred data from {variable_name} to the NetCDF') - - -def write_string_to_netcdf(variable_name, address_of_child, group, list=False, NetCDF=None, verbose = False): - # netcdf and strings dont get along.. we have to do it 'custom': + print(f'Successfully transferred data from {variable_name} to the NCData') + + +def serialize_string(variable_name, address_of_child, group, list=False, NCData=None, verbose = False): + # NCData and strings dont get along.. we have to do it 'custom': # if we hand it an address we need to do it this way: - if list == True: - """ - Save a list of strings to a NetCDF file. - + if list: + """ Convert a list of strings to a numpy.char_array with utf-8 encoded elements - and size rows x cols with each row the same # of cols and save to NetCDF + and size rows x cols with each row the same # of cols and save to NCData as char array. """ @@ -399,5 +498,5 @@ arr[i] = new_list[i] - # save array to netcdf file + # save array to NCData file string_var[:] = arr @@ -431,8 +530,8 @@ -def write_numpy_array_to_netcdf(variable_name, address_of_child, group, NetCDF, verbose = False): - # to make a nested array in netCDF, we have to get the dimensions of the array, - # create corresponding dimensions in the netCDF file, then we can make a variable - # in the netCDF with dimensions identical to those in the original array +def serialize_numpy_array(variable_name, address_of_child, group, NCData, verbose = False): + # to make a nested array in NCData, we have to get the dimensions of the array, + # create corresponding dimensions in the NCData file, then we can make a variable + # in the NCData with dimensions identical to those in the original array # start by getting the data type at the lowest level in the array: