Index: /issm/trunk/src/m/contrib/musselman/read_netCDF.py =================================================================== --- /issm/trunk/src/m/contrib/musselman/read_netCDF.py (revision 27898) +++ /issm/trunk/src/m/contrib/musselman/read_netCDF.py (revision 27899) @@ -28,5 +28,10 @@ def read_netCDF(filename, verbose = False): if verbose: - print('NetCDF42C v1.1.13') + print('NetCDF42C v1.2.0') + + ''' + filename = path and name to save file under + verbose = T/F = show or muted log statements. Naturally muted + ''' # this is a precaution so that data is not lost @@ -43,9 +48,10 @@ NCData.set_auto_mask(False) else: - print('The file you entered does not exist or cannot be found in the current directory') - return print() + return 'The file you entered does not exist or cannot be found in the current directory' - # continuation of band-aid for results class + # in order to handle some subclasses in the results class, we have to utilize this band-aid + # there will likely be more band-aids added unless a class name library is created with all class names that might be added to a md try: + # if results has meaningful data, save the name of the subclass and class instance NCData.groups['results'] make_results_subclasses(NCData, verbose) @@ -76,8 +82,9 @@ return model_copy - except Error: - NCData.close() - return Error - + # just in case something unexpected happens + except Exception as e: + if 'NCData' in locals(): + NCData.close() + raise e def make_results_subclasses(NCData, verbose = False): @@ -123,7 +130,6 @@ # first, we enter the group by: filename.groups['group_name'] # second we search the current level for variables: filename.groups['group_name'].variables.keys() - # at this step we check for multidimensional structure arrays and filter them out + # at this step we check for multidimensional structure arrays/ arrays of objects and filter them out # third we get nested group keys by: filename.groups['group_name'].groups.keys() - # if a variables exists, copy the data to the model framework by calling copy function # if a nested groups exist, repeat all @@ -170,4 +176,53 @@ + +''' + MATLAB has Multidimensional Structure Arrays in 2 known classes: results and qmu. + The python classes results.py and qmu.py emulate this MATLAB object in their own + unique ways. The functions in this script will assign data to either of these + classes such that the final structure is compatible with its parent class. +''' + +def deserialize_nested_results_struct(group_location_in_file, name_of_struct, NCData, verbose = False): + ''' + A common multidimensional array is the 1xn md.results.TransientSolution object. + + The way that this object emulates the MATLAB mutli-dim. struct. array is with + the solution().steps attr. which is a list of solutionstep() instances + The process to recreate is as follows: + 1. Get instance of solution() with solution variable (the instance is made in make_results_subclasses) + 2. For each subgroup, create a solutionstep() class instance + 2a. Populate the instance with the key:value pairs + 2b. Append the instance to the solution().steps list + ''' + # step 1 + class_instance_name = name_of_struct + + # for some reason steps is not already a list + setattr(model_copy.results.__dict__[class_instance_name], 'steps', list()) + + steps = model_copy.results.__dict__[class_instance_name].steps + + # step 2 + layer = 1 + for subgroup in eval(group_location_in_file + ".groups.keys()"): + solutionstep_instance = solutionstep() + # step 2a + subgroup_location_in_file = group_location_in_file + ".groups['" + subgroup + "']" + for key in eval(subgroup_location_in_file + ".variables.keys()"): + value = eval(subgroup_location_in_file + ".variables['" + str(key) + "'][:]") + setattr(solutionstep_instance, key, value) + # step 2b + steps.append(solutionstep_instance) + if verbose: + print('Succesfully loaded layer ' + str(layer) + ' to results.' + str(class_instance_name) + ' struct.') + else: pass + layer += 1 + + if verbose: + print('Successfully recreated results structure ' + str(class_instance_name)) + + + def deserialize_array_of_objects(group_location_in_file, model_copy, NCData, verbose): ''' @@ -193,4 +248,7 @@ original structure when the model was saved ''' + + if verbose: + print(f"Loading array of objects.") # get the name_of_cell_array, rows and cols vars @@ -231,5 +289,5 @@ if "class_is_a" in current_col_vars: class_name = subgroups[str(col)].variables['class_is_a'][:][...].tobytes().decode() - col_data = deserialize_class(class_name, columns.groups[str(col)], NCData, verbose) + col_data = deserialize_class_instance(class_name, columns.groups[str(col)], NCData, verbose) is_object = True elif "this_is_a_nested" in current_col_vars: @@ -267,5 +325,5 @@ if "class_is_a" in current_col_vars: class_name = subgroups[str(col)].variables['class_is_a'][:][...].tobytes().decode() - col_data = deserialize_class(class_name, subgroups[str(col)], NCData, verbose) + col_data = deserialize_class_instance(class_name, subgroups[str(col)], NCData, verbose) is_object = True elif "this_is_a_nested" in current_col_vars: @@ -288,5 +346,4 @@ # finally, add the attribute to the model - # borrow some code from above: pattern = r"\['(.*?)'\]" matches = re.findall(pattern, group_location_in_file) @@ -294,8 +351,16 @@ setattr(model_copy.__dict__[variable_name], name_of_cell_array, array) - - -def deserialize_class(class_name, group, NCData, verbose=False): - + if verbose: + print(f"Successfully loaded array of objects: {name_of_cell_array} to {variable_name}") + + + +def deserialize_class_instance(class_name, group, NCData, verbose=False): + + if verbose: + print(f"Loading class: {class_name}") + + # this function requires the class module to be imported into the namespace of this file. + # we make a custom error in case the class module is not in the list of imported classes. # most ISSM classes are imported by from import class ModuleError(Exception): @@ -333,54 +398,9 @@ # Not supported pass - + + if verbose: + print(f"Successfully loaded class instance {class_name} to model") return class_instance - - -''' - MATLAB has Multidimensional Structure Arrays in 2 known classes: results and qmu. - The python classes results.py and qmu.py emulate this MATLAB object in their own - unique ways. The functions in this script will assign data to either of these - classes such that the final structure is compatible with its parent class. -''' - -def deserialize_nested_results_struct(group_location_in_file, name_of_struct, NCData, verbose = False): - ''' - A common multidimensional array is the 1xn md.results.TransientSolution object. - - The way that this object emulates the MATLAB mutli-dim. struct. array is with - the solution().steps attr. which is a list of solutionstep() instances - The process to recreate is as follows: - 1. Get instance of solution() with solution variable (the instance is made in make_results_subclasses) - 2. For each subgroup, create a solutionstep() class instance - 2a. Populate the instance with the key:value pairs - 2b. Append the instance to the solution().steps list - ''' - # step 1 - class_instance_name = name_of_struct - - # for some reason steps is not already a list - setattr(model_copy.results.__dict__[class_instance_name], 'steps', list()) - - steps = model_copy.results.__dict__[class_instance_name].steps - - # step 2 - layer = 1 - for subgroup in eval(group_location_in_file + ".groups.keys()"): - solutionstep_instance = solutionstep() - # step 2a - subgroup_location_in_file = group_location_in_file + ".groups['" + subgroup + "']" - for key in eval(subgroup_location_in_file + ".variables.keys()"): - value = eval(subgroup_location_in_file + ".variables['" + str(key) + "'][:]") - setattr(solutionstep_instance, key, value) - # step 2b - steps.append(solutionstep_instance) - if verbose: - print('Succesfully loaded layer ' + str(layer) + ' to results.' + str(class_instance_name) + ' struct.') - else: pass - layer += 1 - - if verbose: - print('Successfully recreated results structure ' + str(class_instance_name)) @@ -434,7 +454,4 @@ def deserialize_dict(location_of_variable_in_file, location_of_variable_in_model, NCData, verbose = False): - # as simple as navigating to the location_of_variable_in_model and setting it equal to the location_of_variable_in_file - - # NetCDF4 has a property called "_FillValue" that sometimes saves empty lists, so we have to catch those FillValue = -9223372036854775806 Index: /issm/trunk/src/m/contrib/musselman/write_netCDF.py =================================================================== --- /issm/trunk/src/m/contrib/musselman/write_netCDF.py (revision 27898) +++ /issm/trunk/src/m/contrib/musselman/write_netCDF.py (revision 27899) @@ -1,5 +1,5 @@ # imports -import NCData4 -from NCData4 import Dataset +import netCDF4 +from netCDF4 import Dataset import numpy as np import numpy.ma as ma @@ -23,17 +23,17 @@ -def write_NCData(md, filename: str, verbose = False): +def write_netCDF(md, filename: str, verbose = False): if verbose: - print('Python C2NCData4 v1.1.14') + print('Python C2NetCDF4 v1.2.0') else: pass ''' - md = model() class instance to be saved + md = model class instance to be saved filename = path and name to save file under - verbose = T/F muted or show log statements. Naturally muted + verbose = T/F = show or muted log statements. Naturally muted ''' # this is a precaution so that data is not lost try: # Create a NCData file to write to - NCData = create_NCData(filename, verbose) + NCData = create_NetCDF(filename, verbose) # Create an instance of an empty md class to compare md_var against @@ -55,10 +55,12 @@ NCData.close() if verbose: - print('Model successfully saved as NCData4') + print('Model successfully saved as NetCDF4') else: pass - - except Error: - NCData.close() - return Error + + # just in case something unexpected happens + except Exception as e: + if 'NCData' in locals(): + NCData.close() + raise e @@ -111,5 +113,5 @@ -def create_NCData(filename: str, verbose = False): +def create_NetCDF(filename: str, verbose = False): # If file already exists delete / rename it if os.path.exists(filename): @@ -126,5 +128,5 @@ else: # Otherwise create the file and define it globally so other functions can call it - NCData = Dataset(filename, 'w', format='NCData4') + NCData = Dataset(filename, 'w', format='NETCDF4') NCData.history = 'Created ' + time.ctime(time.time()) NCData.createDimension('Unlim', None) # unlimited dimension @@ -272,9 +274,9 @@ -def (parent_struct_name, address_of_struct, group, NCData, verbose = False): +def serialize_nested_results_struct(parent_struct_name, address_of_struct, group, NCData, verbose = False): ''' - This function takes a solution class instance and saves the solutionstep instances from .steps to the NCData. - - To do this, we get the number of dimensions (substructs) of the parent struct. + This function takes a results.solution class instance and saves the solutionstep instances from .steps to the NCData. + + To do this, we get the number of dimensions (substructs) of the parent struct (list). Next, we iterate through each substruct and record the data. For each substruct, we create a subgroup of the main struct. @@ -313,4 +315,7 @@ def serialize_array_of_objects(list_name, address_of_child, group, NCData, verbose): + if verbose: + print(f"Serializing array of objects.") + # Get the dimensions of the cell array if len(np.shape(address_of_child)) > 1: @@ -341,5 +346,7 @@ else: serialize_objects(address_of_child, subgroup, NCData, cols, verbose) - + + if verbose: + print(f"Successfully serialized array of objects: {list_name}") @@ -358,7 +365,4 @@ pass # this needs more work... - #name_raw = list(address_of_child[col - 1].keys())[0] - #variable_name = name_raw - #serialize_nested_struct(variable_name, variable, subgroup, NCData, verbose) # see if it's a general class -- assume ISSM classes all have __dict__ @@ -373,6 +377,4 @@ - - def serialize_class_instance(instance, group, NCData, verbose): # get parent class name: @@ -418,5 +420,5 @@ #or a bool elif isinstance(address_of_child, bool) or isinstance(address_of_child, np.bool_): - # NCData4 can't handle bool types like True/False so we convert all to int 1/0 and add an attribute named units with value 'bool' + # NetCDF can't handle bool types like True/False so we convert all to int 1/0 and add an attribute named units with value 'bool' variable = group.createVariable(variable_name, int, ('int',)) variable[:] = int(address_of_child) @@ -457,8 +459,6 @@ # NCData and strings dont get along.. we have to do it 'custom': # if we hand it an address we need to do it this way: - if list == True: - """ - Save a list of strings to a NCData file. - + if list: + """ Convert a list of strings to a numpy.char_array with utf-8 encoded elements and size rows x cols with each row the same # of cols and save to NCData @@ -511,5 +511,5 @@ length_of_the_string = len(the_string_to_save) numpy_datatype = 'S' + str(length_of_the_string) - str_out = NCData4.stringtochar(np.array([the_string_to_save], dtype=numpy_datatype)) + str_out = netCDF4.stringtochar(np.array([the_string_to_save], dtype=numpy_datatype)) # we'll need to make a new dimension for the string if it doesn't already exist