Index: /issm/trunk/src/m/contrib/musselman/write_netCDF_commit.py =================================================================== --- /issm/trunk/src/m/contrib/musselman/write_netCDF_commit.py (revision 27880) +++ /issm/trunk/src/m/contrib/musselman/write_netCDF_commit.py (revision 27881) @@ -5,5 +5,5 @@ import numpy.ma as ma import time -from os import path, remove +import os from model import * from results import * @@ -24,5 +24,5 @@ def write_netCDF(model_var, model_name: str, filename: str): - print('Python C2NetCDF4 v1.1.12') + print('Python C2NetCDF4 v1.1.14') ''' model_var = class object to be saved @@ -62,21 +62,44 @@ # nested class instance variable as it appears in the model that we're trying to save quality_control = [] + + # we save lists of instances to the netcdf + solutions = [] + solutionsteps = [] + resultsdakotas = [] + for class_instance_name in model_var.results.__dict__.keys(): + print(class_instance_name) # for each class instance in results, see which class its from and record that info in the netcdf to recreate structure later # check to see if there is a solutionstep class instance if isinstance(model_var.results.__dict__[class_instance_name],solutionstep): quality_control.append(1) - write_string_to_netcdf(variable_name=str('solutionstep'), adress_of_child=str(class_instance_name), group=NetCDF.groups['results']) + solutionsteps.append(class_instance_name) + # check to see if there is a solution class instance if isinstance(model_var.results.__dict__[class_instance_name],solution): quality_control.append(1) - write_string_to_netcdf(variable_name=str('solution'), adress_of_child=str(class_instance_name), group=NetCDF.groups['results']) + solutions.append(class_instance_name) + # check to see if there is a resultsdakota class instance if isinstance(model_var.results.__dict__[class_instance_name],resultsdakota): quality_control.append(1) - write_string_to_netcdf(variable_name=str('resultsdakota'), adress_of_child=str(class_instance_name), group=NetCDF.groups['results']) + resultsdakotas.append(class_instance_name) + + if solutionsteps != []: + write_string_to_netcdf(variable_name=str('solutionstep'), address_of_child=solutionsteps, group=NetCDF.groups['results'], list=True) + + if solutions != []: + write_string_to_netcdf(variable_name=str('solution'), address_of_child=solutions, group=NetCDF.groups['results'], list=True) + + if resultsdakotas != []: + write_string_to_netcdf(variable_name=str('resultsdakota'), address_of_child=resultsdakotas, group=NetCDF.groups['results'], list=True) + + if len(quality_control) != len(model_var.results.__dict__.keys()): print('Error: The class instance within your model.results class is not currently supported by this application') print(type(model_var.results.__dict__[class_instance_name])) + print(solutions) + print(solutionsteps) + print(resultsdakotas) else: print('The results class was successfully stored on disk') @@ -86,5 +109,5 @@ def make_NetCDF(filename: str): # If file already exists delete / rename it - if path.exists(filename): + if os.path.exists(filename): print('File {} allready exist'.format(filename)) @@ -93,5 +116,5 @@ if newname == 'delete': - remove(filename) + os.remove(filename) else: print(('New file name is {}'.format(newname))) @@ -113,11 +136,11 @@ # Iterate over first layer of model_var attributes and assume this first layer is only classes for group in model_var.__dict__.keys(): - adress = str(model_name + '.' + str(group)) + address = str(model_name + '.' + str(group)) # Recursively walk through subclasses - walk_through_subclasses(model_var, adress, model_name) - - - -def walk_through_subclasses(model_var, adress: str, model_name: str): + walk_through_subclasses(model_var, address, model_name) + + + +def walk_through_subclasses(model_var, address: str, model_name: str): # Iterate over each subclass' attributes # Use try/except since it's either a class w/ attributes or it's not, no unknown exceptions @@ -126,27 +149,34 @@ # then compare attributes between models and write to netCDF if they differ # if subclass found, walk through it and repeat - for child in eval(adress + '.__dict__.keys()'): + for child in eval(address + '.__dict__.keys()'): # make a string variable so we can send thru this func again - adress_of_child = str(adress + '.' + str(child)) + address_of_child = str(address + '.' + str(child)) # If the attribute is unchanged, move onto the next layer - adress_of_child_in_empty_class = 'empty_model' + adress_of_child.removeprefix(str(model_name)) + address_of_child_in_empty_class = 'empty_model' + address_of_child.removeprefix(str(model_name)) # using try/except here because sometimes a model can have class instances/attributes that are not # in the framework of an empty model. If this is the case, we move to the except statement try: + # if the current object is a results. object and has the steps attr it needs special treatment + if isinstance(eval(address_of_child), solution) and len(eval(address_of_child + '.steps')) != 0: + create_group(model_var, address_of_child, is_struct = True) # if the variable is an array, assume it has relevant data (this is because the next line cannot evaluate "==" with an array) - if isinstance(eval(adress_of_child), np.ndarray): - create_group(model_var, adress_of_child) + elif isinstance(eval(address_of_child), np.ndarray): + create_group(model_var, address_of_child) # if the attributes are identical we don't need to save anything - elif eval(adress_of_child) == eval(adress_of_child_in_empty_class): - walk_through_subclasses(model_var, adress_of_child, model_name) + elif eval(address_of_child) == eval(address_of_child_in_empty_class): + walk_through_subclasses(model_var, address_of_child, model_name) # If it has been modified, record it in the NetCDF file else: - create_group(model_var, adress_of_child) - walk_through_subclasses(model_var, adress_of_child, model_name) + create_group(model_var, address_of_child) + walk_through_subclasses(model_var, address_of_child, model_name) # AttributeError since the empty_model wouldn't have the same attribute as our model except AttributeError: # THE ORDER OF THESE LINES IS CRITICAL - walk_through_subclasses(model_var, adress_of_child, model_name) - create_group(model_var, adress_of_child) + try: + walk_through_subclasses(model_var, address_of_child, model_name) + create_group(model_var, address_of_child) + except: + pass + except Exception as e: print(e) except AttributeError: pass except Exception as e: print(e) @@ -154,7 +184,7 @@ -def create_group(model_var, adress_of_child): - # start by splitting the adress_of_child into its components - levels_of_class = adress_of_child.split('.') +def create_group(model_var, address_of_child, is_struct = False): + # start by splitting the address_of_child into its components + levels_of_class = address_of_child.split('.') # Handle the first layer of the group(s) @@ -174,6 +204,10 @@ # Lastly, handle the variable(s) - variable_name = levels_of_class[-1] - create_var(variable_name, adress_of_child, group) + if is_struct: + parent_struct_name = address_of_child.split('.')[-1] + copy_nested_results_struct(parent_struct_name, address_of_child, group) + else: + variable_name = levels_of_class[-1] + create_var(variable_name, address_of_child, group) @@ -181,111 +215,205 @@ # need to make sure that we have the right inversion class: inversion, m1qn3inversion, taoinversion if isinstance(model_var.__dict__['inversion'], m1qn3inversion): - write_string_to_netcdf(variable_name=str('inversion_class_name'), adress_of_child=str('m1qn3inversion'), group=NetCDF.groups['inversion']) + write_string_to_netcdf(variable_name=str('inversion_class_name'), address_of_child=str('m1qn3inversion'), group=NetCDF.groups['inversion']) print('Successfully saved inversion class instance ' + 'm1qn3inversion') elif isinstance(model_var.__dict__['inversion'], taoinversion): - write_string_to_netcdf(variable_name=str('inversion_class_name'), adress_of_child=str('taoinversion'), group=NetCDF.groups['inversion']) + write_string_to_netcdf(variable_name=str('inversion_class_name'), address_of_child=str('taoinversion'), group=NetCDF.groups['inversion']) print('Successfully saved inversion class instance ' + 'taoinversion') else: - write_string_to_netcdf(variable_name=str('inversion_class_name'), adress_of_child=str('inversion'), group=NetCDF.groups['inversion']) + write_string_to_netcdf(variable_name=str('inversion_class_name'), address_of_child=str('inversion'), group=NetCDF.groups['inversion']) print('Successfully saved inversion class instance ' + 'inversion') + + + + +def copy_nested_results_struct(parent_struct_name, address_of_struct, group): + ''' + This function takes a solution class instance and saves the solutionstep instances from .steps to the netcdf. + + To do this, we get the number of dimensions (substructs) of the parent struct. + Next, we iterate through each substruct and record the data. + For each substruct, we create a subgroup of the main struct. + For each variable, we create dimensions that are assigned to each subgroup uniquely. + ''' + print("Beginning transfer of nested MATLAB struct to the NetCDF") + + # make a new subgroup to contain all the others: + group = group.createGroup(str(parent_struct_name)) + + # make sure other systems can flag the nested struct type + write_string_to_netcdf('this_is_a_nested', 'struct', group, list=False) + + # other systems know the name of the parent struct because it's covered by the results/qmu functions above + address_of_struct_string = address_of_struct + address_of_struct = eval(address_of_struct) + + no_of_dims = len(address_of_struct) + for substruct in range(0, no_of_dims): + # we start by making subgroups with nice names like "TransientSolution_substruct_44" + name_of_subgroup = '1x' + str(substruct) + subgroup = group.createGroup(str(name_of_subgroup)) + + # do some housekeeping to keep track of the current layer + current_substruct = address_of_struct[substruct] + current_substruct_string = address_of_struct_string + '[' + str(substruct) + ']' + substruct_fields = current_substruct.__dict__.keys() + + # now we need to iterate over each variable of the nested struct and save it to this new subgroup + for variable in substruct_fields: + address_of_child = current_substruct.__dict__[variable] + address_of_child_string = current_substruct_string + '.' + str(variable) + create_var(variable, address_of_child_string, subgroup) + + print(f'Successfully transferred struct {parent_struct_name} to the NetCDF\n') + -def create_var(variable_name, adress_of_child, group): + +def create_var(variable_name, address_of_child, group): # There are lots of different variable types that we need to handle from the model class # This first conditional statement will catch numpy arrays of any dimension and save them - if isinstance(eval(adress_of_child), np.ndarray): - write_numpy_array_to_netcdf(variable_name, adress_of_child, group) + if isinstance(eval(address_of_child), np.ndarray): + write_numpy_array_to_netcdf(variable_name, address_of_child, group) # check if it's an int - elif isinstance(eval(adress_of_child), int) or isinstance(eval(adress_of_child), np.integer): + elif isinstance(eval(address_of_child), int) or isinstance(eval(address_of_child), np.integer): variable = group.createVariable(variable_name, int, ('int',)) - variable[:] = eval(adress_of_child) + variable[:] = eval(address_of_child) # or a float - elif isinstance(eval(adress_of_child), float) or isinstance(eval(adress_of_child), np.floating): + elif isinstance(eval(address_of_child), float) or isinstance(eval(address_of_child), np.floating): variable = group.createVariable(variable_name, float, ('float',)) - variable[:] = eval(adress_of_child) + variable[:] = eval(address_of_child) # or a string - elif isinstance(eval(adress_of_child), str): - write_string_to_netcdf(variable_name, adress_of_child, group) + elif isinstance(eval(address_of_child), str): + write_string_to_netcdf(variable_name, address_of_child, group) #or a bool - elif isinstance(eval(adress_of_child), bool) or isinstance(eval(adress_of_child), np.bool): + elif isinstance(eval(address_of_child), bool) or isinstance(eval(address_of_child), np.bool_): # netcdf4 can't handle bool types like True/False so we convert all to int 1/0 and add an attribute named units with value 'bool' variable = group.createVariable(variable_name, int, ('int',)) - variable[:] = int(eval(adress_of_child)) + variable[:] = int(eval(address_of_child)) variable.units = "bool" # or an empty list - elif isinstance(eval(adress_of_child), list) and len(eval(adress_of_child))==0: + elif isinstance(eval(address_of_child), list) and len(eval(address_of_child))==0: variable = group.createVariable(variable_name, int, ('int',)) # or a list of strings -- this needs work as it can only handle a list of 1 string - elif isinstance(eval(adress_of_child),list) and isinstance(eval(adress_of_child)[0],str): - for string in eval(adress_of_child): + elif isinstance(eval(address_of_child),list) and isinstance(eval(address_of_child)[0],str): + for string in eval(address_of_child): write_string_to_netcdf(variable_name, string, group, list=True) # or a regular list - elif isinstance(eval(adress_of_child), list): + elif isinstance(eval(address_of_child), list): print('made list w/ unlim dim') - variable = group.createVariable(variable_name, type(eval(adress_of_child)[0]), ('Unlim',)) - variable[:] = eval(adress_of_child) + variable = group.createVariable(variable_name, type(eval(address_of_child)[0]), ('Unlim',)) + variable[:] = eval(address_of_child) # anything else... (will likely need to add more cases; ie dict) else: try: - variable = group.createVariable(variable_name, type(eval(adress_of_child)), ('Unlim',)) - variable[:] = eval(adress_of_child) + variable = group.createVariable(variable_name, type(eval(address_of_child)), ('Unlim',)) + variable[:] = eval(address_of_child) print('Used Unlim Dim') - except Exception as e: + except Exception as e: + print(f'There was error with {variable_name} in {address_of_child}') + print("The error message is:") print(e) - print('Datatype given: ' + str(type(eval(adress_of_child)))) - - print('Successfully transferred data from ' + adress_of_child + ' to the NetCDF') - - - -def write_string_to_netcdf(variable_name, adress_of_child, group, list=False): + print('Datatype given: ' + str(type(eval(address_of_child)))) + + print('Successfully transferred data from ' + address_of_child + ' to the NetCDF') + + + +def write_string_to_netcdf(variable_name, address_of_child, group, list=False): # netcdf and strings dont get along.. we have to do it 'custom': - # if we hand it an adress we need to do it this way: - try: - the_string_to_save = eval(adress_of_child) - length_of_the_string = len(the_string_to_save) - numpy_datatype = 'S' + str(length_of_the_string) - str_out = netCDF4.stringtochar(np.array([the_string_to_save], dtype=numpy_datatype)) - #otherwise we need to treat it like a string: - except: - the_string_to_save = adress_of_child - length_of_the_string = len(the_string_to_save) - numpy_datatype = 'S' + str(length_of_the_string) - str_out = netCDF4.stringtochar(np.array([the_string_to_save], dtype=numpy_datatype)) - - # we'll need to make a new dimension for the string if it doesn't already exist - name_of_dimension = 'char' + str(length_of_the_string) - try: - group.createDimension(name_of_dimension, length_of_the_string) - except: pass - # this is another band-aid to the results sub classes... - try: - if list == True: - # now we can make a variable in this dimension: - string = group.createVariable(variable_name, 'S1', (name_of_dimension)) - #finally we can write the variable: - string[:] = [str_out] - else: + # if we hand it an address we need to do it this way: + if list == True: + """ + Save a list of strings to a NetCDF file. + + Convert a list of strings to a numpy.char_array with utf-8 encoded elements + and size rows x cols with each row the same # of cols and save to NetCDF + as char array. + """ + try: + strings = address_of_child + # get dims of array to save + rows = len(strings) + cols = len(max(strings, key = len)) + + # Define dimensions for the strings + rows_name = 'rows' + str(rows) + cols_name = 'cols' + str(cols) + try: + group.createDimension(rows_name, rows) + except: pass + + try: + group.createDimension(cols_name, cols) + except: pass + + # Create a variable to store the strings + string_var = group.createVariable(str(variable_name), 'S1', (rows_name, cols_name)) + + # break the list into a list of lists of words with the same length as the longest word: + # make words same sizes by adding spaces + modded_strings = [word + ' ' * (len(max(strings, key=len)) - len(word)) for word in strings] + # encoded words into list of encoded lists + new_list = [[s.encode('utf-8') for s in word] for word in modded_strings] + + # make numpy char array with dims rows x cols + arr = np.chararray((rows, cols)) + + # fill array with list of encoded lists + for i in range(len(new_list)): + arr[i] = new_list[i] + + # save array to netcdf file + string_var[:] = arr + + print(f'Saved {len(modded_strings)} strings to {variable_name}') + + except Exception as e: + print(f'Error: {e}') + + else: + try: + the_string_to_save = eval(address_of_child) + length_of_the_string = len(the_string_to_save) + numpy_datatype = 'S' + str(length_of_the_string) + str_out = netCDF4.stringtochar(np.array([the_string_to_save], dtype=numpy_datatype)) + #otherwise we need to treat it like a string: + except: + the_string_to_save = address_of_child + length_of_the_string = len(the_string_to_save) + numpy_datatype = 'S' + str(length_of_the_string) + str_out = netCDF4.stringtochar(np.array([the_string_to_save], dtype=numpy_datatype)) + + # we'll need to make a new dimension for the string if it doesn't already exist + name_of_dimension = 'char' + str(length_of_the_string) + try: + group.createDimension(name_of_dimension, length_of_the_string) + except: pass + # this is another band-aid to the results sub classes... + try: # now we can make a variable in this dimension: string = group.createVariable(variable_name, 'S1', (name_of_dimension)) #finally we can write the variable: string[:] = str_out - except RuntimeError: pass - except Exception: - print(Exception) - - -def write_numpy_array_to_netcdf(variable_name, adress_of_child, group): + except RuntimeError: pass + except Exception: + print(Exception) + + + + + +def write_numpy_array_to_netcdf(variable_name, address_of_child, group): # to make a nested array in netCDF, we have to get the dimensions of the array, # create corresponding dimensions in the netCDF file, then we can make a variable @@ -293,17 +421,17 @@ # start by getting the data type at the lowest level in the array: - typeis = eval(adress_of_child + '.dtype') + typeis = eval(address_of_child + '.dtype') # catch boolean arrays here if typeis == bool: # sometimes an array has just 1 element in it, we account for those cases here: - if len(eval(adress_of_child)) == 1: + if len(eval(address_of_child)) == 1: variable = group.createVariable(variable_name, int, ('int',)) - variable[:] = int(eval(adress_of_child)) + variable[:] = int(eval(address_of_child)) variable.units = "bool" else: # make the dimensions dimensions = [] - for dimension in np.shape(eval(adress_of_child)): + for dimension in np.shape(eval(address_of_child)): dimensions.append(str('dim' + str(dimension))) # if the dimension already exists we can't have a duplicate @@ -315,23 +443,21 @@ variable = group.createVariable(variable_name, int, tuple(dimensions)) # write the variable: - variable[:] = eval(adress_of_child + '.astype(int)') + variable[:] = eval(address_of_child + '.astype(int)') variable.units = "bool" - - # handle all other datatypes here else: # sometimes an array has just 1 element in it, we account for those cases here: - if len(eval(adress_of_child)) == 1: + if len(eval(address_of_child)) == 1: if typeis is np.dtype('float64'): variable = group.createVariable(variable_name, typeis, ('float',)) - variable[:] = eval(adress_of_child) + variable[:] = eval(address_of_child + '[0]') elif typeis is np.dtype('int64'): variable = group.createVariable(variable_name, typeis, ('int',)) - variable[:] = eval(adress_of_child) + variable[:] = eval(address_of_child + '[0]') else: print('Encountered single datatype that was not float64 or int64, saving under unlimited dimension, may cause errors.') variable = group.createVariable(variable_name, typeis, ('Unlim',)) - variable[:] = eval(adress_of_child) + variable[:] = eval(address_of_child + '[0]') # This catches all arrays/lists: @@ -339,5 +465,5 @@ # make the dimensions dimensions = [] - for dimension in np.shape(eval(adress_of_child)): + for dimension in np.shape(eval(address_of_child)): dimensions.append(str('dim' + str(dimension))) # if the dimension already exists we can't have a duplicate @@ -350,36 +476,14 @@ # write the variable: - variable[:] = eval(adress_of_child) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + variable[:] = eval(address_of_child) + + + + + + + + + + +