Changeset 27858

Timestamp:

07/26/23 12:04:55 (20 months ago)

Author:

musselman

Message:

Modified read/write_netCDF_commit.py files to work with bool/numpy datatypes

Location:

issm/trunk/src/m/contrib/musselman

Files:

• read_netCDF_commit.py (modified) (3 diffs)
• write_netCDF_commit.py (modified) (9 diffs)

issm/trunk/src/m/contrib/musselman/read_netCDF_commit.py

@@ -33 +33 @@
         # remove masks from numpy arrays for easy conversion
         NCData.set_auto_mask(False)
+    else:
+        print('The file you entered does not exist or cannot be found in the current directory')
     
-
-    # read the contents of the groups
-
-    '''
-    this function navigates like: 
-
-    filename.groups.keys() -> filename.groups['group1'] -> 
-    filename.groups['group1'].groups.keys() -> filename.groups['group1'].groups['group1.1'] ->
-    filename.groups['group1'].groups['group1.1'].groups.keys() ->
-    filename.groups['group1'].groups['group1.1'].groups['group1.1.1'] etc. etc.
-    '''
     # continuation of band-aid for results class
     try:
@@ -63 +54 @@
 def make_results_subclasses():
     for subclass in NCData.groups['results'].variables.keys():
-        class_instance = subclass
+        class_instance = subclass + '()'
         class_instance_name = NCData.groups['results'].variables[subclass][:][...].tobytes().decode()
+        print(class_instance)
+        print(class_instance_name)
         setattr(model_copy.results, class_instance_name, eval(class_instance))
 
@@ -97 +90 @@
     # NetCDF4 has a property called "_FillValue" that sometimes saves empty lists, so we have to catch those
     FillValue = -9223372036854775806
-    
-    # but there are a couple of cases we need to compensate for, like an arrary of a single integer should just be an integer and not an array
-    if len(eval(location_of_variable_in_file))>1:
-        setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]'))
+
+    # results band-aid...
+    if str(location_of_variable_in_model + '.' + variable_name) =='results.solutionstep':
+        pass
     # handle any strings:
-    if 'char' in eval(location_of_variable_in_file + '.dimensions[0]'):
+    elif 'char' in eval(location_of_variable_in_file + '.dimensions[0]'):
         setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:][...].tobytes().decode()'))
-    # catch everything else (lists, 1-D arrays, etc.)
+    # handle ndarrays + lists
+    elif len(eval(location_of_variable_in_file + '[:]'))>1:
+        # check for bool
+        try: # there is only one datatype assigned the attribute 'units' and that is bool, so anything else will go right to except
+            if eval(location_of_variable_in_file + '.units') == 'bool':
+                setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, np.array(eval(location_of_variable_in_file + '[:]'), dtype = bool))
+            else:
+                setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]'))
+        except:
+            setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]'))
+    # catch everything else
     else:
-        # check for FillValue. use try/except because try block will only work on datatypes than like int64, float, single element lists/arrays ect and not nd-arrays/n-lists etc
-        print(eval(location_of_variable_in_file + '[:][0]'))
+        # check for FillValue. use try/except because try block will only work on datatypes like int64, float, single element lists/arrays ect and not nd-arrays/n-lists etc
         try:
+            # this try block will only work on single ints/floats/doubles and will skip to the except block for all other cases
             if FillValue == eval(location_of_variable_in_file + '[:][0]'):
                 setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, [])
+            else:
+                # we have to convert numpy datatypes to native python types with .item()
+                var_to_save = eval(location_of_variable_in_file + '[:][0]')  # note the [0] on the end
+                setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, var_to_save.item())
         except:
-            setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:][0]')) # note the [0] on the end
-        
+            setattr(eval('model_copy.' + location_of_variable_in_model), variable_name, eval(location_of_variable_in_file + '[:]'))
     print('Successfully saved ' + location_of_variable_in_model + '.' + variable_name + ' to model.')
 

issm/trunk/src/m/contrib/musselman/write_netCDF_commit.py

@@ -88 +88 @@
     
         # If so, inqure for a new name or to do delete the existing file
-        newname = input('Give a new name or "delete" to replace: ')
+        newname = input('Give a new name or input "delete" to replace: ')
 
         if newname == 'delete':
@@ -131 +131 @@
             # in the framework of an empty model. If this is the case, we move to the except statement
             try:
-                if type(child) == type(eval(adress_of_child_in_empty_class)):
+                # if the variable is an array, assume it has relevant data
+                if isinstance(eval(adress_of_child), np.ndarray):
+                    create_group(model_var, adress_of_child)
+                    walk_through_subclasses(model_var, adress_of_child, model_name)
+                elif eval(adress_of_child) == eval(adress_of_child_in_empty_class):
                     walk_through_subclasses(model_var, adress_of_child, model_name)
                 # If it has been modified, record it in the NetCDF file
@@ -172 +176 @@
     
     # check if it's an int
-    elif isinstance(eval(adress_of_child), int):
+    elif isinstance(eval(adress_of_child), int) or isinstance(eval(adress_of_child), np.integer):
         variable = group.createVariable(variable_name, int, ('int',))
         variable[:] = eval(adress_of_child)
     
     # or a float
-    elif isinstance(eval(adress_of_child), float):
+    elif isinstance(eval(adress_of_child), float) or isinstance(eval(adress_of_child), np.floating):
         variable = group.createVariable(variable_name, float, ('float',))
         variable[:] = eval(adress_of_child)
@@ -184 +188 @@
     elif isinstance(eval(adress_of_child), str):
         write_string_to_netcdf(variable_name, adress_of_child, group)
+
+    #or a bool
+    elif isinstance(eval(adress_of_child), bool) or isinstance(eval(adress_of_child), np.bool):
+        # netcdf4 can't handle bool types like True/False so we convert all to int 1/0 and add an attribute named units with value 'bool'
+        variable = group.createVariable(variable_name, int, ('int',))
+        variable[:] = int(eval(adress_of_child))
+        variable.units = "bool"
         
     # or an empty list
@@ -192 +203 @@
     elif isinstance(eval(adress_of_child),list) and isinstance(eval(adress_of_child)[0],str):
         for string in eval(adress_of_child):
-            write_string_to_netcdf(variable_name, string, group)
+            write_string_to_netcdf(variable_name, string, group, list=True)
 
     # or a regular list
     elif isinstance(eval(adress_of_child), list):
+        print('made list w/ unlim dim')
         variable = group.createVariable(variable_name, type(eval(adress_of_child)[0]), ('Unlim',))
         variable[:] = eval(adress_of_child)
@@ -204 +216 @@
             variable = group.createVariable(variable_name, type(eval(adress_of_child)), ('Unlim',))
             variable[:] = eval(adress_of_child)
+            print('Used Unlim Dim')
         except Exception as e: 
             print(e)
@@ -212 +225 @@
 
 
-
-def write_string_to_netcdf(variable_name, adress_of_child, group):
+def write_string_to_netcdf(variable_name, adress_of_child, group, list=False):
     # netcdf and strings dont get along.. we have to do it 'custom':
     # if we hand it an adress we need to do it this way:
@@ -233 +245 @@
         group.createDimension(name_of_dimension, length_of_the_string)
     except: pass
-    # now we can make a variable in this dimension:
-    string = group.createVariable(variable_name, 'S1', (name_of_dimension))
-    #finally we can write the variable:
-    string[:] = str_out
+    # this is another band-aid to the results sub classes...
+    try:
+        if list == True:
+            # now we can make a variable in this dimension:
+            string = group.createVariable(variable_name, 'S1', (name_of_dimension))
+            #finally we can write the variable:
+            string[:] = [str_out]
+        else:
+            # now we can make a variable in this dimension:
+            string = group.createVariable(variable_name, 'S1', (name_of_dimension))
+            #finally we can write the variable:
+            string[:] = str_out
+    except RuntimeError: pass
+    except Exception:
+        print(Exception)
 
 
@@ -246 +269 @@
     # start by getting the data type at the lowest level in the array:
     typeis = eval(adress_of_child + '.dtype')
-    
-    # if the array is 1D, we don't need to do anything fancy
-    # sometimes an array has just 1 element in it though, so we need to account for those cases here:
-    if len(eval(adress_of_child)) == 1:
-        if typeis is np.dtype('float64'):
-            variable = group.createVariable(variable_name, typeis, ('float',))
-            variable[:] = eval(adress_of_child)            
-        elif typeis is np.dtype('int64'):
-            variable = group.createVariable(variable_name, typeis, ('int',))
-            variable[:] = eval(adress_of_child)            
+
+    # catch boolean arrays here
+    if typeis == bool:
+        # sometimes an array has just 1 element in it, we account for those cases here:
+        if len(eval(adress_of_child)) == 1:
+            variable = group.createVariable(variable_name, int, ('int',))
+            variable[:] = int(eval(adress_of_child))
+            variable.units = "bool"
         else:
-            variable = group.createVariable(variable_name, typeis, ('Unlim',))
+            # make the dimensions
+            dimensions = []
+            for dimension in np.shape(eval(adress_of_child)):
+                dimensions.append(str('dim' + str(dimension)))
+                # if the dimension already exists we can't have a duplicate
+                try:
+                    group.createDimension(str('dim' + str(dimension)), dimension)
+                except: pass # this would mean that the dimension already exists
+    
+            # create the variable:
+            variable = group.createVariable(variable_name, int, tuple(dimensions))
+            # write the variable:
+            variable[:] = eval(adress_of_child + '.astype(int)')
+            variable.units = "bool"
+
+            
+            
+    # handle all other datatypes here
+    else:
+        # sometimes an array has just 1 element in it, we account for those cases here:
+        if len(eval(adress_of_child)) == 1:
+            if typeis is np.dtype('float64'):
+                variable = group.createVariable(variable_name, typeis, ('float',))
+                variable[:] = eval(adress_of_child)            
+            elif typeis is np.dtype('int64'):
+                variable = group.createVariable(variable_name, typeis, ('int',))
+                variable[:] = eval(adress_of_child)            
+            else:
+                print('Encountered single datatype that was not float64 or int64, saving under unlimited dimension, may cause errors.')
+                variable = group.createVariable(variable_name, typeis, ('Unlim',))
+                variable[:] = eval(adress_of_child)
+    
+        # This catches all arrays/lists:
+        else:
+            # make the dimensions
+            dimensions = []
+            for dimension in np.shape(eval(adress_of_child)):
+                dimensions.append(str('dim' + str(dimension)))
+                # if the dimension already exists we can't have a duplicate
+                try:
+                    group.createDimension(str('dim' + str(dimension)), dimension)
+                except: pass # this would mean that the dimension already exists
+    
+            # create the variable:
+            variable = group.createVariable(variable_name, typeis, tuple(dimensions))
+    
+            # write the variable:
             variable[:] = eval(adress_of_child)
-    
-    # this is the 1D case:
-    elif len(np.shape(eval(adress_of_child))) == 1: 
-        variable = group.createVariable(variable_name, typeis, ('Unlim',))
-        variable[:] = eval(adress_of_child)
-    
-    # But if the array is >1D, we do need to be fancy:
-    else:
-        # make the dimensions
-        dimensions = []
-        for dimension in np.shape(eval(adress_of_child)):
-            dimensions.append(str('dim' + str(dimension)))
-            # if the dimension already exists we can't have a duplicate
-            try:
-                group.createDimension(str('dim' + str(dimension)), dimension)
-            except: pass # this would mean that the dimension already exists
-
-        # create the variable:
-        variable = group.createVariable(variable_name, typeis, tuple(dimensions))
-
-        # write the variable:
-        variable[:] = eval(adress_of_child)