Changeset 25190

Timestamp:

07/01/20 00:39:40 (5 years ago)

Author:

bdef

Message:

CHG: optimizing nc write for longer runs

File:

• issm/trunk-jpl/src/m/contrib/defleurian/netCDF/export_netCDF.py (modified) (10 diffs)

issm/trunk-jpl/src/m/contrib/defleurian/netCDF/export_netCDF.py ¶

@@ -6 +6 @@
 
 
+class ResTable:
+
+    def __init__(self):
+        self.data = []
+
+    def update(self, row):
+        if len(np.shape(row)) == 0:
+            self.data.append(row)
+        else:
+            for r in row:
+                self.data.append(r)
+
+    def finalize(self, rows):
+        print("data of len {} with {} rows".format(len(self.data) ,rows))
+        if len(self.data) > rows:
+            return np.reshape(self.data, newshape=(rows, int(len(self.data) / rows)))
+        else:
+            return np.asarray(self.data)
+
+
 def export_netCDF(md, filename):  # {{{
-    debug = False
+    #verbosity of the code, 0 is no messages, 5 is chatty
+    verbose = 0
     if path.exists(filename):
         print('File {} allready exist'.format(filename))
@@ -35 +56 @@
     dimlist = [2, md.mesh.numberofelements, md.mesh.numberofvertices, np.shape(md.mesh.elements)[1]]
     dimnames = ['DictDummy', 'EltNum', 'VertNum', 'VertPerElt']
-    print('===Creating dimensions ===')
+    if verbose > 0:
+        print('===Creating dimensions ===')
     for i, newdim in enumerate(dimlist):
         if newdim not in list(DimDict.keys()):
@@ -48 +70 @@
     # get all model classes and create respective groups
     groups = dict.keys(md.__dict__)
-    print('===Creating and populating groups===')
+    if verbose > 0:
+        print('===Creating and populating groups===')
     for group in groups:
+        if verbose > 1:
+            print('===Now treating {}==='.format(group))
         NCgroup = NCData.createGroup(str(group))
         # In each group gather the fields of the class
@@ -65 +90 @@
                 NCgroup.__setattr__('classtype', md.__dict__[group].__class__.__name__)
                 if StdList:  # this is a standard or empty list just proceed
+                    if verbose > 4:
+                        print("=££=creating var for {}.{}".format(group, field))
                     Var = md.__dict__[group].__dict__[field]
                     Var = SqueezeVar(Var)
-                    if debug:
-                        print("=££=creating var for {}.{}".format(group, field))
                     DimDict, ncvar = CreateVar(NCData, Var, field, NCgroup, DimDict)
                     if ncvar is not None:
                         FillVar(ncvar, Var)
                 else:  # this is a list of fields, specific treatment needed
-                    if debug:
+                    if verbose > 4:
                         print("list of fields happens for {}.{}".format(group, field))
                     Listsize = len(md.__dict__[group].__dict__[field])
@@ -81 +106 @@
                         Subgroup.__setattr__('classtype', md.__dict__[group].__dict__[field][0].__class__.__name__)
                         subfields = dict.keys(md.__dict__[group].__dict__[field][0].__dict__)
-                    except IndexError:
+                    except (IndexError, AttributeError):
                         Subgroup.__setattr__('classtype', md.__dict__[group].__dict__[field].__class__.__name__)
-                    except AttributeError:
                         subfields = dict.keys(md.__dict__[group].__dict__[field].__getitem__(0))
 
                     for subfield in subfields:
                         if subfield not in['errlog', 'outlog']:
-                            try:
-                                Var = md.__dict__[group].__dict__[field].__getitem__(0).__dict__[subfield]
-                            except AttributeError:
-                                Var = md.__dict__[group].__dict__[field].__getitem__(0)[subfield]
-                            StackedVar = SqueezeVar(Var)
-
-                            for listindex in range(1, Listsize):
+                            StackedVar = ResTable()
+                            for listindex in range(0, Listsize):
                                 try:
                                     Var = md.__dict__[group].__dict__[field].__getitem__(listindex).__dict__[subfield]
+                                    lastindex = listindex + 1
                                 except AttributeError:
                                     Var = md.__dict__[group].__dict__[field].__getitem__(listindex)[subfield]
                                 except KeyError:
                                     #Some fields only exist for the first step
+                                    lastindex = listindex
                                     continue
                                 Var = SqueezeVar(Var)
-                                StackedVar = np.vstack((StackedVar, Var))
-                            if debug:
+                                StackedVar.update(Var)
+                            if verbose > 4:
                                 print("=$$=creating var for {}.{}.{}".format(group, field, subfield))
-                            StackedVar = SqueezeVar(StackedVar)
+                                print(lastindex)
+                            StackedVar = SqueezeVar(StackedVar.finalize(int(lastindex)))
                             DimDict, ncvar = CreateVar(NCData, StackedVar, subfield, Subgroup, DimDict)
                             #and fill it up
@@ -117 +139 @@
                 Var = md.__dict__[group].__dict__[field]
                 Var = SqueezeVar(Var)
-                if debug:
+                if verbose > 4:
                     print("====creating var for {}.{}".format(group, field))
                 DimDict, ncvar = CreateVar(NCData, Var, field, NCgroup, DimDict)
@@ -130 +152 @@
                 Var = md.__dict__[group].__dict__[field].data
                 Var = SqueezeVar(Var)
-                if debug:
+                if verbose > 4:
                     print("=++=creating var for {}.{}".format(group, field))
                 DimDict, ncvar = CreateVar(NCData, Var, field, NCgroup, DimDict)
@@ -144 +166 @@
                         Var = md.__dict__[group].__dict__[field].__dict__[subfield]
                         Var = SqueezeVar(Var)
-                        if debug:
+                        if verbose > 4:
                             print("+==+creating var for {}.{}.{}".format(group, field, subfield))
                         DimDict, ncvar = CreateVar(NCData, Var, subfield, Subgroup, DimDict)
@@ -273 +295 @@
         except TypeError:  # type does not accept nan, get vallue of the variable
             naned = invar
+
         if UnlimIndex:
             if len(val_shape) == 0:
@@ -328 +351 @@
     return Var
 # }}}
+
+
+def grow(self, row):  # {{{
+    np.append(self.data, row)
+# }}}