Changeset 26901 for issm/trunk-jpl/test/NightlyRun/runme.py

Timestamp:

02/25/22 00:11:37 (3 years ago)

Author:

bdef

Message:

CHG: update to nc export and checking procedure in runme

File:

• issm/trunk-jpl/test/NightlyRun/runme.py (modified) (3 diffs)

issm/trunk-jpl/test/NightlyRun/runme.py

@@ -18 +18 @@
 try:
     from arch import archread
-except: # ISSM_DIR is not on path
+except:  # ISSM_DIR is not on path
     import devpath
 
@@ -133 +133 @@
     test_ids = test_ids.difference(exclude_ids)
     # }}}
+    if procedure == 'runFromNC':
+        #That is a bamg test
+        test_ids = test_ids.difference([119, 514])
+        # that is smbGEMB format is weird for the test
+        test_ids = test_ids.difference([243, 244, 252, 253])
+        #those are amr runs where the index is missing from fieldnames
+        test_ids = test_ids.difference([462, 463, 464, 465])
+        #test247 solves for thermal and transient which makes it complex to check
+        test_ids = test_ids.difference([247])
+        #test 902 is running two models with different stepping
+        test_ids = test_ids.difference([902])
+        #I have a size issue in 517 needs investigation
+        test_ids = test_ids.difference([517])
+
     #Process Ids according to benchmarks {{{
     if benchmark == 'nightly':
@@ -200 +214 @@
                     if 'Solution' in key:
                         solvetype = split('Solution', key)[0]
+
+                #we save the results, scrap them and solve.
+                loaded_res = mdl.results
                 mdl.results = []
                 mdl = solve(mdl, solvetype)
+
+                #we loop on the field_names from the nghtly test
                 for k, fieldname in enumerate(Tmod.field_names):
                     try:
+                        #first look for indexing
                         if search(r'\d+$', fieldname):
                             index = int(search(r'\d+$', fieldname).group()) - 1
                             fieldname = fieldname[:search(r'\d+$', fieldname).start()]
-                        else:
+                        elif 'FirstStep' in fieldname:
                             index = 0
-                        #Get field from nc run
+                            fieldname = fieldname[:search('FirstStep', fieldname).start()]
+                        elif 'SecondStep' in fieldname:
+                            index = 1
+                            fieldname = fieldname[:search('SecondStep', fieldname).start()]
+                        elif 'ThirdStep' in fieldname:
+                            index = 2
+                            fieldname = fieldname[:search('ThirdStep', fieldname).start()]
+                        else:
+                            index = 0
+
+                        #Then check if the key exists in the loaded results
+                        try:
+                            reskeys = mdl.results.__dict__[solvetype + 'Solution'][index].__dict__.keys()
+                        except TypeError:
+                            # most probably a steady state so no subscripting
+                            reskeys = mdl.results.__dict__[solvetype + 'Solution'].__dict__.keys()
+                        if fieldname not in reskeys:
+                            sufixes = ["P1bubble", "P1bubbleCondensed", "LliboutryDuval", "CuffeyTemperate", "SSA", "HO", "FS", "P1xP", "P2xP",
+                                       'MINI', 'MINIcondensed', 'TaylorHood', 'XTaylorHood', 'LATaylorHood', 'CrouzeixRaviart', 'LACrouzeixRaviart']
+                            namedifs = {'Misfits': 'J',
+                                        'D': 'DamageDbar',
+                                        'F': 'DamageF',
+                                        'MaterialsRheologyB': 'MaterialsRheologyBbar',
+                                        'SedimentWaterHead': 'SedimentHead',
+                                        'EplWaterHead': 'EplHead',
+                                        'SedimentWaterHeadSubstep': 'SedimentHeadSubstep',
+                                        'EplWaterHeadSubstep': 'EplHeadSubstep',
+                                        'Volume': 'IceVolume',
+                                        'Bed': 'Base',
+                                        'SMB': 'SmbMassBalance'}
+
+                            if fieldname in namedifs.keys():
+                                #Some fields are not consistent
+                                fieldname = namedifs[fieldname]
+                            elif any([suf in fieldname for suf in sufixes]):
+                                #some test have loops that mess up with naming
+                                try:
+                                    sufix = sufixes[np.squeeze(np.where([suf in fieldname for suf in sufixes]))]
+                                except TypeError:
+                                    #probably severalmatches, we take the last one which should be the good one (Needs to be controled in the list above)
+                                    sufix = sufixes[np.squeeze(np.where([suf in fieldname for suf in sufixes]))[-1]]
+                                fieldname = fieldname[:search(sufix, fieldname).start()]
+                            elif fieldname.endswith("P") and index == 1:
+                                #we are looking for P2 but 2 as been considered as an index and so shifted by -1
+                                fieldname = fieldname[:-1]
+                            else:
+                                # could be that the index selected above is part of the name
+                                fieldname = fieldname + str(index + 1)
                         try:
                             field = mdl.results.__dict__[solvetype + 'Solution'][index].__dict__[fieldname]
+                            loaded_field = loaded_res.__dict__[solvetype + 'Solution'][index].__dict__[fieldname]
+                        except TypeError:
+                            # most probably a steady state so no subscripting
+                            try:
+                                field = mdl.results.__dict__[solvetype + 'Solution'].__dict__[fieldname]
+                                loaded_field = loaded_res.__dict__[solvetype + 'Solution'].__dict__[fieldname]
+                            except KeyError:
+                                print("WARNING: {}{} does not exist and checking will be skipped".format(fieldname, index + 1))
+                                continue
                         except KeyError:
-                            print("WARNING: {} does not exist and checking will be skipped".format(fieldname))
+                            print("WARNING: {}{} does not exist and checking will be skipped".format(fieldname, index + 1))
                             continue
-                            #Get reference from std run
+
                         ref = Tmod.field_values[k]
                         #Get tolerance
                         tolerance = Tmod.field_tolerances[k]
+                        #compute differences for the results computed from the nc file
                         error_diff = np.amax(np.abs(ref - field), axis=0) / (np.amax(np.abs(ref), axis=0) + float_info.epsilon)
                         if not np.isscalar(error_diff):
                             error_diff = error_diff[0]
 
+                        #compute the differences for the results of the nc file
+                        load_diff = np.amax(np.abs(np.squeeze(ref) - loaded_field), axis=0) / (np.amax(np.abs(np.squeeze(ref)), axis=0) + float_info.epsilon)
+                        if not np.isscalar(load_diff):
+                            load_diff = load_diff[0]
+
                         #disp test result
-                        if (np.any(error_diff > tolerance) or np.isnan(error_diff)):
-                            print(('ERROR   difference: {:7.2g} > {:7.2g} test id: {} test name: {} field: {}'.format(error_diff, tolerance, id, id_string, fieldname)))
+                        if (np.any(error_diff > tolerance) or np.isnan(error_diff)) and (np.any(load_diff > tolerance) or np.isnan(load_diff)):
+                            if abs(error_diff - load_diff) < tolerance:
+                                print(('WARNING difference: {:7.2g} > {:7.2g} test id: {} field: {}{} differs from computation but equal to saved results'.format(error_diff, tolerance, id, fieldname, index + 1)))
+                            else:
+                                print(('ERROR difference: {:7.2g} > {:7.2g} test id: {} field: {}{} is false in both loaded and computed results'.format(error_diff, tolerance, id, fieldname, index + 1)))
+                                errorcount += 1
+                                erroredtest_list.append(id)
+                        elif (np.any(error_diff > tolerance) or np.isnan(error_diff)):
+                            print(('ERROR   difference: {:7.2g} > {:7.2g} test id: {} test name: {} field: {}{}'.format(error_diff, tolerance, id, id_string, fieldname, index + 1)))
                             errorcount += 1
                             erroredtest_list.append(id)
-                        else:
-                            print(('SUCCESS difference: {:7.2g} < {:7.2g} test id: {} test name: {} field: {}'.format(error_diff, tolerance, id, id_string, fieldname)))
+                        elif (np.any(load_diff > tolerance) or np.isnan(load_diff)):
+                            print(('SAVEERROR difference: {:7.2g} > {:7.2g} test id: {} test name: {} saved result : {}{}'.format(load_diff, tolerance, id, id_string, fieldname, index + 1)))
+                            errorcount += 1
+                            erroredtest_list.append(id)
+                        else:
+                            print(('SUCCESS difference: {:7.2g} < {:7.2g} test id: {} test name: {} field: {}{}'.format(error_diff, tolerance, id, id_string, fieldname, index + 1)))
+                        #disp only if errors for the results
 
                     except Exception as message: