verif: Use new check_result function

pulp-platform · Feb 13, 2024 · 92a659b · 92a659b
1 parent 8dea6b8
commit 92a659b
Show file tree

Hide file tree

Showing 10 changed files with 47 additions and 62 deletions.
diff --git a/sw/blas/axpy/verify.py b/sw/blas/axpy/verify.py
@@ -13,7 +13,7 @@
 sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
 import verification  # noqa: E402
 from elf import Elf  # noqa: E402
-from data_utils import from_buffer  # noqa: E402
+from data_utils import from_buffer, check_result  # noqa: E402
 
 
 ERR_THRESHOLD = 1E-10
@@ -40,10 +40,10 @@ def main():
 
     # Verify results
     z_golden = golden_model(a, x, y)
-    relative_err = np.absolute((z_golden - z_actual) / z_golden)
-    fail = np.any(relative_err > ERR_THRESHOLD)
-    if (fail):
-        verification.dump_results_to_csv([z_golden, z_actual, relative_err],
+    fail, rel_err = check_result(z_golden, z_actual, rtol=ERR_THRESHOLD)
+
+    if fail:
+        verification.dump_results_to_csv([z_golden, z_actual, rel_err],
                                          Path.cwd() / 'axpy_results.csv')
 
     return int(fail)

diff --git a/sw/blas/gemm/verify.py b/sw/blas/gemm/verify.py
@@ -13,7 +13,7 @@
 sys.path.append(str(Path(__file__).parent / "../../../util/sim/"))
 import verification  # noqa: E402
 from elf import Elf  # noqa: E402
-from data_utils import from_buffer, ctype_from_precision_t  # noqa: E402
+from data_utils import from_buffer, ctype_from_precision_t, check_result  # noqa: E402
 
 
 ERR_THRESHOLD = {8: 1e-6, 4: 1e-6, 2: 1e-2, 1: 1e-1}
@@ -58,11 +58,10 @@ def main():
     c_actual = from_buffer(raw_results['c'], ctype_from_precision_t(prec))
     c_golden = golden_model(1, a, b, beta, c).flatten()
 
-    absolute_err = np.absolute(c_golden - c_actual)
-    fail = np.any(absolute_err > ERR_THRESHOLD[prec]) or np.isnan(c_actual).any()
+    fail, abs_err = check_result(c_golden, c_actual, atol=ERR_THRESHOLD[prec])
     if fail:
         print('Simulation results are incorrect.')
-        verification.dump_results_to_csv([c_golden, c_actual, absolute_err],
+        verification.dump_results_to_csv([c_golden, c_actual, abs_err],
                                          Path.cwd() / 'results.csv')
 
     return int(fail)

diff --git a/sw/dnn/concat/verify.py b/sw/dnn/concat/verify.py
@@ -14,7 +14,7 @@
 sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
 import verification  # noqa: E402
 from elf import Elf  # noqa: E402
-from data_utils import from_buffer, ctype_from_precision_t  # noqa: E402
+from data_utils import from_buffer, ctype_from_precision_t, check_result  # noqa: E402
 
 
 ERR_THRESHOLD = 1E-6
@@ -57,12 +57,12 @@ def main():
     output_actual = from_buffer(raw_results['output'], ctype_from_precision_t(prec))
     output_golden = golden_model(inputs).detach().numpy().flatten()
 
-    relative_err = np.absolute((output_golden - output_actual) / output_golden)
-    fail = np.any(relative_err > ERR_THRESHOLD)
-    if (fail):
-        verification.dump_results_to_csv([output_golden, output_actual, relative_err],
+    fail, rel_err = check_result(output_golden, output_actual, rtol=ERR_THRESHOLD)
+
+    if fail:
+        verification.dump_results_to_csv([output_golden, output_actual, rel_err],
                                          Path.cwd() / 'concat_results.csv')
-        print('Maximum relative error:', np.max(relative_err))
+        print('Maximum relative error:', np.max(rel_err))
 
     return int(fail)
 

diff --git a/sw/dnn/conv2d/verify.py b/sw/dnn/conv2d/verify.py
@@ -14,7 +14,7 @@
 sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
 import verification  # noqa: E402
 from elf import Elf  # noqa: E402
-from data_utils import from_buffer, ctype_from_precision_t  # noqa: E402
+from data_utils import from_buffer, ctype_from_precision_t, check_result  # noqa: E402
 
 
 ERR_THRESHOLD = 1E-6
@@ -74,13 +74,12 @@ def main():
     output_golden = golden_model(inputs, filters, padding=1, stride=1)
     output_golden = output_golden.detach().numpy().flatten()
 
-    relative_err = np.absolute((output_golden - output_actual) / output_golden)
-    fail = np.any(relative_err > ERR_THRESHOLD)
-    if (fail):
+    fail, rel_err = check_result(output_golden, output_actual, rtol=ERR_THRESHOLD)
+    if fail:
         verification.dump_results_to_csv(
-                    [output_golden, output_actual, relative_err],
+                    [output_golden, output_actual, rel_err],
                     Path.cwd() / 'results.csv')
-        print('Maximum relative error:', np.max(relative_err))
+        print('Maximum relative error:', np.max(rel_err))
 
     return int(fail)
 

diff --git a/sw/dnn/flashattention_2/verify.py b/sw/dnn/flashattention_2/verify.py
@@ -14,7 +14,7 @@
 sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
 import verification  # noqa: E402
 from elf import Elf  # noqa: E402
-from data_utils import from_buffer, ctype_from_precision_t  # noqa: E402
+from data_utils import from_buffer, ctype_from_precision_t, check_result  # noqa: E402
 
 
 ERR_THRESHOLD = 1E-6
@@ -68,12 +68,11 @@ def main():
     O_golden = exact_golden_model(Q, K, V, B_r, B_c).flatten()
     # O_golden = torch_golden_model(Q, K, V).detach().numpy().flatten()
 
-    relative_err = np.absolute((O_golden - O_actual) / O_golden)
-    fail = np.any(relative_err > ERR_THRESHOLD)
-    if (fail):
-        verification.dump_results_to_csv([O_golden, O_actual, relative_err],
+    fail, rel_err = check_result(O_golden, O_actual, rtol=ERR_THRESHOLD)
+    if fail:
+        verification.dump_results_to_csv([O_golden, O_actual, rel_err],
                                          Path.cwd() / 'flashattention_2_results.csv')
-        print('Maximum relative error:', np.max(relative_err))
+        print('Maximum relative error:', np.max(rel_err))
 
     return int(fail)
 

diff --git a/sw/dnn/fused_concat_linear/verify.py b/sw/dnn/fused_concat_linear/verify.py
@@ -14,7 +14,7 @@
 sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
 import verification  # noqa: E402
 from elf import Elf  # noqa: E402
-from data_utils import from_buffer, ctype_from_precision_t  # noqa: E402
+from data_utils import from_buffer, ctype_from_precision_t, check_result  # noqa: E402
 
 
 ERR_THRESHOLD = 1E-6
@@ -65,12 +65,11 @@ def main():
     output_golden, _ = golden_model(inputs, weights)
     output_golden = output_golden.detach().numpy().flatten()
 
-    relative_err = np.absolute((output_golden - output_actual) / output_golden)
-    fail = np.any(relative_err > ERR_THRESHOLD)
-    if (fail):
-        verification.dump_results_to_csv([output_golden, output_actual, relative_err],
+    fail, rel_err = check_result(output_golden, output_actual, rtol=ERR_THRESHOLD)
+    if fail:
+        verification.dump_results_to_csv([output_golden, output_actual, rel_err],
                                          Path.cwd() / 'results.csv')
-        print('Maximum relative error:', np.max(relative_err))
+        print('Maximum relative error:', np.max(rel_err))
 
     return int(fail)
 

diff --git a/sw/dnn/fusedconv/verify.py b/sw/dnn/fusedconv/verify.py
@@ -14,7 +14,7 @@
 sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
 import verification  # noqa: E402
 from elf import Elf  # noqa: E402
-from data_utils import from_buffer, ctype_from_precision_t  # noqa: E402
+from data_utils import from_buffer, ctype_from_precision_t, check_result  # noqa: E402
 
 
 ERR_THRESHOLD = 1E-6
@@ -107,21 +107,12 @@ def main():
                                        layer['depthwise'])
     output_golden = output_golden.detach().numpy().flatten()
 
-    # relative_err = np.absolute((output_golden - output_actual) / output_golden)
-    # compute relative error only for non-zero elements
-    relative_err = np.zeros_like(output_golden)
-    non_zero = output_golden != 0
-    zero_idx = np.where(output_golden == 0)
-    relative_err[non_zero] = np.absolute((output_golden[non_zero] - output_actual[non_zero])
-                                         / output_golden[non_zero])
-    relative_err[zero_idx] = np.absolute(output_golden[zero_idx] - output_actual[zero_idx])
-
-    fail = np.any(relative_err > ERR_THRESHOLD)
-    if (fail):
+    fail, rel_err = check_result(output_golden, output_actual, rtol=ERR_THRESHOLD)
+    if fail:
         verification.dump_results_to_csv(
-                    [output_golden, output_actual, relative_err],
+                    [output_golden, output_actual, rel_err],
                     Path.cwd() / 'results.csv')
-        print('Maximum relative error:', np.max(relative_err))
+        print('Maximum relative error:', np.max(rel_err))
 
     return int(fail)
 

diff --git a/sw/dnn/gelu/verify.py b/sw/dnn/gelu/verify.py
@@ -14,7 +14,7 @@
 sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
 import verification  # noqa: E402
 from elf import Elf  # noqa: E402
-from data_utils import from_buffer, ctype_from_precision_t  # noqa: E402
+from data_utils import from_buffer, ctype_from_precision_t, check_result  # noqa: E402
 
 
 ERR_THRESHOLD = 1E-0
@@ -50,14 +50,14 @@ def main():
     # Verify results
     ofmap_actual = from_buffer(raw_results['ofmap'], ctype_from_precision_t(prec))
     ofmap_golden = golden_model(ifmap).detach().numpy().flatten()
-    relative_err = np.absolute((ofmap_golden - ofmap_actual) / ofmap_golden)
-    fail = np.any(relative_err > ERR_THRESHOLD)
+
+    fail, rel_err = check_result(ofmap_golden, ofmap_actual, rtol=ERR_THRESHOLD)
 
     # Print results
-    if (fail):
-        verification.dump_results_to_csv([ofmap_golden, ofmap_actual, relative_err],
+    if fail:
+        verification.dump_results_to_csv([ofmap_golden, ofmap_actual, rel_err],
                                          Path.cwd() / 'gelu_results.csv')
-        print('Maximum relative error:', np.max(relative_err))
+        print('Maximum relative error:', np.max(rel_err))
 
     return int(fail)
 

diff --git a/sw/dnn/layernorm/verify.py b/sw/dnn/layernorm/verify.py
@@ -14,7 +14,7 @@
 sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
 import verification  # noqa: E402
 from elf import Elf  # noqa: E402
-from data_utils import from_buffer, ctype_from_precision_t  # noqa: E402
+from data_utils import from_buffer, ctype_from_precision_t, check_result  # noqa: E402
 
 
 ERR_THRESHOLD = 0.001
@@ -60,10 +60,9 @@ def main():
     ofmap_actual = from_buffer(raw_results['ofmap'], ctype_from_precision_t(prec))
     ofmap_golden = golden_model(ifmap, eps, embeddings, prec).detach().numpy().flatten()
 
-    absolute_err = np.absolute(ofmap_golden - ofmap_actual)
-    fail = np.any(absolute_err > ERR_THRESHOLD)
-    if (fail):
-        verification.dump_results_to_csv([ofmap_golden, ofmap_actual, absolute_err],
+    fail, abs_err = check_result(ofmap_golden, ofmap_actual, atol=ERR_THRESHOLD)
+    if fail:
+        verification.dump_results_to_csv([ofmap_golden, ofmap_actual, abs_err],
                                          Path.cwd() / 'layernorm_results.csv')
 
     return int(fail)

diff --git a/sw/dnn/softmax/verify.py b/sw/dnn/softmax/verify.py
@@ -14,7 +14,7 @@
 sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
 import verification  # noqa: E402
 from elf import Elf  # noqa: E402
-from data_utils import from_buffer, ctype_from_precision_t  # noqa: E402
+from data_utils import from_buffer, ctype_from_precision_t, check_result  # noqa: E402
 
 
 ERR_THRESHOLD = 0.003
@@ -59,10 +59,9 @@ def main():
     ofmap_actual = from_buffer(raw_results['ofmap'], ctype_from_precision_t(prec))
     ofmap_golden = golden_model(ifmap, reduce_dim).detach().numpy().flatten()
 
-    absolute_err = np.absolute(ofmap_golden - ofmap_actual)
-    fail = np.any(absolute_err > ERR_THRESHOLD)
+    fail, abs_err = check_result(ofmap_golden, ofmap_actual, atol=ERR_THRESHOLD)
     if (fail):
-        verification.dump_results_to_csv([ofmap_golden, ofmap_actual, absolute_err],
+        verification.dump_results_to_csv([ofmap_golden, ofmap_actual, abs_err],
                                          Path.cwd() / 'softmax_results.csv')
 
     return int(fail)