Enable inner tails in symmetric/triangular matrices

analytics_tools/graphing/kernels_specs.py

@@ -192,8 +192,10 @@ def __init__(self, bench):
         self.bench_type = int(run_dict["bench_type"])
         assert self.bench_type in level_2_bench_types
         self.N = int(run_dict["N"])
-        if self.bench_type != BENCH_TYPE.level_2_sq:
+        if self.bench_type != BENCH_TYPE.level_2_sq.value:
             self.M = int(run_dict["M"])
+        else:
+            self.M = self.N
         self.precision = run_dict["precision"]
         if "TransA" in run_dict:
             self.TransA = int(run_dict["TransA"])
@@ -269,3 +271,53 @@ def get_stored_bytes(self):
         return ((self.N * (self.N + 1)) / 2 * (1 / 4) + self.N) * get_elem_bytes(
             self.precision
         )
+
+
+class symv(level_2):
+    def get_flops(self):
+        return self.N * (self.N + 1) * 2
+
+    def get_input_bytes(self):
+        return (self.N * self.N + 2 * self.N) * get_elem_bytes(self.precision)
+
+    def get_loaded_bytes(self):
+        return (
+            (self.N * (self.N + 1)) / 2 * (1 + 1 / 4 + 1 / 4) + self.N
+        ) * get_elem_bytes(self.precision)
+
+    def get_stored_bytes(self):
+        return ((self.N * (self.N + 1)) / 2 * (1 / 4) + self.N) * get_elem_bytes(
+            self.precision
+        )
+
+
+class syr(level_2):
+    def get_flops(self):
+        return self.N * (self.N + 1)
+
+    def get_input_bytes(self):
+        return (self.N * self.N + self.N) * get_elem_bytes(self.precision)
+
+    def get_loaded_bytes(self):
+        return ((self.N * (self.N + 1)) / 2 * (1 + 1 / 4)) * get_elem_bytes(
+            self.precision
+        )
+
+    def get_stored_bytes(self):
+        return ((self.N * (self.N + 1)) / 2) * get_elem_bytes(self.precision)
+
+
+class syr2(level_2):
+    def get_flops(self):
+        return self.N * (self.N + 1)
+
+    def get_input_bytes(self):
+        return (self.N * self.N + self.N) * get_elem_bytes(self.precision)
+
+    def get_loaded_bytes(self):
+        return ((self.N * (self.N + 1)) / 2 * (1 + 1 / 4)) * get_elem_bytes(
+            self.precision
+        )
+
+    def get_stored_bytes(self):
+        return ((self.N * (self.N + 1)) / 2) * get_elem_bytes(self.precision)

requirements.txt

@@ -1,5 +1,6 @@
 cmake==3.25
-ninja==1.10.1
+ninja==1.11.1.1
 black==22.10.0
 pre-commit==3.2.2
 clang-format==16.0.1
+matplotlib==3.8.3

src/common/stdlib.py

@@ -1247,11 +1247,8 @@ def cse(proc, block, precision):
     for buff, idx_map in buff_map.items():
         for idx, access_list in idx_map.items():
             if len(access_list) > 1:
-                if all(is_read(proc, c) for c in access_list):
-                    proc = bind_and_set_expr(proc, access_list, precision, DRAM)
-                else:
-                    staging_block = get_bounding_block(proc, access_list)
-                    proc = auto_stage_mem(proc, staging_block, buff)
+                staging_block = get_bounding_block(proc, access_list)
+                proc = auto_stage_mem(proc, staging_block, buff)
     return proc
 
 

src/level1/axpy.py

@@ -23,5 +23,5 @@ def axpy_alpha_1(n: size, x: [R][n], y: [R][n]):
 
 ### EXO_LOC SCHEDULE START ###
 for proc in axpy, axpy_alpha_1:
-    variants_generator(optimize_level_1)(proc, "i", 4, globals=globals())
+    variants_generator(optimize_level_1)(proc, "i", 8, globals=globals())
 ### EXO_LOC SCHEDULE END ###

src/level2/symv.py

@@ -42,9 +42,5 @@ def symv_rm_l(n: size, alpha: R, A: [R][n, n], x: [R][n], beta: R, y: [R][n]):
         y[i] += alpha * (dot - A[i, i] * x[i])
 
 
-variants_generator(optimize_level_2)(
-    symv_rm_u, "i #1", 4, 1, round_up=False, globals=globals()
-)
-variants_generator(optimize_level_2)(
-    symv_rm_l, "i #1", 4, 1, round_up=False, globals=globals()
-)
+variants_generator(optimize_level_2)(symv_rm_u, "i #1", 4, 1, globals=globals())
+variants_generator(optimize_level_2)(symv_rm_l, "i #1", 4, 1, globals=globals())

src/level2/syr.py

@@ -8,32 +8,28 @@
 
 ### EXO_LOC ALGORITHM START ###
 @proc
-def syr_rm_u(n: size, alpha: R, x: [R][n], A: [R][n, n]):
+def syr_rm_u(n: size, alpha: R, x: [R][n], x_copy: [R][n], A: [R][n, n]):
     assert stride(A, 1) == 1
 
     for i in seq(0, n):
         for j in seq(0, i + 1):
-            A[n - i - 1, n - j - 1] += (alpha * x[n - i - 1]) * x[n - j - 1]
+            A[n - i - 1, n - j - 1] += (alpha * x[n - i - 1]) * x_copy[n - j - 1]
 
 
 @proc
-def syr_rm_l(n: size, alpha: R, x: [R][n], A: [R][n, n]):
+def syr_rm_l(n: size, alpha: R, x: [R][n], x_copy: [R][n], A: [R][n, n]):
     assert stride(A, 1) == 1
 
     for i in seq(0, n):
         for j in seq(0, i + 1):
-            A[i, j] += alpha * x[i] * x[j]
+            A[i, j] += alpha * x[i] * x_copy[j]
 
 
 ### EXO_LOC ALGORITHM END ###
 
 
 ### EXO_LOC SCHEDULE START ###
-variants_generator(optimize_level_2)(
-    syr_rm_u, "i", 4, 2, round_up=False, globals=globals()
-)
-variants_generator(optimize_level_2)(
-    syr_rm_l, "i", 4, 2, round_up=False, globals=globals()
-)
+variants_generator(optimize_level_2)(syr_rm_u, "i", 4, 2, globals=globals())
+variants_generator(optimize_level_2)(syr_rm_l, "i", 4, 2, globals=globals())
 
 ### EXO_LOC SCHEDULE END ###

src/level2/syr2.py

@@ -8,35 +8,47 @@
 
 ### EXO_LOC ALGORITHM START ###
 @proc
-def syr2_rm_u(n: size, alpha: R, x: [R][n], y: [R][n], A: [R][n, n]):
+def syr2_rm_u(
+    n: size,
+    alpha: R,
+    x: [R][n],
+    x_copy: [R][n],
+    y: [R][n],
+    y_copy: [R][n],
+    A: [R][n, n],
+):
     assert stride(A, 1) == 1
 
     for i in seq(0, n):
         for j in seq(0, i + 1):
             A[n - i - 1, n - j - 1] += (alpha * x[n - i - 1]) * y[n - j - 1] + (
-                alpha * y[n - i - 1]
-            ) * x[n - j - 1]
+                alpha * y_copy[n - i - 1]
+            ) * x_copy[n - j - 1]
 
 
 @proc
-def syr2_rm_l(n: size, alpha: R, x: [R][n], y: [R][n], A: [R][n, n]):
+def syr2_rm_l(
+    n: size,
+    alpha: R,
+    x: [R][n],
+    x_copy: [R][n],
+    y: [R][n],
+    y_copy: [R][n],
+    A: [R][n, n],
+):
     assert stride(A, 1) == 1
 
     for i in seq(0, n):
         for j in seq(0, i + 1):
-            A[i, j] += (alpha * x[i]) * y[j] + (alpha * y[i]) * x[j]
+            A[i, j] += (alpha * x[i]) * y[j] + (alpha * y_copy[i]) * x_copy[j]
 
 
 ### EXO_LOC ALGORITHM END ###
 
 
 ### EXO_LOC SCHEDULE START ###
 
-variants_generator(optimize_level_2)(
-    syr2_rm_u, "i", 4, 1, round_up=False, globals=globals()
-)
-variants_generator(optimize_level_2)(
-    syr2_rm_l, "i", 4, 1, round_up=False, globals=globals()
-)
+variants_generator(optimize_level_2)(syr2_rm_u, "i", 4, 1, globals=globals())
+variants_generator(optimize_level_2)(syr2_rm_l, "i", 4, 1, globals=globals())
 
 ### EXO_LOC SCHEDULE END ###

src/level2/trmv.py

@@ -91,8 +91,6 @@ def trmv_rm_lt(Diag: index, n: size, x: [R][n], A: [R][n, n]):
 ### EXO_LOC SCHEDULE START ###
 
 for proc in trmv_rm_un, trmv_rm_ln, trmv_rm_ut, trmv_rm_lt:
-    variants_generator(optimize_level_2)(
-        proc, "i", 4, 2, round_up=False, globals=globals()
-    )
+    variants_generator(optimize_level_2)(proc, "i", 4, 2, globals=globals())
 
 ### EXO_LOC SCHEDULE END ###

test/codegen/reference/sha256/avx2.json

@@ -1,28 +1,28 @@
 {
-    "exo_asum": "840ecdc334b858e7f672212bdd4a68e73f550ef9209e9563353f0e03efb7b602",
-    "exo_axpy": "6d2f6451c8df0899d2eb1db13ffae5c6ccf72ffe16ee1e64eca62a4b9924bd08",
+    "exo_asum": "d025f72a4a493b0188a2501e5771568c174af5129bb0fffed9c21b727d336d66",
+    "exo_axpy": "d54511e38cf200c8402cc292eb75c15b3cc91813aad41ac26aac43ad9fdabb08",
     "exo_copy": "9251bac4ef34162bacb5c4f304834e362efc315b7071ee181f3f9750bd1e05c6",
-    "exo_dot": "c28d83b9b0d855159d8a2e896eab9d8a48deabcac94883288fa50f44b2f7b98b",
+    "exo_dot": "8e7a71353e80273839bc522cfc8c889cbe25d8701a8c82a3c2559b12d9e90f5f",
     "exo_dsdot": "c901be9d30928e042c35aeb9dab421a34db6d593534f31b4967f9685ecf9628b",
-    "exo_gbmv": "aa3ecf9eb525e4f13ab65a0165c7b7af5ecf60a1d387ef41d95daf68f51297f3",
+    "exo_gbmv": "cb92744337cfdf3aa97d250a18e540c2e8787380ba88bfef790a3e14aeb19f37",
     "exo_gemm": "28f060df6aa28cc7259d8e5379b95f560bb728e9de322d0d752d3113ba9510ea",
-    "exo_gemv": "0b5b2b0f07c3fab2784eb147f23c5ae0540e01b8dc654b99af9a5699d00a169d",
+    "exo_gemv": "80d79a5752c20874fbe3d2c94989f5f5663687d405966065a18224800c931559",
     "exo_ger": "38ba46c410ea9bd1d616add516e9ab82ed811e5785848bce17191d89f81752ce",
     "exo_iamax": "49c60714c479234683166e5651fbe95ed5a43ecd370a391732769588948cc842",
     "exo_nrm2": "858d4f1311d843b2ecb7b26f18fa9fbd5fb5d9b07675043af2f52ca77da3aae6",
     "exo_rot": "4e2bc2d9a7a2dba563f455d45342ca13799ea80f46b9f48e390187daf51143bb",
     "exo_rotm": "2c4ceeaf8090ed38b16cc828f15ea56552ba2c509f211e308687f68d581cb3da",
-    "exo_sbmv": "6cc6b055a63620c036bbc478455f7b1cf22db0f435718bc8e04d7c1b0ea63fd1",
-    "exo_scal": "b45709a77280b6c9f55e2fb4acac3727c93df96d6e91e1fc271408cd162e4c7f",
+    "exo_sbmv": "5fbcbe46cd54adf6c653fa6305281f9b70b7fb61425e6b2d3d981cf08182c207",
+    "exo_scal": "1b907db2aa7ae945c8206f956d2d228c06ec9e19b8cced98d402d063e5feafd3",
     "exo_swap": "908f3bb8eda4065dbaef9ff0c085a9d8b55eb12d893cc53212dfc905f24d290b",
     "exo_symm": "f8192433e2e64b600a573ecd2a77a404f79bb0886316c1e483530fff5bfe0cf4",
-    "exo_symv": "0ba93f9f1150b9da16c0af0fcbf03931f7a0f9d93c9bf0eeb993a7c480a8eaf9",
-    "exo_syr": "7ae09fd4873ee3c3491d70c17f9923640e5c23181906d9bb6dadc615da3e8b5c",
-    "exo_syr2": "366e4191fc5e674a2dc063da89980de07f55fa2b3da8f05979a6b07ef284fa6c",
-    "exo_syrk": "fe61eecd1d907b527dd8279915cf2e7e88265ed21939ad99e87174c9a3fa29de",
-    "exo_tbmv": "dc55c55ecc566cf14a3e59d28884dc900c5476f3294b5b1010f47a0c6fe2b170",
-    "exo_tbsv": "b48649263f7dce7317c24470f7bf8c75d0cb26dbb40915d2745782976f931511",
-    "exo_trmm": "42593a2611d0d2452c5287c81b3c636b146f035e87a77b6b30d549f94dc13d36",
-    "exo_trmv": "13489b438784fec414a5eb0b107e689269e4c191f830fda19cc09521909bf95d",
-    "exo_trsv": "2311c494487ff7e31c4a3c75993ac07d519571c3a4af3b30afa6bb8cf4634222"
+    "exo_symv": "f96e0661b0221c69b43d9e476c3f5ba9a3ee41168149232b6a414145dc0d77f5",
+    "exo_syr": "eb284ae2c8d1424bf32fa791822231a1a1e559117a51beaaafaa4438d10c14ea",
+    "exo_syr2": "9285dc796c9c573cbd974f419563a0c2e3d7507aba3521ee1c89e9e707913332",
+    "exo_syrk": "9894ba92a502df8968c0c4e1e09cb5510a9ad10d77f4e1595570a7d1a2167b4b",
+    "exo_tbmv": "e517f633eeaf1429c2204966a2970e7013054afd1f0bb22795075cfa5e4678db",
+    "exo_tbsv": "faeb1392d2af7dc9cdac9fb707bd7a3273e82c92bcd940e869fb7bc5be14f020",
+    "exo_trmm": "70f7aa84d76fe3be02cbbc5db13fdc9d55b9dd481ba02616f1457a15a653a074",
+    "exo_trmv": "8cab60255358ae985e8e34f719aca6cd60e090f58e5ef53bc3254a094bb8f9ce",
+    "exo_trsv": "f39e76f37f370b48df07c03254f7b4cf0a017887f3cd02708f9efc81c196e83f"
 }

test/common/misc.h

@@ -42,11 +42,24 @@ std::string kernel_name(std::string kernel) {
   return lib_name<lib>() + "_" + type_prefix<T>() + kernel;
 }
 
-template <typename lib, typename T, int order, int TransA>
+template <typename lib, typename T, int order, int TransA, int Uplo>
 std::string level_2_kernel_name(std::string kernel) {
   auto name = lib_name<lib>() + "_" + type_prefix<T>() + kernel;
   name += order == CBLAS_ORDER::CblasRowMajor ? "_rm" : "_col";
-  name += TransA == CBLAS_TRANSPOSE::CblasNoTrans ? "_n" : "_t";
+  if constexpr (TransA + Uplo) {
+    name += "_";
+  }
+  if constexpr (TransA == CBLAS_TRANSPOSE::CblasNoTrans) {
+    name += "n";
+  } else if constexpr (TransA == CBLAS_TRANSPOSE::CblasTrans) {
+    name += "t";
+  }
+
+  if constexpr (Uplo == CBLAS_UPLO::CblasUpper) {
+    name += "u";
+  } else if constexpr (Uplo == CBLAS_UPLO::CblasLower) {
+    name += "l";
+  }
   return name;
 }
 

test/level2/CMakeLists.txt

@@ -9,15 +9,12 @@ add_exo_blas_test(level2 trmv s)
 add_exo_blas_test(level2 trmv d)
 add_exo_blas_test(level2 tbmv s)
 add_exo_blas_test(level2 tbmv d)
-add_exo_blas_test(level2 symv s)
-add_exo_blas_test(level2 symv d)
+add_exo_blas_test(level2 symv "")
 add_exo_blas_test(level2 trsv s)
 add_exo_blas_test(level2 trsv d)
 add_exo_blas_test(level2 tbsv s)
 add_exo_blas_test(level2 tbsv d)
 add_exo_blas_test(level2 sbmv s)
 add_exo_blas_test(level2 sbmv d)
-add_exo_blas_test(level2 syr s)
-add_exo_blas_test(level2 syr d)
-add_exo_blas_test(level2 syr2 s)
-add_exo_blas_test(level2 syr2 d)
+add_exo_blas_test(level2 syr "")
+add_exo_blas_test(level2 syr2 "")