From c013920b8028db39fc0ef52f62dd62088cf54d9f Mon Sep 17 00:00:00 2001 From: "Matthias J. Kannwischer" Date: Tue, 16 Apr 2024 06:55:18 +0800 Subject: [PATCH] update benchmarks --- benchmarks.csv | 32 ++++++++++++++++++++++---------- benchmarks.md | 34 +++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/benchmarks.csv b/benchmarks.csv index 981accb6..e3bb1bbb 100644 --- a/benchmarks.csv +++ b/benchmarks.csv @@ -42,12 +42,15 @@ cross-sha3-r-sdpg-1-fast (10 executions),ref,290136,287742,297758,29963868,29960 cross-sha3-r-sdpg-1-small (10 executions),ref,290135,287741,297757,102853622,102847774,102861948,75137510,75126803,75159685 cross-sha3-r-sdpg-3-fast (10 executions),ref,627948,625525,637639,43573841,43565461,43582933,27513830,27493024,27525746 cross-sha3-r-sdpg-5-fast (10 executions),ref,1146280,1142409,1153794,93557878,93547167,93566329,59948216,59857434,60043852 -dilithium2 (90 executions),clean,1873447,1838554,1903845,7846622,3321671,28761609,2062804,2062332,2063181 -dilithium2 (100 executions),m4f,1427684,1390524,1466437,4219137,1813668,12587382,1417706,1417251,1418128 +dilithium2 (1000 executions),clean,1874167,1827645,1914566,7493877,3321630,40762756,2062795,2062255,2063222 +dilithium2 (1000 executions),m4f,1426036,1379636,1466394,3807970,1813656,18528070,1417745,1417203,1418192 +dilithium2 (1000 executions),m4fstack,1801523,1684895,1902114,12170976,3900911,86281518,3241353,3194028,3281144 dilithium3 (1000 executions),clean,3205551,3204090,3207411,12696585,5097364,74392293,3376992,3376581,3377393 dilithium3 (1000 executions),m4f,2515969,2514498,2517634,5884832,2917322,25268693,2411257,2410858,2411717 -dilithium5 (90 executions),clean,5346066,5287239,5395626,15205929,7953360,49173429,5609664,5609137,5610119 -dilithium5 (100 executions),m4f,4273211,4210308,4329697,8062110,4882708,18398575,4185407,4184878,4185954 +dilithium3 (1000 executions),m4fstack,3412759,3406659,3419247,23673016,6733971,145803146,5733307,5688893,5778120 +dilithium5 (1000 executions),clean,5341477,5286872,5395822,15710371,7953367,75940093,5609679,5609217,5610183 +dilithium5 (1000 executions),m4f,4275029,4210286,4329519,7977781,4882524,25936176,4185417,4184925,4185896 +dilithium5 (1000 executions),m4fstack,5816287,5474236,6115061,33452872,11170780,185259803,9912851,9845789,9981834 falcon-1024 (10 executions),m4-ct,354880005,284902033,635131652,87741288,87506676,87922628,991320,982548,997219 falcon-1024 (10 executions),opt-ct,555202324,284912829,1157528581,87710190,87606677,87841235,993584,983066,997523 falcon-1024 (10 executions),opt-leaktime,438412062,334858742,625013074,80139483,79891200,80551967,994127,984891,997390 @@ -190,11 +193,14 @@ cross-sha3-r-sdpg-1-small,ref,2328,466400,245512,,,,,, cross-sha3-r-sdpg-3-fast,ref,4032,205080,108236,,,,,, cross-sha3-r-sdpg-5-fast,ref,6824,398600,213436,,,,,, dilithium2,clean,38304,51968,36192,,,,,, -dilithium2,m4f,38296,49416,36184,,,,,, +dilithium2,m4f,38296,49416,36220,,,,,, +dilithium2,m4fstack,4408,5072,2704,,,,,, dilithium3,clean,60832,79616,57728,,,,,, dilithium3,m4f,60824,68864,57720,,,,,, +dilithium3,m4fstack,4408,6608,2704,,,,,, dilithium5,clean,97696,122724,92940,,,,,, -dilithium5,m4f,97688,116076,92824,,,,,, +dilithium5,m4f,97688,116076,92932,,,,,, +dilithium5,m4fstack,4408,8136,2712,,,,,, falcon-1024,clean,34988,84604,8784,,,,,, falcon-1024,m4-ct,1156,2508,376,,,,,, falcon-1024,opt-ct,1156,2508,376,,,,,, @@ -339,12 +345,15 @@ cross-sha3-r-sdpg-1-fast,ref,71.8,74.8,77.1,,,,,, cross-sha3-r-sdpg-1-small,ref,71.8,74.7,78.4,,,,,, cross-sha3-r-sdpg-3-fast,ref,71.7,68.2,68.7,,,,,, cross-sha3-r-sdpg-5-fast,ref,71.1,66.1,66.8,,,,,, -dilithium2,clean,60.9,30.2,52.9,,,,,, -dilithium2,m4f,79.9,62.2,76.8,,,,,, +dilithium2,clean,61.0,30.9,52.9,,,,,, +dilithium2,m4f,79.9,60.6,76.8,,,,,, +dilithium2,m4fstack,74.8,55.2,40.8,,,,,, dilithium3,clean,64.7,31.3,56.8,,,,,, dilithium3,m4f,82.3,60.3,79.4,,,,,, -dilithium5,clean,67.0,38.4,61.1,,,,,, -dilithium5,m4f,83.4,63.5,81.7,,,,,, +dilithium3,m4fstack,77.1,54.6,41.0,,,,,, +dilithium5,clean,67.0,35.7,61.1,,,,,, +dilithium5,m4f,83.5,65.0,81.7,,,,,, +dilithium5,m4fstack,76.1,54.5,42.6,,,,,, falcon-1024,clean,6.5,0.3,23.7,,,,,, falcon-1024,m4-ct,7.4,0.4,32.4,,,,,, falcon-1024,opt-ct,11.7,0.4,32.2,,,,,, @@ -490,10 +499,13 @@ cross-sha3-r-sdpg-3-fast,ref,19689,0,208,19897,,,,, cross-sha3-r-sdpg-5-fast,ref,18593,0,208,18801,,,,, dilithium2,clean,8064,0,0,8064,,,,, dilithium2,m4f,18596,0,0,18596,,,,, +dilithium2,m4fstack,24184,0,0,24184,,,,, dilithium3,clean,7580,0,0,7580,,,,, dilithium3,m4f,18588,0,0,18588,,,,, +dilithium3,m4fstack,23448,0,0,23448,,,,, dilithium5,clean,7808,0,0,7808,,,,, dilithium5,m4f,18468,0,0,18468,,,,, +dilithium5,m4fstack,23820,0,0,23820,,,,, falcon-1024,clean,82647,0,0,82647,,,,, falcon-1024,m4-ct,81825,0,79872,161697,,,,, falcon-1024,opt-ct,81825,0,79872,161697,,,,, diff --git a/benchmarks.md b/benchmarks.md index 5574fe2c..5aef4137 100644 --- a/benchmarks.md +++ b/benchmarks.md @@ -44,12 +44,15 @@ | cross-sha3-r-sdpg-1-small (10 executions) | ref | AVG: 290,135
MIN: 287,741
MAX: 297,757 | AVG: 102,853,622
MIN: 102,847,774
MAX: 102,861,948 | AVG: 75,137,510
MIN: 75,126,803
MAX: 75,159,685 | | cross-sha3-r-sdpg-3-fast (10 executions) | ref | AVG: 627,948
MIN: 625,525
MAX: 637,639 | AVG: 43,573,841
MIN: 43,565,461
MAX: 43,582,933 | AVG: 27,513,830
MIN: 27,493,024
MAX: 27,525,746 | | cross-sha3-r-sdpg-5-fast (10 executions) | ref | AVG: 1,146,280
MIN: 1,142,409
MAX: 1,153,794 | AVG: 93,557,878
MIN: 93,547,167
MAX: 93,566,329 | AVG: 59,948,216
MIN: 59,857,434
MAX: 60,043,852 | -| dilithium2 (90 executions) | clean | AVG: 1,873,447
MIN: 1,838,554
MAX: 1,903,845 | AVG: 7,846,622
MIN: 3,321,671
MAX: 28,761,609 | AVG: 2,062,804
MIN: 2,062,332
MAX: 2,063,181 | -| dilithium2 (100 executions) | m4f | AVG: 1,427,684
MIN: 1,390,524
MAX: 1,466,437 | AVG: 4,219,137
MIN: 1,813,668
MAX: 12,587,382 | AVG: 1,417,706
MIN: 1,417,251
MAX: 1,418,128 | +| dilithium2 (1000 executions) | clean | AVG: 1,874,167
MIN: 1,827,645
MAX: 1,914,566 | AVG: 7,493,877
MIN: 3,321,630
MAX: 40,762,756 | AVG: 2,062,795
MIN: 2,062,255
MAX: 2,063,222 | +| dilithium2 (1000 executions) | m4f | AVG: 1,426,036
MIN: 1,379,636
MAX: 1,466,394 | AVG: 3,807,970
MIN: 1,813,656
MAX: 18,528,070 | AVG: 1,417,745
MIN: 1,417,203
MAX: 1,418,192 | +| dilithium2 (1000 executions) | m4fstack | AVG: 1,801,523
MIN: 1,684,895
MAX: 1,902,114 | AVG: 12,170,976
MIN: 3,900,911
MAX: 86,281,518 | AVG: 3,241,353
MIN: 3,194,028
MAX: 3,281,144 | | dilithium3 (1000 executions) | clean | AVG: 3,205,551
MIN: 3,204,090
MAX: 3,207,411 | AVG: 12,696,585
MIN: 5,097,364
MAX: 74,392,293 | AVG: 3,376,992
MIN: 3,376,581
MAX: 3,377,393 | | dilithium3 (1000 executions) | m4f | AVG: 2,515,969
MIN: 2,514,498
MAX: 2,517,634 | AVG: 5,884,832
MIN: 2,917,322
MAX: 25,268,693 | AVG: 2,411,257
MIN: 2,410,858
MAX: 2,411,717 | -| dilithium5 (90 executions) | clean | AVG: 5,346,066
MIN: 5,287,239
MAX: 5,395,626 | AVG: 15,205,929
MIN: 7,953,360
MAX: 49,173,429 | AVG: 5,609,664
MIN: 5,609,137
MAX: 5,610,119 | -| dilithium5 (100 executions) | m4f | AVG: 4,273,211
MIN: 4,210,308
MAX: 4,329,697 | AVG: 8,062,110
MIN: 4,882,708
MAX: 18,398,575 | AVG: 4,185,407
MIN: 4,184,878
MAX: 4,185,954 | +| dilithium3 (1000 executions) | m4fstack | AVG: 3,412,759
MIN: 3,406,659
MAX: 3,419,247 | AVG: 23,673,016
MIN: 6,733,971
MAX: 145,803,146 | AVG: 5,733,307
MIN: 5,688,893
MAX: 5,778,120 | +| dilithium5 (1000 executions) | clean | AVG: 5,341,477
MIN: 5,286,872
MAX: 5,395,822 | AVG: 15,710,371
MIN: 7,953,367
MAX: 75,940,093 | AVG: 5,609,679
MIN: 5,609,217
MAX: 5,610,183 | +| dilithium5 (1000 executions) | m4f | AVG: 4,275,029
MIN: 4,210,286
MAX: 4,329,519 | AVG: 7,977,781
MIN: 4,882,524
MAX: 25,936,176 | AVG: 4,185,417
MIN: 4,184,925
MAX: 4,185,896 | +| dilithium5 (1000 executions) | m4fstack | AVG: 5,816,287
MIN: 5,474,236
MAX: 6,115,061 | AVG: 33,452,872
MIN: 11,170,780
MAX: 185,259,803 | AVG: 9,912,851
MIN: 9,845,789
MAX: 9,981,834 | | falcon-1024 (10 executions) | m4-ct | AVG: 354,880,005
MIN: 284,902,033
MAX: 635,131,652 | AVG: 87,741,288
MIN: 87,506,676
MAX: 87,922,628 | AVG: 991,320
MIN: 982,548
MAX: 997,219 | | falcon-1024 (10 executions) | opt-ct | AVG: 555,202,324
MIN: 284,912,829
MAX: 1,157,528,581 | AVG: 87,710,190
MIN: 87,606,677
MAX: 87,841,235 | AVG: 993,584
MIN: 983,066
MAX: 997,523 | | falcon-1024 (10 executions) | opt-leaktime | AVG: 438,412,062
MIN: 334,858,742
MAX: 625,013,074 | AVG: 80,139,483
MIN: 79,891,200
MAX: 80,551,967 | AVG: 994,127
MIN: 984,891
MAX: 997,390 | @@ -194,11 +197,14 @@ | cross-sha3-r-sdpg-3-fast | ref | 4,032 | 205,080 | 108,236 | | cross-sha3-r-sdpg-5-fast | ref | 6,824 | 398,600 | 213,436 | | dilithium2 | clean | 38,304 | 51,968 | 36,192 | -| dilithium2 | m4f | 38,296 | 49,416 | 36,184 | +| dilithium2 | m4f | 38,296 | 49,416 | 36,220 | +| dilithium2 | m4fstack | 4,408 | 5,072 | 2,704 | | dilithium3 | clean | 60,832 | 79,616 | 57,728 | | dilithium3 | m4f | 60,824 | 68,864 | 57,720 | +| dilithium3 | m4fstack | 4,408 | 6,608 | 2,704 | | dilithium5 | clean | 97,696 | 122,724 | 92,940 | -| dilithium5 | m4f | 97,688 | 116,076 | 92,824 | +| dilithium5 | m4f | 97,688 | 116,076 | 92,932 | +| dilithium5 | m4fstack | 4,408 | 8,136 | 2,712 | | falcon-1024 | clean | 34,988 | 84,604 | 8,784 | | falcon-1024 | m4-ct | 1,156 | 2,508 | 376 | | falcon-1024 | opt-ct | 1,156 | 2,508 | 376 | @@ -345,12 +351,15 @@ | cross-sha3-r-sdpg-1-small | ref | 71.8% | 74.7% | 78.4% | | cross-sha3-r-sdpg-3-fast | ref | 71.7% | 68.2% | 68.7% | | cross-sha3-r-sdpg-5-fast | ref | 71.1% | 66.1% | 66.8% | -| dilithium2 | clean | 60.9% | 30.2% | 52.9% | -| dilithium2 | m4f | 79.9% | 62.2% | 76.8% | +| dilithium2 | clean | 61.0% | 30.9% | 52.9% | +| dilithium2 | m4f | 79.9% | 60.6% | 76.8% | +| dilithium2 | m4fstack | 74.8% | 55.2% | 40.8% | | dilithium3 | clean | 64.7% | 31.3% | 56.8% | -| dilithium3 | m4f | 82.3% | 60.3% | 79.4% | -| dilithium5 | clean | 67.0% | 38.4% | 61.1% | -| dilithium5 | m4f | 83.4% | 63.5% | 81.7% | +| dilithium3 | m4f | 82.3% | 61.4% | 79.4% | +| dilithium3 | m4fstack | 77.1% | 54.6% | 41.0% | +| dilithium5 | clean | 67.0% | 35.7% | 61.1% | +| dilithium5 | m4f | 83.5% | 65.0% | 81.7% | +| dilithium5 | m4fstack | 76.1% | 54.5% | 42.6% | | falcon-1024 | clean | 6.5% | 0.3% | 23.7% | | falcon-1024 | m4-ct | 7.4% | 0.4% | 32.4% | | falcon-1024 | opt-ct | 11.7% | 0.4% | 32.2% | @@ -498,10 +507,13 @@ | cross-sha3-r-sdpg-5-fast | ref | 18,593 | 0 | 208 | 18,801 | | dilithium2 | clean | 8,064 | 0 | 0 | 8,064 | | dilithium2 | m4f | 18,596 | 0 | 0 | 18,596 | +| dilithium2 | m4fstack | 24,184 | 0 | 0 | 24,184 | | dilithium3 | clean | 7,580 | 0 | 0 | 7,580 | | dilithium3 | m4f | 18,588 | 0 | 0 | 18,588 | +| dilithium3 | m4fstack | 23,448 | 0 | 0 | 23,448 | | dilithium5 | clean | 7,808 | 0 | 0 | 7,808 | | dilithium5 | m4f | 18,468 | 0 | 0 | 18,468 | +| dilithium5 | m4fstack | 23,820 | 0 | 0 | 23,820 | | falcon-1024 | clean | 82,647 | 0 | 0 | 82,647 | | falcon-1024 | m4-ct | 81,825 | 0 | 79,872 | 161,697 | | falcon-1024 | opt-ct | 81,825 | 0 | 79,872 | 161,697 |