From 6c0ce889ae7c7b02fa529c2e25468a3c8b678dd7 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Fri, 8 Mar 2024 14:47:26 +0100 Subject: [PATCH 01/15] Add a script to generate submissions (blockchain mock). --- __init__.py | 0 e2e_test/data.py | 67 ++++++++++++++++++++++++++++++++ e2e_test/generate_submissions.py | 37 ++++++++++++++++++ e2e_test/network.py | 29 ++++++++++++++ 4 files changed, 133 insertions(+) create mode 100644 __init__.py create mode 100644 e2e_test/data.py create mode 100644 e2e_test/generate_submissions.py create mode 100644 e2e_test/network.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/e2e_test/data.py b/e2e_test/data.py new file mode 100644 index 0000000..3fd2b17 --- /dev/null +++ b/e2e_test/data.py @@ -0,0 +1,67 @@ +"This module holds some hard-coded testing data." + +# Some dummy public keys. Their corresponding private keys are +# irrelevant for this test. +BP_KEYS = [ + "B62qrx9d59WXHARNxQjMy4Eb1i9SRpwuH8gcuxM6dkHnAgTXcN6dDzf", + "B62qrhKjqf3jMWbtoM4VHqUAY5M3gE2v6Wm4L2dpw36GxtxPBzJPsyV", + "B62qo5uVVat4XfqWUk9EKE18ZHUpxiDn7zoksrZxrZSXYQxFAroSTzu", + "B62qiqkuPiZyNNJ2vcxx9m85dhGYzoSUCtnwL9YnqvSiK2TJMuXjvP5", + "B62qksawzNzjzn9CfqwqgsuJgf23aDUu5d6i8mYLTDEE4cXEjALEYxK", + "B62qr6pi7s78Kk9WNvPWPdSUSY6TTs84DzAzKKkLwsKkgbfudmN7Wd3", + "B62qpqhAdPrtMos2bmghGaKF9xjcdXiXFnxj51rqgFynv1bKbJuD8U7", + "B62qqywGkLD9TGMh9bxG9yJHpoZVbg1jSbY6gdzYejAZwSv42MAMrj5", + "B62qrAxCoBbtwDVn3SBkhwLeQqKK31xdnGuNkxPCZURnU2f3n3njqWJ", + "B62qk3T3xFms7iG5Qh5jE2xdESvCzVnfbxSMPKzhoj5eUiUdsa6k2eS", + "B62qkEqfYYa1o6uw7chgF4VcoqJJXjMriybGuARHR3vr2Ny3Nvz3UTM", + "B62qioo2qyrK4VEoubjba6mTM5GKTEcBAuqjetb4ACs9gH9VgJMDfYS", + "B62qjZaNVRxHTF763F15NQxnx9wqBcu5AHpaV2BvzUgEvi7zMRdP8fv", + "B62qnQHMbVt4ZqNinuigazUEGwYdQeuiAvBPu3NcuwPd6PZ3PdzkEfa", + "B62qnPaJTWs2ZcJwn5g53w7NvT3kEuK87z7pzuNyPyGShHkRmKBw1w9", + "B62qoj5CqvomLKMbocS8PupSkB69gHCEE8HN27ToqZQuNSAHao9GUZa" +] + +# Some dummy libp2p peer ids. +LIBP2P_PEER_IDS = [ + "12D3KooWAmMKPQKfF8bbUN7gjNSHvbrQ8NY5JhJ1qWCnMBzvJdh5", + "12D3KooWRuvBs2QNyE1TD1vfVqAXKbU3qwUvSJTDRWesgpvT4sxw", + "12D3KooWDe8Sq3CEp7HXJ9aAPghuvKT32PWBWpeSiQwXModSZ2A5", + "12D3KooWFG3XauGMv6xtAodKraS9FWGMPRMrTvjKeuHU21axerYW", + "12D3KooWJjna99EYehJgpYGUffTtgSMjiSeZ5UHXfuBdh9yGSnjN", + "12D3KooWCEcj6994dvi8nHofHgvHdK6PRi9USqE8dNSVCfaXLQdC", + "12D3KooWRBzi4zpPjssakh2uXADRMMk7X8vBtpCEsPuZW6spwgib", + "12D3KooWBg12daDfEZoASq7jAQAQn4E1d5RNhhD9MQN1AocpQR6x", + "12D3KooWLHyNwbiyKiTHTJTL28jYVzQ24zUKNSiiLYVZpufX4ubs", + "12D3KooWCtt2WMYnJsty27Bn17Q9vqHFFCKYWYew5WXhmf35X49i", + "12D3KooWMsepsE9zXaCcFTeyQsH1TYRJg4J5hVBKripXKRfU6vWm", + "12D3KooWDTxtEJ5PmcA4kTeBtDFyVhSaizdUKzx4NKFjhctUrKPV", + "12D3KooWPMvmxXtKydU9T52sXPFTpj1d9Pq2qvs6j3qisqsetJ7K", + "12D3KooWRxzEAisB3sXuDbsn3ZhBgxMHohFsTjk1wsBxk5J96JhB", + "12D3KooWHecvaEeAimF5gJ6FKBEcB2VcyLk3L7ynh7PgFX2VkPAJ", + "12D3KooWEQQxABEeYyX7DGLjDuEWTtNvEmN9UAPWWKTzxfMaFZQJ" +] + +BLOCKS = [ + "3NLkXPFWhTAZYLRPNzmGHQaZ1A2wRuccvBuRiepBS5EtunTY4vGE", + "3NKpXDdgm9z55ucSZyN2nUhDVGYik7dVYjrBk3QDfoK1PBWzjZ8m", + "3NLYxeJBvN9Tknvd2SaKDdGiCUsqygPnfoNRiaBPQNM8wByK5FDn", + "3NKmkgh7owDSPkKjAuag6BS4VRVSakKbzbSMPCHYDLoajGNmCsBs", + "3NKkRhrx1zzQjwNxpozmezXTk6iQ424f5H7FckQ9kr3Z3CTXVwSK", + "3NL22xiofNqjK2HaUMffrJSFLFKZpk9xY2K6FEPWfL6wxx61Tc3G", + "3NLVR9avC5qPbpTkjFziXpbtBRne9EtEavKhaeWyxYNW1JNwJbLk", + "3NKjvrYw2Ym5ci397qJytKcKZvkWT6oA1zx911L4G5DUHpoHCSRD", + "3NK5kq9BUQbNxw7c2DHtijagDHqjjjtEpb3aaBoCJZahuijhM8xD", + "3NK6T4ZkCDW6wFGA8dhUzKBGaLcvEBe4mwMg5AqUrxsbymoRLwNR", + "3NKHjNUBbkMbsM8BtZCkzXCNc7V3VL1iYZv1bhuievYsg7kPj7y9", + "3NLB1mxDwST489u4S6NNZPc8T1UCHnKNvXTtwrqM9F5oqUEaf1nK", + "3NKBfRSjygmZctUKNwCEfGnJfwn3RkArVi5ZydMRHeDu5ZYJCKCR", + "3NKABeDUHq9vTVp8ra5N2GTyhN4FvckRbFpcS7RjiLCPuWPCVB26", + "3NL88psq93f8U7DcpKQcSJgMWujEotrLtL8vsvyXDjH5FMmm6nS8", + "3NLKiSvicQbQKVu5c5uxmeQ53tT9vdsdkDLfnno74xwLipHgmSit", + "3NKTuhtYamf7gYS6jbj2Gv353kDkEtdTE1af83erw4Ye8h6t2Db3", + "3NLbZ5dgJ7yAiSbAfYYNEFp5cV9NGvNKfBsxLggn7BKKJfJ3EVvy", + "3NKi2Tj9LdCS8HYadFFjzEaeMBkFETQEo1UtKnuK75DP7RwDsq9c", + "3NKGTH7XLTjTXMm3wWYZ8wuvabNEam6GNqwjH5KCztgWWyDs7hYT" +] + +SNARK_WORK = "XLjrgk2GplfoMYyG2nfpqByY2HTSYGM3R7mQ5SUByBF3sEnFXqM+qkknTCn6Q4DTmhlnHHq4tj7fot7RAFfmApCKHRNPbaF6NQDV/QroOfDH7PXaEFmE64BQD2EeQJkYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACBXHWfqATKbtc83BoR1/K1C2Xc6VHpaCGJC+B6/sGNOGzGcOUmHvvZzEjFiM2i9LspJ9BZwK/QcMmO1pQuZkEGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAXewScVeoz6qSSdMKfpDgNOaGWcceri2Pt+i3tEAV+YCd7BJxV6jPqpJJ0wp+kOA05oZZxx6uLY+36Le0QBX5gK868TbqxTTN8lUN7AMrspfsSGwQ1kaxpGoVkb1x0pWDgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADf+DSaAFaJIKiWuExLdDUe9OUtRXOhGj+O5wi/3kDjVsxnDlJh772cxIxYjNovS7KSfQWcCv0HDJjtaULmZBBgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAF3sEnFXqM+qkknTCn6Q4DTmhlnHHq4tj7fot7RAFfmAnewScVeoz6qSSdMKfpDgNOaGWcceri2Pt+i3tEAV+YC/AAgWKOnAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIHyq3kAx7rVV72qiUYdHnxzRCgtSmdRM+BFnr/KM3APS/HyYhJcX78DJ/BZtAXW5phSxAPzEfTMqa53WhvyXe0LL3eKMQwD865U+dH8cy0L8qtedrdbWYRQA/KRgR+4GBvC2/ODPtc+3NiqdAAAAAAAAAAAAAPyOfzpP156BxPy6AFSkKVIkMgD8iMlAb1Ee52T85tERreHOUbcA/Fa1M2WuTFsU/Kbkh+ozYsgsAPzFPY/gqcTlgvwjaWG7Crm0ewD8dkpjujXKlGn8pg+fL7vVGfcA/NkK2w/aGW35/Fb9JKqQEjauAPzASTPPVpARl/zC4fOX/nYL9QD8L6Yk/KCTyF78iZVlWSZwPsgA/PLVu2YCz5oN/I57xopcIgvzAPxA4SRr2RpVZvwr6zUzD/uD/gD818f//Yqv8a78NF/D8rycFpoA/ESDzPufZpcA/MJSYYkVmvt7APzRKn4AyPzGevwfwfJ03pnc+QD82GECXrKyiR78CExY49qYMWcA/Px/gRpxi2J6/Lwu9zKOfZ0PAPxOAlA9diXiPvzTuyU5wvJ5lQAAAA/8lPUV59Wxlx78X0YgDu1N1p/8Z0bescI1eP786OejD5erAwMA3qkFMpqUtPYDKXBVHn3yibZ841DUgREo8eAe6rKgVzA0sDkCHp4+FkhAOhxDmgX0fTUGnLx6cc3a7p6QPUvrDPxvv+asybOCM/xjGHb5WEOXeQD8l4eI6QYrOt38x6FEKUDmet0A/MufnPQw5ejG/N2coM1lu90HAPwTGmmHolksU/x7b2UqsLwhqQD8iwcQj7F3nOL87gxr3wBfXPgA/IecsActp70d/KAmX+nilxtNAPwRX4BWfHR1nPzo8c76aWP+oQD8TWDp29+KK1z8m/cQ8oxxjFoA/Ehr4FFcs8Ai/O1tqUBzi4imAPxuZHZetdcHkPwSjk7bOYvGwQD8ySs/N17jRUT85c2M/BXHQJ0A/E6qvEuEgphC/Ly3r9DXJ6mXAPx3bv3/Wz3KmfyUQlwVVWrm7wD8VJmXIXGyfUv8QMiTYeCiH5UA/LNHB7K+zNEs/B0CZPI83tFbAAD8b7/mrMmzgjP8Yxh2+VhDl3kA/JeHiOkGKzrd/MehRClA5nrdAPzLn5z0MOXoxvzdnKDNZbvdBwD8Expph6JZLFP8e29lKrC8IakA/IsHEI+xd5zi/O4Ma98AX1z4APyHnLAHLae9HfygJl/p4pcbTQD8EV+AVnx0dZz86PHO+mlj/qEA/E1g6dvfiitc/Jv3EPKMcYxaAPxIa+BRXLPAIvztbalAc4uIpgD8bmR2XrXXB5D8Eo5O2zmLxsEA/MkrPzde40VE/OXNjPwVx0CdAPxOqrxLhIKYQvy8t6/Q1yeplwD8d279/1s9ypn8lEJcFVVq5u8A/FSZlyFxsn1L/EDIk2Hgoh+VAPyzRweyvszRLPwdAmTyPN7RWwAAAAAAAPR7+S7aTllngy3ML5fTzEeXClZPplgN2ON82fiNL+Un7Qmnm4q4j0/wWcRss/DnHspmW9lRa9esuf/PlwlSAz8Bue9zAtMPrJn6IhG/tuQfyGbeJ8/4ooCnRAH33XFI2w4BB+Wh/EA29FcA5NFqi9UaShGmPTH+gwnLrHlrvxSS7gUBJomUYEf690lver0Cy5x9QLAyhpDiXMLB/ZjbLWgPRjYBI3XWOFBwdXkgTIjN5sH5/XzbBXwJf0YqEsMKqkkzsCABRpTjJhe72g3vTlFvU+XbQZzI5yZDW40/bl+6xpe0xjMBF+jygMkiB/aHhLuKT7fMiRCR7P2tbrO1RISSOpCPEg4BLlRl615r2a/r47rTskI/hJ+lHratuyPszeZQv1E4fwkBbe0eUzVGgqhET6/wdoUWvWGebKHevWDPl/Mvht/oWyIBZ8FIOphQNx/YFdZLJNBo8ZiQ41JZCxMJRLCm1ZEBqQYB80dncxlncrKWDyiMtxMIeL5TYm20huKFFL4cBXNb7TEB+hKp4d+SccBGA8v+ygisjzqA3a95IvF7Va9X5cArKzEBcVlALoeLLbrr+l8IUG5wydGEjSlO8V824TBu91LuTTQBvB1+yW2XkEYUNuFLWuZbdjlSJ/EmwnewEytjPNr2ZBoBK/BXcqMYHKCVUsyj5ROI2ufcChVBrVfUeozn1XOCcSQB1GI8x11uFKU/PVNG6IXEcYvWfJWjwxDOynMARa1BZykBl/4z+BNHG5fJm84ktmjPZkCh34ajWUTfezDZVCZP4AoBTb4xJ2WKjtf9xhQd/7Xc7rjusS+30K+YyDuFPsL38BABLN3GHe2sDblY0/CEU97H/3Psw+ZPCw7+nksas516oTAB/xZmCjgvRmm3LmhvdrBHzry8x7DBRipOZkvoMCtQbykBGZdWjdHN+cvjPWqwWfy/Uobvy8p7mU7+fYLWglzqdDcBqJm90w58LdQ5XiYdrKBy7AkNWZwi1g5TJ1NUfteGhx8BvuxVpToHbIOa7uvXe4c6veRaRiVI6zcdlpi0W1A93g4BnKS8YCnqLDMlh63QJiveiLiMNoWM0NiDOgJ6YtHLYy4B1/ZGSQTGN3N31JlJKPsLr07vdLC70cmxj3ErHsyNHjIBVSM3IxdbBGD6p4rRk/jhWszyLpjEdRzGw2YduzniMiUBDaEiNYnH17/buEkL49j19OmvrKIzCzm5lxB+9uMD/woB7kPY7NZyloBRIdRiqQVnjBs5E50ZyZoWinSxpy7DRwoBZf4xgMyNFOjDxX5kbAIiWcJmMi68Mns71iafOXlgHwgBozJxkWeYBH9Sb3yikxH8/gGC7BEVk65VNskgABFrkwoBWE5/3w3pL99gT3c1bPJ2309gHs8HaVIw9BpUfb/ayh0AAZ8VINp/gWlmzTef4gSH7rQHQEEOLseV1fE9bmljCMswAUAXGQI6BynvkelHgjbRLgOTrAuJNqfmO2r17zAuy4kMAX5hm4d3vOL9vFgCkT8RwlPdCsrg46M53U1s9tJNOC8XAep1Rl5Ra7rYB/UTlMuv+A1y2qBbhKRWoruOjk6Xa+QuAWZwXMmZsQCfmUJDSFCD8biKM/HUWyKl5TKwfTRiRAE4AXVpJmPO0wdOoslhomBdZHzha9toox9fM0gcX7gFHv8cAabjJteQhnkXhMcL9eF/JTfQvfT5+lqXLgrl8+H6+ysCASqmq5/vx4Q1BmOLQENe3k/gbpUQSlGQArTK9gajNaMMAaYLHlkVTh/mVcS8Tg+5uTpTtcOEinCNkZqKSAq98oIHAdt/Zq2Q2Br4Xdr1cw8QQJKsuDns0BBQHDBO3pn6A+0BAQpdPnmGJNBdq290pQjUA/n/ovioDy0IGCVwT0d89akyAVL2T2VOteGP+7rJ4bb01gJLa4lCQjNnoTlW3E5VjN4PAS3rVckuz/LgJdJCVFNloQsKpy78ZsKoTh55qUzOL68DATV9oXc1bwPNQFRDOiFnBLmptiXCeoViM/GdFTLHSdQEATtW9uoLLcJb1auwxROzvQQbqIKzU6aB+E/9poIgk7UaAePt5QJuzwh9NYNHkp3UkI+j82A2I5Xfzn3VOeDM/eABAZ04ssXJiynHQ5HK+o9VsxIjRZGZv3d7Z8AMOvzy9NsXAUhM0EnXi2FYQoWLxxb5Vs09FFOuPJr6nywOP4G03L8VAU06fHuZMeNNYGW6bYcxOICRQ0H2dN7+xbgNxf/1rrQsAdv3In2QycijsVHKpsBbdXwfioFYwxKFb2wX9ILcxFMnASQuIxTUYPDb11eGFKrMhm1oRbYZWtKCSEuF5/3oSjAPAf302ZkN6ugpNcYz581PFFBdZgaBz1TGpj3qkbTuwtArAfF+g1x78DqnPjev3w2RQDyRXBf8OTiTv/JsaZe0Z545Ae1rubgiuj9QgLabjoqIRYUXD23seZahWiZcRtzHKJo8AWO34TbkfcwhAOSrYHFk95xYgL6L/pd9Idh5rBbZb8sLAQZ6GNwSMYeHjUxbRJR2aBVnJB08FHYGvw+JSsXbLnUbAQDiK3R5AWokOV3BOlZwJpXxDQl/uY6hhwG0WhLVxSYVAZoYG526rFbhHsfDnJBV6n38R5zfEhukwmN6Un/0gVsDAUgvmedHgcWxXxjx8Q1/pu2LkHMEsMJGPyzo9ROJDOISAd2tnhHFONbeBu6Bjo2WH4oyLPtjR2I6m7ZEJYIDgQcnAAGJmdctrdMAtOXti8fLyDG9YxVuNVKAiHm3OK6St7rlDAE7bxxTN9ETDez13nqBHPinSC6kn3qs2gBdjAJTZMTkHgG6eMdu8cngt53mwy3TnFUVM/HYYgOGYehuRCk+6e6/JgEC/biLOUiwKRNVWs6tDG/B+JWdNxEf7J/AyEttzpvbMAFpHtDZ0tGzJpJkQS+a0WZYH+q+N2qx+UQWn2AHpJybPgHCZk0sNCZ09+iuMvCbqAxDrjhDHcprt8LvijLU9veaAwFrQCVklSugRGOCG6bqA1FMd1hK1KQgD92JvzSU9EJAFQHF2Jf7SkfrNHCWzAt2/s+uLVDQWFezjIJIew/SEv89AAGoOwvE8Vyb1ldAfvvm8A2rrHdA//8NjyvBwk95pjkUGwE1koKRjAabiwejE1SHTctX4HLo+xLPNt2OY38ZFDPSBwEebNgEYaZLoW7AVfrQI46gMDDxp1fDvDauNePj+4kVJQF1PSwL96Eq0RQZL7MSdRnfPoqjy5Hpi0RnPYM3pvPFOwEqyinll97/41p37jlgAicPP74p+tpzHBQtiUlAwI7YBQFnP2yYqGlMXk0dE+0pnufwWNr3XyMmGfPX4R/u7+dWGwABYob/Mc68J4sbWtxxfCGjb2S4FaJ7NI4OHd2X8u9IzwwBds61pr5rarE4UVD1YWJE6VFGyxlWtmoWoQ0bsBG01TYBFupEXWtFrQMK1NwcZUp1byiPi4eDxzDO+lHECQvEkyoBAZJnhAeiyy6KLTGO1udCKpR+OtWXzzzAShkosIAzBzEBYOAZMD3Liv+flvb5FUz0bdktyv+49ErF3NU+xbUa7jQBoC/b8QmR2FAzAhWAFAGHena3bzzYy1gNa91Oc9rf/S8BSBQKBbOdER0dyPwlZC+0Y6nqO4Lur0TGa62i148JmRYBUL5L9zUsLy9lpsIfH178FKlmOo6ZCLsBSTQDdoHbODMBIb2t5YIc3DY6kPVwqoiSJEVpCT/q96NyRgZHcY8nAyQB7R5y2WAf0VhJsRK0nrhPKoTKba0jS+ZVurJDjhDt9TIBv9bPYKJDLz7M7BVR2AQDqe/ofwWr38Jsb0XYuVmpNSoBtNfZ3H4SFQN7pi7HlmLkKeQE6BqkjAx2871wRxt6bBwAAAAAAAAAAAAAAAAAAAAAAAAAAItPrrSDqvfkPX0ovt3g8WYAt2O7b/9mQ7rKXOPLnY4+2hgN0Tu1QeDw0cAXVI/JXu1g5qKxJk8WKdyCsV3Npw+JF8OEVAaQ3w6mYgRd0FdCFrlk2dXXMhozRBjC/MoiKYKUL90FKJEGyLyZJYdxqNYORiig+RwptD9gHpgML3kUasWMPgkAEPZiLAw74nS74Gbr2p0VyGIWpZKoak4/XCYVG9POVsqBcTZbtIzJXz7MnrcjwdHKuUEkUhw+ZofACxf+L8UQhVAIaDJm9KDEFuCseB6fZdZbjh5B/1sYB/Yy3rsg3U9NL01Gv4W5mlrSmetfRMkGpPxd1D7M1NZPuRTsfE6HzQI2WlTjF1Ut//2ZNdnoKaNNNTdyO7aV9towHeFnn1fCC7YS/gQsxicQM4trNmoBd2DJ9HFANzT1D3M0xu8CwNdKaUt68ClHrPWnqHSwM/ND0HQMKIs2pTe3bwvYgYh94rsJmS8SMLmUjw1E9TTp6oKECZ3ZdqnUdEA3AucKOm3LCLollOnteQIAeKTEeIvm5FgTaGOh7a5NDxwrOUdVkZ7fdEHHR1PZ7fk6J/Dmwm8fkpooe6fvtGXaFj/qJ0oZk103eNDW5Iq/snJyEJ5Uugpwt+u10yFc+FBxMYR87DOSTQBjipBn5nPsvda7eCiZ6vp0dHOfjnA/VNgFt7FmaV+3eKYuS1zq0k9ROmh8CTPVTO4AaVtpi9jE6xgumVUBmnv62RAcPKOCuOzesttQ9Juzwe2IANA38V+IE1hYq/1uhK0aV1kheXwEikLpNVjGvIfywQB8XzbK6mcLXnIzXjVY3jvSlCcaLoCkC3Fm8Qzf6l51hw5fyWOgkDu2+4xXM5p0Psq2RddvNokC25GQoPfgOK1tvxJcP0+pCyss/hlPSF+c9nKriwPq8SjH/5VLw4kzGgvtr7PT16g91SN7ZgBu/Ybc7zUx9o81q1aH6L9K7HKnSAflFOwgdScOFvhd0EZYTnzytVwLjPQfPYUULwsjzMp4XcA9Q2kyJA1WmehKI+EBrtVPaMEbe9BjrqoEuBmvhJ/XYMlLN6M/VJdztxshLHj/4OAR6RpORD1Y+TrIdmedkgXIWsAJrjIqMCFFfnmX0t3/oJh4+l6HQSgm9x33QzFaZx0NZM24E+WohPgmHG7Dk8DxN06BNvsG56d6EVCA5trfDKqtNrMrtsrsP7GxYC4cX24UgYT6QYkkFliKIu/Bp5s6j/D/GRS77hsIwJV9ciNRkudW5ROSt7uOyPMWJgeeuY13d3vwMbOHvDAX3zLZ9RBu0aWQ7MlQ5iik4E2A/58S2xj8968gAIA/X7KU1vyF9VtRETox9vS6LbtIe32YITm7mzuVL/QFKOqIUzYoCmum1Hjab4isl0ce4jn7FJiU5Oa3uZKIAw1brPXXr0Wvx4Q5Piy1gR28iPRt5ouNOnJdCEPSFg8qOyxoTUi/hI4zr0AW3J9R+lT2rxBjMkfUNV0pmUC0EHAk8e6AC1TT5RTUsEY/OdEHtS9nFitexN8cu1orYD869RQn6dlcVX3YJOwvCqE45GkrGX2ZU+V4rECPEG+JKwYnJTCGbKBwkZaeYDyhVtggj8ZbQjHVWzwygUhhIiOuTKAYOAcYmL4deH2eyEtE0q0zkyTMVPtLzox6pk6BOCis9AJaW/g1z77m7FzoT59h7i1ae9vyfNrsR4S3Vyb24+l/B/ptcsdYGObBf2bwp3odAyAlPLhfR51m3y1sKTvMdPMI023zztUOK43HMNk0AiuSwBEu3iNyCSLqa1VsWH23MwTeQAqkgocM4T9NoJo0rxmN2/o4bVzDrA/OkoCXx7E9Fuzz5QQF+xW5pELEzj0zvEaxI8/t/Mx5utdVzxcZQI093A4uYNrO6YIHv3RV4VeWHxHWJTPDTxvh5a4dFD31JQoy/g0gGMDv2htlUxYIn/myU2GhSVNQ/On0psM5t8emONls4KoJCRgXrAK720aaWnCS+WhLMEoS6ls3mEwzhmQQAL/E/M9Qm3tMn5Jk7owcjegyPY47vkyLv1mjht3YIYw5TvkWf+o1wHTwuE+/6gfe4UAde6YHDepw9dMA2kq/HD1z8xV5orL3RFYYb187Q80nEcCsFfqFtOwC1jqXU4kYJI+NptFpVcwYAyD5IOnsO5bDQEsfmNOD5nTiYIQ7U2YmRwjNmO4hxsa6ZEUT2T7d+XjV2bAJqt1fPx/CAmLRKhzq58wqmKbrRXUdyb2Mz2zD0nChZWgCJuITNf6R4Ho6ORQTVxtyBNrv/Twx/wD9blx/bpnmPc+Qf5eerhQ5e38Eelu01OCKrHobNjrA+q2pes9nI6Aac3Ge7WPFVTGWbR5ZsWl1rnLp7vmoA1wK+GUs92DOFx7d04j+hfafjky5PJeH+g3SEPc+qNTTBKHjNWbH2GKWIl4rh8pH9hfP9ZIdbe/MUPe3aU4Gin9XfO98eHx/StJoYGdZetQshsW59i5Zeuc7bt1ddQRpN9H5jeaR1lsZkjTEAXk3PRGgZMNsA7WXL+wuA6rxvNwYuGI6b6HsvMNuCPnkbKlpKLhzP6IWOz1UokUV3CE4zH7xEptKIopHoI5l9/5A2ZCCcMlgbz+zK5Qc7rH00tDIlZwDn42nTQ409TiVOS/Ruv4X0Iz6Gln0f5bPGAgu6AWjrN+R7MYOFlxsPRqYvtI5i8otgukYGf/WcSqfEpqrG135YKbPlSZ5SAWHM3OBi0gQiLRtcDhZ21hCosaNOL0otKmrMXUr02LL5mMEyPKCZqaYf3WPPr9RcDMtmYO+AVkRCw+aqkoYxLM0IAnSxZBHxBVhXngBnY+9Orp7puhYE9L+5+NTMUPzmek9LvCafjdenyMUlA0pkgqkiWPRAzHxJBWD4M1MmIGU8+g5vyvltGF8fGclCPiLACQSIdbypFLd0at0qTmunUbEcGoyqE7fcih/T4IsdgToivWsd+BwWZl7XdodTQ5bCv2ApoGVlXGFRRnjwQmr8sxNkJOKXc2TTsnULEkqbbRHusRDWsulUQvbppFPHx+a9HiSpF5V5OPaZQ7s2es7DdT6bfc191PqQMUAS34DC3RqRwLgQgOtREklJ22kjzihU9p+V659JlVtKsPpGh0VsLEdG2Ax1PwjxNRdMGUkOi4nlu+1B/ucmHhpPnEHBQgVNsZuqT19teWiy6B+a2FqQ0qDElsw0b9lDfusO4QgSMJMjd4qRIk9NpvAPZ6BXZlj5rsjs++DctU4WKYF9zy1ukJ+3IfZlDa/QLM2EO9W+SPJWhhf0hmvOjb8BCOyEADP2TfsLs9Mj/wMlPi16N+6F77u7+iazG/WvaTBmHk0PYO7dgFwLRCq2XIKXQWAaAy0h+c/OnU5JEBDI28Eb8oXlVeSbR506jtIiqcQ5i/1eZITEEg5ogMnFOwBamjL+Bejk9buDairospPUuMoCszXQuRtzaw6aUr32jls9y8+GNib4N+kKaw72Ad+/ymXz8oWgd+4bRoM8KogrQY093wOgdJvpGoFG9dY+nlB8/Y4TlQA6KIev03F6OXPJtxM1wLFAbS6/9qjazQ6NCNnsQIQliJ0S727MdA2g48QxOiXNsK8OMrrZElUMVNjT8iemF5P2szzXHqKgHuIdx/OV0oJ3ON8i1ppIHSV2NzccLJpwbmXYLSyLw/6+PY9441AwBdt4GYVbk6A31zzjTIWWwSbdIzBie2ynuXCXRjeNKw8O2ylC1Y0ClEWKMk1iH907q3F77dbqV4MURG+4JopbpgFjZzHiW3jC+PgjSRq9f5qxrhv9VDiFJnZ9t5xMp3VSwM6lab6cjXIsVxLzfgvNHtUbd08ozE3H9p3PqhGdxr4FmgmNwn6ELQm6jyet8dkgA5LPa+DeLuSgA0vl1BQHuQP6NOmLYM2IBf24E1dn5KRfme7kbAdnWhXgFplzj6ocQysydrVV7+WiEAVo2ypcpXcC5A6bqIaCkj+FEMf47EEMMdIqEF8YIldHtz3B3gUA03v+039UGk8IpRdXH92L+Uk/5q4dHcZkzymm9MlkPr04YsbW+0Gadpob3+p5hBdpx6QHGCFXAUGbafhwE1rUK3NT6w53MNMvcoPv0zvkewvOfhhFIL7EvyL27LF/SUSK05tl4/fhsuTTHUDeo0jjdU24dlaVMXsR2c5PiuwVsTqO+tnfq8EjsGoyCLa1bcDXBwsayIaxhjG+jk1BibaJNCYAPGaBQMKt18/93+xHqGuAoT0H+mryUWYMid1Yb4khnLwZwUevmFKwcBGgaPR9BI8/OwR9TfwyaTC/EgcFxsNmY7kwsXAhwFQ2bCgRIREux3Mpa9g1cRbd4gGeHXOJddSmp7R/orrfEjZQ8YVxBkTBc1mwOeBMNlp4I4tXFXPchNRSuufUOPlnj2KtjWwaGkBZM8oxmHcmEwwd8ui0NuALjEh7rXeOZYssvIzXQVGsSqFSQW0n1pVY7PAhZQyZFnEasDZ9SJqBEIrselEfwdxGo066Cwre/8tT7e4WMyJZIDCpmf/dO2IzxsHoOXj7pYPPgQYxNJay5dn8UUEmyPnAUqIcRytdGnCVNNOXXxUpxUAlDPW+TPG/TlfI59QVDlMqM6a7sf6MqsG3icAlAgz0BwaOjt73P2HCwXqTY5FLRjbFKX6RLtbbjz+/uoGUrkCOR+lNR42w4N+043GuJTEyWKkk2+r/PCQBaQslAC2q1oxGWZxqOTszAx+kmLGEjT/BCLW8dhTlu16R0Ztxupy9ii3D40fk66jHDkSA1i+dl4BfiYcRy1iTlfjxJKvwgngLH6Hn0rqH9h2ep7CAiAuAAxWilmZWxtJd3t3vbjnuWo2gF37WIYODgiwGeXVQJA2yYawtknHJvPvi87GbOXqeTHR9M+FlouY9eP4w5EwlOYM3EzjnRy0MCy+FnQb/Mm7CZEQnX9mjpDc4w8SUa/GYFeH2q6RYKPnuAeSOBGWwIYAOyTX3AFN/iEaegOHRGTF1638JRz9O3V8V5vUDyrUuibcm6tEn45EaJrwwYKcspdeUrSX3XCOfZTJ2FFb5fXDFD4CSsQ3zDNMe3ks0sZRHTw8g4rdJtCYuBBScFWtBhsYDkc1XpBfTO0m0GjvonSa+yfZLUagImL96W75QSDVQjbSkl8zDQE/yQFsV5WLJCz9ucG+A0AZywZVSW47XbZsNAA0Wv/bt2db0ZP8t545G5kJkj7m5XPwqsFS6EAb8weLBXic+6LxUs5ad0q7KWym1diFAwMDJQtX3ldikEFeaLEbOTcWiQfnF5i2QCxgsApl9u65/mpVRb8RoVP0lhtcxQ+Rog6T5PfGm9PXgwKQzOiKJ5Lgk3DQZC+kS30x+mzSBRv4E1dyLpZtjkpUFVV6sEMNGnMMLjSeGxw//g6vaGUroM0xNfSfedzA7lDgHDqOidrwOMA/hU8ZCjZBsgvFGQqrFCBPASHjhsACvI60JCqzTTA1AljM2sXRevS7232lGcy+HP7PAzvV9J1tArvFnS9T2ztPT3prLejye0md7ocYQJb8y63QN4ZiEpmG7TzF4nd+ZvWnlmnFVhsiwuegmBsABS6Zx4Hg3bdMVSyuPNMSafqxkHJRPWc+kBK2J1ppJEFLryd6T4ie9rV5gGXsexYkr5YJSPr4q5ha/PslDjsoZ9ZXbGAQ14ElpiYKacBT+M7vNkVKDJ6VXbb4oMdCWikF294/LKDfI99aXaYQ/lx58dICSns0KA2P2T1K+fspKg+euiDTyubAdhCIhb1Fh562gfD2eJoQ+PE8C+G28RdKNQwwsHyJNjF8KVrU2ivhtHVFoAYvzrdgZzAiDS+cDzEc6WWzIRNnWReNVED8MnhVB1x7N19anZFCCjUSCnym9QJJyz5BBCV0WknyQ31UtVCNsqNaWNueGxNnA5+/LSbHGOcn9juFKmDZ+LiVrvgSjycJYFuw66edq34+HOsLL+EcvL9v2JnyWsKy+yy2pNmGnkyM39sy/aHqjetKPvO5KSpyZ6a2y8UMxMSrNo6tQBUyp4gn6WbuZilnjN7wH32LGdTVCR9xszffZJXWI5FfL4wQ9jwFu3jCHGz1oCWqAIkL0Qs5gmUHfriCgiSvQ3aQyldpeqR9H5AAj47yaobIdQWkaC71svlp3cTujDnEV5N5c6RJM7Y254AjIchF8OT+B2glbceL4P92akW4LG72X+OzU/0YrtD3XVkwoJWeUt4TODhFzZMH1Ee2vJ3IuloSWxy0G/dIR7y7wj97V4nDUT3tjytz2SUsbnFnZxywzE8aMfH4HVJfl8u0gH6t/5RBB6BgmpI8g/3tqaRfGc/aBXqscJC0owKuzVIo5bariZEOr+toEpI3ekMWWWmfIfKvE28NGdQl8R4DQDkDwqaPezWupbYyP2OWeUZgy4vdIEOFHEEgtY0+jt5DlWIe6wpNIItfj0bBzYwG0rUvYFrQ9ro3ZE44/UwvJNZkHLuevfs7lWmkAuNUhFS0aah9Ry5rUc5SyJP8jUL53sAUkEZ/pi0AhktEm8ch7QckUecM+wACABjj8Ig0KImmV+u9vh0bLpmeF4SLFZW6TzKPt8QHwqzYG0jbyuZUbtpYcZN4PWcPHqfuFEzKMHiakE3V/yqcLNNyB0MWOUZRtW1ZL0U1tRIbYcvkBCmRJW55QSWiXkMkmg0CmSyaOWG64O6voripMjBtK+r4xFxrjYfc9YTn5KGT/SWGGwkrbTDdpcHxCgwxPrsUBzuay0alG6PhaxJoi/j1stM3MjIpkYDTzXHvoROxKzR7t/7snCrA8Zm3dyrYq8m6Nsyev3jm9xoJSFVNMl86orqBmzDTIBzL1Ueh+6h9ZMQTCp49nrIbn2caDmwdlC6Z6Gt7h2GIOt+vMEKyYhnUNNPCyFvr7dPjY4Cq1R2XqM5VxrJWSI0IerUMpsWP4R0Wsc8jpHmnERJZznKhPpODhlCSvGhEocyd35kugPTQ37yrkvvWXNkQofZp+VQMvZFvWJd4/GSoFl4jIGu74DjvAkk8iZfgkoFKc2z72w+kxHOnSIxtz6nPfXKI24nN9OhVulV3M7NxBeNh7o59FBrVhPU3FuO907tbZcuMJoiCgtyr/IUwn0oaxYVcfgARreaAVLJstisdAzu3XlbxCL97Y8vO1tzAfsOqCF/WajZjyA2vIhhVDOEt5l9fymREG6TkdACjDu/iHvw4MRJYLMcFs3nxn31IOix/fFoomuwk4PvqAWbXFjngJq5mnXcZRG74OQjfpTY2f86JEy+rj0M22d8cgBVQISjwwVtFeDmMc2ZG+bdqE+hPTuWfzOTF70wkPZgmAT+QYYChBCU+ClnMSXt+ZuyP5BOuUlgFcM6r4ZVZpN5UVbAvHye02J4nh1uOFy61ZsGtlwdaVHET7gmdn5FONLz+rF08SiWgNQcknSNRvGuSAkYWjv3JKfFnszcPCbvLrdOA7qC3a4DvL+cn+hseEEj5xQf4SsG3xZkD2m/yyZV1oE+sfMzluX4yB3jZCitdKCV9oYGRJSWXNUmjKIDvCBLTh1JTJ4kWNjKCRKTRk4bKCQwZQrvZQdPmr5YuiD21v49zvDTwu58KNh5q7J/hrJdzh3QlSdYD3txlF4IW7BbHjQd/Oa1+wLUIi7YKZP468BhExfUUa4wCe+JebWrZOESp838Y7yITBB8OpnNXJedFhzkVNegisglhUjpG2UqGJJ3QH6Jsd8vYAM/k+t+kbwQBA/HeTkZjNOHXMO29XVPoiAcGh60EorAbF4vGsEmk2ULU241CJ40c8NyXF+5C4jkbaRMZHivQqzdjmqD1z+WLIvUhuSNo0q2kLw+qQY6u7S7S3F8uyF2QNF4u+EOmiH3q/5sSqWTSC9HWJNSxWGKQGfk6aDxdkVYlJc9WaFpeh++5wHeyp39j6SfZ5dIDoGojN4cK1BIH9hSr2X5MNcveY7cxkRvfTDXY3WWTzP0/v+9mX8G4+VRYL1WyHgBQVR14Vd9PIZkSB4iaHpBI6cZusF5D/Oy/DtMlPGyajJHSdx/SPjTfuA4MYb9RrdLOS8BCTaAdBTuZgQvYTN0W6bs4PZnfQOX/nyYoGnaLJQdLp0hfTD1YVzvZG5UeOMYKEtCJLy02QyMOvMqIf+Uq5Z3BxwM+jShqink9+wOCk+0vJPqEs13ePYbmHO6iPcHMwR281m8cNyTWVyHTYzYmQDVrf4TOVEf6qBnDWosQz/LQPQMn7VO54siWE89R2dh2MmjXf5qNwiTiCg0rB8j3/+P0EDGO9CFIlU0e10CxMitJYihn1VRe0w76D/djGeLPnJ/ciZ/C9nI2yAtjjYmxRNciXyu3bJyddMQdAfHjI1SpVeCcm4gvqZewPoZgkUuq2zjtH30frTEnODW6+PTBHIxZMk5OqhiTsBc9Dp5elHzYWcF2VPYx5QG7BAIq2TYugEAiLJAsEG2fmzpf8pzFNTl29AdhRVED5dJn6xpBptUBajDlbaIuwOdgJBYEdRJ5B52X1jrqiYPpAOY56bdy87XgCxpev3CudiYkkQDm55oxs5Y74/Wv7xhDpKW8Xbz9YKsLVfcSRgHFID4AAAC4smwnQP0A4fUF" diff --git a/e2e_test/generate_submissions.py b/e2e_test/generate_submissions.py new file mode 100644 index 0000000..d46dedc --- /dev/null +++ b/e2e_test/generate_submissions.py @@ -0,0 +1,37 @@ +"""This module generates submissions to the uptime service from an +imaginary blockchain. There are 15 block producers sending submissions +in 1-minute intervals from one another. This way every block on the +blockchain is submitted 3 times. These blocks are pre-generated and +SNARK work proofs in submissions are dummy, as they are not produced +on a real blockchain. The goal is to run the uptime service validation +against these blocks and submissions and see that: + - every BP scores 100% of available points; + - blocks form a smooth, uninterrupted chain and there are no forks.""" + +import argparse +import itertools +import requests + +from data import BLOCKS, BP_KEYS, LIBP2P_PEER_IDS, SNARK_WORK +from network import NODES + + +def parse_args(): + "Parse command line options." + p = argparse.ArgumentParser() + p.add_argument("uptime_service_url") + p.parse_args() + +def main(args): + """Generate submissions for the uptime service.""" + nodes = itertools.cycle(NODES) + for statehash in BLOCKS: + for _ in range(3): + node = next(nodes) + sub = node.submission(statehash) + print("Sending submission:", sub) + requests.post(args.uptime_service_url, json=sub, timeout=15.0) + + +if __name__ == "__main__": + main(parse_args()) diff --git a/e2e_test/network.py b/e2e_test/network.py new file mode 100644 index 0000000..304c38c --- /dev/null +++ b/e2e_test/network.py @@ -0,0 +1,29 @@ +"A mock for a real Mina network." + +from dataclasses import dataclass +from datetime import datetime, timezone + +from data import BP_KEYS, LIBP2P_PEER_IDS, SNARK_WORK + + +@dataclass +class Node: + """A Node is a simple bundle of a block producer's key and corresponding + lilp2p peer id. It's capable of generating submission in the name od that + node.""" + peer_id: str + public_key: str + + def submission(self, block): + """Create a new submission. Actually make it a method of an object + containing a BP pub key and a peer_id.""" + now = datetime.now(timezone.utc) + return { + "created_at": now.strftime('%Y-%m-%dT%H:%M:%SZ'), + "peer_id": self.peer_id, + "snark_work": SNARK_WORK, # this is a dummy proof. + "submitter": self.public_key, + "block_hash": block + } + +NODES = list(Node(peer_id, bp) for bp, peer_id in zip(BP_KEYS, LIBP2P_PEER_IDS)) From b10b7fa41496c46898104537c8110d31fe6e2979 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 19 Mar 2024 09:40:04 +0100 Subject: [PATCH 02/15] Add a shell script to generate dummy blockchains for testing. --- e2e_test/dummy-submission.json | 8 ++++ e2e_test/gen_blocks.sh | 71 ++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 e2e_test/dummy-submission.json create mode 100755 e2e_test/gen_blocks.sh diff --git a/e2e_test/dummy-submission.json b/e2e_test/dummy-submission.json new file mode 100644 index 0000000..f28d06b --- /dev/null +++ b/e2e_test/dummy-submission.json @@ -0,0 +1,8 @@ +{ + "created_at": "2024-02-21T09:21:17Z", + "peer_id": "12D3KooWGFjBhvkwj6jGTfhQxWMq4rTvw22vgbgnc2KPF4xhNhAv", + "snark_work": null, + "remote_addr": "193.164.254.212", + "submitter": "B62qiZv3JwF9wRGtqhNbLsfWMm6zYd71RD6Akfq9AUK8MY4EC5Ye9i6", + "block_hash": "dummy" +} diff --git a/e2e_test/gen_blocks.sh b/e2e_test/gen_blocks.sh new file mode 100755 index 0000000..b763914 --- /dev/null +++ b/e2e_test/gen_blocks.sh @@ -0,0 +1,71 @@ +#!/bin/env bash +# The following env variables need to be set for this script to work properly, +# as the defaults provided are unlikely to work: +# MINA_DIR - the path to the mina source code +# BLOCK_DIR - the path to the directory where the generated blocks will be output +# SUBMISSION - the path to the dummy submission file +# A dummy submission required to pass blocks through stateless verifier +# in order to obtain state hashes. +# NOTE: all paths should be absolute, or else the stateless verifier gets +# confused and fails to load files. + +if [[ -n "$MINA_DIR" ]]; then + MINA_DIR="$(realpath "$MINA_DIR")" +else + MINA_DIR="$(dirname "$0")/../mina" +fi +if [[ -n "$BLOCK_DIR" ]]; then + BLOCK_DIR="$(realpath "$BLOCK_DIR")" +else + BLOCK_DIR="$(realpath blocks)" +fi +if [[ -n "$SUBMISSION" ]]; then + SUBMISSION="$(realpath "$SUBMISSION")" +else + SUBMISSION="$(realpath "$(dirname "$0")/dummy-submission.json")" +fi + +if [[ -z "$1" ]]; then + block_count=20 +else + block_count="$1" +fi +dummy_hash="dummy" + +# If an argument is provided, it's assumed to be the state hash of +# the parent block. Otherwise, the parent state hash is generated at +# random. +function generate_block_after() { + if [[ -z "$1" ]]; then + args=() + else + args=("--parent" "$1") + fi + $MINA_DIR/_build/default/src/app/dump_blocks/dump_blocks.exe \ + -o bin:"$BLOCK_DIR/$dummy_hash.dat" --full "${args[@]}" +} + +function get_state_hash() { + $MINA_DIR/_build/default/src/app/delegation_verify/delegation_verify.exe \ + fs --block-dir "$BLOCK_DIR" --no-check \ + "$SUBMISSION" \ + | jq -r .state_hash +} + +mkdir -p "$BLOCK_DIR" +cd $MINA_DIR +generate_block_after # first block in the chain + +current_block="$(get_state_hash)" +echo "$current_block" > "$BLOCK_DIR/block_list.txt" + +while [[ "$block_count" -gt 0 ]]; do + generate_block_after "$current_block" + current_block="$(get_state_hash)" + echo "$current_block" >> "$BLOCK_DIR/block_list.txt" + mv -v "$BLOCK_DIR/$dummy_hash.dat" "$BLOCK_DIR/$current_block.dat" + block_count="$((block_count - 1))" +done + +echo "*** Generated blocks: ***" +cat "$BLOCK_DIR/block_list.txt" From 7b3c4b61ac2ad9820eb23bd1accd30451845dcb8 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 19 Mar 2024 09:40:25 +0100 Subject: [PATCH 03/15] Improve the submission generator. --- e2e_test/data.py | 41 +++++++++-------- e2e_test/generate_submissions.py | 78 ++++++++++++++++++++++++++++---- e2e_test/network.py | 11 +++-- 3 files changed, 98 insertions(+), 32 deletions(-) diff --git a/e2e_test/data.py b/e2e_test/data.py index 3fd2b17..8222611 100644 --- a/e2e_test/data.py +++ b/e2e_test/data.py @@ -42,26 +42,27 @@ ] BLOCKS = [ - "3NLkXPFWhTAZYLRPNzmGHQaZ1A2wRuccvBuRiepBS5EtunTY4vGE", - "3NKpXDdgm9z55ucSZyN2nUhDVGYik7dVYjrBk3QDfoK1PBWzjZ8m", - "3NLYxeJBvN9Tknvd2SaKDdGiCUsqygPnfoNRiaBPQNM8wByK5FDn", - "3NKmkgh7owDSPkKjAuag6BS4VRVSakKbzbSMPCHYDLoajGNmCsBs", - "3NKkRhrx1zzQjwNxpozmezXTk6iQ424f5H7FckQ9kr3Z3CTXVwSK", - "3NL22xiofNqjK2HaUMffrJSFLFKZpk9xY2K6FEPWfL6wxx61Tc3G", - "3NLVR9avC5qPbpTkjFziXpbtBRne9EtEavKhaeWyxYNW1JNwJbLk", - "3NKjvrYw2Ym5ci397qJytKcKZvkWT6oA1zx911L4G5DUHpoHCSRD", - "3NK5kq9BUQbNxw7c2DHtijagDHqjjjtEpb3aaBoCJZahuijhM8xD", - "3NK6T4ZkCDW6wFGA8dhUzKBGaLcvEBe4mwMg5AqUrxsbymoRLwNR", - "3NKHjNUBbkMbsM8BtZCkzXCNc7V3VL1iYZv1bhuievYsg7kPj7y9", - "3NLB1mxDwST489u4S6NNZPc8T1UCHnKNvXTtwrqM9F5oqUEaf1nK", - "3NKBfRSjygmZctUKNwCEfGnJfwn3RkArVi5ZydMRHeDu5ZYJCKCR", - "3NKABeDUHq9vTVp8ra5N2GTyhN4FvckRbFpcS7RjiLCPuWPCVB26", - "3NL88psq93f8U7DcpKQcSJgMWujEotrLtL8vsvyXDjH5FMmm6nS8", - "3NLKiSvicQbQKVu5c5uxmeQ53tT9vdsdkDLfnno74xwLipHgmSit", - "3NKTuhtYamf7gYS6jbj2Gv353kDkEtdTE1af83erw4Ye8h6t2Db3", - "3NLbZ5dgJ7yAiSbAfYYNEFp5cV9NGvNKfBsxLggn7BKKJfJ3EVvy", - "3NKi2Tj9LdCS8HYadFFjzEaeMBkFETQEo1UtKnuK75DP7RwDsq9c", - "3NKGTH7XLTjTXMm3wWYZ8wuvabNEam6GNqwjH5KCztgWWyDs7hYT" + "3NLhSLrCTSXkH2Wfx2AxizqpCHzGxafxkxcSFV2mWsUdH4Vi6Ejv", + "3NKWXoh4skEqmowiGgdUvDciWCbDtJXJYwDAKGXFZUXbJ5EL9qfU", + "3NKWb3jqqVshKRheitns8nUyzUMTbT2ZkRj1j2DK3exzc24iXjpb", + "3NKNVuYCcJ3LJJHVocqVs9AeeUrYz3X2p7MQqjiRT91Hf157YVHH", + "3NKNvWurFAFhctpdvCCSSd15gizGVbae2T9vxGEzWCF2dWgDiHvC", + "3NKzie3F7WMUG2csKVpeBPF3vZKFYW26sChft1kfqWg4S7sdPY1i", + "3NLtX9dUx9Kuq7hXuQAGmwN5ZaHyFXg8R5Asm1V1QXk5gqqoChxt", + "3NKtkDEfvYXYh4MRAeTwWtqkygQ9HSKw9TZNH2A2CTt5CGR5xNQx", + "3NLbtXPrVVhFNgsDQRzZUhqPWNdWoNvhwj6KvVFnEx35ncstmCwe", + "3NLdjKCz8kR1UKUgP4EbobSb7nHQFcNaFLp6PXpNpo7oZ7mGRLkT", + "3NLhi3B6ks7MwtEszL5yQy5YaFS5wj2kADNwnBxUNbCa9UqVr2DM", + "3NKHLSgGp2V7PtdEjygAGcYMxDjsKk8TbkYparT8JHYoRNpnXR4u", + "3NLCTiYLX7EV4Pvv8733qCL6DqMA2CDZkhmJ9hcuGTSB411ruAzd", + "3NKC2U9Y4MJrjx9Kn5Y99BDvPq1G9rHJR9pqHYyy67KgGMuqUpvd", + "3NLdG85uVvn7PQ2cP5E2b8fNKyyvbpFxALdCcp1hKHLLA5bUute4", + "3NLjjLr2PWe7z7ct3b8somx7Hsjh94e415uSh3pBh9kdBgHWe852", + "3NKVjVtknrYGjqSqFWV6ytqaMEzzhFjxDNpy7HFjAEWLE4BnsYRq", + "3NLcVSrK9SbGyH48irdUo6E5pjSuVHcKJYJy93UXyXBWAkSKajdx", + "3NLxZKC38vXsASfwKnM6FDQ1zuJsdrYYnbFjGfa4cidLgTR4ZaGp", + "3NKWKooiXivDAwyQ66ebboJmbFkSTkDBdtcz6vcXapwWZkTt7yQV", + "3NKnj6x9BMYe3Mcms4V8JjPbqFp3mpUefW59tRYRvQ21Q3s2ysx1" ] SNARK_WORK = "XLjrgk2GplfoMYyG2nfpqByY2HTSYGM3R7mQ5SUByBF3sEnFXqM+qkknTCn6Q4DTmhlnHHq4tj7fot7RAFfmApCKHRNPbaF6NQDV/QroOfDH7PXaEFmE64BQD2EeQJkYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACBXHWfqATKbtc83BoR1/K1C2Xc6VHpaCGJC+B6/sGNOGzGcOUmHvvZzEjFiM2i9LspJ9BZwK/QcMmO1pQuZkEGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAXewScVeoz6qSSdMKfpDgNOaGWcceri2Pt+i3tEAV+YCd7BJxV6jPqpJJ0wp+kOA05oZZxx6uLY+36Le0QBX5gK868TbqxTTN8lUN7AMrspfsSGwQ1kaxpGoVkb1x0pWDgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADf+DSaAFaJIKiWuExLdDUe9OUtRXOhGj+O5wi/3kDjVsxnDlJh772cxIxYjNovS7KSfQWcCv0HDJjtaULmZBBgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAF3sEnFXqM+qkknTCn6Q4DTmhlnHHq4tj7fot7RAFfmAnewScVeoz6qSSdMKfpDgNOaGWcceri2Pt+i3tEAV+YC/AAgWKOnAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIHyq3kAx7rVV72qiUYdHnxzRCgtSmdRM+BFnr/KM3APS/HyYhJcX78DJ/BZtAXW5phSxAPzEfTMqa53WhvyXe0LL3eKMQwD865U+dH8cy0L8qtedrdbWYRQA/KRgR+4GBvC2/ODPtc+3NiqdAAAAAAAAAAAAAPyOfzpP156BxPy6AFSkKVIkMgD8iMlAb1Ee52T85tERreHOUbcA/Fa1M2WuTFsU/Kbkh+ozYsgsAPzFPY/gqcTlgvwjaWG7Crm0ewD8dkpjujXKlGn8pg+fL7vVGfcA/NkK2w/aGW35/Fb9JKqQEjauAPzASTPPVpARl/zC4fOX/nYL9QD8L6Yk/KCTyF78iZVlWSZwPsgA/PLVu2YCz5oN/I57xopcIgvzAPxA4SRr2RpVZvwr6zUzD/uD/gD818f//Yqv8a78NF/D8rycFpoA/ESDzPufZpcA/MJSYYkVmvt7APzRKn4AyPzGevwfwfJ03pnc+QD82GECXrKyiR78CExY49qYMWcA/Px/gRpxi2J6/Lwu9zKOfZ0PAPxOAlA9diXiPvzTuyU5wvJ5lQAAAA/8lPUV59Wxlx78X0YgDu1N1p/8Z0bescI1eP786OejD5erAwMA3qkFMpqUtPYDKXBVHn3yibZ841DUgREo8eAe6rKgVzA0sDkCHp4+FkhAOhxDmgX0fTUGnLx6cc3a7p6QPUvrDPxvv+asybOCM/xjGHb5WEOXeQD8l4eI6QYrOt38x6FEKUDmet0A/MufnPQw5ejG/N2coM1lu90HAPwTGmmHolksU/x7b2UqsLwhqQD8iwcQj7F3nOL87gxr3wBfXPgA/IecsActp70d/KAmX+nilxtNAPwRX4BWfHR1nPzo8c76aWP+oQD8TWDp29+KK1z8m/cQ8oxxjFoA/Ehr4FFcs8Ai/O1tqUBzi4imAPxuZHZetdcHkPwSjk7bOYvGwQD8ySs/N17jRUT85c2M/BXHQJ0A/E6qvEuEgphC/Ly3r9DXJ6mXAPx3bv3/Wz3KmfyUQlwVVWrm7wD8VJmXIXGyfUv8QMiTYeCiH5UA/LNHB7K+zNEs/B0CZPI83tFbAAD8b7/mrMmzgjP8Yxh2+VhDl3kA/JeHiOkGKzrd/MehRClA5nrdAPzLn5z0MOXoxvzdnKDNZbvdBwD8Expph6JZLFP8e29lKrC8IakA/IsHEI+xd5zi/O4Ma98AX1z4APyHnLAHLae9HfygJl/p4pcbTQD8EV+AVnx0dZz86PHO+mlj/qEA/E1g6dvfiitc/Jv3EPKMcYxaAPxIa+BRXLPAIvztbalAc4uIpgD8bmR2XrXXB5D8Eo5O2zmLxsEA/MkrPzde40VE/OXNjPwVx0CdAPxOqrxLhIKYQvy8t6/Q1yeplwD8d279/1s9ypn8lEJcFVVq5u8A/FSZlyFxsn1L/EDIk2Hgoh+VAPyzRweyvszRLPwdAmTyPN7RWwAAAAAAAPR7+S7aTllngy3ML5fTzEeXClZPplgN2ON82fiNL+Un7Qmnm4q4j0/wWcRss/DnHspmW9lRa9esuf/PlwlSAz8Bue9zAtMPrJn6IhG/tuQfyGbeJ8/4ooCnRAH33XFI2w4BB+Wh/EA29FcA5NFqi9UaShGmPTH+gwnLrHlrvxSS7gUBJomUYEf690lver0Cy5x9QLAyhpDiXMLB/ZjbLWgPRjYBI3XWOFBwdXkgTIjN5sH5/XzbBXwJf0YqEsMKqkkzsCABRpTjJhe72g3vTlFvU+XbQZzI5yZDW40/bl+6xpe0xjMBF+jygMkiB/aHhLuKT7fMiRCR7P2tbrO1RISSOpCPEg4BLlRl615r2a/r47rTskI/hJ+lHratuyPszeZQv1E4fwkBbe0eUzVGgqhET6/wdoUWvWGebKHevWDPl/Mvht/oWyIBZ8FIOphQNx/YFdZLJNBo8ZiQ41JZCxMJRLCm1ZEBqQYB80dncxlncrKWDyiMtxMIeL5TYm20huKFFL4cBXNb7TEB+hKp4d+SccBGA8v+ygisjzqA3a95IvF7Va9X5cArKzEBcVlALoeLLbrr+l8IUG5wydGEjSlO8V824TBu91LuTTQBvB1+yW2XkEYUNuFLWuZbdjlSJ/EmwnewEytjPNr2ZBoBK/BXcqMYHKCVUsyj5ROI2ufcChVBrVfUeozn1XOCcSQB1GI8x11uFKU/PVNG6IXEcYvWfJWjwxDOynMARa1BZykBl/4z+BNHG5fJm84ktmjPZkCh34ajWUTfezDZVCZP4AoBTb4xJ2WKjtf9xhQd/7Xc7rjusS+30K+YyDuFPsL38BABLN3GHe2sDblY0/CEU97H/3Psw+ZPCw7+nksas516oTAB/xZmCjgvRmm3LmhvdrBHzry8x7DBRipOZkvoMCtQbykBGZdWjdHN+cvjPWqwWfy/Uobvy8p7mU7+fYLWglzqdDcBqJm90w58LdQ5XiYdrKBy7AkNWZwi1g5TJ1NUfteGhx8BvuxVpToHbIOa7uvXe4c6veRaRiVI6zcdlpi0W1A93g4BnKS8YCnqLDMlh63QJiveiLiMNoWM0NiDOgJ6YtHLYy4B1/ZGSQTGN3N31JlJKPsLr07vdLC70cmxj3ErHsyNHjIBVSM3IxdbBGD6p4rRk/jhWszyLpjEdRzGw2YduzniMiUBDaEiNYnH17/buEkL49j19OmvrKIzCzm5lxB+9uMD/woB7kPY7NZyloBRIdRiqQVnjBs5E50ZyZoWinSxpy7DRwoBZf4xgMyNFOjDxX5kbAIiWcJmMi68Mns71iafOXlgHwgBozJxkWeYBH9Sb3yikxH8/gGC7BEVk65VNskgABFrkwoBWE5/3w3pL99gT3c1bPJ2309gHs8HaVIw9BpUfb/ayh0AAZ8VINp/gWlmzTef4gSH7rQHQEEOLseV1fE9bmljCMswAUAXGQI6BynvkelHgjbRLgOTrAuJNqfmO2r17zAuy4kMAX5hm4d3vOL9vFgCkT8RwlPdCsrg46M53U1s9tJNOC8XAep1Rl5Ra7rYB/UTlMuv+A1y2qBbhKRWoruOjk6Xa+QuAWZwXMmZsQCfmUJDSFCD8biKM/HUWyKl5TKwfTRiRAE4AXVpJmPO0wdOoslhomBdZHzha9toox9fM0gcX7gFHv8cAabjJteQhnkXhMcL9eF/JTfQvfT5+lqXLgrl8+H6+ysCASqmq5/vx4Q1BmOLQENe3k/gbpUQSlGQArTK9gajNaMMAaYLHlkVTh/mVcS8Tg+5uTpTtcOEinCNkZqKSAq98oIHAdt/Zq2Q2Br4Xdr1cw8QQJKsuDns0BBQHDBO3pn6A+0BAQpdPnmGJNBdq290pQjUA/n/ovioDy0IGCVwT0d89akyAVL2T2VOteGP+7rJ4bb01gJLa4lCQjNnoTlW3E5VjN4PAS3rVckuz/LgJdJCVFNloQsKpy78ZsKoTh55qUzOL68DATV9oXc1bwPNQFRDOiFnBLmptiXCeoViM/GdFTLHSdQEATtW9uoLLcJb1auwxROzvQQbqIKzU6aB+E/9poIgk7UaAePt5QJuzwh9NYNHkp3UkI+j82A2I5Xfzn3VOeDM/eABAZ04ssXJiynHQ5HK+o9VsxIjRZGZv3d7Z8AMOvzy9NsXAUhM0EnXi2FYQoWLxxb5Vs09FFOuPJr6nywOP4G03L8VAU06fHuZMeNNYGW6bYcxOICRQ0H2dN7+xbgNxf/1rrQsAdv3In2QycijsVHKpsBbdXwfioFYwxKFb2wX9ILcxFMnASQuIxTUYPDb11eGFKrMhm1oRbYZWtKCSEuF5/3oSjAPAf302ZkN6ugpNcYz581PFFBdZgaBz1TGpj3qkbTuwtArAfF+g1x78DqnPjev3w2RQDyRXBf8OTiTv/JsaZe0Z545Ae1rubgiuj9QgLabjoqIRYUXD23seZahWiZcRtzHKJo8AWO34TbkfcwhAOSrYHFk95xYgL6L/pd9Idh5rBbZb8sLAQZ6GNwSMYeHjUxbRJR2aBVnJB08FHYGvw+JSsXbLnUbAQDiK3R5AWokOV3BOlZwJpXxDQl/uY6hhwG0WhLVxSYVAZoYG526rFbhHsfDnJBV6n38R5zfEhukwmN6Un/0gVsDAUgvmedHgcWxXxjx8Q1/pu2LkHMEsMJGPyzo9ROJDOISAd2tnhHFONbeBu6Bjo2WH4oyLPtjR2I6m7ZEJYIDgQcnAAGJmdctrdMAtOXti8fLyDG9YxVuNVKAiHm3OK6St7rlDAE7bxxTN9ETDez13nqBHPinSC6kn3qs2gBdjAJTZMTkHgG6eMdu8cngt53mwy3TnFUVM/HYYgOGYehuRCk+6e6/JgEC/biLOUiwKRNVWs6tDG/B+JWdNxEf7J/AyEttzpvbMAFpHtDZ0tGzJpJkQS+a0WZYH+q+N2qx+UQWn2AHpJybPgHCZk0sNCZ09+iuMvCbqAxDrjhDHcprt8LvijLU9veaAwFrQCVklSugRGOCG6bqA1FMd1hK1KQgD92JvzSU9EJAFQHF2Jf7SkfrNHCWzAt2/s+uLVDQWFezjIJIew/SEv89AAGoOwvE8Vyb1ldAfvvm8A2rrHdA//8NjyvBwk95pjkUGwE1koKRjAabiwejE1SHTctX4HLo+xLPNt2OY38ZFDPSBwEebNgEYaZLoW7AVfrQI46gMDDxp1fDvDauNePj+4kVJQF1PSwL96Eq0RQZL7MSdRnfPoqjy5Hpi0RnPYM3pvPFOwEqyinll97/41p37jlgAicPP74p+tpzHBQtiUlAwI7YBQFnP2yYqGlMXk0dE+0pnufwWNr3XyMmGfPX4R/u7+dWGwABYob/Mc68J4sbWtxxfCGjb2S4FaJ7NI4OHd2X8u9IzwwBds61pr5rarE4UVD1YWJE6VFGyxlWtmoWoQ0bsBG01TYBFupEXWtFrQMK1NwcZUp1byiPi4eDxzDO+lHECQvEkyoBAZJnhAeiyy6KLTGO1udCKpR+OtWXzzzAShkosIAzBzEBYOAZMD3Liv+flvb5FUz0bdktyv+49ErF3NU+xbUa7jQBoC/b8QmR2FAzAhWAFAGHena3bzzYy1gNa91Oc9rf/S8BSBQKBbOdER0dyPwlZC+0Y6nqO4Lur0TGa62i148JmRYBUL5L9zUsLy9lpsIfH178FKlmOo6ZCLsBSTQDdoHbODMBIb2t5YIc3DY6kPVwqoiSJEVpCT/q96NyRgZHcY8nAyQB7R5y2WAf0VhJsRK0nrhPKoTKba0jS+ZVurJDjhDt9TIBv9bPYKJDLz7M7BVR2AQDqe/ofwWr38Jsb0XYuVmpNSoBtNfZ3H4SFQN7pi7HlmLkKeQE6BqkjAx2871wRxt6bBwAAAAAAAAAAAAAAAAAAAAAAAAAAItPrrSDqvfkPX0ovt3g8WYAt2O7b/9mQ7rKXOPLnY4+2hgN0Tu1QeDw0cAXVI/JXu1g5qKxJk8WKdyCsV3Npw+JF8OEVAaQ3w6mYgRd0FdCFrlk2dXXMhozRBjC/MoiKYKUL90FKJEGyLyZJYdxqNYORiig+RwptD9gHpgML3kUasWMPgkAEPZiLAw74nS74Gbr2p0VyGIWpZKoak4/XCYVG9POVsqBcTZbtIzJXz7MnrcjwdHKuUEkUhw+ZofACxf+L8UQhVAIaDJm9KDEFuCseB6fZdZbjh5B/1sYB/Yy3rsg3U9NL01Gv4W5mlrSmetfRMkGpPxd1D7M1NZPuRTsfE6HzQI2WlTjF1Ut//2ZNdnoKaNNNTdyO7aV9towHeFnn1fCC7YS/gQsxicQM4trNmoBd2DJ9HFANzT1D3M0xu8CwNdKaUt68ClHrPWnqHSwM/ND0HQMKIs2pTe3bwvYgYh94rsJmS8SMLmUjw1E9TTp6oKECZ3ZdqnUdEA3AucKOm3LCLollOnteQIAeKTEeIvm5FgTaGOh7a5NDxwrOUdVkZ7fdEHHR1PZ7fk6J/Dmwm8fkpooe6fvtGXaFj/qJ0oZk103eNDW5Iq/snJyEJ5Uugpwt+u10yFc+FBxMYR87DOSTQBjipBn5nPsvda7eCiZ6vp0dHOfjnA/VNgFt7FmaV+3eKYuS1zq0k9ROmh8CTPVTO4AaVtpi9jE6xgumVUBmnv62RAcPKOCuOzesttQ9Juzwe2IANA38V+IE1hYq/1uhK0aV1kheXwEikLpNVjGvIfywQB8XzbK6mcLXnIzXjVY3jvSlCcaLoCkC3Fm8Qzf6l51hw5fyWOgkDu2+4xXM5p0Psq2RddvNokC25GQoPfgOK1tvxJcP0+pCyss/hlPSF+c9nKriwPq8SjH/5VLw4kzGgvtr7PT16g91SN7ZgBu/Ybc7zUx9o81q1aH6L9K7HKnSAflFOwgdScOFvhd0EZYTnzytVwLjPQfPYUULwsjzMp4XcA9Q2kyJA1WmehKI+EBrtVPaMEbe9BjrqoEuBmvhJ/XYMlLN6M/VJdztxshLHj/4OAR6RpORD1Y+TrIdmedkgXIWsAJrjIqMCFFfnmX0t3/oJh4+l6HQSgm9x33QzFaZx0NZM24E+WohPgmHG7Dk8DxN06BNvsG56d6EVCA5trfDKqtNrMrtsrsP7GxYC4cX24UgYT6QYkkFliKIu/Bp5s6j/D/GRS77hsIwJV9ciNRkudW5ROSt7uOyPMWJgeeuY13d3vwMbOHvDAX3zLZ9RBu0aWQ7MlQ5iik4E2A/58S2xj8968gAIA/X7KU1vyF9VtRETox9vS6LbtIe32YITm7mzuVL/QFKOqIUzYoCmum1Hjab4isl0ce4jn7FJiU5Oa3uZKIAw1brPXXr0Wvx4Q5Piy1gR28iPRt5ouNOnJdCEPSFg8qOyxoTUi/hI4zr0AW3J9R+lT2rxBjMkfUNV0pmUC0EHAk8e6AC1TT5RTUsEY/OdEHtS9nFitexN8cu1orYD869RQn6dlcVX3YJOwvCqE45GkrGX2ZU+V4rECPEG+JKwYnJTCGbKBwkZaeYDyhVtggj8ZbQjHVWzwygUhhIiOuTKAYOAcYmL4deH2eyEtE0q0zkyTMVPtLzox6pk6BOCis9AJaW/g1z77m7FzoT59h7i1ae9vyfNrsR4S3Vyb24+l/B/ptcsdYGObBf2bwp3odAyAlPLhfR51m3y1sKTvMdPMI023zztUOK43HMNk0AiuSwBEu3iNyCSLqa1VsWH23MwTeQAqkgocM4T9NoJo0rxmN2/o4bVzDrA/OkoCXx7E9Fuzz5QQF+xW5pELEzj0zvEaxI8/t/Mx5utdVzxcZQI093A4uYNrO6YIHv3RV4VeWHxHWJTPDTxvh5a4dFD31JQoy/g0gGMDv2htlUxYIn/myU2GhSVNQ/On0psM5t8emONls4KoJCRgXrAK720aaWnCS+WhLMEoS6ls3mEwzhmQQAL/E/M9Qm3tMn5Jk7owcjegyPY47vkyLv1mjht3YIYw5TvkWf+o1wHTwuE+/6gfe4UAde6YHDepw9dMA2kq/HD1z8xV5orL3RFYYb187Q80nEcCsFfqFtOwC1jqXU4kYJI+NptFpVcwYAyD5IOnsO5bDQEsfmNOD5nTiYIQ7U2YmRwjNmO4hxsa6ZEUT2T7d+XjV2bAJqt1fPx/CAmLRKhzq58wqmKbrRXUdyb2Mz2zD0nChZWgCJuITNf6R4Ho6ORQTVxtyBNrv/Twx/wD9blx/bpnmPc+Qf5eerhQ5e38Eelu01OCKrHobNjrA+q2pes9nI6Aac3Ge7WPFVTGWbR5ZsWl1rnLp7vmoA1wK+GUs92DOFx7d04j+hfafjky5PJeH+g3SEPc+qNTTBKHjNWbH2GKWIl4rh8pH9hfP9ZIdbe/MUPe3aU4Gin9XfO98eHx/StJoYGdZetQshsW59i5Zeuc7bt1ddQRpN9H5jeaR1lsZkjTEAXk3PRGgZMNsA7WXL+wuA6rxvNwYuGI6b6HsvMNuCPnkbKlpKLhzP6IWOz1UokUV3CE4zH7xEptKIopHoI5l9/5A2ZCCcMlgbz+zK5Qc7rH00tDIlZwDn42nTQ409TiVOS/Ruv4X0Iz6Gln0f5bPGAgu6AWjrN+R7MYOFlxsPRqYvtI5i8otgukYGf/WcSqfEpqrG135YKbPlSZ5SAWHM3OBi0gQiLRtcDhZ21hCosaNOL0otKmrMXUr02LL5mMEyPKCZqaYf3WPPr9RcDMtmYO+AVkRCw+aqkoYxLM0IAnSxZBHxBVhXngBnY+9Orp7puhYE9L+5+NTMUPzmek9LvCafjdenyMUlA0pkgqkiWPRAzHxJBWD4M1MmIGU8+g5vyvltGF8fGclCPiLACQSIdbypFLd0at0qTmunUbEcGoyqE7fcih/T4IsdgToivWsd+BwWZl7XdodTQ5bCv2ApoGVlXGFRRnjwQmr8sxNkJOKXc2TTsnULEkqbbRHusRDWsulUQvbppFPHx+a9HiSpF5V5OPaZQ7s2es7DdT6bfc191PqQMUAS34DC3RqRwLgQgOtREklJ22kjzihU9p+V659JlVtKsPpGh0VsLEdG2Ax1PwjxNRdMGUkOi4nlu+1B/ucmHhpPnEHBQgVNsZuqT19teWiy6B+a2FqQ0qDElsw0b9lDfusO4QgSMJMjd4qRIk9NpvAPZ6BXZlj5rsjs++DctU4WKYF9zy1ukJ+3IfZlDa/QLM2EO9W+SPJWhhf0hmvOjb8BCOyEADP2TfsLs9Mj/wMlPi16N+6F77u7+iazG/WvaTBmHk0PYO7dgFwLRCq2XIKXQWAaAy0h+c/OnU5JEBDI28Eb8oXlVeSbR506jtIiqcQ5i/1eZITEEg5ogMnFOwBamjL+Bejk9buDairospPUuMoCszXQuRtzaw6aUr32jls9y8+GNib4N+kKaw72Ad+/ymXz8oWgd+4bRoM8KogrQY093wOgdJvpGoFG9dY+nlB8/Y4TlQA6KIev03F6OXPJtxM1wLFAbS6/9qjazQ6NCNnsQIQliJ0S727MdA2g48QxOiXNsK8OMrrZElUMVNjT8iemF5P2szzXHqKgHuIdx/OV0oJ3ON8i1ppIHSV2NzccLJpwbmXYLSyLw/6+PY9441AwBdt4GYVbk6A31zzjTIWWwSbdIzBie2ynuXCXRjeNKw8O2ylC1Y0ClEWKMk1iH907q3F77dbqV4MURG+4JopbpgFjZzHiW3jC+PgjSRq9f5qxrhv9VDiFJnZ9t5xMp3VSwM6lab6cjXIsVxLzfgvNHtUbd08ozE3H9p3PqhGdxr4FmgmNwn6ELQm6jyet8dkgA5LPa+DeLuSgA0vl1BQHuQP6NOmLYM2IBf24E1dn5KRfme7kbAdnWhXgFplzj6ocQysydrVV7+WiEAVo2ypcpXcC5A6bqIaCkj+FEMf47EEMMdIqEF8YIldHtz3B3gUA03v+039UGk8IpRdXH92L+Uk/5q4dHcZkzymm9MlkPr04YsbW+0Gadpob3+p5hBdpx6QHGCFXAUGbafhwE1rUK3NT6w53MNMvcoPv0zvkewvOfhhFIL7EvyL27LF/SUSK05tl4/fhsuTTHUDeo0jjdU24dlaVMXsR2c5PiuwVsTqO+tnfq8EjsGoyCLa1bcDXBwsayIaxhjG+jk1BibaJNCYAPGaBQMKt18/93+xHqGuAoT0H+mryUWYMid1Yb4khnLwZwUevmFKwcBGgaPR9BI8/OwR9TfwyaTC/EgcFxsNmY7kwsXAhwFQ2bCgRIREux3Mpa9g1cRbd4gGeHXOJddSmp7R/orrfEjZQ8YVxBkTBc1mwOeBMNlp4I4tXFXPchNRSuufUOPlnj2KtjWwaGkBZM8oxmHcmEwwd8ui0NuALjEh7rXeOZYssvIzXQVGsSqFSQW0n1pVY7PAhZQyZFnEasDZ9SJqBEIrselEfwdxGo066Cwre/8tT7e4WMyJZIDCpmf/dO2IzxsHoOXj7pYPPgQYxNJay5dn8UUEmyPnAUqIcRytdGnCVNNOXXxUpxUAlDPW+TPG/TlfI59QVDlMqM6a7sf6MqsG3icAlAgz0BwaOjt73P2HCwXqTY5FLRjbFKX6RLtbbjz+/uoGUrkCOR+lNR42w4N+043GuJTEyWKkk2+r/PCQBaQslAC2q1oxGWZxqOTszAx+kmLGEjT/BCLW8dhTlu16R0Ztxupy9ii3D40fk66jHDkSA1i+dl4BfiYcRy1iTlfjxJKvwgngLH6Hn0rqH9h2ep7CAiAuAAxWilmZWxtJd3t3vbjnuWo2gF37WIYODgiwGeXVQJA2yYawtknHJvPvi87GbOXqeTHR9M+FlouY9eP4w5EwlOYM3EzjnRy0MCy+FnQb/Mm7CZEQnX9mjpDc4w8SUa/GYFeH2q6RYKPnuAeSOBGWwIYAOyTX3AFN/iEaegOHRGTF1638JRz9O3V8V5vUDyrUuibcm6tEn45EaJrwwYKcspdeUrSX3XCOfZTJ2FFb5fXDFD4CSsQ3zDNMe3ks0sZRHTw8g4rdJtCYuBBScFWtBhsYDkc1XpBfTO0m0GjvonSa+yfZLUagImL96W75QSDVQjbSkl8zDQE/yQFsV5WLJCz9ucG+A0AZywZVSW47XbZsNAA0Wv/bt2db0ZP8t545G5kJkj7m5XPwqsFS6EAb8weLBXic+6LxUs5ad0q7KWym1diFAwMDJQtX3ldikEFeaLEbOTcWiQfnF5i2QCxgsApl9u65/mpVRb8RoVP0lhtcxQ+Rog6T5PfGm9PXgwKQzOiKJ5Lgk3DQZC+kS30x+mzSBRv4E1dyLpZtjkpUFVV6sEMNGnMMLjSeGxw//g6vaGUroM0xNfSfedzA7lDgHDqOidrwOMA/hU8ZCjZBsgvFGQqrFCBPASHjhsACvI60JCqzTTA1AljM2sXRevS7232lGcy+HP7PAzvV9J1tArvFnS9T2ztPT3prLejye0md7ocYQJb8y63QN4ZiEpmG7TzF4nd+ZvWnlmnFVhsiwuegmBsABS6Zx4Hg3bdMVSyuPNMSafqxkHJRPWc+kBK2J1ppJEFLryd6T4ie9rV5gGXsexYkr5YJSPr4q5ha/PslDjsoZ9ZXbGAQ14ElpiYKacBT+M7vNkVKDJ6VXbb4oMdCWikF294/LKDfI99aXaYQ/lx58dICSns0KA2P2T1K+fspKg+euiDTyubAdhCIhb1Fh562gfD2eJoQ+PE8C+G28RdKNQwwsHyJNjF8KVrU2ivhtHVFoAYvzrdgZzAiDS+cDzEc6WWzIRNnWReNVED8MnhVB1x7N19anZFCCjUSCnym9QJJyz5BBCV0WknyQ31UtVCNsqNaWNueGxNnA5+/LSbHGOcn9juFKmDZ+LiVrvgSjycJYFuw66edq34+HOsLL+EcvL9v2JnyWsKy+yy2pNmGnkyM39sy/aHqjetKPvO5KSpyZ6a2y8UMxMSrNo6tQBUyp4gn6WbuZilnjN7wH32LGdTVCR9xszffZJXWI5FfL4wQ9jwFu3jCHGz1oCWqAIkL0Qs5gmUHfriCgiSvQ3aQyldpeqR9H5AAj47yaobIdQWkaC71svlp3cTujDnEV5N5c6RJM7Y254AjIchF8OT+B2glbceL4P92akW4LG72X+OzU/0YrtD3XVkwoJWeUt4TODhFzZMH1Ee2vJ3IuloSWxy0G/dIR7y7wj97V4nDUT3tjytz2SUsbnFnZxywzE8aMfH4HVJfl8u0gH6t/5RBB6BgmpI8g/3tqaRfGc/aBXqscJC0owKuzVIo5bariZEOr+toEpI3ekMWWWmfIfKvE28NGdQl8R4DQDkDwqaPezWupbYyP2OWeUZgy4vdIEOFHEEgtY0+jt5DlWIe6wpNIItfj0bBzYwG0rUvYFrQ9ro3ZE44/UwvJNZkHLuevfs7lWmkAuNUhFS0aah9Ry5rUc5SyJP8jUL53sAUkEZ/pi0AhktEm8ch7QckUecM+wACABjj8Ig0KImmV+u9vh0bLpmeF4SLFZW6TzKPt8QHwqzYG0jbyuZUbtpYcZN4PWcPHqfuFEzKMHiakE3V/yqcLNNyB0MWOUZRtW1ZL0U1tRIbYcvkBCmRJW55QSWiXkMkmg0CmSyaOWG64O6voripMjBtK+r4xFxrjYfc9YTn5KGT/SWGGwkrbTDdpcHxCgwxPrsUBzuay0alG6PhaxJoi/j1stM3MjIpkYDTzXHvoROxKzR7t/7snCrA8Zm3dyrYq8m6Nsyev3jm9xoJSFVNMl86orqBmzDTIBzL1Ueh+6h9ZMQTCp49nrIbn2caDmwdlC6Z6Gt7h2GIOt+vMEKyYhnUNNPCyFvr7dPjY4Cq1R2XqM5VxrJWSI0IerUMpsWP4R0Wsc8jpHmnERJZznKhPpODhlCSvGhEocyd35kugPTQ37yrkvvWXNkQofZp+VQMvZFvWJd4/GSoFl4jIGu74DjvAkk8iZfgkoFKc2z72w+kxHOnSIxtz6nPfXKI24nN9OhVulV3M7NxBeNh7o59FBrVhPU3FuO907tbZcuMJoiCgtyr/IUwn0oaxYVcfgARreaAVLJstisdAzu3XlbxCL97Y8vO1tzAfsOqCF/WajZjyA2vIhhVDOEt5l9fymREG6TkdACjDu/iHvw4MRJYLMcFs3nxn31IOix/fFoomuwk4PvqAWbXFjngJq5mnXcZRG74OQjfpTY2f86JEy+rj0M22d8cgBVQISjwwVtFeDmMc2ZG+bdqE+hPTuWfzOTF70wkPZgmAT+QYYChBCU+ClnMSXt+ZuyP5BOuUlgFcM6r4ZVZpN5UVbAvHye02J4nh1uOFy61ZsGtlwdaVHET7gmdn5FONLz+rF08SiWgNQcknSNRvGuSAkYWjv3JKfFnszcPCbvLrdOA7qC3a4DvL+cn+hseEEj5xQf4SsG3xZkD2m/yyZV1oE+sfMzluX4yB3jZCitdKCV9oYGRJSWXNUmjKIDvCBLTh1JTJ4kWNjKCRKTRk4bKCQwZQrvZQdPmr5YuiD21v49zvDTwu58KNh5q7J/hrJdzh3QlSdYD3txlF4IW7BbHjQd/Oa1+wLUIi7YKZP468BhExfUUa4wCe+JebWrZOESp838Y7yITBB8OpnNXJedFhzkVNegisglhUjpG2UqGJJ3QH6Jsd8vYAM/k+t+kbwQBA/HeTkZjNOHXMO29XVPoiAcGh60EorAbF4vGsEmk2ULU241CJ40c8NyXF+5C4jkbaRMZHivQqzdjmqD1z+WLIvUhuSNo0q2kLw+qQY6u7S7S3F8uyF2QNF4u+EOmiH3q/5sSqWTSC9HWJNSxWGKQGfk6aDxdkVYlJc9WaFpeh++5wHeyp39j6SfZ5dIDoGojN4cK1BIH9hSr2X5MNcveY7cxkRvfTDXY3WWTzP0/v+9mX8G4+VRYL1WyHgBQVR14Vd9PIZkSB4iaHpBI6cZusF5D/Oy/DtMlPGyajJHSdx/SPjTfuA4MYb9RrdLOS8BCTaAdBTuZgQvYTN0W6bs4PZnfQOX/nyYoGnaLJQdLp0hfTD1YVzvZG5UeOMYKEtCJLy02QyMOvMqIf+Uq5Z3BxwM+jShqink9+wOCk+0vJPqEs13ePYbmHO6iPcHMwR281m8cNyTWVyHTYzYmQDVrf4TOVEf6qBnDWosQz/LQPQMn7VO54siWE89R2dh2MmjXf5qNwiTiCg0rB8j3/+P0EDGO9CFIlU0e10CxMitJYihn1VRe0w76D/djGeLPnJ/ciZ/C9nI2yAtjjYmxRNciXyu3bJyddMQdAfHjI1SpVeCcm4gvqZewPoZgkUuq2zjtH30frTEnODW6+PTBHIxZMk5OqhiTsBc9Dp5elHzYWcF2VPYx5QG7BAIq2TYugEAiLJAsEG2fmzpf8pzFNTl29AdhRVED5dJn6xpBptUBajDlbaIuwOdgJBYEdRJ5B52X1jrqiYPpAOY56bdy87XgCxpev3CudiYkkQDm55oxs5Y74/Wv7xhDpKW8Xbz9YKsLVfcSRgHFID4AAAC4smwnQP0A4fUF" diff --git a/e2e_test/generate_submissions.py b/e2e_test/generate_submissions.py index d46dedc..e5b0c1c 100644 --- a/e2e_test/generate_submissions.py +++ b/e2e_test/generate_submissions.py @@ -9,28 +9,90 @@ - blocks form a smooth, uninterrupted chain and there are no forks.""" import argparse +import base64 +from datetime import datetime, timedelta, timezone import itertools +import json +import os import requests +import sys +import time from data import BLOCKS, BP_KEYS, LIBP2P_PEER_IDS, SNARK_WORK from network import NODES +class Scheduler: + """The scheduler mocks the behaviour of a real blockchain. It is + given a list of block producers and a list of blocks. It then + cycles over the block producers, returning them one by one, in + intervals of the submission_time. It also keeps track of the + current block, switching it every block_time, taking the next + block from the provided list. When that list is exhausted, + iteration stops.""" + + def __init__(self, blocks, nodes, block_dir, + block_time=timedelta(minutes=3), + submission_time=timedelta(minutes=1)): + self.block_dir = block_dir + self.blocks = iter(blocks) + self.nodes = itertools.cycle(nodes) + self.current_block = next(self.blocks) + self.block_time = block_time + self.submission_time = submission_time + self.next_block = None + self.next_submission = None + self.block_data = None + + def __iter__(self): + now = datetime.now(timezone.utc) + now.replace(second=0, microsecond=0) + self.next_block = now + self.block_time + self.next_submission = now + self.submission_time + return self + + def __next__(self): + now = datetime.now(timezone.utc) + if now >= self.next_block: + self.next_block += self.block_time + # at some point this will raise StopIteration + # which we allow to propagate to terminate the + # scheduling + self.current_block = next(self.blocks) + self.block_data = None + + if now < self.next_submission: + time.sleep((self.next_submission - now).total_seconds()) + + self.next_submission += timedelta(seconds=60) + return next(self.nodes) + + def read_block(self): + if self.block_data is None: + filename = f"{self.current_block}.dat" + with open(os.path.join(self.block_dir, filename), "rb") as f: + block = f.read() + self.block_data = base64.b64encode(block).decode("ascii") + return self.block_data + + def parse_args(): "Parse command line options." p = argparse.ArgumentParser() + p.add_argument("--block-dir", required=True, help="Directory with block files.") p.add_argument("uptime_service_url") - p.parse_args() + return p.parse_args() def main(args): """Generate submissions for the uptime service.""" - nodes = itertools.cycle(NODES) - for statehash in BLOCKS: - for _ in range(3): - node = next(nodes) - sub = node.submission(statehash) - print("Sending submission:", sub) - requests.post(args.uptime_service_url, json=sub, timeout=15.0) + scheduler = Scheduler(BLOCKS, NODES, args.block_dir) + for node in scheduler: + sub = node.submission(scheduler.read_block()) + now = datetime.now(timezone.utc) + print(f"{now}: Submitting block {scheduler.current_block} for {node.public_key}...") + r = requests.post(args.uptime_service_url, json=sub, timeout=15.0) + json.dump(r.json(), sys.stdout, indent=2) + print("Done.") if __name__ == "__main__": diff --git a/e2e_test/network.py b/e2e_test/network.py index 304c38c..5714018 100644 --- a/e2e_test/network.py +++ b/e2e_test/network.py @@ -19,11 +19,14 @@ def submission(self, block): containing a BP pub key and a peer_id.""" now = datetime.now(timezone.utc) return { - "created_at": now.strftime('%Y-%m-%dT%H:%M:%SZ'), - "peer_id": self.peer_id, - "snark_work": SNARK_WORK, # this is a dummy proof. "submitter": self.public_key, - "block_hash": block + "signature": "7mX1kSj74K1FVnNrRhDMabMshRA2iNadA5Q5ikqh95FAE3Hi4o6fQUQzgHmuacLk7ZZh9evh1FwAzMe1JwCycr5PZQ3RoXZf", + "data": { + "block": block, + "created_at": now.strftime('%Y-%m-%dT%H:%M:%SZ'), + "peer_id": self.peer_id, + "snark_work": None + } } NODES = list(Node(peer_id, bp) for bp, peer_id in zip(BP_KEYS, LIBP2P_PEER_IDS)) From 1ba41bc252ce2c2d4327378c7b49aa02d1008cb9 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 19 Mar 2024 09:40:40 +0100 Subject: [PATCH 04/15] Add a README. --- e2e_test/README.md | 85 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 e2e_test/README.md diff --git a/e2e_test/README.md b/e2e_test/README.md new file mode 100644 index 0000000..468bb98 --- /dev/null +++ b/e2e_test/README.md @@ -0,0 +1,85 @@ +Uptime service end-to-end test +============================== + +Step 1. Block generation +------------------------ + +A blockchain is a hairy process, involving a lot of peer-to-peer +networking, distributing blocks and transactions, selecting +block producers and so forth. Any process involving a real +blockchain is inherently irreproducible. On top of that, Mina +nodes take a considerable time to start and synchronise. All this +would slow down the test. + +In order to avoid these issues, we mock the network with a Python +script which will send submissions on behalf of some imaginary block +producers, containing blocks of some imaginary, dummy blockchain. +Because we don't want to depend on a node to generate these blocks, +we need to generate them beforehand. For this step we require +Mina repository. In particular we will use: + +* `dump_blocks` app to generate dummy blocks +* `delegation_verify` app to extract state hashes from generated + blocks + +Both these apps are handled automatically by the provided `gen_blocks.sh` +script (they must be compiled manually, though). For the script to work +the following env variables should be set: + +* `MINA_DIR` - the path to the mina repository +* `BLOCK_DIR` - the path to the directory where the generated blocks will be output +* `SUBMISSION` - the path to the dummy submission file + +The script can be used as follows: + + $ BLOCK_DIR=/home/user/blocks ./gen_block.sh + +Step 2. Upload +-------------- + +Pre-generated blocks should be uploaded to s3, where the mock will be able +to access them. An example package of 120 consecutive blocks can be found +here: https://s3.console.aws.amazon.com/s3/buckets/673156464838-mina-precomputed-blocks?region=us-west-2&bucketType=general&prefix=uptime-service-e2e-test/. + +The folder should contain any number of `*.dat` files, each of which +should contain a binary block, and whose name should consist of the +block's state hash and `.dat` filename extension. Additionally, the +directory should also contain a special file `block_list.txt`, which +lists all the block state hashes in order, that is each block on the list +is the parent of the block on the next line. This file is used by the +mock to submit blocks in the right order, so that the uptime service +can reconstruct the dummy blockchain correctly and award uptime points +accordingly. + +Step 3. Running the mock +------------------------ + +This step requires an uptime-service already running. It should be configured +such that to does not verify signatures on submissions. We want to test the +logic of the uptime service, not signature verification. The mock is run +as follows: + + $ python generate_submissions.py --block-s3-dir + +The mock contains a list of 15 hard-coded public keys, which serve as +block producers' addresses. It loads the list of blocks at the given +address and forms the chain of state hashes. It also rotates the list +of block producers so that it never ends. It sets the block pointer to +the first block on the list. + +Then every minute it picks up the next node and sends a submission +with the block currently pointed to by the block pointer. Every 3 minutes +it also moves the block pointer to the next block. This way every block +producer appears to make a submission every 15 minutes and each block +gets submitted by 3 distinct block producers. + +These parameters can be tweaked using command line parameters: + +* `--block-time` followed by an integer defines the interval in seconds + after which the system proceeds to the next block. +* `--submission-time` followed by an integer defines the interval + in seconds after which the system proceeds with the next submission. + +The mock can also be run without downloading blocks from s3. In this case +`--block-s3-dir` parameter should be replaces with `--block-dir` pointing +to a local directory containing blocks as described above. From c6b1421a41767c00f37b0a4aff5a8e6ff0253ad8 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 19 Mar 2024 09:57:26 +0100 Subject: [PATCH 05/15] Improve the UI for the blockchain generator. --- e2e_test/README.md | 3 ++- e2e_test/gen_blocks.sh | 29 ++++++++++++++++++++++------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/e2e_test/README.md b/e2e_test/README.md index 468bb98..c18b081 100644 --- a/e2e_test/README.md +++ b/e2e_test/README.md @@ -26,7 +26,8 @@ Both these apps are handled automatically by the provided `gen_blocks.sh` script (they must be compiled manually, though). For the script to work the following env variables should be set: -* `MINA_DIR` - the path to the mina repository +* `DFELEGATION_VERIFY` - the path to the stateless verifier binary +* `DUMP_BLOCKS` - the path to the dump blocks tool. * `BLOCK_DIR` - the path to the directory where the generated blocks will be output * `SUBMISSION` - the path to the dummy submission file diff --git a/e2e_test/gen_blocks.sh b/e2e_test/gen_blocks.sh index b763914..0e44841 100755 --- a/e2e_test/gen_blocks.sh +++ b/e2e_test/gen_blocks.sh @@ -1,19 +1,35 @@ #!/bin/env bash # The following env variables need to be set for this script to work properly, # as the defaults provided are unlikely to work: -# MINA_DIR - the path to the mina source code +# DELEGATION_VERIFY - the path to the stateless verifier binary +# DUMP_BLOCKS - the path to the dump blocks tool # BLOCK_DIR - the path to the directory where the generated blocks will be output # SUBMISSION - the path to the dummy submission file # A dummy submission required to pass blocks through stateless verifier # in order to obtain state hashes. +# +# If there is a source code for Mina on the machine and it has delegation_verify +# and dump_blocks apps compiled, it is possible to provide MINA_DIR variable +# pointing to the root of the source directory rather than specify binary paths +# directly. + # NOTE: all paths should be absolute, or else the stateless verifier gets # confused and fails to load files. - if [[ -n "$MINA_DIR" ]]; then MINA_DIR="$(realpath "$MINA_DIR")" else MINA_DIR="$(dirname "$0")/../mina" fi +if [[ -n "$DELEGATION_VERIFY" ]]; then + DELEGATION_VERIFY="$(realpath "$DELEGATION_VERIFY")" +else + DELEGATION_VERIFY="$MINA_DIR/_build/default/src/app/delegation_verify/delegation_verify.exe" +fi +if [[ -n "$DUMP_BLOCKS" ]]; then + DUMP_BLOCKS="$(realpath "$DUMP_BLOCKS")" +else + DUMP_BLOCKS="$MINA_DIR/_build/default/src/app/dump_blocks/dump_blocks.exe" +fi if [[ -n "$BLOCK_DIR" ]]; then BLOCK_DIR="$(realpath "$BLOCK_DIR")" else @@ -41,14 +57,11 @@ function generate_block_after() { else args=("--parent" "$1") fi - $MINA_DIR/_build/default/src/app/dump_blocks/dump_blocks.exe \ - -o bin:"$BLOCK_DIR/$dummy_hash.dat" --full "${args[@]}" + $DUMP_BLOCKS -o bin:"$BLOCK_DIR/$dummy_hash.dat" --full "${args[@]}" } function get_state_hash() { - $MINA_DIR/_build/default/src/app/delegation_verify/delegation_verify.exe \ - fs --block-dir "$BLOCK_DIR" --no-check \ - "$SUBMISSION" \ + $DELEGATION_VERIFY fs --block-dir "$BLOCK_DIR" --no-check "$SUBMISSION" \ | jq -r .state_hash } @@ -57,7 +70,9 @@ cd $MINA_DIR generate_block_after # first block in the chain current_block="$(get_state_hash)" +mv -v "$BLOCK_DIR/$dummy_hash.dat" "$BLOCK_DIR/$current_block.dat" echo "$current_block" > "$BLOCK_DIR/block_list.txt" +block_count="$((block_count - 1))" while [[ "$block_count" -gt 0 ]]; do generate_block_after "$current_block" From fd3df0d0e9b6d16dce961861451b2d9b6b9e98ba Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 19 Mar 2024 10:15:02 +0100 Subject: [PATCH 06/15] Add class docstrings. --- e2e_test/generate_submissions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/e2e_test/generate_submissions.py b/e2e_test/generate_submissions.py index e5b0c1c..98933bc 100644 --- a/e2e_test/generate_submissions.py +++ b/e2e_test/generate_submissions.py @@ -34,6 +34,7 @@ class Scheduler: def __init__(self, blocks, nodes, block_dir, block_time=timedelta(minutes=3), submission_time=timedelta(minutes=1)): + "Initialize the scheduler." self.block_dir = block_dir self.blocks = iter(blocks) self.nodes = itertools.cycle(nodes) @@ -45,6 +46,7 @@ def __init__(self, blocks, nodes, block_dir, self.block_data = None def __iter__(self): + "Initialize an iteration." now = datetime.now(timezone.utc) now.replace(second=0, microsecond=0) self.next_block = now + self.block_time @@ -52,6 +54,7 @@ def __iter__(self): return self def __next__(self): + "Return the next scheduled submission." now = datetime.now(timezone.utc) if now >= self.next_block: self.next_block += self.block_time @@ -68,6 +71,7 @@ def __next__(self): return next(self.nodes) def read_block(self): + "Read block data from disk and cache." if self.block_data is None: filename = f"{self.current_block}.dat" with open(os.path.join(self.block_dir, filename), "rb") as f: From 58148a2ec42cb1ed7a20c17afc68257dd08cd68c Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 19 Mar 2024 14:31:56 +0100 Subject: [PATCH 07/15] Refactor the submission generator. Extract loading blocks from file system. Make submission time and block time configurable. --- e2e_test/data.py | 26 ----------------- e2e_test/generate_submissions.py | 48 +++++++++++++++++++------------- e2e_test/local_block_reader.py | 36 ++++++++++++++++++++++++ e2e_test/network.py | 2 +- 4 files changed, 65 insertions(+), 47 deletions(-) create mode 100644 e2e_test/local_block_reader.py diff --git a/e2e_test/data.py b/e2e_test/data.py index 8222611..b928bf2 100644 --- a/e2e_test/data.py +++ b/e2e_test/data.py @@ -40,29 +40,3 @@ "12D3KooWHecvaEeAimF5gJ6FKBEcB2VcyLk3L7ynh7PgFX2VkPAJ", "12D3KooWEQQxABEeYyX7DGLjDuEWTtNvEmN9UAPWWKTzxfMaFZQJ" ] - -BLOCKS = [ - "3NLhSLrCTSXkH2Wfx2AxizqpCHzGxafxkxcSFV2mWsUdH4Vi6Ejv", - "3NKWXoh4skEqmowiGgdUvDciWCbDtJXJYwDAKGXFZUXbJ5EL9qfU", - "3NKWb3jqqVshKRheitns8nUyzUMTbT2ZkRj1j2DK3exzc24iXjpb", - "3NKNVuYCcJ3LJJHVocqVs9AeeUrYz3X2p7MQqjiRT91Hf157YVHH", - "3NKNvWurFAFhctpdvCCSSd15gizGVbae2T9vxGEzWCF2dWgDiHvC", - "3NKzie3F7WMUG2csKVpeBPF3vZKFYW26sChft1kfqWg4S7sdPY1i", - "3NLtX9dUx9Kuq7hXuQAGmwN5ZaHyFXg8R5Asm1V1QXk5gqqoChxt", - "3NKtkDEfvYXYh4MRAeTwWtqkygQ9HSKw9TZNH2A2CTt5CGR5xNQx", - "3NLbtXPrVVhFNgsDQRzZUhqPWNdWoNvhwj6KvVFnEx35ncstmCwe", - "3NLdjKCz8kR1UKUgP4EbobSb7nHQFcNaFLp6PXpNpo7oZ7mGRLkT", - "3NLhi3B6ks7MwtEszL5yQy5YaFS5wj2kADNwnBxUNbCa9UqVr2DM", - "3NKHLSgGp2V7PtdEjygAGcYMxDjsKk8TbkYparT8JHYoRNpnXR4u", - "3NLCTiYLX7EV4Pvv8733qCL6DqMA2CDZkhmJ9hcuGTSB411ruAzd", - "3NKC2U9Y4MJrjx9Kn5Y99BDvPq1G9rHJR9pqHYyy67KgGMuqUpvd", - "3NLdG85uVvn7PQ2cP5E2b8fNKyyvbpFxALdCcp1hKHLLA5bUute4", - "3NLjjLr2PWe7z7ct3b8somx7Hsjh94e415uSh3pBh9kdBgHWe852", - "3NKVjVtknrYGjqSqFWV6ytqaMEzzhFjxDNpy7HFjAEWLE4BnsYRq", - "3NLcVSrK9SbGyH48irdUo6E5pjSuVHcKJYJy93UXyXBWAkSKajdx", - "3NLxZKC38vXsASfwKnM6FDQ1zuJsdrYYnbFjGfa4cidLgTR4ZaGp", - "3NKWKooiXivDAwyQ66ebboJmbFkSTkDBdtcz6vcXapwWZkTt7yQV", - "3NKnj6x9BMYe3Mcms4V8JjPbqFp3mpUefW59tRYRvQ21Q3s2ysx1" -] - -SNARK_WORK = "XLjrgk2GplfoMYyG2nfpqByY2HTSYGM3R7mQ5SUByBF3sEnFXqM+qkknTCn6Q4DTmhlnHHq4tj7fot7RAFfmApCKHRNPbaF6NQDV/QroOfDH7PXaEFmE64BQD2EeQJkYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACBXHWfqATKbtc83BoR1/K1C2Xc6VHpaCGJC+B6/sGNOGzGcOUmHvvZzEjFiM2i9LspJ9BZwK/QcMmO1pQuZkEGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAXewScVeoz6qSSdMKfpDgNOaGWcceri2Pt+i3tEAV+YCd7BJxV6jPqpJJ0wp+kOA05oZZxx6uLY+36Le0QBX5gK868TbqxTTN8lUN7AMrspfsSGwQ1kaxpGoVkb1x0pWDgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADf+DSaAFaJIKiWuExLdDUe9OUtRXOhGj+O5wi/3kDjVsxnDlJh772cxIxYjNovS7KSfQWcCv0HDJjtaULmZBBgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAF3sEnFXqM+qkknTCn6Q4DTmhlnHHq4tj7fot7RAFfmAnewScVeoz6qSSdMKfpDgNOaGWcceri2Pt+i3tEAV+YC/AAgWKOnAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIHyq3kAx7rVV72qiUYdHnxzRCgtSmdRM+BFnr/KM3APS/HyYhJcX78DJ/BZtAXW5phSxAPzEfTMqa53WhvyXe0LL3eKMQwD865U+dH8cy0L8qtedrdbWYRQA/KRgR+4GBvC2/ODPtc+3NiqdAAAAAAAAAAAAAPyOfzpP156BxPy6AFSkKVIkMgD8iMlAb1Ee52T85tERreHOUbcA/Fa1M2WuTFsU/Kbkh+ozYsgsAPzFPY/gqcTlgvwjaWG7Crm0ewD8dkpjujXKlGn8pg+fL7vVGfcA/NkK2w/aGW35/Fb9JKqQEjauAPzASTPPVpARl/zC4fOX/nYL9QD8L6Yk/KCTyF78iZVlWSZwPsgA/PLVu2YCz5oN/I57xopcIgvzAPxA4SRr2RpVZvwr6zUzD/uD/gD818f//Yqv8a78NF/D8rycFpoA/ESDzPufZpcA/MJSYYkVmvt7APzRKn4AyPzGevwfwfJ03pnc+QD82GECXrKyiR78CExY49qYMWcA/Px/gRpxi2J6/Lwu9zKOfZ0PAPxOAlA9diXiPvzTuyU5wvJ5lQAAAA/8lPUV59Wxlx78X0YgDu1N1p/8Z0bescI1eP786OejD5erAwMA3qkFMpqUtPYDKXBVHn3yibZ841DUgREo8eAe6rKgVzA0sDkCHp4+FkhAOhxDmgX0fTUGnLx6cc3a7p6QPUvrDPxvv+asybOCM/xjGHb5WEOXeQD8l4eI6QYrOt38x6FEKUDmet0A/MufnPQw5ejG/N2coM1lu90HAPwTGmmHolksU/x7b2UqsLwhqQD8iwcQj7F3nOL87gxr3wBfXPgA/IecsActp70d/KAmX+nilxtNAPwRX4BWfHR1nPzo8c76aWP+oQD8TWDp29+KK1z8m/cQ8oxxjFoA/Ehr4FFcs8Ai/O1tqUBzi4imAPxuZHZetdcHkPwSjk7bOYvGwQD8ySs/N17jRUT85c2M/BXHQJ0A/E6qvEuEgphC/Ly3r9DXJ6mXAPx3bv3/Wz3KmfyUQlwVVWrm7wD8VJmXIXGyfUv8QMiTYeCiH5UA/LNHB7K+zNEs/B0CZPI83tFbAAD8b7/mrMmzgjP8Yxh2+VhDl3kA/JeHiOkGKzrd/MehRClA5nrdAPzLn5z0MOXoxvzdnKDNZbvdBwD8Expph6JZLFP8e29lKrC8IakA/IsHEI+xd5zi/O4Ma98AX1z4APyHnLAHLae9HfygJl/p4pcbTQD8EV+AVnx0dZz86PHO+mlj/qEA/E1g6dvfiitc/Jv3EPKMcYxaAPxIa+BRXLPAIvztbalAc4uIpgD8bmR2XrXXB5D8Eo5O2zmLxsEA/MkrPzde40VE/OXNjPwVx0CdAPxOqrxLhIKYQvy8t6/Q1yeplwD8d279/1s9ypn8lEJcFVVq5u8A/FSZlyFxsn1L/EDIk2Hgoh+VAPyzRweyvszRLPwdAmTyPN7RWwAAAAAAAPR7+S7aTllngy3ML5fTzEeXClZPplgN2ON82fiNL+Un7Qmnm4q4j0/wWcRss/DnHspmW9lRa9esuf/PlwlSAz8Bue9zAtMPrJn6IhG/tuQfyGbeJ8/4ooCnRAH33XFI2w4BB+Wh/EA29FcA5NFqi9UaShGmPTH+gwnLrHlrvxSS7gUBJomUYEf690lver0Cy5x9QLAyhpDiXMLB/ZjbLWgPRjYBI3XWOFBwdXkgTIjN5sH5/XzbBXwJf0YqEsMKqkkzsCABRpTjJhe72g3vTlFvU+XbQZzI5yZDW40/bl+6xpe0xjMBF+jygMkiB/aHhLuKT7fMiRCR7P2tbrO1RISSOpCPEg4BLlRl615r2a/r47rTskI/hJ+lHratuyPszeZQv1E4fwkBbe0eUzVGgqhET6/wdoUWvWGebKHevWDPl/Mvht/oWyIBZ8FIOphQNx/YFdZLJNBo8ZiQ41JZCxMJRLCm1ZEBqQYB80dncxlncrKWDyiMtxMIeL5TYm20huKFFL4cBXNb7TEB+hKp4d+SccBGA8v+ygisjzqA3a95IvF7Va9X5cArKzEBcVlALoeLLbrr+l8IUG5wydGEjSlO8V824TBu91LuTTQBvB1+yW2XkEYUNuFLWuZbdjlSJ/EmwnewEytjPNr2ZBoBK/BXcqMYHKCVUsyj5ROI2ufcChVBrVfUeozn1XOCcSQB1GI8x11uFKU/PVNG6IXEcYvWfJWjwxDOynMARa1BZykBl/4z+BNHG5fJm84ktmjPZkCh34ajWUTfezDZVCZP4AoBTb4xJ2WKjtf9xhQd/7Xc7rjusS+30K+YyDuFPsL38BABLN3GHe2sDblY0/CEU97H/3Psw+ZPCw7+nksas516oTAB/xZmCjgvRmm3LmhvdrBHzry8x7DBRipOZkvoMCtQbykBGZdWjdHN+cvjPWqwWfy/Uobvy8p7mU7+fYLWglzqdDcBqJm90w58LdQ5XiYdrKBy7AkNWZwi1g5TJ1NUfteGhx8BvuxVpToHbIOa7uvXe4c6veRaRiVI6zcdlpi0W1A93g4BnKS8YCnqLDMlh63QJiveiLiMNoWM0NiDOgJ6YtHLYy4B1/ZGSQTGN3N31JlJKPsLr07vdLC70cmxj3ErHsyNHjIBVSM3IxdbBGD6p4rRk/jhWszyLpjEdRzGw2YduzniMiUBDaEiNYnH17/buEkL49j19OmvrKIzCzm5lxB+9uMD/woB7kPY7NZyloBRIdRiqQVnjBs5E50ZyZoWinSxpy7DRwoBZf4xgMyNFOjDxX5kbAIiWcJmMi68Mns71iafOXlgHwgBozJxkWeYBH9Sb3yikxH8/gGC7BEVk65VNskgABFrkwoBWE5/3w3pL99gT3c1bPJ2309gHs8HaVIw9BpUfb/ayh0AAZ8VINp/gWlmzTef4gSH7rQHQEEOLseV1fE9bmljCMswAUAXGQI6BynvkelHgjbRLgOTrAuJNqfmO2r17zAuy4kMAX5hm4d3vOL9vFgCkT8RwlPdCsrg46M53U1s9tJNOC8XAep1Rl5Ra7rYB/UTlMuv+A1y2qBbhKRWoruOjk6Xa+QuAWZwXMmZsQCfmUJDSFCD8biKM/HUWyKl5TKwfTRiRAE4AXVpJmPO0wdOoslhomBdZHzha9toox9fM0gcX7gFHv8cAabjJteQhnkXhMcL9eF/JTfQvfT5+lqXLgrl8+H6+ysCASqmq5/vx4Q1BmOLQENe3k/gbpUQSlGQArTK9gajNaMMAaYLHlkVTh/mVcS8Tg+5uTpTtcOEinCNkZqKSAq98oIHAdt/Zq2Q2Br4Xdr1cw8QQJKsuDns0BBQHDBO3pn6A+0BAQpdPnmGJNBdq290pQjUA/n/ovioDy0IGCVwT0d89akyAVL2T2VOteGP+7rJ4bb01gJLa4lCQjNnoTlW3E5VjN4PAS3rVckuz/LgJdJCVFNloQsKpy78ZsKoTh55qUzOL68DATV9oXc1bwPNQFRDOiFnBLmptiXCeoViM/GdFTLHSdQEATtW9uoLLcJb1auwxROzvQQbqIKzU6aB+E/9poIgk7UaAePt5QJuzwh9NYNHkp3UkI+j82A2I5Xfzn3VOeDM/eABAZ04ssXJiynHQ5HK+o9VsxIjRZGZv3d7Z8AMOvzy9NsXAUhM0EnXi2FYQoWLxxb5Vs09FFOuPJr6nywOP4G03L8VAU06fHuZMeNNYGW6bYcxOICRQ0H2dN7+xbgNxf/1rrQsAdv3In2QycijsVHKpsBbdXwfioFYwxKFb2wX9ILcxFMnASQuIxTUYPDb11eGFKrMhm1oRbYZWtKCSEuF5/3oSjAPAf302ZkN6ugpNcYz581PFFBdZgaBz1TGpj3qkbTuwtArAfF+g1x78DqnPjev3w2RQDyRXBf8OTiTv/JsaZe0Z545Ae1rubgiuj9QgLabjoqIRYUXD23seZahWiZcRtzHKJo8AWO34TbkfcwhAOSrYHFk95xYgL6L/pd9Idh5rBbZb8sLAQZ6GNwSMYeHjUxbRJR2aBVnJB08FHYGvw+JSsXbLnUbAQDiK3R5AWokOV3BOlZwJpXxDQl/uY6hhwG0WhLVxSYVAZoYG526rFbhHsfDnJBV6n38R5zfEhukwmN6Un/0gVsDAUgvmedHgcWxXxjx8Q1/pu2LkHMEsMJGPyzo9ROJDOISAd2tnhHFONbeBu6Bjo2WH4oyLPtjR2I6m7ZEJYIDgQcnAAGJmdctrdMAtOXti8fLyDG9YxVuNVKAiHm3OK6St7rlDAE7bxxTN9ETDez13nqBHPinSC6kn3qs2gBdjAJTZMTkHgG6eMdu8cngt53mwy3TnFUVM/HYYgOGYehuRCk+6e6/JgEC/biLOUiwKRNVWs6tDG/B+JWdNxEf7J/AyEttzpvbMAFpHtDZ0tGzJpJkQS+a0WZYH+q+N2qx+UQWn2AHpJybPgHCZk0sNCZ09+iuMvCbqAxDrjhDHcprt8LvijLU9veaAwFrQCVklSugRGOCG6bqA1FMd1hK1KQgD92JvzSU9EJAFQHF2Jf7SkfrNHCWzAt2/s+uLVDQWFezjIJIew/SEv89AAGoOwvE8Vyb1ldAfvvm8A2rrHdA//8NjyvBwk95pjkUGwE1koKRjAabiwejE1SHTctX4HLo+xLPNt2OY38ZFDPSBwEebNgEYaZLoW7AVfrQI46gMDDxp1fDvDauNePj+4kVJQF1PSwL96Eq0RQZL7MSdRnfPoqjy5Hpi0RnPYM3pvPFOwEqyinll97/41p37jlgAicPP74p+tpzHBQtiUlAwI7YBQFnP2yYqGlMXk0dE+0pnufwWNr3XyMmGfPX4R/u7+dWGwABYob/Mc68J4sbWtxxfCGjb2S4FaJ7NI4OHd2X8u9IzwwBds61pr5rarE4UVD1YWJE6VFGyxlWtmoWoQ0bsBG01TYBFupEXWtFrQMK1NwcZUp1byiPi4eDxzDO+lHECQvEkyoBAZJnhAeiyy6KLTGO1udCKpR+OtWXzzzAShkosIAzBzEBYOAZMD3Liv+flvb5FUz0bdktyv+49ErF3NU+xbUa7jQBoC/b8QmR2FAzAhWAFAGHena3bzzYy1gNa91Oc9rf/S8BSBQKBbOdER0dyPwlZC+0Y6nqO4Lur0TGa62i148JmRYBUL5L9zUsLy9lpsIfH178FKlmOo6ZCLsBSTQDdoHbODMBIb2t5YIc3DY6kPVwqoiSJEVpCT/q96NyRgZHcY8nAyQB7R5y2WAf0VhJsRK0nrhPKoTKba0jS+ZVurJDjhDt9TIBv9bPYKJDLz7M7BVR2AQDqe/ofwWr38Jsb0XYuVmpNSoBtNfZ3H4SFQN7pi7HlmLkKeQE6BqkjAx2871wRxt6bBwAAAAAAAAAAAAAAAAAAAAAAAAAAItPrrSDqvfkPX0ovt3g8WYAt2O7b/9mQ7rKXOPLnY4+2hgN0Tu1QeDw0cAXVI/JXu1g5qKxJk8WKdyCsV3Npw+JF8OEVAaQ3w6mYgRd0FdCFrlk2dXXMhozRBjC/MoiKYKUL90FKJEGyLyZJYdxqNYORiig+RwptD9gHpgML3kUasWMPgkAEPZiLAw74nS74Gbr2p0VyGIWpZKoak4/XCYVG9POVsqBcTZbtIzJXz7MnrcjwdHKuUEkUhw+ZofACxf+L8UQhVAIaDJm9KDEFuCseB6fZdZbjh5B/1sYB/Yy3rsg3U9NL01Gv4W5mlrSmetfRMkGpPxd1D7M1NZPuRTsfE6HzQI2WlTjF1Ut//2ZNdnoKaNNNTdyO7aV9towHeFnn1fCC7YS/gQsxicQM4trNmoBd2DJ9HFANzT1D3M0xu8CwNdKaUt68ClHrPWnqHSwM/ND0HQMKIs2pTe3bwvYgYh94rsJmS8SMLmUjw1E9TTp6oKECZ3ZdqnUdEA3AucKOm3LCLollOnteQIAeKTEeIvm5FgTaGOh7a5NDxwrOUdVkZ7fdEHHR1PZ7fk6J/Dmwm8fkpooe6fvtGXaFj/qJ0oZk103eNDW5Iq/snJyEJ5Uugpwt+u10yFc+FBxMYR87DOSTQBjipBn5nPsvda7eCiZ6vp0dHOfjnA/VNgFt7FmaV+3eKYuS1zq0k9ROmh8CTPVTO4AaVtpi9jE6xgumVUBmnv62RAcPKOCuOzesttQ9Juzwe2IANA38V+IE1hYq/1uhK0aV1kheXwEikLpNVjGvIfywQB8XzbK6mcLXnIzXjVY3jvSlCcaLoCkC3Fm8Qzf6l51hw5fyWOgkDu2+4xXM5p0Psq2RddvNokC25GQoPfgOK1tvxJcP0+pCyss/hlPSF+c9nKriwPq8SjH/5VLw4kzGgvtr7PT16g91SN7ZgBu/Ybc7zUx9o81q1aH6L9K7HKnSAflFOwgdScOFvhd0EZYTnzytVwLjPQfPYUULwsjzMp4XcA9Q2kyJA1WmehKI+EBrtVPaMEbe9BjrqoEuBmvhJ/XYMlLN6M/VJdztxshLHj/4OAR6RpORD1Y+TrIdmedkgXIWsAJrjIqMCFFfnmX0t3/oJh4+l6HQSgm9x33QzFaZx0NZM24E+WohPgmHG7Dk8DxN06BNvsG56d6EVCA5trfDKqtNrMrtsrsP7GxYC4cX24UgYT6QYkkFliKIu/Bp5s6j/D/GRS77hsIwJV9ciNRkudW5ROSt7uOyPMWJgeeuY13d3vwMbOHvDAX3zLZ9RBu0aWQ7MlQ5iik4E2A/58S2xj8968gAIA/X7KU1vyF9VtRETox9vS6LbtIe32YITm7mzuVL/QFKOqIUzYoCmum1Hjab4isl0ce4jn7FJiU5Oa3uZKIAw1brPXXr0Wvx4Q5Piy1gR28iPRt5ouNOnJdCEPSFg8qOyxoTUi/hI4zr0AW3J9R+lT2rxBjMkfUNV0pmUC0EHAk8e6AC1TT5RTUsEY/OdEHtS9nFitexN8cu1orYD869RQn6dlcVX3YJOwvCqE45GkrGX2ZU+V4rECPEG+JKwYnJTCGbKBwkZaeYDyhVtggj8ZbQjHVWzwygUhhIiOuTKAYOAcYmL4deH2eyEtE0q0zkyTMVPtLzox6pk6BOCis9AJaW/g1z77m7FzoT59h7i1ae9vyfNrsR4S3Vyb24+l/B/ptcsdYGObBf2bwp3odAyAlPLhfR51m3y1sKTvMdPMI023zztUOK43HMNk0AiuSwBEu3iNyCSLqa1VsWH23MwTeQAqkgocM4T9NoJo0rxmN2/o4bVzDrA/OkoCXx7E9Fuzz5QQF+xW5pELEzj0zvEaxI8/t/Mx5utdVzxcZQI093A4uYNrO6YIHv3RV4VeWHxHWJTPDTxvh5a4dFD31JQoy/g0gGMDv2htlUxYIn/myU2GhSVNQ/On0psM5t8emONls4KoJCRgXrAK720aaWnCS+WhLMEoS6ls3mEwzhmQQAL/E/M9Qm3tMn5Jk7owcjegyPY47vkyLv1mjht3YIYw5TvkWf+o1wHTwuE+/6gfe4UAde6YHDepw9dMA2kq/HD1z8xV5orL3RFYYb187Q80nEcCsFfqFtOwC1jqXU4kYJI+NptFpVcwYAyD5IOnsO5bDQEsfmNOD5nTiYIQ7U2YmRwjNmO4hxsa6ZEUT2T7d+XjV2bAJqt1fPx/CAmLRKhzq58wqmKbrRXUdyb2Mz2zD0nChZWgCJuITNf6R4Ho6ORQTVxtyBNrv/Twx/wD9blx/bpnmPc+Qf5eerhQ5e38Eelu01OCKrHobNjrA+q2pes9nI6Aac3Ge7WPFVTGWbR5ZsWl1rnLp7vmoA1wK+GUs92DOFx7d04j+hfafjky5PJeH+g3SEPc+qNTTBKHjNWbH2GKWIl4rh8pH9hfP9ZIdbe/MUPe3aU4Gin9XfO98eHx/StJoYGdZetQshsW59i5Zeuc7bt1ddQRpN9H5jeaR1lsZkjTEAXk3PRGgZMNsA7WXL+wuA6rxvNwYuGI6b6HsvMNuCPnkbKlpKLhzP6IWOz1UokUV3CE4zH7xEptKIopHoI5l9/5A2ZCCcMlgbz+zK5Qc7rH00tDIlZwDn42nTQ409TiVOS/Ruv4X0Iz6Gln0f5bPGAgu6AWjrN+R7MYOFlxsPRqYvtI5i8otgukYGf/WcSqfEpqrG135YKbPlSZ5SAWHM3OBi0gQiLRtcDhZ21hCosaNOL0otKmrMXUr02LL5mMEyPKCZqaYf3WPPr9RcDMtmYO+AVkRCw+aqkoYxLM0IAnSxZBHxBVhXngBnY+9Orp7puhYE9L+5+NTMUPzmek9LvCafjdenyMUlA0pkgqkiWPRAzHxJBWD4M1MmIGU8+g5vyvltGF8fGclCPiLACQSIdbypFLd0at0qTmunUbEcGoyqE7fcih/T4IsdgToivWsd+BwWZl7XdodTQ5bCv2ApoGVlXGFRRnjwQmr8sxNkJOKXc2TTsnULEkqbbRHusRDWsulUQvbppFPHx+a9HiSpF5V5OPaZQ7s2es7DdT6bfc191PqQMUAS34DC3RqRwLgQgOtREklJ22kjzihU9p+V659JlVtKsPpGh0VsLEdG2Ax1PwjxNRdMGUkOi4nlu+1B/ucmHhpPnEHBQgVNsZuqT19teWiy6B+a2FqQ0qDElsw0b9lDfusO4QgSMJMjd4qRIk9NpvAPZ6BXZlj5rsjs++DctU4WKYF9zy1ukJ+3IfZlDa/QLM2EO9W+SPJWhhf0hmvOjb8BCOyEADP2TfsLs9Mj/wMlPi16N+6F77u7+iazG/WvaTBmHk0PYO7dgFwLRCq2XIKXQWAaAy0h+c/OnU5JEBDI28Eb8oXlVeSbR506jtIiqcQ5i/1eZITEEg5ogMnFOwBamjL+Bejk9buDairospPUuMoCszXQuRtzaw6aUr32jls9y8+GNib4N+kKaw72Ad+/ymXz8oWgd+4bRoM8KogrQY093wOgdJvpGoFG9dY+nlB8/Y4TlQA6KIev03F6OXPJtxM1wLFAbS6/9qjazQ6NCNnsQIQliJ0S727MdA2g48QxOiXNsK8OMrrZElUMVNjT8iemF5P2szzXHqKgHuIdx/OV0oJ3ON8i1ppIHSV2NzccLJpwbmXYLSyLw/6+PY9441AwBdt4GYVbk6A31zzjTIWWwSbdIzBie2ynuXCXRjeNKw8O2ylC1Y0ClEWKMk1iH907q3F77dbqV4MURG+4JopbpgFjZzHiW3jC+PgjSRq9f5qxrhv9VDiFJnZ9t5xMp3VSwM6lab6cjXIsVxLzfgvNHtUbd08ozE3H9p3PqhGdxr4FmgmNwn6ELQm6jyet8dkgA5LPa+DeLuSgA0vl1BQHuQP6NOmLYM2IBf24E1dn5KRfme7kbAdnWhXgFplzj6ocQysydrVV7+WiEAVo2ypcpXcC5A6bqIaCkj+FEMf47EEMMdIqEF8YIldHtz3B3gUA03v+039UGk8IpRdXH92L+Uk/5q4dHcZkzymm9MlkPr04YsbW+0Gadpob3+p5hBdpx6QHGCFXAUGbafhwE1rUK3NT6w53MNMvcoPv0zvkewvOfhhFIL7EvyL27LF/SUSK05tl4/fhsuTTHUDeo0jjdU24dlaVMXsR2c5PiuwVsTqO+tnfq8EjsGoyCLa1bcDXBwsayIaxhjG+jk1BibaJNCYAPGaBQMKt18/93+xHqGuAoT0H+mryUWYMid1Yb4khnLwZwUevmFKwcBGgaPR9BI8/OwR9TfwyaTC/EgcFxsNmY7kwsXAhwFQ2bCgRIREux3Mpa9g1cRbd4gGeHXOJddSmp7R/orrfEjZQ8YVxBkTBc1mwOeBMNlp4I4tXFXPchNRSuufUOPlnj2KtjWwaGkBZM8oxmHcmEwwd8ui0NuALjEh7rXeOZYssvIzXQVGsSqFSQW0n1pVY7PAhZQyZFnEasDZ9SJqBEIrselEfwdxGo066Cwre/8tT7e4WMyJZIDCpmf/dO2IzxsHoOXj7pYPPgQYxNJay5dn8UUEmyPnAUqIcRytdGnCVNNOXXxUpxUAlDPW+TPG/TlfI59QVDlMqM6a7sf6MqsG3icAlAgz0BwaOjt73P2HCwXqTY5FLRjbFKX6RLtbbjz+/uoGUrkCOR+lNR42w4N+043GuJTEyWKkk2+r/PCQBaQslAC2q1oxGWZxqOTszAx+kmLGEjT/BCLW8dhTlu16R0Ztxupy9ii3D40fk66jHDkSA1i+dl4BfiYcRy1iTlfjxJKvwgngLH6Hn0rqH9h2ep7CAiAuAAxWilmZWxtJd3t3vbjnuWo2gF37WIYODgiwGeXVQJA2yYawtknHJvPvi87GbOXqeTHR9M+FlouY9eP4w5EwlOYM3EzjnRy0MCy+FnQb/Mm7CZEQnX9mjpDc4w8SUa/GYFeH2q6RYKPnuAeSOBGWwIYAOyTX3AFN/iEaegOHRGTF1638JRz9O3V8V5vUDyrUuibcm6tEn45EaJrwwYKcspdeUrSX3XCOfZTJ2FFb5fXDFD4CSsQ3zDNMe3ks0sZRHTw8g4rdJtCYuBBScFWtBhsYDkc1XpBfTO0m0GjvonSa+yfZLUagImL96W75QSDVQjbSkl8zDQE/yQFsV5WLJCz9ucG+A0AZywZVSW47XbZsNAA0Wv/bt2db0ZP8t545G5kJkj7m5XPwqsFS6EAb8weLBXic+6LxUs5ad0q7KWym1diFAwMDJQtX3ldikEFeaLEbOTcWiQfnF5i2QCxgsApl9u65/mpVRb8RoVP0lhtcxQ+Rog6T5PfGm9PXgwKQzOiKJ5Lgk3DQZC+kS30x+mzSBRv4E1dyLpZtjkpUFVV6sEMNGnMMLjSeGxw//g6vaGUroM0xNfSfedzA7lDgHDqOidrwOMA/hU8ZCjZBsgvFGQqrFCBPASHjhsACvI60JCqzTTA1AljM2sXRevS7232lGcy+HP7PAzvV9J1tArvFnS9T2ztPT3prLejye0md7ocYQJb8y63QN4ZiEpmG7TzF4nd+ZvWnlmnFVhsiwuegmBsABS6Zx4Hg3bdMVSyuPNMSafqxkHJRPWc+kBK2J1ppJEFLryd6T4ie9rV5gGXsexYkr5YJSPr4q5ha/PslDjsoZ9ZXbGAQ14ElpiYKacBT+M7vNkVKDJ6VXbb4oMdCWikF294/LKDfI99aXaYQ/lx58dICSns0KA2P2T1K+fspKg+euiDTyubAdhCIhb1Fh562gfD2eJoQ+PE8C+G28RdKNQwwsHyJNjF8KVrU2ivhtHVFoAYvzrdgZzAiDS+cDzEc6WWzIRNnWReNVED8MnhVB1x7N19anZFCCjUSCnym9QJJyz5BBCV0WknyQ31UtVCNsqNaWNueGxNnA5+/LSbHGOcn9juFKmDZ+LiVrvgSjycJYFuw66edq34+HOsLL+EcvL9v2JnyWsKy+yy2pNmGnkyM39sy/aHqjetKPvO5KSpyZ6a2y8UMxMSrNo6tQBUyp4gn6WbuZilnjN7wH32LGdTVCR9xszffZJXWI5FfL4wQ9jwFu3jCHGz1oCWqAIkL0Qs5gmUHfriCgiSvQ3aQyldpeqR9H5AAj47yaobIdQWkaC71svlp3cTujDnEV5N5c6RJM7Y254AjIchF8OT+B2glbceL4P92akW4LG72X+OzU/0YrtD3XVkwoJWeUt4TODhFzZMH1Ee2vJ3IuloSWxy0G/dIR7y7wj97V4nDUT3tjytz2SUsbnFnZxywzE8aMfH4HVJfl8u0gH6t/5RBB6BgmpI8g/3tqaRfGc/aBXqscJC0owKuzVIo5bariZEOr+toEpI3ekMWWWmfIfKvE28NGdQl8R4DQDkDwqaPezWupbYyP2OWeUZgy4vdIEOFHEEgtY0+jt5DlWIe6wpNIItfj0bBzYwG0rUvYFrQ9ro3ZE44/UwvJNZkHLuevfs7lWmkAuNUhFS0aah9Ry5rUc5SyJP8jUL53sAUkEZ/pi0AhktEm8ch7QckUecM+wACABjj8Ig0KImmV+u9vh0bLpmeF4SLFZW6TzKPt8QHwqzYG0jbyuZUbtpYcZN4PWcPHqfuFEzKMHiakE3V/yqcLNNyB0MWOUZRtW1ZL0U1tRIbYcvkBCmRJW55QSWiXkMkmg0CmSyaOWG64O6voripMjBtK+r4xFxrjYfc9YTn5KGT/SWGGwkrbTDdpcHxCgwxPrsUBzuay0alG6PhaxJoi/j1stM3MjIpkYDTzXHvoROxKzR7t/7snCrA8Zm3dyrYq8m6Nsyev3jm9xoJSFVNMl86orqBmzDTIBzL1Ueh+6h9ZMQTCp49nrIbn2caDmwdlC6Z6Gt7h2GIOt+vMEKyYhnUNNPCyFvr7dPjY4Cq1R2XqM5VxrJWSI0IerUMpsWP4R0Wsc8jpHmnERJZznKhPpODhlCSvGhEocyd35kugPTQ37yrkvvWXNkQofZp+VQMvZFvWJd4/GSoFl4jIGu74DjvAkk8iZfgkoFKc2z72w+kxHOnSIxtz6nPfXKI24nN9OhVulV3M7NxBeNh7o59FBrVhPU3FuO907tbZcuMJoiCgtyr/IUwn0oaxYVcfgARreaAVLJstisdAzu3XlbxCL97Y8vO1tzAfsOqCF/WajZjyA2vIhhVDOEt5l9fymREG6TkdACjDu/iHvw4MRJYLMcFs3nxn31IOix/fFoomuwk4PvqAWbXFjngJq5mnXcZRG74OQjfpTY2f86JEy+rj0M22d8cgBVQISjwwVtFeDmMc2ZG+bdqE+hPTuWfzOTF70wkPZgmAT+QYYChBCU+ClnMSXt+ZuyP5BOuUlgFcM6r4ZVZpN5UVbAvHye02J4nh1uOFy61ZsGtlwdaVHET7gmdn5FONLz+rF08SiWgNQcknSNRvGuSAkYWjv3JKfFnszcPCbvLrdOA7qC3a4DvL+cn+hseEEj5xQf4SsG3xZkD2m/yyZV1oE+sfMzluX4yB3jZCitdKCV9oYGRJSWXNUmjKIDvCBLTh1JTJ4kWNjKCRKTRk4bKCQwZQrvZQdPmr5YuiD21v49zvDTwu58KNh5q7J/hrJdzh3QlSdYD3txlF4IW7BbHjQd/Oa1+wLUIi7YKZP468BhExfUUa4wCe+JebWrZOESp838Y7yITBB8OpnNXJedFhzkVNegisglhUjpG2UqGJJ3QH6Jsd8vYAM/k+t+kbwQBA/HeTkZjNOHXMO29XVPoiAcGh60EorAbF4vGsEmk2ULU241CJ40c8NyXF+5C4jkbaRMZHivQqzdjmqD1z+WLIvUhuSNo0q2kLw+qQY6u7S7S3F8uyF2QNF4u+EOmiH3q/5sSqWTSC9HWJNSxWGKQGfk6aDxdkVYlJc9WaFpeh++5wHeyp39j6SfZ5dIDoGojN4cK1BIH9hSr2X5MNcveY7cxkRvfTDXY3WWTzP0/v+9mX8G4+VRYL1WyHgBQVR14Vd9PIZkSB4iaHpBI6cZusF5D/Oy/DtMlPGyajJHSdx/SPjTfuA4MYb9RrdLOS8BCTaAdBTuZgQvYTN0W6bs4PZnfQOX/nyYoGnaLJQdLp0hfTD1YVzvZG5UeOMYKEtCJLy02QyMOvMqIf+Uq5Z3BxwM+jShqink9+wOCk+0vJPqEs13ePYbmHO6iPcHMwR281m8cNyTWVyHTYzYmQDVrf4TOVEf6qBnDWosQz/LQPQMn7VO54siWE89R2dh2MmjXf5qNwiTiCg0rB8j3/+P0EDGO9CFIlU0e10CxMitJYihn1VRe0w76D/djGeLPnJ/ciZ/C9nI2yAtjjYmxRNciXyu3bJyddMQdAfHjI1SpVeCcm4gvqZewPoZgkUuq2zjtH30frTEnODW6+PTBHIxZMk5OqhiTsBc9Dp5elHzYWcF2VPYx5QG7BAIq2TYugEAiLJAsEG2fmzpf8pzFNTl29AdhRVED5dJn6xpBptUBajDlbaIuwOdgJBYEdRJ5B52X1jrqiYPpAOY56bdy87XgCxpev3CudiYkkQDm55oxs5Y74/Wv7xhDpKW8Xbz9YKsLVfcSRgHFID4AAAC4smwnQP0A4fUF" diff --git a/e2e_test/generate_submissions.py b/e2e_test/generate_submissions.py index 98933bc..e4941ca 100644 --- a/e2e_test/generate_submissions.py +++ b/e2e_test/generate_submissions.py @@ -9,16 +9,16 @@ - blocks form a smooth, uninterrupted chain and there are no forks.""" import argparse -import base64 from datetime import datetime, timedelta, timezone import itertools import json -import os -import requests import sys import time -from data import BLOCKS, BP_KEYS, LIBP2P_PEER_IDS, SNARK_WORK +import requests + +from data import BP_KEYS, LIBP2P_PEER_IDS +from local_block_reader import LocalBlockReader from network import NODES @@ -31,19 +31,16 @@ class Scheduler: block from the provided list. When that list is exhausted, iteration stops.""" - def __init__(self, blocks, nodes, block_dir, + def __init__(self, nodes, block_reader, block_time=timedelta(minutes=3), submission_time=timedelta(minutes=1)): "Initialize the scheduler." - self.block_dir = block_dir - self.blocks = iter(blocks) + self.block_reader = block_reader self.nodes = itertools.cycle(nodes) - self.current_block = next(self.blocks) self.block_time = block_time self.submission_time = submission_time self.next_block = None self.next_submission = None - self.block_data = None def __iter__(self): "Initialize an iteration." @@ -51,6 +48,9 @@ def __iter__(self): now.replace(second=0, microsecond=0) self.next_block = now + self.block_time self.next_submission = now + self.submission_time + # initialize iteration on block reader and select the first block + iter(self.block_reader) + next(self.block_reader) return self def __next__(self): @@ -61,35 +61,43 @@ def __next__(self): # at some point this will raise StopIteration # which we allow to propagate to terminate the # scheduling - self.current_block = next(self.blocks) - self.block_data = None + next(self.block_reader) if now < self.next_submission: time.sleep((self.next_submission - now).total_seconds()) - self.next_submission += timedelta(seconds=60) + self.next_submission += self.submission_time return next(self.nodes) def read_block(self): - "Read block data from disk and cache." - if self.block_data is None: - filename = f"{self.current_block}.dat" - with open(os.path.join(self.block_dir, filename), "rb") as f: - block = f.read() - self.block_data = base64.b64encode(block).decode("ascii") - return self.block_data + "Use the block reader to extract more block data." + return self.block_reader.read_block() + + @property + def current_block(self): + "Return the state hash of the current block." + return self.block_reader.current_state_hash def parse_args(): "Parse command line options." p = argparse.ArgumentParser() p.add_argument("--block-dir", required=True, help="Directory with block files.") + p.add_argument("--block-time", default=180, type=int, help="Block time in seconds.") + p.add_argument("--submission-time", default=60, type=int, + help="Interval between subsequent submissions.") p.add_argument("uptime_service_url") return p.parse_args() def main(args): """Generate submissions for the uptime service.""" - scheduler = Scheduler(BLOCKS, NODES, args.block_dir) + block_reader = LocalBlockReader(args.block_dir) + scheduler = Scheduler( + NODES, + block_reader, + block_time=timedelta(seconds=args.block_time), + submission_time=timedelta(seconds=args.submission_time) + ) for node in scheduler: sub = node.submission(scheduler.read_block()) now = datetime.now(timezone.utc) diff --git a/e2e_test/local_block_reader.py b/e2e_test/local_block_reader.py new file mode 100644 index 0000000..1b80370 --- /dev/null +++ b/e2e_test/local_block_reader.py @@ -0,0 +1,36 @@ +"This module is concerned with reading blocks from as local storage (file system)." + +import base64 +import os + + +class LocalBlockReader: + "Read blocks from local file system." + + def __init__(self, block_dir): + "Initialize." + self.block_dir = block_dir + self.blocks = None + self.current_state_hash = None + self.current_block_data = None + + def __iter__(self): + block_list = os.path.join(self.block_dir, "block_list.txt") + with open(block_list, "r", encoding="utf-8") as fp: + self.blocks = (l.strip() for l in fp.readlines()) + return self + + def __next__(self): + self.current_state_hash = next(self.blocks) + self.current_block_data = None + return self.current_state_hash + + def read_block(self): + """Read the current block's data from disk and cache for + future reuse.""" + if self.current_block_data is None: + filename = f"{self.current_state_hash}.dat" + path = os.path.join(self.block_dir, filename) + with open(path, "rb") as fp: + self.current_block_data = base64.b64encode(fp.read()).decode("utf-8") + return self.current_block_data diff --git a/e2e_test/network.py b/e2e_test/network.py index 5714018..70e3a59 100644 --- a/e2e_test/network.py +++ b/e2e_test/network.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from datetime import datetime, timezone -from data import BP_KEYS, LIBP2P_PEER_IDS, SNARK_WORK +from data import BP_KEYS, LIBP2P_PEER_IDS @dataclass From ce13fab8b7b00266c8829b9fbd84ef89b94d033f Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 19 Mar 2024 14:32:56 +0100 Subject: [PATCH 08/15] Check if submission file exists in blocks generator. --- e2e_test/gen_blocks.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/e2e_test/gen_blocks.sh b/e2e_test/gen_blocks.sh index 0e44841..9ab4399 100755 --- a/e2e_test/gen_blocks.sh +++ b/e2e_test/gen_blocks.sh @@ -40,6 +40,10 @@ if [[ -n "$SUBMISSION" ]]; then else SUBMISSION="$(realpath "$(dirname "$0")/dummy-submission.json")" fi +if ! [[ -f "$SUBMISSION" ]]; then + echo "Submission file not found!" > /dev/stderr + exit 1 +fi if [[ -z "$1" ]]; then block_count=20 From 2f0f40af989d6eea1fd7817d5beb5f32d2ffb628 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 19 Mar 2024 16:03:59 +0100 Subject: [PATCH 09/15] Add an option to load blocks from s3. --- e2e_test/block_reader.py | 28 ++++++++++++++++++++++++ e2e_test/generate_submissions.py | 13 +++++++++-- e2e_test/local_block_reader.py | 19 ++++++---------- e2e_test/s3_block_reader.py | 37 ++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 14 deletions(-) create mode 100644 e2e_test/block_reader.py create mode 100644 e2e_test/s3_block_reader.py diff --git a/e2e_test/block_reader.py b/e2e_test/block_reader.py new file mode 100644 index 0000000..a32150a --- /dev/null +++ b/e2e_test/block_reader.py @@ -0,0 +1,28 @@ +"""This module defines a generic block reader, which cannot read blocks, +just contains the general policy of iterating over them.""" + + +class BlockReader: + """General logic of returning blocks in sequence, regardless of + how they're stored or read.""" + + def __init__(self): + "Initialize." + self.blocks = None + self.current_state_hash = None + self.current_block_data = None + + def __iter__(self): + self.blocks = self.read_block_list() + return self + + def __next__(self): + self.current_state_hash = next(self.blocks) + self.current_block_data = None + return self.current_state_hash + + def read_block_list(self): + """Read the list of block state hashes to process. + This is a dummy implementation, returning an empty + iterator. To be overridden in subclasses.""" + return iter(()) diff --git a/e2e_test/generate_submissions.py b/e2e_test/generate_submissions.py index e4941ca..0d9e436 100644 --- a/e2e_test/generate_submissions.py +++ b/e2e_test/generate_submissions.py @@ -19,6 +19,7 @@ from data import BP_KEYS, LIBP2P_PEER_IDS from local_block_reader import LocalBlockReader +from s3_block_reader import S3BlockReader from network import NODES @@ -82,7 +83,9 @@ def current_block(self): def parse_args(): "Parse command line options." p = argparse.ArgumentParser() - p.add_argument("--block-dir", required=True, help="Directory with block files.") + p.add_argument("--block-dir", help="Directory with block files.") + p.add_argument("--block-s3-bucket", help="S3 bucket where blocks are stored.") + p.add_argument("--block-s3-dir", help="S3 directory where blocks are stored.") p.add_argument("--block-time", default=180, type=int, help="Block time in seconds.") p.add_argument("--submission-time", default=60, type=int, help="Interval between subsequent submissions.") @@ -91,7 +94,13 @@ def parse_args(): def main(args): """Generate submissions for the uptime service.""" - block_reader = LocalBlockReader(args.block_dir) + if args.block_dir is not None: + block_reader = LocalBlockReader(args.block_dir) + elif args.block_s3_dir is not None and args.block_s3_bucket is not None: + block_reader = S3BlockReader(args.block_s3_bucket, args.block_s3_dir) + else: + raise RuntimeError("No block storage provided!") + scheduler = Scheduler( NODES, block_reader, diff --git a/e2e_test/local_block_reader.py b/e2e_test/local_block_reader.py index 1b80370..8cdfaca 100644 --- a/e2e_test/local_block_reader.py +++ b/e2e_test/local_block_reader.py @@ -3,27 +3,22 @@ import base64 import os +from block_reader import BlockReader -class LocalBlockReader: + +class LocalBlockReader(BlockReader): "Read blocks from local file system." def __init__(self, block_dir): "Initialize." + super().__init__() self.block_dir = block_dir - self.blocks = None - self.current_state_hash = None - self.current_block_data = None - def __iter__(self): + def read_block_list(self): + "Read the list of block state hashes to process." block_list = os.path.join(self.block_dir, "block_list.txt") with open(block_list, "r", encoding="utf-8") as fp: - self.blocks = (l.strip() for l in fp.readlines()) - return self - - def __next__(self): - self.current_state_hash = next(self.blocks) - self.current_block_data = None - return self.current_state_hash + return (l.strip() for l in fp.readlines()) def read_block(self): """Read the current block's data from disk and cache for diff --git a/e2e_test/s3_block_reader.py b/e2e_test/s3_block_reader.py new file mode 100644 index 0000000..810b799 --- /dev/null +++ b/e2e_test/s3_block_reader.py @@ -0,0 +1,37 @@ +"""This module is concerned with downloading blocks to submit from S3.""" + +import base64 +import boto3 + +from block_reader import BlockReader + + +class S3BlockReader(BlockReader): + "Read blocks from local file system." + + def __init__(self, s3_bucket, prefix): + "Initialize." + super().__init__() + self.bucket = s3_bucket + self.prefix = prefix + self.client = boto3.client("s3") + + def read_block_list(self): + "Read the list of block state hashes to process." + block_list_resp = self.client.get_object( + Bucket=self.bucket, + Key=f"{self.prefix}/block_list.txt" + ) + return (bs.decode("utf8").strip() for bs in block_list_resp["Body"].readlines()) + + def read_block(self): + """Read the current block's data from disk and cache for + future reuse.""" + if self.current_block_data is None: + block_resp = self.client.get_object( + Bucket=self.bucket, + Key=f"{self.prefix}/{self.current_state_hash}.dat" + ) + self.current_block_data = base64.b64encode(block_resp["Body"].read()).decode("utf-8") + + return self.current_block_data From d1fcf2bc1531aac53f5e246fb02606027035c4da Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 19 Mar 2024 16:09:02 +0100 Subject: [PATCH 10/15] Update the README. --- e2e_test/README.md | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/e2e_test/README.md b/e2e_test/README.md index c18b081..bf14dff 100644 --- a/e2e_test/README.md +++ b/e2e_test/README.md @@ -1,22 +1,23 @@ Uptime service end-to-end test ============================== -Step 1. Block generation ------------------------- - A blockchain is a hairy process, involving a lot of peer-to-peer -networking, distributing blocks and transactions, selecting -block producers and so forth. Any process involving a real -blockchain is inherently irreproducible. On top of that, Mina -nodes take a considerable time to start and synchronise. All this -would slow down the test. +networking, distributing blocks and transactions, selecting block +producers, resolving forks and so forth. Any process involving a real +blockchain is inherently irreproducible. On top of that, Mina nodes +take a considerable time to start and synchronise. All this would slow +down the test. In order to avoid these issues, we mock the network with a Python script which will send submissions on behalf of some imaginary block producers, containing blocks of some imaginary, dummy blockchain. Because we don't want to depend on a node to generate these blocks, -we need to generate them beforehand. For this step we require -Mina repository. In particular we will use: +we need to generate them beforehand. + +Step 1. Block generation +------------------------ + +For this step we require Mina repository. In particular we will use: * `dump_blocks` app to generate dummy blocks * `delegation_verify` app to extract state hashes from generated @@ -35,6 +36,11 @@ The script can be used as follows: $ BLOCK_DIR=/home/user/blocks ./gen_block.sh +If source code for Mina is present in the file system, `DELEGATION_VERIFY` +and `DUMP_BLOCKS` paths can be replaced with `MINA_DIR` containing a path +to the root of the source code repository. Required apps still have to be +compiled by hand. + Step 2. Upload -------------- @@ -56,11 +62,12 @@ Step 3. Running the mock ------------------------ This step requires an uptime-service already running. It should be configured -such that to does not verify signatures on submissions. We want to test the +such that it does not verify signatures on submissions. We want to test the logic of the uptime service, not signature verification. The mock is run as follows: - $ python generate_submissions.py --block-s3-dir + $ python generate_submissions.py --block-s3-bucket \ + --block-s3-dir The mock contains a list of 15 hard-coded public keys, which serve as block producers' addresses. It loads the list of blocks at the given @@ -81,6 +88,7 @@ These parameters can be tweaked using command line parameters: * `--submission-time` followed by an integer defines the interval in seconds after which the system proceeds with the next submission. -The mock can also be run without downloading blocks from s3. In this case -`--block-s3-dir` parameter should be replaces with `--block-dir` pointing -to a local directory containing blocks as described above. +The mock can also be run without downloading blocks from s3. In this +case `--block-s3-bucket` and `--block-s3-dir` parameters should be +replaces with `--block-dir` pointing to a local directory containing +blocks as described above. From 14e1fd19df7dc1c94fffd47c10187386bed8eaeb Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 20 Mar 2024 09:44:05 +0100 Subject: [PATCH 11/15] Move the dummy blockchain generator and mock to their own dirs. --- blockchain_mock/README.md | 34 +++++++++++ {e2e_test => blockchain_mock}/__init__.py | 0 {e2e_test => blockchain_mock}/block_reader.py | 0 {e2e_test => blockchain_mock}/data.py | 0 .../generate_submissions.py | 0 .../local_block_reader.py | 0 {e2e_test => blockchain_mock}/network.py | 0 .../s3_block_reader.py | 0 dummy_blockchain/README.md | 27 +++++++++ .../dummy-submission.json | 0 {e2e_test => dummy_blockchain}/gen_blocks.sh | 0 e2e_test/README.md | 58 +------------------ 12 files changed, 64 insertions(+), 55 deletions(-) create mode 100644 blockchain_mock/README.md rename {e2e_test => blockchain_mock}/__init__.py (100%) rename {e2e_test => blockchain_mock}/block_reader.py (100%) rename {e2e_test => blockchain_mock}/data.py (100%) rename {e2e_test => blockchain_mock}/generate_submissions.py (100%) rename {e2e_test => blockchain_mock}/local_block_reader.py (100%) rename {e2e_test => blockchain_mock}/network.py (100%) rename {e2e_test => blockchain_mock}/s3_block_reader.py (100%) create mode 100644 dummy_blockchain/README.md rename {e2e_test => dummy_blockchain}/dummy-submission.json (100%) rename {e2e_test => dummy_blockchain}/gen_blocks.sh (100%) diff --git a/blockchain_mock/README.md b/blockchain_mock/README.md new file mode 100644 index 0000000..25116ca --- /dev/null +++ b/blockchain_mock/README.md @@ -0,0 +1,34 @@ +Blockchain mock +=============== + +This script requires an uptime-service already running. It should be +configured such that it does not verify signatures on submissions. We +want to test the logic of the uptime service, not signature +verification. The mock is run as follows: + + $ python generate_submissions.py --block-s3-bucket \ + --block-s3-dir + +The mock contains a list of 15 hard-coded public keys, which serve as +block producers' addresses. It loads the list of blocks at the given +address and forms the chain of state hashes. It also rotates the list +of block producers so that it never ends. It sets the block pointer to +the first block on the list. + +Then every minute it picks up the next node and sends a submission +with the block currently pointed to by the block pointer. Every 3 minutes +it also moves the block pointer to the next block. This way every block +producer appears to make a submission every 15 minutes and each block +gets submitted by 3 distinct block producers. + +These parameters can be tweaked using command line parameters: + +* `--block-time` followed by an integer defines the interval in seconds + after which the system proceeds to the next block. +* `--submission-time` followed by an integer defines the interval + in seconds after which the system proceeds with the next submission. + +The mock can also be run without downloading blocks from s3. In this +case `--block-s3-bucket` and `--block-s3-dir` parameters should be +replaces with `--block-dir` pointing to a local directory containing +blocks as described above. diff --git a/e2e_test/__init__.py b/blockchain_mock/__init__.py similarity index 100% rename from e2e_test/__init__.py rename to blockchain_mock/__init__.py diff --git a/e2e_test/block_reader.py b/blockchain_mock/block_reader.py similarity index 100% rename from e2e_test/block_reader.py rename to blockchain_mock/block_reader.py diff --git a/e2e_test/data.py b/blockchain_mock/data.py similarity index 100% rename from e2e_test/data.py rename to blockchain_mock/data.py diff --git a/e2e_test/generate_submissions.py b/blockchain_mock/generate_submissions.py similarity index 100% rename from e2e_test/generate_submissions.py rename to blockchain_mock/generate_submissions.py diff --git a/e2e_test/local_block_reader.py b/blockchain_mock/local_block_reader.py similarity index 100% rename from e2e_test/local_block_reader.py rename to blockchain_mock/local_block_reader.py diff --git a/e2e_test/network.py b/blockchain_mock/network.py similarity index 100% rename from e2e_test/network.py rename to blockchain_mock/network.py diff --git a/e2e_test/s3_block_reader.py b/blockchain_mock/s3_block_reader.py similarity index 100% rename from e2e_test/s3_block_reader.py rename to blockchain_mock/s3_block_reader.py diff --git a/dummy_blockchain/README.md b/dummy_blockchain/README.md new file mode 100644 index 0000000..e41cc6d --- /dev/null +++ b/dummy_blockchain/README.md @@ -0,0 +1,27 @@ +Generate dummy blockchain +========================= + +For this script we require 2 apps from Mina repository. In particular +we will use: + +* `dump_blocks` app to generate dummy blocks +* `delegation_verify` app to extract state hashes from generated + blocks + +Both these apps are handled automatically by the provided `gen_blocks.sh` +script (they must be compiled manually, though). For the script to work +the following env variables should be set: + +* `DELEGATION_VERIFY` - the path to the stateless verifier binary +* `DUMP_BLOCKS` - the path to the dump blocks tool. +* `BLOCK_DIR` - the path to the directory where the generated blocks will be output +* `SUBMISSION` - the path to the dummy submission file + +The script can be used as follows: + + $ BLOCK_DIR=/home/user/blocks ./gen_block.sh + +If source code for Mina is present in the file system, `DELEGATION_VERIFY` +and `DUMP_BLOCKS` paths can be replaced with `MINA_DIR` containing a path +to the root of the source code repository. Required apps still have to be +compiled by hand. diff --git a/e2e_test/dummy-submission.json b/dummy_blockchain/dummy-submission.json similarity index 100% rename from e2e_test/dummy-submission.json rename to dummy_blockchain/dummy-submission.json diff --git a/e2e_test/gen_blocks.sh b/dummy_blockchain/gen_blocks.sh similarity index 100% rename from e2e_test/gen_blocks.sh rename to dummy_blockchain/gen_blocks.sh diff --git a/e2e_test/README.md b/e2e_test/README.md index bf14dff..76f6f06 100644 --- a/e2e_test/README.md +++ b/e2e_test/README.md @@ -12,35 +12,13 @@ In order to avoid these issues, we mock the network with a Python script which will send submissions on behalf of some imaginary block producers, containing blocks of some imaginary, dummy blockchain. Because we don't want to depend on a node to generate these blocks, -we need to generate them beforehand. +we need to generate them beforehand. Step 1. Block generation ------------------------ -For this step we require Mina repository. In particular we will use: +Use the script in `dummy_blockchain` directory to generate a dummy blockchain. -* `dump_blocks` app to generate dummy blocks -* `delegation_verify` app to extract state hashes from generated - blocks - -Both these apps are handled automatically by the provided `gen_blocks.sh` -script (they must be compiled manually, though). For the script to work -the following env variables should be set: - -* `DFELEGATION_VERIFY` - the path to the stateless verifier binary -* `DUMP_BLOCKS` - the path to the dump blocks tool. -* `BLOCK_DIR` - the path to the directory where the generated blocks will be output -* `SUBMISSION` - the path to the dummy submission file - -The script can be used as follows: - - $ BLOCK_DIR=/home/user/blocks ./gen_block.sh - -If source code for Mina is present in the file system, `DELEGATION_VERIFY` -and `DUMP_BLOCKS` paths can be replaced with `MINA_DIR` containing a path -to the root of the source code repository. Required apps still have to be -compiled by hand. - Step 2. Upload -------------- @@ -61,34 +39,4 @@ accordingly. Step 3. Running the mock ------------------------ -This step requires an uptime-service already running. It should be configured -such that it does not verify signatures on submissions. We want to test the -logic of the uptime service, not signature verification. The mock is run -as follows: - - $ python generate_submissions.py --block-s3-bucket \ - --block-s3-dir - -The mock contains a list of 15 hard-coded public keys, which serve as -block producers' addresses. It loads the list of blocks at the given -address and forms the chain of state hashes. It also rotates the list -of block producers so that it never ends. It sets the block pointer to -the first block on the list. - -Then every minute it picks up the next node and sends a submission -with the block currently pointed to by the block pointer. Every 3 minutes -it also moves the block pointer to the next block. This way every block -producer appears to make a submission every 15 minutes and each block -gets submitted by 3 distinct block producers. - -These parameters can be tweaked using command line parameters: - -* `--block-time` followed by an integer defines the interval in seconds - after which the system proceeds to the next block. -* `--submission-time` followed by an integer defines the interval - in seconds after which the system proceeds with the next submission. - -The mock can also be run without downloading blocks from s3. In this -case `--block-s3-bucket` and `--block-s3-dir` parameters should be -replaces with `--block-dir` pointing to a local directory containing -blocks as described above. +Use the `generate_submissions.py` script in `blockchain_mock` directory. From f792a8be53e2514cd40fa75cc8c9752389466af0 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 20 Mar 2024 09:51:40 +0100 Subject: [PATCH 12/15] Change the default Mina path in dummy blockchain script. --- dummy_blockchain/gen_blocks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dummy_blockchain/gen_blocks.sh b/dummy_blockchain/gen_blocks.sh index 9ab4399..a9e5fc3 100755 --- a/dummy_blockchain/gen_blocks.sh +++ b/dummy_blockchain/gen_blocks.sh @@ -18,7 +18,7 @@ if [[ -n "$MINA_DIR" ]]; then MINA_DIR="$(realpath "$MINA_DIR")" else - MINA_DIR="$(dirname "$0")/../mina" + MINA_DIR="$HOME/work/mina" fi if [[ -n "$DELEGATION_VERIFY" ]]; then DELEGATION_VERIFY="$(realpath "$DELEGATION_VERIFY")" From 7b4e417fbc6080e52c2d1ba0ca72b912057f8c19 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 20 Mar 2024 11:09:20 +0100 Subject: [PATCH 13/15] Expand the README on blockchain mock. --- blockchain_mock/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/blockchain_mock/README.md b/blockchain_mock/README.md index 25116ca..68c0676 100644 --- a/blockchain_mock/README.md +++ b/blockchain_mock/README.md @@ -32,3 +32,23 @@ The mock can also be run without downloading blocks from s3. In this case `--block-s3-bucket` and `--block-s3-dir` parameters should be replaces with `--block-dir` pointing to a local directory containing blocks as described above. + +The values of `--block-time` and `--submission-time` do not have any +formal constraints assigned to them, but they are expressed as an +integral number of seconds. Setting the `--submission-time` to 0 +will cause the script to send submissions as fast as possible and +actual submission times will depend on machine's and network +connection's throughput, so it's not recommended. + +`--block-time` is only checked before the next submission is going to +be made. For this reason setting it to a value smaller or equal to +`--submission-time` will have the effect that for every submission a +new block is picked. Every block will be submitted at least once +nonetheless, even if `--block-time` is much smaller than +`--submission-time`, so there is not really any point in choosing a +value smaller than `--submission-time`. + +Note that choosing too low value of `--block-time` relative to +`--submission-time` may result in uptime service refusing to score any +points, because it expects to see the same blocks submitted by +multiple peers. From 8c244887709b69d2b567babcd7675fe3e0524c90 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 20 Mar 2024 11:09:40 +0100 Subject: [PATCH 14/15] Refactor dummy blockchain generator. Include generation of the first block in the loop. --- dummy_blockchain/gen_blocks.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dummy_blockchain/gen_blocks.sh b/dummy_blockchain/gen_blocks.sh index a9e5fc3..9a9fd47 100755 --- a/dummy_blockchain/gen_blocks.sh +++ b/dummy_blockchain/gen_blocks.sh @@ -51,6 +51,7 @@ else block_count="$1" fi dummy_hash="dummy" +current_block= # If an argument is provided, it's assumed to be the state hash of # the parent block. Otherwise, the parent state hash is generated at @@ -71,14 +72,10 @@ function get_state_hash() { mkdir -p "$BLOCK_DIR" cd $MINA_DIR -generate_block_after # first block in the chain - -current_block="$(get_state_hash)" -mv -v "$BLOCK_DIR/$dummy_hash.dat" "$BLOCK_DIR/$current_block.dat" -echo "$current_block" > "$BLOCK_DIR/block_list.txt" -block_count="$((block_count - 1))" while [[ "$block_count" -gt 0 ]]; do + # NOTE: in the first pass $current_block is empty, resulting in no + # parent being passed to dump_blocks. generate_block_after "$current_block" current_block="$(get_state_hash)" echo "$current_block" >> "$BLOCK_DIR/block_list.txt" From e41c015d168c0ac848afa36c3afb00af6aa61cf2 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 20 Mar 2024 16:16:29 +0100 Subject: [PATCH 15/15] Add scripts for DB migration. --- db_migration/README.md | 36 ++++++++ db_migration/db_diff.py | 177 +++++++++++++++++++++++++++++++++++++++ db_migration/migrate.sql | 104 +++++++++++++++++++++++ 3 files changed, 317 insertions(+) create mode 100644 db_migration/README.md create mode 100644 db_migration/db_diff.py create mode 100644 db_migration/migrate.sql diff --git a/db_migration/README.md b/db_migration/README.md new file mode 100644 index 0000000..a0e681b --- /dev/null +++ b/db_migration/README.md @@ -0,0 +1,36 @@ +# Migrate the uptime service database. + +The delegation program that the Foundation ran before the hard fork will continue after the hard fork. Because computations of the uptime service are recursive in nature (output from the previous round influences how the next round will go), we need to migrate some data from the old service to the new one to serve as input for the first round. We’re not so much concerned with keeping historic record – such a record will be kept in the form of final database snapshot. Rather, we’re concerned with providing initial state of the service to kick it off properly. To this end we run a specialised migration script, whose task is to: + +- Trim the database, leaving behind only data relevant for the past 3 months (after 1st December 2023). +- Migrate it to the format expected by the new uptime service. + +Difference between the old database schema and the new one are not large. There’s a couple of tables that have been dropped as well as a few columns in existing tables. One column needs to be added in an existing table – the score_history table lacks a primary key. The migration script linked below takes care of all that. + +[migrate.sql](./migrate.sql) + +This script is intended to be run on a **copy** of the ontab database. Because it performs DELETE commands on some tables, it’s best to preserve a backup of the original database for safety, but also to preserve historic data, which might yet be needed in the future. This copy of the Ontab database will be provided as a database snapshot from AWS RDS service. It is important to have the snapshot encrypted with a key MF has access to so that we can restore the database from this snapshot on our end to run the script above. + +NOTE: this will take several hours to execute (a lot of rows to drop and constraints to check). When the script is done, take a snapshot of the resulting database as a backup and you can hook the uptime service coordinator up to it, + +## Taking a db diff + +Under some circumstances (like e.g. an emergency hard fork) there might not be enough time to take a full Ontab database’s snapshot quickly (it takes several hours to produce) and then trim it (which takes another several hours). In that case a quicker approach can be used to synchronise the databases – the script linked below can be used to take a partial db dump (for increased speed) from the original database and move relevant records to the new database. The script can be run like this: + +```bash +$ python db_diff.py -H $HOST -p $PORT -U $USERNAME -w $PASSWORD -d leaderboard_snark $DATE +``` + +[db_diff.py](./db_diff.py) + +Where: + +- `$HOST` is the hostname of the original ontab database (delegation-uptime-ontab-4.c14zvdudnyw7.us-west-2.rds.amazonaws.com at the moment of writing this document) +- `$PORT` is port of the service (defaults to 5432) +- `$USERNAME` is the user that can read the database (we’ve graciously been given `ro_user` user to access the database). +- `$PASSWORD` is the USERNAME’s password for the database. +- `$DATE` is the day from which we want to take the diff (e.g. 2024-02-01). Only records added after that date will be dumped. + +NOTE: the script requires psycopg2 package downloaded from pip. + +This script will produce some SQL commands on the standard output. Write it to a file and run the script against the target database in order to load the data. Or you can simply pipe it to the `psql` command. diff --git a/db_migration/db_diff.py b/db_migration/db_diff.py new file mode 100644 index 0000000..a3c053d --- /dev/null +++ b/db_migration/db_diff.py @@ -0,0 +1,177 @@ +import argparse +from datetime import datetime +import psycopg2 +from psycopg2.sql import Literal + + +def batched(iterable, n): + """Yield successive n-sized chunks from iterable.""" + items = [] + count = 0 + for item in iterable: + if count < n: + items.append(item) + count += 1 + else: + yield items + items = [item] + count = 1 + if items: + yield items + +class Insert: + + def __init__(self, connection, table, columns): + self.connection = connection + self.table = table + self.columns = columns + self.results = [] + + def fetch(self, condition, args, joins=()): + cols = ', '.join('{}.{}'.format(self.table, col) for col in self.columns) + j = ' '.join('JOIN {tbl} AS {as} ON {as}.{col} = {val}'.format(**join) for join in joins) + q = 'SELECT DISTINCT {} FROM {} {} WHERE {} ORDER BY {}.{} LIMIT ALL' + with self.connection.cursor() as cursor: + cursor.execute(q.format(cols, self.table, j, condition, self.table, self.columns[0]), args) + self.results += cursor.fetchall() + + def print(self): + for batch in batched(self.results, 1000): + cols = ', '.join(self.columns) + print('INSERT INTO {}'.format(self.table)) + print(' ({})'.format(cols)) + print('VALUES') + print(' ', ',\n '.join('({})'.format(', '.join(Literal(item).as_string(self.connection) for item in row)) for row in batch)) + print('ON CONFLICT(id)') + print('DO UPDATE SET') + print(' ', ',\n '.join('{col} = EXCLUDED.{col}'.format(col=col) for col in self.columns)) + print(';') + + +def parse_args(): + p = argparse.ArgumentParser(description='Diff two uptime service databases.') + p.add_argument('-H', '--host', help='Database hostname', default='localhost') + p.add_argument('-p', '--port', help='Database port', default=5432, type=int) + p.add_argument('-U', '--user', help='Database username', default='postgres') + p.add_argument('-w', '--password', help='Database password', required=True) + p.add_argument('-d', '--database', help='Database name', required=True) + p.add_argument('last_update', help='Last update time', type=datetime.fromisoformat) + return p.parse_args() + +def main(args): + conn = psycopg2.connect( + user=args.user, + password=args.password, + host=args.host, + port=args.port, + database=args.database + ) + + print('BEGIN;\n') + bot_logs = Insert( + conn, + 'bot_logs', + ('id', + 'files_processed', + 'file_timestamps', + 'batch_start_epoch', + 'batch_end_epoch', + 'processing_time') + ) + bot_logs.fetch( + 'bot_logs.batch_end_epoch >= trunc(extract(epoch from (%s)))', + (args.last_update, ) + ) + bot_logs.print() + + statehash = Insert(conn, 'statehash', ('id', 'value')) + statehash.fetch( + 'bl.batch_end_epoch >= trunc(extract(epoch from (%s)))', + (args.last_update, ), + joins=({'tbl': 'bot_logs_statehash', 'as': 'bls', 'col': 'statehash_id', 'val': 'statehash.id'}, + {'tbl': 'bot_logs', 'as': 'bl', 'col': 'id', 'val': 'bls.bot_log_id'}) + ) + statehash.fetch( + 'bl.batch_end_epoch >= trunc(extract(epoch from (%s)))', + (args.last_update, ), + joins=({'tbl': 'bot_logs_statehash', 'as': 'bls', 'col': 'parent_statehash_id', 'val': 'statehash.id'}, + {'tbl': 'bot_logs', 'as': 'bl', 'col': 'id', 'val': 'bls.bot_log_id'}) + ) + statehash.print() + + bot_logs_statehash = Insert( + conn, + 'bot_logs_statehash', + ('id', + 'bot_log_id', + 'statehash_id', + 'parent_statehash_id', + 'weight') + ) + bot_logs_statehash.fetch( + 'bl.batch_end_epoch >= trunc(extract(epoch from (%s)))', + (args.last_update, ), + joins=({'tbl': 'bot_logs', 'as': 'bl', 'col': 'id', 'val': 'bot_logs_statehash.bot_log_id'}, ) + ) + bot_logs_statehash.print() + + nodes = Insert( + conn, + 'nodes', + ('id', + 'block_producer_key', + 'score', + 'score_percent', + 'updated_at', + 'email_id', + 'application_status') + ) + nodes.fetch( + 'bl.batch_end_epoch >= trunc(extract(epoch from (%s))) AND updated_at IS NOT NULL', + (args.last_update, ), + joins=({'tbl': 'points', 'as': 'p', 'col': 'node_id', 'val': 'nodes.id'}, + {'tbl': 'bot_logs', 'as': 'bl', 'col': 'id', 'val': 'p.bot_log_id'}) + ) + for row in nodes.results: + if row[4] is None: + # we don't want NULLs in this column, so we have to insert something + # it doesn't really matter much anyways. + row[4] = 'now()' + nodes.print() + + points = Insert( + conn, + 'points', + ('id', + 'file_name', + 'blockchain_epoch', + 'blockchain_height', + 'created_at', + 'amount', + 'node_id', + 'bot_log_id', + 'file_timestamps', + 'statehash_id') + ) + points.fetch( + 'bl.batch_end_epoch >= trunc(extract(epoch from (%s)))', + (args.last_update, ), + joins=({'tbl': 'bot_logs', 'as': 'bl', 'col': 'id', 'val': 'points.bot_log_id'}, ) + ) + points.print() + + score_history = Insert( + conn, + 'score_history', + ('node_id', + 'score_at', + 'score', + 'score_percent') + ) + score_history.fetch('score_at >= %s AND score_at IS NOT NULL', (args.last_update, )) + score_history.print() + + print('COMMIT;') + +if __name__ == '__main__': + main(parse_args()) diff --git a/db_migration/migrate.sql b/db_migration/migrate.sql new file mode 100644 index 0000000..425c9e5 --- /dev/null +++ b/db_migration/migrate.sql @@ -0,0 +1,104 @@ +BEGIN; + +DROP TABLE uptime_file_history; + +DROP TABLE bp_ip_address_2022_12; + +DROP TABLE nodes_sidecar; + +DROP TABLE points_summary; + +DELETE FROM bot_logs +WHERE batch_start_epoch <= trunc(extract(epoch from ('2023-12-01 00:00:00+00' :: timestamp))); + +DELETE FROM nodes +WHERE updated_at IS NULL; + +DELETE FROM points +WHERE bot_log_id IS NULL +OR node_id IS NULL +OR statehash_id IS NULL +OR bot_log_id NOT IN (SELECT id FROM bot_logs) +OR node_id NOT IN (SELECT id FROM nodes); + +DELETE FROM bot_logs_statehash +WHERE bot_log_id IS NULL +OR statehash_id IS NULL +OR bot_log_id NOT IN (SELECT id FROM bot_logs); + +DELETE FROM score_history +WHERE node_id IS NULL +OR node_id NOT IN (SELECT id FROM nodes); + +DELETE FROM statehash +WHERE id NOT IN ( + SELECT DISTINCT statehash_id + FROM bot_logs_statehash + WHERE statehash_id IS NOT NULL + UNION + SELECT DISTINCT parent_statehash_id + FROM bot_logs_statehash + WHERE parent_statehash_id IS NOT NULL + UNION + SELECT DISTINCT statehash_id + FROM points + WHERE statehash_id IS NOT NULL +); + +ALTER TABLE bot_logs +DROP COLUMN status; + +ALTER TABLE bot_logs +DROP COLUMN number_of_threads; + +ALTER TABLE bot_logs_statehash +ALTER COLUMN statehash_id SET NOT NULL; + +ALTER TABLE bot_logs_statehash +ALTER COLUMN parent_statehash_id SET NOT NULL; + +ALTER TABLE bot_logs_statehash +ALTER COLUMN bot_log_id SET NOT NULL; + +ALTER TABLE bot_logs_statehash +ADD FOREIGN KEY (bot_log_id) REFERENCES bot_logs(id); + +ALTER TABLE bot_logs_statehash +ADD FOREIGN KEY (statehash_id) REFERENCES statehash(id); + +ALTER TABLE bot_logs_statehash +ADD FOREIGN KEY (parent_statehash_id) REFERENCES statehash(id); + +ALTER TABLE points +ALTER COLUMN node_id SET NOT NULL; + +ALTER TABLE points +ALTER COLUMN bot_log_id SET NOT NULL; + +ALTER TABLE points +ADD FOREIGN KEY (bot_log_id) REFERENCES bot_logs(id); + +ALTER TABLE points +ADD FOREIGN KEY (node_id) REFERENCES nodes(id); + +ALTER TABLE score_history +ALTER COLUMN node_id SET NOT NULL; + +ALTER TABLE score_history +ADD FOREIGN KEY (node_id) REFERENCES nodes(id); + +ALTER TABLE score_history +ADD COLUMN id SERIAL PRIMARY KEY; + +ALTER TABLE score_history +ALTER COLUMN score_percent TYPE NUMERIC(10, 2); + +ALTER TABLE nodes +ALTER COLUMN updated_at SET NOT NULL; + +ALTER TABLE nodess +ALTER COLUMN score_percent TYPE NUMERIC(10, 2); + +DROP TRIGGER trg_update_point_summary ON points; + +COMMIT;