Large wasm module support

dfinity · Jan 17, 2024 · f5787ca · f5787ca
1 parent f290401
commit f5787ca
Show file tree

Hide file tree

Showing 14 changed files with 177 additions and 120 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 # UNRELEASED
 
+### feat: large canister modules now supported
+
+When using `dfx deploy` or `dfx canister install`, previously WASM modules larger than 2MiB would be rejected.
+They are now automatically submitted via the chunking API if they are large enough.
+From a user perspective the limitation will simply have been lifted.
+
 ### feat!: update `dfx cycles` commands with mainnet `cycles-ledger` canister ID
 
 The `dfx cycles` command no longer needs nor accepts the `--cycles-ledger-canister-id <canister id>` parameter.

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -20,11 +20,11 @@ license = "Apache-2.0"
 [workspace.dependencies]
 candid = "0.10.0"
 candid_parser = "0.1.2"
-ic-agent = { git = "https://github.com/dfinity/agent-rs.git", rev = "a7f44ad05e77fc89b8447dd65b345e7a62fd1042" }
+ic-agent = { git = "https://github.com/dfinity/agent-rs.git", rev = "3bf221238ea0e038aa23c6749151534a3f0bfe85" }
 ic-asset = { path = "src/canisters/frontend/ic-asset" }
 ic-cdk = "0.12.0"
-ic-identity-hsm = { git = "https://github.com/dfinity/agent-rs.git", rev = "a7f44ad05e77fc89b8447dd65b345e7a62fd1042" }
-ic-utils = { git = "https://github.com/dfinity/agent-rs.git", rev = "a7f44ad05e77fc89b8447dd65b345e7a62fd1042" }
+ic-identity-hsm = { git = "https://github.com/dfinity/agent-rs.git", rev = "3bf221238ea0e038aa23c6749151534a3f0bfe85" }
+ic-utils = { git = "https://github.com/dfinity/agent-rs.git", rev = "3bf221238ea0e038aa23c6749151534a3f0bfe85" }
 
 aes-gcm = "0.10.3"
 anyhow = "1.0.56"

diff --git a/e2e/assets/large_canister/dfx.json b/e2e/assets/large_canister/dfx.json
@@ -0,0 +1,10 @@
+{
+    "canisters": {
+        "large": {
+            "type": "custom",
+            "wasm": "large.wasm",
+            "candid": "large.did",
+            "build": []
+        } 
+    }
+}
diff --git a/e2e/assets/large_canister/large.did b/e2e/assets/large_canister/large.did
@@ -0,0 +1 @@
+service : {}
diff --git a/e2e/assets/large_canister/large.wasm b/e2e/assets/large_canister/large.wasm
diff --git a/e2e/tests-dfx/install.bash b/e2e/tests-dfx/install.bash
@@ -152,6 +152,16 @@ teardown() {
   assert_eq "$BUILD_HASH" "$ONCHAIN_HASH"
 }
 
+@test "can install >2MiB wasm" {
+  install_asset large_canister
+  dfx_start
+  dfx canister create --all
+  assert_command dfx build
+  assert_command dfx canister install --all
+  assert_command dfx canister info large
+  assert_match "Module hash: 0x$(sha2sum large.wasm | head -c 64)"
+}
+
 @test "--mode=auto selects install or upgrade automatically" {
   dfx_start
   assert_command dfx canister create e2e_project_backend

diff --git a/nix/sources.json b/nix/sources.json
@@ -15,53 +15,53 @@
     "canister_sandbox-x86_64-darwin": {
         "builtin": false,
         "description": "The canister_sandbox binary. It must be updated together with the replica binary.",
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "1c9lfnajc3mvqq76rcgjvjmbqlm02f2pjw6gpxj5d0248sa82pkc",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "0b6ykxs3cfzp0dr4b3ahcl2ps9g6paw55rmgmcr38wljqfw4r03r",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-darwin/canister_sandbox.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-darwin/canister_sandbox.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-darwin/canister_sandbox.gz"
     },
     "canister_sandbox-x86_64-linux": {
         "builtin": false,
         "description": "The canister_sandbox binary. It must be updated together with the replica binary.",
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "13pss6m6kgaxsy7n0kz0v639s3qycri9hdcka660mf4341zkiz6w",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "06j0i8v690az5p7v7wj1gmrxhggmr1bas05dk8d6mdx5fxq2sxf6",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-linux/canister_sandbox.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-linux/canister_sandbox.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-linux/canister_sandbox.gz"
     },
     "ic-admin-x86_64-darwin": {
         "builtin": false,
         "description": "The ic-admin binary.",
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "16ipbiwzl5g9zmfb7nn76hzyibagskhw1p3qp1jmr4ymcmm43g7f",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "08537h5g92in58142g4cjwj0g7nvhl5bpi4b9d0p0liim9brzzq3",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-darwin/ic-admin.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-darwin/ic-admin.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-darwin/ic-admin.gz"
     },
     "ic-admin-x86_64-linux": {
         "builtin": false,
         "description": "The ic-admin binary.",
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "0r0pfmxa5h04d2d1dyb7hwfmiyga9fvqwnjxghk0knwrpmrd5mmb",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "1xb2g2gib89rnhblqa05mhwiblkq08vcdfpr4hqzpfv0ik4m8kqn",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-linux/ic-admin.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-linux/ic-admin.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-linux/ic-admin.gz"
     },
     "ic-btc-adapter-x86_64-darwin": {
         "builtin": false,
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "0fa5qik4im6krnq9cr7mdwidvrl2fbmfqydvvfb72zrznxbmad1x",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "1xj5h7bgpnagrhfckr0zdjvndh2ckdfjg6vpyxyf7l0jsvx8l0w1",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-darwin/ic-btc-adapter.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-darwin/ic-btc-adapter.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-darwin/ic-btc-adapter.gz"
     },
     "ic-btc-adapter-x86_64-linux": {
         "builtin": false,
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "1i3v72v37viczz5irbb0z0i6yrva78hzdhbg4qny0wfclkjp0gpx",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "1l2ya82kd01j9c9v3vkjh8wzqwqkd206n3d4azv6ffrcjk2byj07",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-linux/ic-btc-adapter.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-linux/ic-btc-adapter.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-linux/ic-btc-adapter.gz"
     },
     "ic-btc-canister": {
@@ -74,68 +74,68 @@
     },
     "ic-https-outcalls-adapter-x86_64-darwin": {
         "builtin": false,
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "1s532b8xp0h68lcnc1jw8gbvaxky4z55z0f1r0pg6zxy9qsd2sps",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "019cfp108qgqz4bdvsp92jd02yc33srlq4qs1kfslng2d7367594",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-darwin/ic-https-outcalls-adapter.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-darwin/ic-https-outcalls-adapter.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-darwin/ic-https-outcalls-adapter.gz"
     },
     "ic-https-outcalls-adapter-x86_64-linux": {
         "builtin": false,
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "0q1k2gf5jix8zdgnci6h4nl8smzcfax3pl4rm3br4pn4jd59zh8n",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "1y5zzi4255wp2zqghgkyhwy7acqc3g54pwqnn96lnvqfz80kbvk5",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-linux/ic-https-outcalls-adapter.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-linux/ic-https-outcalls-adapter.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-linux/ic-https-outcalls-adapter.gz"
     },
     "ic-nns-init-x86_64-darwin": {
         "builtin": false,
         "description": "The ic-nns-init binary.",
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "0imihws14znvqpfq7r7g7sf9xl05lsjfy2xgwbamplpvp684gpj3",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "192i56x37j04r1z1znrkvkg5jjs5bs2jmlm7lb2b95sx1idz0y0y",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-darwin/ic-nns-init.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-darwin/ic-nns-init.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-darwin/ic-nns-init.gz"
     },
     "ic-nns-init-x86_64-linux": {
         "builtin": false,
         "description": "The ic-nns-init binary.",
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "0ddjympvvgrbs3q6lk21257iy1xxrj28vqcj74mdhy6mdmb3zmlc",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "1591praw6q1allnpfw9fiz6arjr108wfzwv71kq8224v9p8vj2mh",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-linux/ic-nns-init.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-linux/ic-nns-init.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-linux/ic-nns-init.gz"
     },
     "ic-starter-x86_64-darwin": {
         "builtin": false,
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "1vpn7xj2v3hlfflklca6z90dlxl5r4qhd0m60yk2a9pjb2pl77p8",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "0519yfk1phbwj5iplcbjphs48lacshy5bi3zrv084hf1y1ylm0w6",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-darwin/ic-starter.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-darwin/ic-starter.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-darwin/ic-starter.gz"
     },
     "ic-starter-x86_64-linux": {
         "builtin": false,
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "0qlql6kslynbay2yi2blqvrn87f1f1qsiwk1ygr8qn723zz9brwi",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "0qc2q0r0bi05qd8jfr1j0ngghll80cg8fb7nzg7bffj4zbrlqp76",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-linux/ic-starter.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-linux/ic-starter.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-linux/ic-starter.gz"
     },
     "icx-proxy-x86_64-darwin": {
         "builtin": false,
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "0s5gkpd8m8zdxnd3dy6450yybnqpw5l8yl5j01hx41zxg0yk1aws",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "0grdnw67yvi82rx91dqsqy2hvbbp9nn7ca8j2bsdrwfwys528js0",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-darwin/icx-proxy-dev.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-darwin/icx-proxy-dev.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-darwin/icx-proxy-dev.gz"
     },
     "icx-proxy-x86_64-linux": {
         "builtin": false,
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "1d337q9wi6d6z0hf8by945jvb42xvl7s4cs8lr1cf2iqqgayqf2i",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "05w34miq9hzzn5nfjhcvk56ihcgs90cmqdciwqgnh9sqr2j8wcci",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-linux/icx-proxy-dev.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-linux/icx-proxy-dev.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-linux/icx-proxy-dev.gz"
     },
     "motoko-base": {
@@ -167,55 +167,55 @@
     "replica-x86_64-darwin": {
         "builtin": false,
         "description": "The replica binary. It must be updated together with the canister_sandbox binary.",
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "08ndv6c26x86021jf47fv9hvn22zzxgd2js9v8k0xcjpn2x3z6f3",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "1z71dglwchqj9gj3vkrmg86dg7bw2b5h7n4mh395l2pzg56am246",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-darwin/replica.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-darwin/replica.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-darwin/replica.gz"
     },
     "replica-x86_64-linux": {
         "builtin": false,
         "description": "The replica binary. It must be updated together with the canister_sandbox binary.",
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "1a3wh4jsjjha722lrb3cdym7hl47w5ixbnljawn7jqxj6kf8p4h5",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "1k5gpvqzw30y5g3pdxixqvbsx3jrkdwn9wqc2wp2p4rf38h79hix",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-linux/replica.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-linux/replica.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-linux/replica.gz"
     },
     "sandbox_launcher-x86_64-darwin": {
         "builtin": false,
         "description": "The sandbox_launcher binary. It must be updated together with the replica binary.",
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "1mnmm9kjljng8gj4g9wak9abm9iyzcf2z2awvlm2bkrnh936m401",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "1pvv0b83p3jh7h41rw0kq9wgv44bspakv75aisxs7shc5hjqf51p",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-darwin/sandbox_launcher.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-darwin/sandbox_launcher.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-darwin/sandbox_launcher.gz"
     },
     "sandbox_launcher-x86_64-linux": {
         "builtin": false,
         "description": "The sandbox_launcher binary. It must be updated together with the replica binary.",
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "15qxsfkva8hg2xay5lngaqnnwg224pc28abhn63902znbx96sbzs",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "1a8cicb2xaha4b4dm5cwq886r5izxc3akkg3gz13gf385x2pbx95",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-linux/sandbox_launcher.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-linux/sandbox_launcher.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-linux/sandbox_launcher.gz"
     },
     "sns-x86_64-darwin": {
         "builtin": false,
         "description": "The sns binary.",
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "1q53sm2976k3h1x9w2szr3pk6rvj5mdrcrzlg3y2khlh632n0sva",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "0gvvxkhc9bznc7w3jj4vinr62ms9j8wk6s9iwx0xwh9a6s2maixc",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-darwin/sns.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-darwin/sns.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-darwin/sns.gz"
     },
     "sns-x86_64-linux": {
         "builtin": false,
         "description": "The sns binary.",
-        "rev": "69239b33174639bb9febe7905ca9d90751b0988a",
-        "sha256": "1h1p0ls93arsbgrc02jvcjaai1m4habg9xnza4x3nri71ck9f6v1",
+        "rev": "072b2a6586c409efa88f2244d658307ff3a645d8",
+        "sha256": "1ppkipx2zbvak7kl3hrw9c143dkfh5jhmxb50c7bv0q4w6x08vyf",
         "type": "file",
-        "url": "https://download.dfinity.systems/ic/69239b33174639bb9febe7905ca9d90751b0988a/binaries/x86_64-linux/sns.gz",
+        "url": "https://download.dfinity.systems/ic/072b2a6586c409efa88f2244d658307ff3a645d8/binaries/x86_64-linux/sns.gz",
         "url_template": "https://download.dfinity.systems/ic/<rev>/binaries/x86_64-linux/sns.gz"
     }
 }