Merge branch 'main' into kai/sync-time

docs/CHANGELOG.md

@@ -4,9 +4,20 @@
 
 ### Added
 
+- feat: sync_call support in HttpAgent and Actor
+  - Skips polling if the sync call succeeds and provides a certificate
+  - Falls back to v2 api if the v3 endpoint 404's
+  - Adds certificate to SubmitResponse endpoint
+  - adds callSync option to `HttpAgent.call`, which defaults to `true`
 - feat: management canister interface updates for schnorr signatures
 - feat: ensure that identity-secp256k1 seed phrase must produce a 64 byte seed
 - docs: documentation and metadata for use-auth-client
+- feat: adds optional `rootKey` to `HttpAgentOptions` to allow for a custom root key to be used for verifying signatures from other networks
+- chore: npm audit bumping micromatch
+- feat: exports polling utilities from `@dfinity/agent` for use in other packages
+  - `pollForResponse` now uses the default strategy by default
+  - Updated the `bls-verify` jsdoc comment to accurately reflect that the default strategy now uses @noble/curves
+- docs: clarifies meaning of `effectiveCanisterId` in `CallOptions`
 
 ### Changed
 - feat: replaces hdkey and bip32 implementations with `@scure/bip39` and `@scure/bip32` due to vulnerability and lack of maintenance for `elliptic`
@@ -17,6 +28,8 @@
 
 ### Changed
 
+- fix: passing `request` correctly during pollForResponse `Processing` status
+  - credit: [Senior Joinu](https://forum.dfinity.org/t/timestamp-failed-to-pass-the-watermark-after-retrying-the-configured-3-times/29180/11?)
 - ci: removing headless browser tests pending a rewrite
 - ci: changing token for creating release
 

e2e/node/basic/counter.test.ts

@@ -1,4 +1,5 @@
-import counterCanister, { createActor } from '../canisters/counter';
+import { Actor, HttpAgent } from '@dfinity/agent';
+import counterCanister, { idl } from '../canisters/counter';
 import { it, expect, describe, vi } from 'vitest';
 
 describe('counter', () => {
@@ -37,26 +38,49 @@ describe('counter', () => {
 describe('retrytimes', () => {
   it('should retry after a failure', async () => {
     let count = 0;
+    const { canisterId } = await counterCanister();
     const fetchMock = vi.fn(function (...args) {
-      if (count <= 1) {
-        count += 1;
+      count += 1;
+      // let the first 3 requests pass, then throw an error on the call
+      if (count === 3) {
         return new Response('Test error - ignore', {
           status: 500,
           statusText: 'Internal Server Error',
         });
       }
+
       // eslint-disable-next-line prefer-spread
-      return fetch.apply(
-        null,
-        args as [input: string | Request, init?: RequestInit | CMRequestInit | undefined],
-      );
+      return fetch.apply(null, args as [input: string | Request, init?: RequestInit | undefined]);
     });
 
-    const counter = await createActor({ fetch: fetchMock as typeof fetch, retryTimes: 3 });
-    try {
-      expect(await counter.greet('counter')).toEqual('Hello, counter!');
-    } catch (error) {
-      console.error(error);
+    const counter = await Actor.createActor(idl, {
+      canisterId,
+      agent: await HttpAgent.create({
+        fetch: fetchMock as typeof fetch,
+        retryTimes: 3,
+        host: 'http://localhost:4943',
+        shouldFetchRootKey: true,
+      }),
+    });
+
+    const result = await counter.greet('counter');
+    expect(result).toEqual('Hello, counter!');
+
+    // The number of calls should be 4 or more, depending on whether the test environment is using v3 or v2
+    if (findV2inCalls(fetchMock.mock.calls as [string, Request][]) === -1) {
+      // TODO - pin to 4 once dfx v0.23.0 is released
+      expect(fetchMock.mock.calls.length).toBe(4);
+    } else {
+      expect(fetchMock.mock.calls.length).toBeGreaterThanOrEqual(4);
     }
   }, 40000);
 });
+
+const findV2inCalls = (calls: [string, Request][]) => {
+  for (let i = 0; i < calls.length; i++) {
+    if (calls[i][0].includes('v2')) {
+      return i;
+    }
+  }
+  return -1;
+};

e2e/node/basic/mainnet.test.ts

@@ -141,3 +141,21 @@ describe('call forwarding', () => {
     expect(reply).toBeTruthy();
   }, 15_000);
 });
+
+
+test('it should allow you to set an incorrect root key', async () => {
+  const agent = HttpAgent.createSync({
+    rootKey: new Uint8Array(31),
+  });
+  const idlFactory = ({ IDL }) =>
+    IDL.Service({
+      whoami: IDL.Func([], [IDL.Principal], ['query']),
+    });
+
+  const actor = Actor.createActor(idlFactory, {
+    agent,
+    canisterId: Principal.fromText('rrkah-fqaaa-aaaaa-aaaaq-cai'),
+  });
+
+  expect(actor.whoami).rejects.toThrowError(`Invalid certificate:`);
+});

e2e/node/basic/watermark.test.ts

@@ -114,16 +114,44 @@ test('replay attack', async () => {
   expect(startValue3).toBe(1n);
 
   const queryResponseIndex = indexOfQueryResponse(fetchProxy.history);
+  console.log(queryResponseIndex);
 
   fetchProxy.replayFromHistory(queryResponseIndex);
 
-  // the replayed request should throw an error
-  expect(fetchProxy.calls).toBe(7);
+  // The number of calls should be 4 or more, depending on whether the test environment is using v3 or v2
+  const usingV2 =
+    findV2inCalls(
+      fetchProxy.history.map(response => {
+        return [response.url];
+      }),
+    ) !== -1;
+  if (usingV2) {
+    // TODO - pin to 5 once dfx v0.23.0 is released
+    // the replayed request should throw an error
+    expect(fetchProxy.calls).toBe(5);
+  } else {
+    expect(fetchProxy.calls).toBeGreaterThanOrEqual(5);
+  }
 
   await expect(actor.read()).rejects.toThrowError(
     'Timestamp failed to pass the watermark after retrying the configured 3 times. We cannot guarantee the integrity of the response since it could be a replay attack.',
   );
 
-  // The agent should should have made 4 additional requests (3 retries + 1 original request)
-  expect(fetchProxy.calls).toBe(11);
+  // TODO - pin to 9 once dfx v0.23.0 is released
+  if (usingV2) {
+    // the replayed request should throw an error
+    // The agent should should have made 4 additional requests (3 retries + 1 original request)
+    expect(fetchProxy.calls).toBe(9);
+  } else {
+    expect(fetchProxy.calls).toBeGreaterThanOrEqual(9);
+  }
 }, 10_000);
+
+const findV2inCalls = (calls: [string][]) => {
+  for (let i = 0; i < calls.length; i++) {
+    if (calls[i][0].includes('v2')) {
+      return i;
+    }
+  }
+  return -1;
+};

e2e/node/canisters/counter.ts

@@ -11,7 +11,7 @@ let cache: {
   actor: any;
 } | null = null;
 
-const idl = ({ IDL }) => {
+export const idl = ({ IDL }) => {
   return IDL.Service({
     inc: IDL.Func([], [], []),
     inc_read: IDL.Func([], [IDL.Nat], []),

e2e/node/integration/actor.test.ts

@@ -14,9 +14,9 @@ test("Legacy Agent interface should be accepted by Actor's createActor", async (
   );
 
   // Verify that update calls work
-  await actor.write(8n); //?
+  await actor.write(8n);
   // Verify that query calls work
-  const count = await actor.read(); //?
+  const count = await actor.read();
   expect(count).toBe(8n);
 }, 15_000);
 // TODO: tests for rejected, unknown time out

packages/agent/src/actor.test.ts

@@ -6,7 +6,7 @@ import { CallRequest, SubmitRequestType, UnSigned } from './agent/http/types';
 import * as cbor from './cbor';
 import { requestIdOf } from './request_id';
 import * as pollingImport from './polling';
-import { ActorConfig } from './actor';
+import { Actor, ActorConfig } from './actor';
 
 const importActor = async (mockUpdatePolling?: () => void) => {
   jest.dontMock('./polling');
@@ -329,7 +329,7 @@ describe('makeActor', () => {
     `);
     expect(replyUpdateWithHttpDetails.result).toEqual(canisterDecodedReturnValue);
 
-    replyUpdateWithHttpDetails.httpDetails['requestDetails']['nonce'] = new Uint8Array(); //?
+    replyUpdateWithHttpDetails.httpDetails['requestDetails']['nonce'] = new Uint8Array();
 
     expect(replyUpdateWithHttpDetails.httpDetails).toMatchSnapshot();
   });

packages/agent/src/actor.ts

@@ -9,12 +9,12 @@ import {
   SubmitResponse,
 } from './agent';
 import { AgentError } from './errors';
-import { IDL } from '@dfinity/candid';
+import { bufFromBufLike, IDL } from '@dfinity/candid';
 import { pollForResponse, PollStrategyFactory, strategy } from './polling';
 import { Principal } from '@dfinity/principal';
 import { RequestId } from './request_id';
 import { toHex } from './utils/buffer';
-import { Certificate, CreateCertificateOptions } from './certificate';
+import { Certificate, CreateCertificateOptions, lookupResultToBuffer } from './certificate';
 import managementCanisterIdl from './canisters/management_idl';
 import _SERVICE, { canister_install_mode, canister_settings } from './canisters/management_service';
 
@@ -525,35 +525,49 @@ function _createActorMethod(
       const ecid = effectiveCanisterId !== undefined ? Principal.from(effectiveCanisterId) : cid;
       const arg = IDL.encode(func.argTypes, args);
 
+      if (agent.rootKey == null)
+        throw new AgentError('Agent root key not initialized before making call');
+
       const { requestId, response, requestDetails } = await agent.call(cid, {
         methodName,
         arg,
         effectiveCanisterId: ecid,
       });
-
-      requestId;
-      response;
-      requestDetails;
-
-      if (!response.ok || response.body /* IC-1462 */) {
-        throw new UpdateCallRejectedError(cid, methodName, requestId, response);
+      let reply: ArrayBuffer | undefined;
+      let certificate: Certificate | undefined;
+      if (response.body && response.body.certificate) {
+        const cert = response.body.certificate;
+        certificate = await Certificate.create({
+          certificate: bufFromBufLike(cert),
+          rootKey: agent.rootKey,
+          canisterId: Principal.from(canisterId),
+          blsVerify,
+        });
+        const path = [new TextEncoder().encode('request_status'), requestId];
+        const status = new TextDecoder().decode(
+          lookupResultToBuffer(certificate.lookup([...path, 'status'])),
+        );
+
+        switch (status) {
+          case 'replied':
+            reply = lookupResultToBuffer(certificate.lookup([...path, 'reply']));
+            break;
+          case 'rejected':
+            throw new UpdateCallRejectedError(cid, methodName, requestId, response);
+        }
+      }
+      // Fall back to polling if we recieve an Accepted response code
+      if (response.status === 202) {
+        const pollStrategy = pollingStrategyFactory();
+        // Contains the certificate and the reply from the boundary node
+        const response = await pollForResponse(agent, ecid, requestId, pollStrategy, blsVerify);
+        certificate = response.certificate;
+        reply = response.reply;
       }
-
-      const pollStrategy = pollingStrategyFactory();
-      // Contains the certificate and the reply from the boundary node
-      const { certificate, reply } = await pollForResponse(
-        agent,
-        ecid,
-        requestId,
-        pollStrategy,
-        blsVerify,
-      );
-      reply;
       const shouldIncludeHttpDetails = func.annotations.includes(ACTOR_METHOD_WITH_HTTP_DETAILS);
       const shouldIncludeCertificate = func.annotations.includes(ACTOR_METHOD_WITH_CERTIFICATE);
 
       const httpDetails = { ...response, requestDetails } as HttpDetailsResponse;
-
       if (reply !== undefined) {
         if (shouldIncludeHttpDetails && shouldIncludeCertificate) {
           return {

packages/agent/src/agent/api.ts

@@ -111,8 +111,8 @@ export interface CallOptions {
   arg: ArrayBuffer;
 
   /**
-   * An effective canister ID, used for routing. This should only be mentioned if
-   * it's different from the canister ID.
+   * An effective canister ID, used for routing. Usually the canister ID, except for management canister calls.
+   * @see https://internetcomputer.org/docs/current/references/ic-interface-spec/#http-effective-canister-id
    */
   effectiveCanisterId: Principal | string;
 }
@@ -132,6 +132,8 @@ export interface SubmitResponse {
       error_code?: string;
       reject_code: number;
       reject_message: string;
+      // Available in a v3 call response
+      certificate?: ArrayBuffer;
     } | null;
     headers: HttpHeaderField[];
   };

packages/agent/src/agent/http/http.test.ts

@@ -107,7 +107,7 @@ test('call', async () => {
   expect(requestId).toEqual(expectedRequestId);
   const call1 = calls[0][0];
   const call2 = calls[0][1];
-  expect(call1).toBe(`http://127.0.0.1/api/v2/canister/${canisterId.toText()}/call`);
+  expect(call1).toBe(`http://127.0.0.1/api/v3/canister/${canisterId.toText()}/call`);
   expect(call2.method).toEqual('POST');
   expect(call2.body).toEqual(cbor.encode(expectedRequest));
   expect(call2.headers['Content-Type']).toEqual('application/cbor');
@@ -321,7 +321,7 @@ test('use anonymous principal if unspecified', async () => {
   expect(calls.length).toBe(1);
   expect(requestId).toEqual(expectedRequestId);
 
-  expect(calls[0][0]).toBe(`http://127.0.0.1/api/v2/canister/${canisterId.toText()}/call`);
+  expect(calls[0][0]).toBe(`http://127.0.0.1/api/v3/canister/${canisterId.toText()}/call`);
   const call2 = calls[0][1];
   expect(call2.method).toEqual('POST');
   expect(call2.body).toEqual(cbor.encode(expectedRequest));
@@ -794,6 +794,8 @@ test('retry requests that fail due to a network failure', async () => {
     fetch: mockFetch,
   });
 
+  agent.rootKey = new Uint8Array(32);
+
   try {
     await agent.call(Principal.managementCanister(), {
       methodName: 'test',
@@ -811,6 +813,7 @@ test('it should log errors to console if the option is set', async () => {
   const agent = new HttpAgent({ host: HTTP_AGENT_HOST, fetch: jest.fn(), logToConsole: true });
   await agent.syncTime();
 });
+
 jest.setTimeout(5000);
 test('it should sync time with the replica for a query', async () => {
   const canisterId = 'ivcos-eqaaa-aaaab-qablq-cai';