Switch frontend to using MessagePack for a few canisters (#6254)

backend/canisters/user_index/api/src/main.rs

@@ -50,6 +50,7 @@ fn main() {
     generate_ts_method!(user_index, check_username);
     generate_ts_method!(user_index, chit_leaderboard);
     generate_ts_method!(user_index, current_user);
+    generate_ts_method!(user_index, external_achievements);
     generate_ts_method!(user_index, diamond_membership_fees);
     generate_ts_method!(user_index, platform_moderators);
     generate_ts_method!(user_index, platform_moderators_group);

frontend/openchat-agent/codegen.sh

@@ -8,9 +8,6 @@ didc bind ../../backend/canisters/community/api/can.did -t js > ./src/services/c
 didc bind ../../backend/canisters/group/api/can.did -t ts > ./src/services/group/candid/types.d.ts
 didc bind ../../backend/canisters/group/api/can.did -t js > ./src/services/group/candid/idl.js
 
-didc bind ../../backend/canisters/group_index/api/can.did -t ts > ./src/services/groupIndex/candid/types.d.ts
-didc bind ../../backend/canisters/group_index/api/can.did -t js > ./src/services/groupIndex/candid/idl.js
-
 didc bind ../../backend/canisters/identity/api/can.did -t ts > ./src/services/identity/candid/types.d.ts
 didc bind ../../backend/canisters/identity/api/can.did -t js > ./src/services/identity/candid/idl.js
 
@@ -23,15 +20,9 @@ didc bind ../../backend/canisters/market_maker/api/can.did -t js > ./src/service
 didc bind ../../backend/canisters/notifications_index/api/can.did -t ts > ./src/services/notifications/candid/types.d.ts
 didc bind ../../backend/canisters/notifications_index/api/can.did -t js > ./src/services/notifications/candid/idl.js
 
-didc bind ../../backend/canisters/online_users/api/can.did -t ts > ./src/services/online/candid/types.d.ts
-didc bind ../../backend/canisters/online_users/api/can.did -t js > ./src/services/online/candid/idl.js
-
 didc bind ../../backend/canisters/proposals_bot/api/can.did -t ts > ./src/services/proposalsBot/candid/types.d.ts
 didc bind ../../backend/canisters/proposals_bot/api/can.did -t js > ./src/services/proposalsBot/candid/idl.js
 
-didc bind ../../backend/canisters/registry/api/can.did -t ts > ./src/services/registry/candid/types.d.ts
-didc bind ../../backend/canisters/registry/api/can.did -t js > ./src/services/registry/candid/idl.js
-
 didc bind ../../backend/canisters/storage_bucket/api/can.did -t ts > ./src/services/storageBucket/candid/types.d.ts
 didc bind ../../backend/canisters/storage_bucket/api/can.did -t js > ./src/services/storageBucket/candid/idl.js
 
@@ -44,9 +35,6 @@ didc bind ../../backend/canisters/translations/api/can.did -t js > ./src/service
 didc bind ../../backend/canisters/user/api/can.did -t ts > ./src/services/user/candid/types.d.ts
 didc bind ../../backend/canisters/user/api/can.did -t js > ./src/services/user/candid/idl.js
 
-didc bind ../../backend/canisters/user_index/api/can.did -t ts > ./src/services/userIndex/candid/types.d.ts
-didc bind ../../backend/canisters/user_index/api/can.did -t js > ./src/services/userIndex/candid/idl.js
-
 didc bind ./src/services/dexes/icpSwap/index/candid/can.did -t ts > ./src/services/dexes/icpSwap/index/candid/types.d.ts
 didc bind ./src/services/dexes/icpSwap/index/candid/can.did -t js > ./src/services/dexes/icpSwap/index/candid/idl.js
 

frontend/openchat-agent/package.json

@@ -23,16 +23,19 @@
         "@dfinity/candid": "^2.0.0",
         "@dfinity/identity": "^2.0.0",
         "@dfinity/principal": "^2.0.0",
+        "@sinclair/typebox": "^0.33.9",
         "idb": "^7.1.1",
         "identicon.js": "^2.3.3",
         "js-sha3": "^0.8.0",
         "md5": "^2.3.0",
+        "msgpackr": "^1.11.0",
         "openchat-shared": "*",
         "uuid": "^9.0.0"
     },
     "devDependencies": {
         "@babel/core": "^7.22.10",
         "@babel/preset-env": "^7.22.10",
+        "@sinclair/typebox-codegen": "^0.8.13",
         "@testing-library/jest-dom": "^6.0.0",
         "@types/identicon.js": "^2.3.1",
         "@types/jest": "^29.5.3",

frontend/openchat-agent/src/services/candidService.ts

@@ -1,9 +1,17 @@
-import { Actor, HttpAgent, type Identity } from "@dfinity/agent";
+import { Actor, HttpAgent, type Identity, polling, UpdateCallRejectedError } from "@dfinity/agent";
 import type { IDL } from "@dfinity/candid";
-import type { Principal } from "@dfinity/principal";
-import { AuthError, DestinationInvalidError, SessionExpiryError } from "openchat-shared";
+import { Principal } from "@dfinity/principal";
+import {
+    AuthError,
+    DestinationInvalidError,
+    ResponseTooLargeError,
+    SessionExpiryError,
+} from "openchat-shared";
 import { ReplicaNotUpToDateError, toCanisterResponseError } from "./error";
-import { ResponseTooLargeError } from "openchat-shared";
+import { type Options, Packr } from "msgpackr";
+import { identity } from "../utils/mapping";
+import type { Static, TSchema } from "@sinclair/typebox";
+import { Value } from "@sinclair/typebox/value";
 
 const MAX_RETRIES = process.env.NODE_ENV === "production" ? 7 : 3;
 const RETRY_DELAY = 100;
@@ -12,6 +20,8 @@ function debug(msg: string): void {
     console.log(msg);
 }
 
+const Packer = new Packr({ useRecords: false, skipValues: [undefined] } as unknown as Options);
+
 export abstract class CandidService {
     protected createServiceClient<T>(factory: IDL.InterfaceFactory): T {
         return Actor.createActor<T>(factory, {
@@ -24,6 +34,68 @@ export abstract class CandidService {
         return this.identity.getPrincipal();
     }
 
+    protected async executeMsgpackQuery<In extends TSchema, Resp extends TSchema, Out>(
+        methodName: string,
+        args: Static<In>,
+        mapper: (from: Static<Resp>) => Out,
+        requestValidator: In,
+        responseValidator: Resp,
+    ): Promise<Out> {
+        const payload = CandidService.prepareMsgpackArgs(args, requestValidator);
+
+        const response = await this.handleQueryResponse(
+            () =>
+                this.agent.query(this.canisterId, {
+                    methodName: methodName + "_msgpack",
+                    arg: payload,
+                }),
+            identity,
+            args,
+        );
+        if (response.status === "replied") {
+            return CandidService.processMsgpackResponse(
+                response.reply.arg,
+                mapper,
+                responseValidator,
+            );
+        } else {
+            throw new Error(
+                `query rejected. Code: ${response.reject_code}. Message: ${response.reject_message}`,
+            );
+        }
+    }
+
+    protected async executeMsgpackUpdate<In extends TSchema, Resp extends TSchema, Out>(
+        methodName: string,
+        args: Static<In>,
+        mapper: (from: Static<Resp>) => Out,
+        requestValidator: In,
+        responseValidator: Resp,
+    ): Promise<Out> {
+        const payload = CandidService.prepareMsgpackArgs(args, requestValidator);
+
+        try {
+            const { requestId, response } = await this.agent.call(this.canisterId, {
+                methodName: methodName + "_msgpack",
+                arg: payload,
+            });
+            const canisterId = Principal.fromText(this.canisterId);
+            if (!response.ok || response.body) {
+                throw new UpdateCallRejectedError(canisterId, methodName, requestId, response);
+            }
+            const { reply } = await polling.pollForResponse(
+                this.agent,
+                canisterId,
+                requestId,
+                polling.defaultStrategy(),
+            );
+            return CandidService.processMsgpackResponse(reply, mapper, responseValidator);
+        } catch (err) {
+            console.log(err, args);
+            throw toCanisterResponseError(err as Error, this.identity);
+        }
+    }
+
     protected handleResponse<From, To>(
         service: Promise<From>,
         mapper: (from: From) => To,
@@ -84,6 +156,32 @@ export abstract class CandidService {
             });
     }
 
+    private static validate<T extends TSchema>(value: unknown, validator: T): T {
+        try {
+            return Value.Parse(validator, value);
+        } catch (err) {
+            throw new Error("Validation failed: " + JSON.stringify(err));
+        }
+    }
+
+    private static prepareMsgpackArgs<T extends TSchema>(
+        value: Static<T>,
+        validator: T,
+    ): ArrayBuffer {
+        const validated = CandidService.validate(value, validator);
+        return Packer.pack(validated);
+    }
+
+    private static processMsgpackResponse<Resp extends TSchema, Out>(
+        responseBytes: ArrayBuffer,
+        mapper: (from: Resp) => Out,
+        validator: Resp,
+    ): Out {
+        const response = Packer.unpack(new Uint8Array(responseBytes));
+        const validated = CandidService.validate(response, validator);
+        return mapper(validated);
+    }
+
     constructor(
         protected identity: Identity,
         protected agent: HttpAgent,