Added support for uploading big censuses in chunks

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- Added support for uploading big censuses in chunks.
+
 ## [0.3.1] - 2023-09-20
 
 ### Added

src/api/census.ts

@@ -22,7 +22,7 @@ interface ICensusCreateResponse {
 
 interface ICensusAddResponse {}
 
-interface ICensusPublishResponse {
+export interface ICensusPublishResponse {
   /**
    * The identifier of the published census
    */

src/client.ts

@@ -23,7 +23,14 @@ import {
   Vote,
   WeightedCensus,
 } from './types';
-import { API_URL, EXPLORER_URL, FAUCET_AUTH_TOKEN, FAUCET_URL, TX_WAIT_OPTIONS } from './util/constants';
+import {
+  API_URL,
+  CENSUS_CHUNK_SIZE,
+  EXPLORER_URL,
+  FAUCET_AUTH_TOKEN,
+  FAUCET_URL,
+  TX_WAIT_OPTIONS,
+} from './util/constants';
 import {
   AccountData,
   AccountService,
@@ -120,7 +127,7 @@ export class VocdoniSDKClient {
     this.wallet = opts.wallet;
     this.electionId = opts.electionId;
     this.explorerUrl = EXPLORER_URL[opts.env];
-    this.censusService = new CensusService({ url: this.url });
+    this.censusService = new CensusService({ url: this.url, chunk_size: CENSUS_CHUNK_SIZE });
     this.fileService = new FileService({ url: this.url });
     this.chainService = new ChainService({
       url: this.url,

src/services/census.ts

@@ -1,11 +1,12 @@
 import { Service, ServiceProperties } from './service';
-import { CensusType, PlainCensus, WeightedCensus } from '../types';
-import { CensusAPI, WalletAPI } from '../api';
+import { CensusType, ICensusParticipant, PlainCensus, WeightedCensus } from '../types';
+import { CensusAPI, ICensusPublishResponse, WalletAPI } from '../api';
 import { Wallet } from '@ethersproject/wallet';
 import invariant from 'tiny-invariant';
 
 interface CensusServiceProperties {
   auth: CensusAuth;
+  chunk_size: number;
 }
 
 type CensusServiceParameters = ServiceProperties & CensusServiceProperties;
@@ -46,6 +47,7 @@ export type CspCensusProof = {
 
 export class CensusService extends Service implements CensusServiceProperties {
   public auth: CensusAuth;
+  public chunk_size: number;
 
   /**
    * Instantiate the census service.
@@ -65,6 +67,7 @@ export class CensusService extends Service implements CensusServiceProperties {
    */
   fetchCensusInfo(censusId: string): Promise<{ size: number; weight: bigint; type: CensusType }> {
     invariant(this.url, 'No URL set');
+
     return Promise.all([
       CensusAPI.size(this.url, censusId),
       CensusAPI.weight(this.url, censusId),
@@ -91,6 +94,7 @@ export class CensusService extends Service implements CensusServiceProperties {
    */
   async fetchProof(censusId: string, key: string): Promise<CensusProof> {
     invariant(this.url, 'No URL set');
+
     return CensusAPI.proof(this.url, censusId, key).then((censusProof) => ({
       type: censusProof.type,
       weight: censusProof.weight,
@@ -101,24 +105,80 @@ export class CensusService extends Service implements CensusServiceProperties {
     }));
   }
 
+  create(censusType: CensusType): Promise<string> {
+    invariant(this.url, 'No URL set');
+
+    return this.fetchAccountToken()
+      .then(() => CensusAPI.create(this.url, this.auth.identifier, censusType))
+      .then((response) => response.censusID);
+  }
+
+  async add(censusId: string, participants: ICensusParticipant[]) {
+    invariant(this.url, 'No URL set');
+    invariant(this.auth, 'No census auth set');
+    invariant(this.chunk_size, 'No chunk size set');
+
+    const participantsChunked = participants.reduce((result, item, index) => {
+      const chunkIndex = Math.floor(index / this.chunk_size);
+
+      if (!result[chunkIndex]) {
+        result[chunkIndex] = [];
+      }
+
+      result[chunkIndex].push(item);
+
+      return result;
+    }, []);
+
+    for (const chunk of participantsChunked) {
+      await CensusAPI.add(this.url, this.auth.identifier, censusId, chunk);
+    }
+
+    return censusId;
+  }
+
+  private addParallel(censusId: string, participants: ICensusParticipant[]) {
+    invariant(this.url, 'No URL set');
+    invariant(this.auth, 'No census auth set');
+    invariant(this.chunk_size, 'No chunk size set');
+
+    const participantsChunked = participants.reduce((result, item, index) => {
+      const chunkIndex = Math.floor(index / this.chunk_size);
+
+      if (!result[chunkIndex]) {
+        result[chunkIndex] = [];
+      }
+
+      result[chunkIndex].push(item);
+
+      return result;
+    }, []);
+
+    return participantsChunked.map((chunk) => CensusAPI.add(this.url, this.auth.identifier, censusId, chunk));
+  }
+
+  /**
+   * Publishes the given census identifier.
+   *
+   * @param {string} censusId The census identifier
+   */
+  publish(censusId: string): Promise<ICensusPublishResponse> {
+    invariant(this.url, 'No URL set');
+    invariant(this.auth, 'No census auth set');
+
+    return CensusAPI.publish(this.url, this.auth.identifier, censusId);
+  }
+
   /**
    * Publishes the given census.
    *
    * @param {PlainCensus | WeightedCensus} census The census to be published.
    * @returns {Promise<void>}
    */
   createCensus(census: PlainCensus | WeightedCensus): Promise<void> {
-    invariant(this.url, 'No URL set');
-    const censusCreation = this.fetchAccountToken().then(() =>
-      CensusAPI.create(this.url, this.auth.identifier, census.type)
-    );
-
-    const censusAdding = censusCreation.then((censusCreateResponse) =>
-      CensusAPI.add(this.url, this.auth.identifier, censusCreateResponse.censusID, census.participants)
-    );
-
-    return Promise.all([censusCreation, censusAdding])
-      .then((censusData) => CensusAPI.publish(this.url, this.auth.identifier, censusData[0].censusID))
+    return this.create(census.type)
+      .then((censusId) => this.add(censusId, census.participants))
+      .then((censusId) => this.publish(censusId))
       .then((censusPublish) => {
         census.censusId = censusPublish.censusID;
         census.censusURI = censusPublish.uri;
@@ -130,6 +190,29 @@ export class CensusService extends Service implements CensusServiceProperties {
       });
   }
 
+  /**
+   * Publishes the given census.
+   *
+   * @param {PlainCensus | WeightedCensus} census The census to be published.
+   * @returns {Promise<void>}
+   */
+  // @ts-ignore
+  private createCensusParallel(census: PlainCensus | WeightedCensus): Promise<void> {
+    return this.create(census.type).then((censusId) =>
+      Promise.all(this.addParallel(censusId, census.participants))
+        .then(() => this.publish(censusId))
+        .then((censusPublish) => {
+          census.censusId = censusPublish.censusID;
+          census.censusURI = censusPublish.uri;
+          census.size = census.participants.length;
+          census.weight = census.participants.reduce(
+            (currentValue, participant) => currentValue + participant.weight,
+            BigInt(0)
+          );
+        })
+    );
+  }
+
   /**
    * Fetches the specific account token auth and sets it to the current instance.
    *

src/util/constants.ts

@@ -35,3 +35,5 @@ export const VOCDONI_SIK_SIGNATURE_LENGTH = 64;
 export const VOCDONI_SIK_PAYLOAD =
   'This signature request is used to create your own secret identity key (SIK) for the Vocdoni protocol and generate your anonymous account.\n' +
   'Only accept this signature request if you fully trust the application. This request will not trigger a blockchain transaction or cost any gas fees.';
+
+export const CENSUS_CHUNK_SIZE = 8192;

test/services/census.test.ts

@@ -2,7 +2,8 @@ import { CensusService, CensusType } from '../../src';
 // @ts-ignore
 import { URL } from './util/client.params';
 import { Wallet } from '@ethersproject/wallet';
-import { WeightedCensus } from '@vocdoni/sdk';
+import { PlainCensus, WeightedCensus } from '@vocdoni/sdk';
+import { CENSUS_CHUNK_SIZE } from '../../src/util/constants';
 
 describe('Census Service tests', () => {
   it('should have the correct type and properties', () => {
@@ -45,12 +46,62 @@ describe('Census Service tests', () => {
     expect(census.censusId).toMatch(/^[0-9a-fA-F]{64}$/);
     expect(census.censusURI).toBeDefined();
     expect(census.type).toEqual(CensusType.WEIGHTED);
-    expect(census.size).toEqual(10);
+    expect(census.size).toEqual(numVotes);
     expect(census.weight).toEqual(BigInt(55));
 
     const censusInfo = await service.fetchCensusInfo(census.censusId);
     expect(censusInfo.type).toEqual(CensusType.WEIGHTED);
     expect(censusInfo.size).toEqual(10);
     expect(censusInfo.weight).toEqual(BigInt(55));
   }, 30000);
+  it('should create a census by batches and return the correct information', async () => {
+    const numVotes = 63;
+    const service = new CensusService({ url: URL, chunk_size: 9 });
+    const census = new PlainCensus();
+    const participants: Wallet[] = [...new Array(numVotes)].map(() => Wallet.createRandom());
+    census.add(participants.map((participant) => participant.address));
+
+    await service.createCensus(census);
+
+    expect(census.censusId).toMatch(/^[0-9a-fA-F]{64}$/);
+    expect(census.censusURI).toBeDefined();
+    expect(census.type).toEqual(CensusType.WEIGHTED);
+    expect(census.size).toEqual(numVotes);
+    expect(census.weight).toEqual(BigInt(numVotes));
+
+    const censusInfo = await service.fetchCensusInfo(census.censusId);
+    expect(censusInfo.type).toEqual(CensusType.WEIGHTED);
+    expect(censusInfo.size).toEqual(numVotes);
+    expect(censusInfo.weight).toEqual(BigInt(numVotes));
+  }, 30000);
+  it('should create a big census by batches and return the correct information', async () => {
+    const numVotes = 10000;
+    const service = new CensusService({ url: URL, chunk_size: CENSUS_CHUNK_SIZE });
+    const census = new PlainCensus();
+
+    const pad = (num, size) => {
+      num = num.toString();
+      while (num.length < size) num = '0' + num;
+      return num;
+    };
+
+    // Adding not random addresses for testing purposes
+    census.participants = [...new Array(numVotes)].map((_v, i) => ({
+      key: '0x' + pad(++i, 40),
+      weight: 1n,
+    }));
+
+    await service.createCensus(census);
+
+    expect(census.censusId).toMatch(/^[0-9a-fA-F]{64}$/);
+    expect(census.censusURI).toBeDefined();
+    expect(census.type).toEqual(CensusType.WEIGHTED);
+    expect(census.size).toEqual(numVotes);
+    expect(census.weight).toEqual(BigInt(numVotes));
+
+    const censusInfo = await service.fetchCensusInfo(census.censusId);
+    expect(censusInfo.type).toEqual(CensusType.WEIGHTED);
+    expect(censusInfo.size).toEqual(numVotes);
+    expect(censusInfo.weight).toEqual(BigInt(numVotes));
+  }, 30000);
 });