diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 44a0644b141..ca75e34f5ec 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -13,4 +13,5 @@ jobs: - name: 'Dependency Review' uses: actions/dependency-review-action@v3 with: - allow-licenses: AGPL-3.0-only, LGPL-3.0, MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, ISC, X11, 0BSD, GPL-3.0, Unlicense + allow-licenses: AGPL-3.0-only, LGPL-3.0, MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, ISC, X11, 0BSD, GPL-3.0, BSD-3-Clause AND BSD-3-Clause-Clear, Unlicense + allow-dependencies-licenses: 'pkg:npm/parse-mongo-url' diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a5688568696..d2c149af35b 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -133,7 +133,7 @@ jobs: needs: - build_and_push - branch_meta - uses: hpi-schul-cloud/dof_app_deploy/.github/workflows/deploy.yml@main + uses: hpi-schul-cloud/dof_app_deploy/.github/workflows/deploy.yml@BC-4256-Integration-tldraw with: branch: ${{ needs.branch_meta.outputs.branch }} secrets: diff --git a/ansible/roles/schulcloud-server-tldraw/meta/main.yml b/ansible/roles/schulcloud-server-tldraw/meta/main.yml new file mode 100644 index 00000000000..895ccb2bc29 --- /dev/null +++ b/ansible/roles/schulcloud-server-tldraw/meta/main.yml @@ -0,0 +1,9 @@ +galaxy_info: + role_name: schulcloud-server-tldraw + author: Schul-Cloud Verbund + description: tldraw role for the schulcloud-server + company: Schul-Cloud Verbund + license: license (AGPLv3) + min_ansible_version: 2.8 + galaxy_tags: [] +dependencies: [] diff --git a/ansible/roles/schulcloud-server-tldraw/tasks/main.yml b/ansible/roles/schulcloud-server-tldraw/tasks/main.yml new file mode 100644 index 00000000000..ac0f2c551f5 --- /dev/null +++ b/ansible/roles/schulcloud-server-tldraw/tasks/main.yml @@ -0,0 +1,11 @@ +- name: TlDrawServerDeployment + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: api-tldraw-deployment.yml.j2 + +- name: TlDrawWsService + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: api-tldraw-svc.yml.j2 diff --git a/ansible/roles/schulcloud-server-tldraw/templates/api-tldraw-deployment.yml.j2 b/ansible/roles/schulcloud-server-tldraw/templates/api-tldraw-deployment.yml.j2 new file mode 100644 index 00000000000..4d226a7d776 --- /dev/null +++ b/ansible/roles/schulcloud-server-tldraw/templates/api-tldraw-deployment.yml.j2 @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api-tldraw-deployment + namespace: {{ NAMESPACE }} + labels: + app: tldraw +spec: + replicas: {{ TLDRAW_EDITOR_REPLICAS|default("1", true) }} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + #maxUnavailable: 1 + revisionHistoryLimit: 4 + paused: false + selector: + matchLabels: + app: tldraw + template: + metadata: + labels: + app: tldraw + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + runAsNonRoot: true + containers: + - name: tldraw + image: {{ SCHULCLOUD_SERVER_IMAGE }}:{{ SCHULCLOUD_SERVER_IMAGE_TAG }} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3345 + name: tldraw + protocol: TCP + envFrom: + - configMapRef: + name: api-configmap + - secretRef: + name: api-secret + command: ['npm', 'run', 'nest:start:tldraw:prod'] + resources: + limits: + cpu: {{ TLDRAW_EDITOR_CPU_LIMITS|default("2000m", true) }} + memory: {{ TLDRAW_EDITOR_MEMORY_LIMITS|default("500Mi", true) }} + requests: + cpu: {{ TLDRAW_EDITOR_CPU_REQUESTS|default("100m", true) }} + memory: {{ TLDRAW_EDITOR_MEMORY_REQUESTS|default("50Mi", true) }} diff --git a/ansible/roles/schulcloud-server-tldraw/templates/api-tldraw-svc.yml.j2 b/ansible/roles/schulcloud-server-tldraw/templates/api-tldraw-svc.yml.j2 new file mode 100644 index 00000000000..e2d55c2a317 --- /dev/null +++ b/ansible/roles/schulcloud-server-tldraw/templates/api-tldraw-svc.yml.j2 @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: api-tldraw-svc + namespace: {{ NAMESPACE }} + labels: + app: tldraw +spec: + type: ClusterIP + ports: + - port: 3345 + targetPort: 3345 + protocol: TCP + name: tldraw + selector: + app: tldraw diff --git a/apps/server/src/apps/tldraw.app.ts b/apps/server/src/apps/tldraw.app.ts new file mode 100644 index 00000000000..e541222205b --- /dev/null +++ b/apps/server/src/apps/tldraw.app.ts @@ -0,0 +1,31 @@ +/* istanbul ignore file */ +/* eslint-disable no-console */ +import { NestFactory } from '@nestjs/core'; +import { install as sourceMapInstall } from 'source-map-support'; +import { TldrawModule } from '@src/modules/tldraw'; +import { Logger } from '@src/core/logger'; +import * as WebSocket from 'ws'; +import { WsAdapter } from '@nestjs/platform-ws'; +import { enableOpenApiDocs } from '@shared/controller/swagger'; +import { AppStartLoggable } from '@src/apps/helpers/app-start-loggable'; + +async function bootstrap() { + sourceMapInstall(); + + const nestApp = await NestFactory.create(TldrawModule); + const wss = new WebSocket.Server({ noServer: true }); + nestApp.useWebSocketAdapter(new WsAdapter(wss)); + nestApp.enableCors(); + enableOpenApiDocs(nestApp, 'docs'); + + const logger = await nestApp.resolve(Logger); + await nestApp.init(); + + logger.info( + new AppStartLoggable({ + appName: 'Tldraw server app', + }) + ); +} + +void bootstrap(); diff --git a/apps/server/src/modules/tldraw/config.ts b/apps/server/src/modules/tldraw/config.ts new file mode 100644 index 00000000000..a892ee6c843 --- /dev/null +++ b/apps/server/src/modules/tldraw/config.ts @@ -0,0 +1,30 @@ +import { Configuration } from '@hpi-schul-cloud/commons'; + +export interface TldrawConfig { + NEST_LOG_LEVEL: string; + INCOMING_REQUEST_TIMEOUT: number; + TLDRAW_DB_COLLECTION_NAME: string; + TLDRAW_DB_FLUSH_SIZE: string; + TLDRAW_DB_MULTIPLE_COLLECTIONS: boolean; + CONNECTION_STRING: string; + FEATURE_TLDRAW_ENABLED: boolean; + TLDRAW_PING_TIMEOUT: number; + TLDRAW_GC_ENABLED: number; +} + +const tldrawConnectionString: string = Configuration.get('TLDRAW_DB_URL') as string; + +const tldrawConfig = { + NEST_LOG_LEVEL: Configuration.get('NEST_LOG_LEVEL') as string, + INCOMING_REQUEST_TIMEOUT: Configuration.get('INCOMING_REQUEST_TIMEOUT_API') as number, + TLDRAW_DB_COLLECTION_NAME: Configuration.get('TLDRAW__DB_COLLECTION_NAME') as string, + TLDRAW_DB_FLUSH_SIZE: Configuration.get('TLDRAW__DB_FLUSH_SIZE') as number, + TLDRAW_DB_MULTIPLE_COLLECTIONS: Configuration.get('TLDRAW__DB_MULTIPLE_COLLECTIONS') as boolean, + FEATURE_TLDRAW_ENABLED: Configuration.get('FEATURE_TLDRAW_ENABLED') as boolean, + CONNECTION_STRING: tldrawConnectionString, + TLDRAW_PING_TIMEOUT: Configuration.get('TLDRAW__PING_TIMEOUT') as number, + TLDRAW_GC_ENABLED: Configuration.get('TLDRAW__GC_ENABLED') as boolean, +}; + +export const SOCKET_PORT = Configuration.get('TLDRAW__SOCKET_PORT') as number; +export const config = () => tldrawConfig; diff --git a/apps/server/src/modules/tldraw/controller/api-test/tldraw.ws.api.spec.ts b/apps/server/src/modules/tldraw/controller/api-test/tldraw.ws.api.spec.ts new file mode 100644 index 00000000000..feb924a8931 --- /dev/null +++ b/apps/server/src/modules/tldraw/controller/api-test/tldraw.ws.api.spec.ts @@ -0,0 +1,129 @@ +import { WsAdapter } from '@nestjs/platform-ws'; +import { Test } from '@nestjs/testing'; +import WebSocket from 'ws'; +import { TextEncoder } from 'util'; +import { INestApplication } from '@nestjs/common'; +import { TldrawTestModule } from '../..'; +import { TldrawWs } from '../tldraw.ws'; +import { TestConnection } from '../../testing/test-connection'; + +describe('WebSocketController (WsAdapter)', () => { + let app: INestApplication; + let gateway: TldrawWs; + let ws: WebSocket; + + const gatewayPort = 3346; + const wsUrl = TestConnection.getWsUrl(gatewayPort); + const clientMessageMock = 'test-message'; + + const getMessage = () => new TextEncoder().encode(clientMessageMock); + + beforeAll(async () => { + const testingModule = await Test.createTestingModule({ + imports: [TldrawTestModule], + }).compile(); + gateway = testingModule.get(TldrawWs); + app = testingModule.createNestApplication(); + app.useWebSocketAdapter(new WsAdapter(app)); + await app.init(); + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(() => { + jest.useFakeTimers({ advanceTimers: true, doNotFake: ['setInterval', 'clearInterval', 'setTimeout'] }); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('when tldraw is correctly setup', () => { + const setup = async () => { + const handleConnectionSpy = jest.spyOn(gateway, 'handleConnection'); + jest.spyOn(Uint8Array.prototype, 'reduce').mockReturnValueOnce(1); + + ws = await TestConnection.setupWs(wsUrl, 'TEST'); + + const { buffer } = getMessage(); + + return { handleConnectionSpy, buffer }; + }; + + it(`should handle connection and data transfer`, async () => { + const { handleConnectionSpy, buffer } = await setup(); + ws.send(buffer, () => {}); + + expect(handleConnectionSpy).toHaveBeenCalledTimes(1); + ws.close(); + }); + + it(`check if client will receive message`, async () => { + const { buffer } = await setup(); + ws.send(buffer, () => {}); + + gateway.server.on('connection', (client) => { + client.on('message', (payload) => { + expect(payload).toBeInstanceOf(ArrayBuffer); + }); + }); + + ws.close(); + }); + }); + + describe('when tldraw doc has multiple clients', () => { + const setup = async () => { + const handleConnectionSpy = jest.spyOn(gateway, 'handleConnection'); + ws = await TestConnection.setupWs(wsUrl, 'TEST2'); + const ws2 = await TestConnection.setupWs(wsUrl, 'TEST2'); + + const { buffer } = getMessage(); + + return { + handleConnectionSpy, + ws2, + buffer, + }; + }; + + it(`should handle 2 connections at same doc and data transfer`, async () => { + const { handleConnectionSpy, ws2, buffer } = await setup(); + ws.send(buffer); + ws2.send(buffer); + + expect(handleConnectionSpy).toHaveBeenCalled(); + expect(handleConnectionSpy).toHaveBeenCalledTimes(2); + + ws.close(); + ws2.close(); + }); + }); + + describe('when tldraw is not correctly setup', () => { + const setup = async () => { + const handleConnectionSpy = jest.spyOn(gateway, 'handleConnection'); + + ws = await TestConnection.setupWs(wsUrl); + + return { + handleConnectionSpy, + }; + }; + + it(`should refuse connection if there is no docName`, async () => { + const { handleConnectionSpy } = await setup(); + + const { buffer } = getMessage(); + ws.send(buffer); + + expect(gateway.server).toBeDefined(); + expect(handleConnectionSpy).toHaveBeenCalled(); + expect(handleConnectionSpy).toHaveBeenCalledTimes(1); + + ws.close(); + }); + }); +}); diff --git a/apps/server/src/modules/tldraw/controller/index.ts b/apps/server/src/modules/tldraw/controller/index.ts new file mode 100644 index 00000000000..0b0cf7d103b --- /dev/null +++ b/apps/server/src/modules/tldraw/controller/index.ts @@ -0,0 +1 @@ +export * from './tldraw.ws'; diff --git a/apps/server/src/modules/tldraw/controller/tldraw.ws.ts b/apps/server/src/modules/tldraw/controller/tldraw.ws.ts new file mode 100644 index 00000000000..1851b1d565d --- /dev/null +++ b/apps/server/src/modules/tldraw/controller/tldraw.ws.ts @@ -0,0 +1,48 @@ +import { WebSocketGateway, WebSocketServer, OnGatewayInit, OnGatewayConnection } from '@nestjs/websockets'; +import { Server, WebSocket } from 'ws'; +import { ConfigService } from '@nestjs/config'; +import { TldrawConfig, SOCKET_PORT } from '../config'; +import { WsCloseCodeEnum } from '../types/ws-close-code-enum'; +import { TldrawWsService } from '../service'; + +@WebSocketGateway(SOCKET_PORT) +export class TldrawWs implements OnGatewayInit, OnGatewayConnection { + @WebSocketServer() + server!: Server; + + constructor( + private readonly configService: ConfigService, + private readonly tldrawWsService: TldrawWsService + ) {} + + public handleConnection(client: WebSocket, request: Request): void { + const docName = this.getDocNameFromRequest(request); + + if (docName.length > 0 && this.configService.get('FEATURE_TLDRAW_ENABLED')) { + this.tldrawWsService.setupWSConnection(client, docName); + } else { + client.close( + WsCloseCodeEnum.WS_CLIENT_BAD_REQUEST_CODE, + 'Document name is mandatory in url or Tldraw Tool is turned off.' + ); + } + } + + public afterInit(): void { + this.tldrawWsService.setPersistence({ + bindState: async (docName, ydoc) => { + await this.tldrawWsService.updateDocument(docName, ydoc); + }, + writeState: async (docName) => { + // This is called when all connections to the document are closed. + // eslint-disable-next-line @typescript-eslint/no-unsafe-call,@typescript-eslint/no-unsafe-member-access + await this.tldrawWsService.flushDocument(docName); + }, + }); + } + + private getDocNameFromRequest(request: Request): string { + const urlStripped = request.url.replace('/', ''); + return urlStripped; + } +} diff --git a/apps/server/src/modules/tldraw/domain/ws-shared-doc.do.spec.ts b/apps/server/src/modules/tldraw/domain/ws-shared-doc.do.spec.ts new file mode 100644 index 00000000000..78cf9ea9428 --- /dev/null +++ b/apps/server/src/modules/tldraw/domain/ws-shared-doc.do.spec.ts @@ -0,0 +1,165 @@ +import { Test } from '@nestjs/testing'; +import { INestApplication } from '@nestjs/common'; +import WebSocket from 'ws'; +import { WsAdapter } from '@nestjs/platform-ws'; +import { CoreModule } from '@src/core'; +import { ConfigModule } from '@nestjs/config'; +import { createConfigModuleOptions } from '@src/config'; +import { createMock } from '@golevelup/ts-jest'; +import * as AwarenessProtocol from 'y-protocols/awareness'; +import { config } from '../config'; +import { TldrawBoardRepo } from '../repo/tldraw-board.repo'; +import { TldrawWsService } from '../service'; +import { WsSharedDocDo } from './ws-shared-doc.do'; +import { TldrawWs } from '../controller'; +import { TestConnection } from '../testing/test-connection'; + +describe('WsSharedDocDo', () => { + let app: INestApplication; + let ws: WebSocket; + let service: TldrawWsService; + + const gatewayPort = 3346; + const wsUrl = TestConnection.getWsUrl(gatewayPort); + + jest.useFakeTimers(); + + beforeAll(async () => { + const imports = [CoreModule, ConfigModule.forRoot(createConfigModuleOptions(config))]; + const testingModule = await Test.createTestingModule({ + imports, + providers: [ + TldrawWs, + TldrawBoardRepo, + { + provide: TldrawWsService, + useValue: createMock(), + }, + ], + }).compile(); + + service = testingModule.get(TldrawWsService); + app = testingModule.createNestApplication(); + app.useWebSocketAdapter(new WsAdapter(app)); + jest.useFakeTimers({ advanceTimers: true, doNotFake: ['setInterval', 'clearInterval', 'setTimeout'] }); + await app.init(); + }); + + afterAll(async () => { + await app.close(); + }); + + describe('ydoc client awareness change handler', () => { + const setup = async () => { + ws = await TestConnection.setupWs(wsUrl); + + class MockAwareness { + on = jest.fn(); + } + const doc = new WsSharedDocDo('TEST', service); + doc.awareness = new MockAwareness() as unknown as AwarenessProtocol.Awareness; + const awarenessMetaMock = new Map(); + awarenessMetaMock.set(1, { clock: 11, lastUpdated: 21 }); + awarenessMetaMock.set(2, { clock: 12, lastUpdated: 22 }); + awarenessMetaMock.set(3, { clock: 13, lastUpdated: 23 }); + const awarenessStatesMock = new Map(); + awarenessStatesMock.set(1, { updating: '21' }); + awarenessStatesMock.set(2, { updating: '22' }); + awarenessStatesMock.set(3, { updating: '23' }); + doc.awareness.states = awarenessStatesMock; + doc.awareness.meta = awarenessMetaMock; + + const sendSpy = jest.spyOn(service, 'send').mockReturnValue(); + + const mockIDs = new Set(); + const mockConns = new Map>(); + mockConns.set(ws, mockIDs); + doc.conns = mockConns; + + return { + sendSpy, + doc, + mockIDs, + mockConns, + }; + }; + + describe('when adding two clients states', () => { + it('should have two registered clients states', async () => { + const { sendSpy, doc, mockIDs } = await setup(); + const awarenessUpdate = { + added: [1, 3], + updated: [], + removed: [], + }; + doc.awarenessChangeHandler(awarenessUpdate, ws); + + expect(mockIDs.size).toBe(2); + expect(mockIDs.has(1)).toBe(true); + expect(mockIDs.has(3)).toBe(true); + expect(mockIDs.has(2)).toBe(false); + expect(sendSpy).toBeCalled(); + + ws.close(); + sendSpy.mockRestore(); + }); + }); + + describe('when removing one of two existing clients states', () => { + it('should have one registered client state', async () => { + const { sendSpy, doc, mockIDs } = await setup(); + let awarenessUpdate: { added: number[]; updated: number[]; removed: number[] } = { + added: [1, 3], + updated: [], + removed: [], + }; + + doc.awarenessChangeHandler(awarenessUpdate, ws); + + awarenessUpdate = { + added: [], + updated: [], + removed: [1], + }; + + doc.awarenessChangeHandler(awarenessUpdate, ws); + + expect(mockIDs.size).toBe(1); + expect(mockIDs.has(1)).toBe(false); + expect(mockIDs.has(3)).toBe(true); + expect(sendSpy).toBeCalled(); + + ws.close(); + sendSpy.mockRestore(); + }); + }); + + describe('when updating client state', () => { + it('should not change number of states', async () => { + const { sendSpy, doc, mockIDs } = await setup(); + let awarenessUpdate: { added: number[]; updated: number[]; removed: number[] } = { + added: [1], + updated: [], + removed: [], + }; + + doc.awarenessChangeHandler(awarenessUpdate, ws); + + awarenessUpdate = { + added: [], + updated: [1], + removed: [], + }; + + doc.awarenessChangeHandler(awarenessUpdate, ws); + + expect(mockIDs.size).toBe(1); + expect(mockIDs.has(1)).toBe(true); + expect(sendSpy).toBeCalled(); + + ws.close(); + sendSpy.mockRestore(); + }); + }); + }); +}); diff --git a/apps/server/src/modules/tldraw/domain/ws-shared-doc.do.ts b/apps/server/src/modules/tldraw/domain/ws-shared-doc.do.ts new file mode 100644 index 00000000000..a84c0e7a6b5 --- /dev/null +++ b/apps/server/src/modules/tldraw/domain/ws-shared-doc.do.ts @@ -0,0 +1,88 @@ +import { Doc } from 'yjs'; +import WebSocket from 'ws'; +import { Awareness, encodeAwarenessUpdate } from 'y-protocols/awareness'; +import { encoding } from 'lib0'; +import { WSMessageType } from '../types/connection-enum'; +import { TldrawWsService } from '../service'; + +export class WsSharedDocDo extends Doc { + public name: string; + + public conns: Map>; + + public awareness: Awareness; + + /** + * @param {string} name + * @param {TldrawWsService} tldrawService + * @param {boolean} gcEnabled + */ + constructor(name: string, private tldrawService: TldrawWsService, gcEnabled = true) { + super({ gc: gcEnabled }); + this.name = name; + this.conns = new Map(); + this.awareness = new Awareness(this); + this.awareness.setLocalState(null); + + this.awareness.on('update', this.awarenessChangeHandler); + this.on('update', (update: Uint8Array, origin, doc: WsSharedDocDo) => { + this.tldrawService.updateHandler(update, origin, doc); + }); + } + + /** + * @param {{ added: Array, updated: Array, removed: Array }} changes + * @param {WebSocket | null} wsConnection Origin is the connection that made the change + */ + public awarenessChangeHandler = ( + { added, updated, removed }: { added: Array; updated: Array; removed: Array }, + wsConnection: WebSocket | null + ): void => { + const changedClients = this.manageClientsConnections({ added, updated, removed }, wsConnection); + const buff = this.prepareAwarenessMessage(changedClients); + this.sendAwarenessMessage(buff); + }; + + /** + * @param {{ added: Array, updated: Array, removed: Array }} changes + * @param {WebSocket | null} wsConnection Origin is the connection that made the change + */ + private manageClientsConnections( + { added, updated, removed }: { added: Array; updated: Array; removed: Array }, + wsConnection: WebSocket | null + ): number[] { + const changedClients = added.concat(updated, removed); + if (wsConnection !== null) { + const connControlledIDs = this.conns.get(wsConnection); + if (connControlledIDs !== undefined) { + added.forEach((clientID) => { + connControlledIDs.add(clientID); + }); + removed.forEach((clientID) => { + connControlledIDs.delete(clientID); + }); + } + } + return changedClients; + } + + /** + * @param changedClients array of changed clients + */ + private prepareAwarenessMessage(changedClients: number[]): Uint8Array { + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, WSMessageType.AWARENESS); + encoding.writeVarUint8Array(encoder, encodeAwarenessUpdate(this.awareness, changedClients)); + const message = encoding.toUint8Array(encoder); + return message; + } + + /** + * @param {{ Uint8Array }} buff encoded message about changes + */ + private sendAwarenessMessage(buff: Uint8Array): void { + this.conns.forEach((_, c) => { + this.tldrawService.send(this, c, buff); + }); + } +} diff --git a/apps/server/src/modules/tldraw/index.ts b/apps/server/src/modules/tldraw/index.ts new file mode 100644 index 00000000000..b398fab4a5c --- /dev/null +++ b/apps/server/src/modules/tldraw/index.ts @@ -0,0 +1,2 @@ +export * from './tldraw.module'; +export * from './tldraw-test.module'; diff --git a/apps/server/src/modules/tldraw/repo/index.ts b/apps/server/src/modules/tldraw/repo/index.ts new file mode 100644 index 00000000000..0c1ae29e62f --- /dev/null +++ b/apps/server/src/modules/tldraw/repo/index.ts @@ -0,0 +1 @@ +export * from './tldraw-board.repo'; diff --git a/apps/server/src/modules/tldraw/repo/tldraw-board.repo.spec.ts b/apps/server/src/modules/tldraw/repo/tldraw-board.repo.spec.ts new file mode 100644 index 00000000000..6d9b3c799bb --- /dev/null +++ b/apps/server/src/modules/tldraw/repo/tldraw-board.repo.spec.ts @@ -0,0 +1,221 @@ +import { Test } from '@nestjs/testing'; +import { INestApplication } from '@nestjs/common'; +import WebSocket from 'ws'; +import { WsAdapter } from '@nestjs/platform-ws'; +import { CoreModule } from '@src/core'; +import { ConfigModule } from '@nestjs/config'; +import { createConfigModuleOptions } from '@src/config'; +import { createMock } from '@golevelup/ts-jest'; +import { Doc } from 'yjs'; +import * as YjsUtils from '../utils/ydoc-utils'; +import { config } from '../config'; +import { TldrawBoardRepo } from './tldraw-board.repo'; +import { WsSharedDocDo } from '../domain/ws-shared-doc.do'; +import { TldrawWsService } from '../service'; +import { TldrawWs } from '../controller'; +import { TestConnection } from '../testing/test-connection'; + +describe('TldrawBoardRepo', () => { + let app: INestApplication; + let repo: TldrawBoardRepo; + let ws: WebSocket; + let service: TldrawWsService; + + const gatewayPort = 3346; + const wsUrl = TestConnection.getWsUrl(gatewayPort); + + jest.useFakeTimers(); + + beforeAll(async () => { + const imports = [CoreModule, ConfigModule.forRoot(createConfigModuleOptions(config))]; + const testingModule = await Test.createTestingModule({ + imports, + providers: [ + TldrawWs, + TldrawBoardRepo, + { + provide: TldrawWsService, + useValue: createMock(), + }, + ], + }).compile(); + + service = testingModule.get(TldrawWsService); + repo = testingModule.get(TldrawBoardRepo); + app = testingModule.createNestApplication(); + app.useWebSocketAdapter(new WsAdapter(app)); + jest.useFakeTimers({ advanceTimers: true, doNotFake: ['setInterval', 'clearInterval', 'setTimeout'] }); + await app.init(); + }); + + afterAll(async () => { + await app.close(); + }); + + it('should check if repo and its properties are set correctly', () => { + expect(repo).toBeDefined(); + expect(repo.mdb).toBeDefined(); + expect(repo.configService).toBeDefined(); + expect(repo.flushSize).toBeDefined(); + expect(repo.multipleCollections).toBeDefined(); + expect(repo.connectionString).toBeDefined(); + expect(repo.collectionName).toBeDefined(); + }); + + describe('updateDocument', () => { + describe('when document receives empty update', () => { + const setup = async () => { + const doc = new WsSharedDocDo('TEST2', service); + ws = await TestConnection.setupWs(wsUrl, 'TEST2'); + const wsSet = new Set(); + wsSet.add(ws); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + doc.conns.set(ws, wsSet); + const storeGetYDocSpy = jest + .spyOn(repo.mdb, 'getYDoc') + .mockImplementation(() => Promise.resolve(new WsSharedDocDo('TEST', service))); + const storeUpdateSpy = jest.spyOn(repo.mdb, 'storeUpdate').mockImplementation(() => Promise.resolve(1)); + + return { + doc, + storeUpdateSpy, + storeGetYDocSpy, + }; + }; + + it('should not update db with diff', async () => { + const { doc, storeUpdateSpy, storeGetYDocSpy } = await setup(); + + await repo.updateDocument('TEST2', doc); + expect(storeUpdateSpy).toHaveBeenCalledTimes(0); + storeUpdateSpy.mockRestore(); + storeGetYDocSpy.mockRestore(); + ws.close(); + }); + }); + + describe('when document receive update', () => { + const setup = async () => { + const clientMessageMock = 'test-message'; + const doc = new WsSharedDocDo('TEST', service); + ws = await TestConnection.setupWs(wsUrl, 'TEST'); + const wsSet = new Set(); + wsSet.add(ws); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + doc.conns.set(ws, wsSet); + const storeUpdateSpy = jest.spyOn(repo.mdb, 'storeUpdate').mockImplementation(() => Promise.resolve(1)); + const storeGetYDocSpy = jest + .spyOn(repo.mdb, 'getYDoc') + .mockImplementation(() => Promise.resolve(new WsSharedDocDo('TEST', service))); + const byteArray = new TextEncoder().encode(clientMessageMock); + + return { + doc, + byteArray, + storeUpdateSpy, + storeGetYDocSpy, + }; + }; + + it('should update db with diff', async () => { + const { doc, byteArray, storeUpdateSpy, storeGetYDocSpy } = await setup(); + + await repo.updateDocument('TEST', doc); + doc.emit('update', [byteArray, undefined, doc]); + expect(storeUpdateSpy).toHaveBeenCalled(); + expect(storeUpdateSpy).toHaveBeenCalledTimes(1); + storeUpdateSpy.mockRestore(); + storeGetYDocSpy.mockRestore(); + ws.close(); + }); + }); + }); + + describe('getYDocFromMdb', () => { + describe('when taking doc data from db', () => { + const setup = () => { + const storeGetYDocSpy = jest + .spyOn(repo.mdb, 'getYDoc') + .mockImplementation(() => Promise.resolve(new WsSharedDocDo('TEST', service))); + + return { + storeGetYDocSpy, + }; + }; + + it('should return ydoc', async () => { + const { storeGetYDocSpy } = setup(); + expect(await repo.getYDocFromMdb('test')).toBeInstanceOf(Doc); + + storeGetYDocSpy.mockRestore(); + }); + }); + }); + + describe('updateStoredDocWithDiff', () => { + describe('when the difference between update and current drawing is more than 0', () => { + const setup = () => { + const calculateDiffSpy = jest.spyOn(YjsUtils, 'calculateDiff').mockImplementationOnce(() => 1); + const storeUpdateSpy = jest.spyOn(repo.mdb, 'storeUpdate').mockResolvedValueOnce(Promise.resolve(1)); + + return { + calculateDiffSpy, + storeUpdateSpy, + }; + }; + + it('should call store update method', () => { + const { storeUpdateSpy, calculateDiffSpy } = setup(); + const diffArray = new Uint8Array(); + repo.updateStoredDocWithDiff('test', diffArray); + + expect(storeUpdateSpy).toHaveBeenCalled(); + + calculateDiffSpy.mockRestore(); + storeUpdateSpy.mockRestore(); + }); + }); + + describe('when the difference between update and current drawing is 0', () => { + const setup = () => { + const calculateDiffSpy = jest.spyOn(YjsUtils, 'calculateDiff').mockImplementationOnce(() => 0); + const storeUpdateSpy = jest.spyOn(repo.mdb, 'storeUpdate'); + + return { + calculateDiffSpy, + storeUpdateSpy, + }; + }; + + it('should not call store update method', () => { + const { storeUpdateSpy, calculateDiffSpy } = setup(); + const diffArray = new Uint8Array(); + repo.updateStoredDocWithDiff('test', diffArray); + + expect(storeUpdateSpy).not.toHaveBeenCalled(); + + calculateDiffSpy.mockRestore(); + storeUpdateSpy.mockRestore(); + }); + }); + }); + + describe('flushDocument', () => { + const setup = () => { + const flushDocumentSpy = jest.spyOn(repo.mdb, 'flushDocument').mockResolvedValueOnce(Promise.resolve()); + + return { flushDocumentSpy }; + }; + + it('should call flush method on mdbPersistence', async () => { + const { flushDocumentSpy } = setup(); + await repo.flushDocument('test'); + + expect(flushDocumentSpy).toHaveBeenCalled(); + + flushDocumentSpy.mockRestore(); + }); + }); +}); diff --git a/apps/server/src/modules/tldraw/repo/tldraw-board.repo.ts b/apps/server/src/modules/tldraw/repo/tldraw-board.repo.ts new file mode 100644 index 00000000000..b505218d990 --- /dev/null +++ b/apps/server/src/modules/tldraw/repo/tldraw-board.repo.ts @@ -0,0 +1,72 @@ +import { Injectable } from '@nestjs/common'; +import { MongodbPersistence } from 'y-mongodb-provider'; +import { ConfigService } from '@nestjs/config'; +import { TldrawConfig } from '@src/modules/tldraw/config'; +import { applyUpdate, Doc, encodeStateAsUpdate, encodeStateVector } from 'yjs'; +import { calculateDiff } from '../utils'; +import { WsSharedDocDo } from '../types'; + +@Injectable() +export class TldrawBoardRepo { + public connectionString: string; + + public collectionName: string; + + public flushSize: number; + + public multipleCollections: boolean; + + public mdb: MongodbPersistence; + + constructor(public readonly configService: ConfigService) { + this.connectionString = this.configService.get('CONNECTION_STRING'); + this.collectionName = this.configService.get('TLDRAW_DB_COLLECTION_NAME') ?? 'drawings'; + this.flushSize = this.configService.get('TLDRAW_DB_FLUSH_SIZE') ?? 400; + this.multipleCollections = this.configService.get('TLDRAW_DB_MULTIPLE_COLLECTIONS'); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-call,@typescript-eslint/no-unsafe-assignment + this.mdb = new MongodbPersistence(this.connectionString, { + collectionName: this.collectionName, + flushSize: this.flushSize, + multipleCollections: this.multipleCollections, + }); + } + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + // eslint-disable-next-line consistent-return + public async getYDocFromMdb(docName: string): Promise { + // eslint-disable-next-line @typescript-eslint/no-unsafe-call,@typescript-eslint/no-unsafe-assignment,@typescript-eslint/no-unsafe-member-access + const yDoc = await this.mdb.getYDoc(docName); + if (yDoc instanceof Doc) { + return yDoc; + } + } + + public updateStoredDocWithDiff(docName: string, diff: Uint8Array): void { + const calc = calculateDiff(diff); + if (calc > 0) { + void this.mdb.storeUpdate(docName, diff); + } + } + + public async updateDocument(docName: string, ydoc: WsSharedDocDo): Promise { + const persistedYdoc = await this.getYDocFromMdb(docName); + const persistedStateVector = encodeStateVector(persistedYdoc); + const diff = encodeStateAsUpdate(ydoc, persistedStateVector); + this.updateStoredDocWithDiff(docName, diff); + + applyUpdate(ydoc, encodeStateAsUpdate(persistedYdoc)); + + ydoc.on('update', (update: Uint8Array) => { + void this.mdb.storeUpdate(docName, update); + }); + + persistedYdoc.destroy(); + } + + public async flushDocument(docName: string): Promise { + // eslint-disable-next-line @typescript-eslint/no-unsafe-call,@typescript-eslint/no-unsafe-member-access + await this.mdb.flushDocument(docName); + } +} diff --git a/apps/server/src/modules/tldraw/service/index.ts b/apps/server/src/modules/tldraw/service/index.ts new file mode 100644 index 00000000000..a056b2ece10 --- /dev/null +++ b/apps/server/src/modules/tldraw/service/index.ts @@ -0,0 +1 @@ +export * from './tldraw.ws.service'; diff --git a/apps/server/src/modules/tldraw/service/tldraw.ws.service.spec.ts b/apps/server/src/modules/tldraw/service/tldraw.ws.service.spec.ts new file mode 100644 index 00000000000..7eebb7ccc4e --- /dev/null +++ b/apps/server/src/modules/tldraw/service/tldraw.ws.service.spec.ts @@ -0,0 +1,449 @@ +import { Test } from '@nestjs/testing'; +import { INestApplication } from '@nestjs/common'; +import WebSocket from 'ws'; +import { WsAdapter } from '@nestjs/platform-ws'; +import { CoreModule } from '@src/core'; +import { ConfigModule } from '@nestjs/config'; +import { createConfigModuleOptions } from '@src/config'; +import { config } from '@src/modules/tldraw/config'; +import { WsSharedDocDo } from '@src/modules/tldraw/types'; +import { TextEncoder } from 'util'; +import * as SyncProtocols from 'y-protocols/sync'; +import * as AwarenessProtocol from 'y-protocols/awareness'; +import { encoding } from 'lib0'; +import { TldrawBoardRepo } from '@src/modules/tldraw/repo'; +import { TldrawWs } from '@src/modules/tldraw/controller'; +import { TldrawWsFactory } from '@shared/testing/factory/tldraw.ws.factory'; +import { TldrawWsService } from '.'; +import { TestConnection } from '../testing/test-connection'; + +jest.mock('y-protocols/awareness', () => { + const moduleMock: unknown = { + __esModule: true, + ...jest.requireActual('y-protocols/awareness'), + }; + return moduleMock; +}); +jest.mock('y-protocols/sync', () => { + const moduleMock: unknown = { + __esModule: true, + ...jest.requireActual('y-protocols/sync'), + }; + return moduleMock; +}); + +describe('TldrawWSService', () => { + let app: INestApplication; + let ws: WebSocket; + let service: TldrawWsService; + + const gatewayPort = 3346; + const wsUrl = TestConnection.getWsUrl(gatewayPort); + + const delay = (ms: number) => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + + jest.useFakeTimers(); + + beforeAll(async () => { + const imports = [CoreModule, ConfigModule.forRoot(createConfigModuleOptions(config))]; + const testingModule = await Test.createTestingModule({ + imports, + providers: [TldrawWs, TldrawBoardRepo, TldrawWsService], + }).compile(); + + service = testingModule.get(TldrawWsService); + app = testingModule.createNestApplication(); + app.useWebSocketAdapter(new WsAdapter(app)); + jest.useFakeTimers({ advanceTimers: true, doNotFake: ['setInterval', 'clearInterval', 'setTimeout'] }); + await app.init(); + }); + + afterAll(async () => { + await app.close(); + }); + + const createMessage = (values: number[]) => { + const encoder = encoding.createEncoder(); + values.forEach((val) => { + encoding.writeVarUint(encoder, val); + }); + encoding.writeVarUint(encoder, 0); + encoding.writeVarUint(encoder, 1); + const msg = encoding.toUint8Array(encoder); + return { + msg, + }; + }; + + it('should chcek if service properties are set correctly', () => { + expect(service).toBeDefined(); + expect(service.pingTimeout).toBeDefined(); + expect(service.persistence).toBeDefined(); + }); + + describe('send', () => { + describe('when client is not connected to WS', () => { + const setup = async () => { + ws = await TestConnection.setupWs(wsUrl); + const clientMessageMock = 'test-message'; + + const closeConSpy = jest.spyOn(service, 'closeConn').mockImplementationOnce(() => {}); + const sendSpy = jest.spyOn(service, 'send'); + const doc = TldrawWsFactory.createWsSharedDocDo(); + const byteArray = new TextEncoder().encode(clientMessageMock); + + return { + closeConSpy, + sendSpy, + doc, + byteArray, + }; + }; + + it('should throw error for send message', async () => { + const { closeConSpy, sendSpy, doc, byteArray } = await setup(); + + service.send(doc, ws, byteArray); + + expect(sendSpy).toThrow(); + expect(sendSpy).toHaveBeenCalledWith(doc, ws, byteArray); + expect(closeConSpy).toHaveBeenCalled(); + + ws.close(); + sendSpy.mockRestore(); + }); + }); + + describe('when websocket has ready state different than 0 or 1', () => { + const setup = () => { + const clientMessageMock = 'test-message'; + const closeConSpy = jest.spyOn(service, 'closeConn'); + const sendSpy = jest.spyOn(service, 'send'); + const doc = TldrawWsFactory.createWsSharedDocDo(); + const socketMock = TldrawWsFactory.createWebsocket(3); + const byteArray = new TextEncoder().encode(clientMessageMock); + + return { + closeConSpy, + sendSpy, + doc, + socketMock, + byteArray, + }; + }; + + it('should close connection', () => { + const { closeConSpy, sendSpy, doc, socketMock, byteArray } = setup(); + + service.send(doc, socketMock, byteArray); + + expect(sendSpy).toHaveBeenCalledWith(doc, socketMock, byteArray); + expect(sendSpy).toHaveBeenCalledTimes(1); + expect(closeConSpy).toHaveBeenCalled(); + + closeConSpy.mockRestore(); + sendSpy.mockRestore(); + }); + }); + + describe('when websocket has ready state 0', () => { + const setup = async () => { + ws = await TestConnection.setupWs(wsUrl); + const clientMessageMock = 'test-message'; + + const sendSpy = jest.spyOn(service, 'send'); + const doc = TldrawWsFactory.createWsSharedDocDo(); + const socketMock = TldrawWsFactory.createWebsocket(0); + doc.conns.set(socketMock, new Set()); + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, 2); + const updateByteArray = new TextEncoder().encode(clientMessageMock); + encoding.writeVarUint8Array(encoder, updateByteArray); + const msg = encoding.toUint8Array(encoder); + return { + sendSpy, + doc, + msg, + }; + }; + + it('should call send in updateHandler', async () => { + const { sendSpy, doc, msg } = await setup(); + + service.updateHandler(msg, {}, doc); + + expect(sendSpy).toHaveBeenCalled(); + + ws.close(); + sendSpy.mockRestore(); + }); + }); + + describe('when received message of specific type', () => { + const setup = async (messageValues: number[]) => { + ws = await TestConnection.setupWs(wsUrl, 'TEST'); + + const sendSpy = jest.spyOn(service, 'send'); + const applyAwarenessUpdateSpy = jest.spyOn(AwarenessProtocol, 'applyAwarenessUpdate'); + const syncProtocolUpdateSpy = jest + .spyOn(SyncProtocols, 'readSyncMessage') + .mockImplementationOnce((dec, enc) => { + enc.bufs = [new Uint8Array(2), new Uint8Array(2)]; + return 1; + }); + const doc = new WsSharedDocDo('TEST', service); + const { msg } = createMessage(messageValues); + + return { + sendSpy, + applyAwarenessUpdateSpy, + syncProtocolUpdateSpy, + doc, + msg, + }; + }; + + it('should call send method when received message of type SYNC', async () => { + const { sendSpy, applyAwarenessUpdateSpy, syncProtocolUpdateSpy, doc, msg } = await setup([0, 1]); + + service.messageHandler(ws, doc, msg); + + expect(sendSpy).toHaveBeenCalledTimes(1); + + ws.close(); + sendSpy.mockRestore(); + applyAwarenessUpdateSpy.mockRestore(); + syncProtocolUpdateSpy.mockRestore(); + }); + + it('should not call send method when received message of type AWARENESS', async () => { + const { sendSpy, applyAwarenessUpdateSpy, syncProtocolUpdateSpy, doc, msg } = await setup([1, 1, 0]); + service.messageHandler(ws, doc, msg); + + expect(sendSpy).toHaveBeenCalledTimes(0); + expect(applyAwarenessUpdateSpy).toHaveBeenCalledTimes(1); + + ws.close(); + sendSpy.mockRestore(); + applyAwarenessUpdateSpy.mockRestore(); + syncProtocolUpdateSpy.mockRestore(); + }); + + it('should do nothing when received message unknown type', async () => { + const { sendSpy, applyAwarenessUpdateSpy, syncProtocolUpdateSpy, doc, msg } = await setup([2]); + service.messageHandler(ws, doc, msg); + + expect(sendSpy).toHaveBeenCalledTimes(0); + expect(applyAwarenessUpdateSpy).toHaveBeenCalledTimes(0); + + ws.close(); + sendSpy.mockRestore(); + applyAwarenessUpdateSpy.mockRestore(); + syncProtocolUpdateSpy.mockRestore(); + }); + }); + + describe('when error is thrown during receiving message', () => { + const setup = async () => { + ws = await TestConnection.setupWs(wsUrl); + + const sendSpy = jest.spyOn(service, 'send'); + jest.spyOn(SyncProtocols, 'readSyncMessage').mockImplementationOnce(() => { + throw new Error('error'); + }); + const doc = new WsSharedDocDo('TEST', service); + const { msg } = createMessage([0]); + + return { + sendSpy, + doc, + msg, + }; + }; + + it('should not call send method', async () => { + const { sendSpy, doc, msg } = await setup(); + + service.messageHandler(ws, doc, msg); + + expect(sendSpy).toHaveBeenCalledTimes(0); + + ws.close(); + sendSpy.mockRestore(); + }); + }); + + describe('when awareness states (clients) size is greater then one', () => { + const setup = async () => { + ws = await TestConnection.setupWs(wsUrl, 'TEST'); + + const doc = new WsSharedDocDo('TEST', service); + doc.awareness.states = new Map(); + doc.awareness.states.set(1, ['test1']); + doc.awareness.states.set(2, ['test2']); + + const messageHandlerSpy = jest.spyOn(service, 'messageHandler').mockImplementationOnce(() => {}); + const sendSpy = jest.spyOn(service, 'send'); + const getYDocSpy = jest.spyOn(service, 'getYDoc').mockImplementationOnce(() => doc); + const { msg } = createMessage([0]); + jest.spyOn(AwarenessProtocol, 'encodeAwarenessUpdate').mockImplementationOnce(() => msg); + + return { + messageHandlerSpy, + sendSpy, + getYDocSpy, + }; + }; + + it('should send to every client', async () => { + const { messageHandlerSpy, sendSpy, getYDocSpy } = await setup(); + + service.setupWSConnection(ws); + + expect(sendSpy).toHaveBeenCalledTimes(2); + + ws.close(); + messageHandlerSpy.mockRestore(); + sendSpy.mockRestore(); + getYDocSpy.mockRestore(); + }); + }); + }); + + describe('closeConn', () => { + describe('when trying to close already closed connection', () => { + const setup = async () => { + ws = await TestConnection.setupWs(wsUrl); + + jest.spyOn(ws, 'close').mockImplementationOnce(() => { + throw new Error('some error'); + }); + }; + + it('should throw error', async () => { + await setup(); + try { + const doc = TldrawWsFactory.createWsSharedDocDo(); + service.closeConn(doc, ws); + } catch (err) { + expect(err).toBeDefined(); + } + + ws.close(); + }); + }); + + describe('when ping failed', () => { + const setup = async () => { + ws = await TestConnection.setupWs(wsUrl, 'TEST'); + + const messageHandlerSpy = jest.spyOn(service, 'messageHandler').mockImplementationOnce(() => {}); + const closeConnSpy = jest.spyOn(service, 'closeConn'); + jest.spyOn(ws, 'ping').mockImplementationOnce(() => { + throw new Error('error'); + }); + + return { + messageHandlerSpy, + closeConnSpy, + }; + }; + + it('should close connection', async () => { + const { messageHandlerSpy, closeConnSpy } = await setup(); + + service.setupWSConnection(ws); + + await delay(50); + + expect(closeConnSpy).toHaveBeenCalled(); + + ws.close(); + messageHandlerSpy.mockRestore(); + closeConnSpy.mockRestore(); + }); + }); + }); + + describe('messageHandler', () => { + describe('when message is received', () => { + const setup = async (messageValues: number[]) => { + ws = await TestConnection.setupWs(wsUrl, 'TEST'); + + const messageHandlerSpy = jest.spyOn(service, 'messageHandler'); + const readSyncMessageSpy = jest.spyOn(SyncProtocols, 'readSyncMessage').mockImplementationOnce((dec, enc) => { + enc.bufs = [new Uint8Array(2), new Uint8Array(2)]; + return 1; + }); + const { msg } = createMessage(messageValues); + + return { + messageHandlerSpy, + msg, + readSyncMessageSpy, + }; + }; + + it('should handle message', async () => { + const { messageHandlerSpy, msg, readSyncMessageSpy } = await setup([0, 1]); + + service.setupWSConnection(ws); + ws.emit('message', msg); + + expect(messageHandlerSpy).toHaveBeenCalledTimes(1); + + ws.close(); + messageHandlerSpy.mockRestore(); + readSyncMessageSpy.mockRestore(); + }); + }); + }); + + describe('getYDoc', () => { + describe('when getting yDoc by name', () => { + it('should assign to service.doc and return instance', () => { + const docName = 'get-test'; + const doc = service.getYDoc(docName); + expect(doc).toBeInstanceOf(WsSharedDocDo); + expect(service.docs.get(docName)).not.toBeUndefined(); + }); + }); + }); + + describe('updateDocument', () => { + const setup = () => { + const updateDocumentSpy = jest.spyOn(service, 'updateDocument').mockImplementation(() => Promise.resolve()); + + return { updateDocumentSpy }; + }; + + it('should call update method', async () => { + const { updateDocumentSpy } = setup(); + await service.updateDocument('test', TldrawWsFactory.createWsSharedDocDo()); + + expect(updateDocumentSpy).toHaveBeenCalled(); + + updateDocumentSpy.mockRestore(); + }); + }); + + describe('flushDocument', () => { + const setup = () => { + const flushDocumentSpy = jest.spyOn(service, 'flushDocument').mockResolvedValueOnce(Promise.resolve()); + + return { flushDocumentSpy }; + }; + + it('should call flush method', async () => { + const { flushDocumentSpy } = setup(); + await service.flushDocument('test'); + + expect(flushDocumentSpy).toHaveBeenCalled(); + + flushDocumentSpy.mockRestore(); + }); + }); +}); diff --git a/apps/server/src/modules/tldraw/service/tldraw.ws.service.ts b/apps/server/src/modules/tldraw/service/tldraw.ws.service.ts new file mode 100644 index 00000000000..77d6bd9afbd --- /dev/null +++ b/apps/server/src/modules/tldraw/service/tldraw.ws.service.ts @@ -0,0 +1,208 @@ +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { TldrawConfig } from '@src/modules/tldraw/config'; +import { Persitence, WSConnectionState, WSMessageType, WsSharedDocDo } from '@src/modules/tldraw/types'; +import WebSocket from 'ws'; +import { applyAwarenessUpdate, encodeAwarenessUpdate, removeAwarenessStates } from 'y-protocols/awareness'; +import { encoding, decoding, map } from 'lib0'; +import { readSyncMessage, writeSyncStep1, writeUpdate } from 'y-protocols/sync'; +import { TldrawBoardRepo } from '@src/modules/tldraw/repo'; + +@Injectable() +export class TldrawWsService { + public pingTimeout: number; + + public persistence: Persitence | null = null; + + public docs = new Map(); + + constructor( + private readonly configService: ConfigService, + private readonly tldrawBoardRepo: TldrawBoardRepo + ) { + this.pingTimeout = this.configService.get('TLDRAW_PING_TIMEOUT'); + } + + public setPersistence(persistence_: Persitence): void { + this.persistence = persistence_; + } + + /** + * @param {WsSharedDocDo} doc + * @param {WebSocket} ws + */ + public closeConn(doc: WsSharedDocDo, ws: WebSocket): void { + if (doc.conns.has(ws)) { + const controlledIds = doc.conns.get(ws) as Set; + doc.conns.delete(ws); + removeAwarenessStates(doc.awareness, Array.from(controlledIds), null); + if (doc.conns.size === 0 && this.persistence !== null) { + // if persisted, we store state and destroy ydocument + this.persistence + .writeState(doc.name, doc) + .then(() => { + doc.destroy(); + return null; + }) + .catch(() => {}); + this.docs.delete(doc.name); + } + } + + try { + ws.close(); + } catch (err) { + throw new Error('Cannot close the connection. It is possible that connection is already closed.'); + } + } + + /** + * @param {WsSharedDocDo} doc + * @param {WebSocket} conn + * @param {Uint8Array} message + */ + public send(doc: WsSharedDocDo, conn: WebSocket, message: Uint8Array): void { + if (conn.readyState !== WSConnectionState.CONNECTING && conn.readyState !== WSConnectionState.OPEN) { + this.closeConn(doc, conn); + } + try { + conn.send(message, (err: Error | undefined) => { + if (err != null) { + this.closeConn(doc, conn); + } + }); + } catch (e) { + this.closeConn(doc, conn); + } + } + + /** + * @param {Uint8Array} update + * @param {any} origin + * @param {WsSharedDocDo} doc + */ + public updateHandler(update: Uint8Array, origin, doc: WsSharedDocDo): void { + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, WSMessageType.SYNC); + writeUpdate(encoder, update); + const message = encoding.toUint8Array(encoder); + doc.conns.forEach((_, conn) => { + this.send(doc, conn, message); + }); + } + + /** + * Gets a Y.Doc by name, whether in memory or on disk + * + * @param {string} docName - the name of the Y.Doc to find or create + * @param {boolean} gc - whether to allow gc on the doc (applies only when created) + * @return {WsSharedDocDo} + */ + getYDoc(docName: string, gc = true): WsSharedDocDo { + return map.setIfUndefined(this.docs, docName, () => { + const doc = new WsSharedDocDo(docName, this, gc); + if (this.persistence !== null) { + this.persistence.bindState(docName, doc).catch(() => {}); + } + this.docs.set(docName, doc); + return doc; + }); + } + + public messageHandler(conn: WebSocket, doc: WsSharedDocDo, message: Uint8Array): void { + try { + const encoder = encoding.createEncoder(); + const decoder = decoding.createDecoder(message); + const messageType = decoding.readVarUint(decoder); + switch (messageType) { + case WSMessageType.SYNC: + encoding.writeVarUint(encoder, WSMessageType.SYNC); + readSyncMessage(decoder, encoder, doc, conn); + + // If the `encoder` only contains the type of reply message and no + // message, there is no need to send the message. When `encoder` only + // contains the type of reply, its length is 1. + if (encoding.length(encoder) > 1) { + this.send(doc, conn, encoding.toUint8Array(encoder)); + } + break; + case WSMessageType.AWARENESS: { + applyAwarenessUpdate(doc.awareness, decoding.readVarUint8Array(decoder), conn); + break; + } + default: + break; + } + } catch (err) { + doc.emit('error', [err]); + } + } + + /** + * @param {WebSocket} ws + * @param {string} docName + */ + public setupWSConnection(ws: WebSocket, docName = 'GLOBAL'): void { + ws.binaryType = 'arraybuffer'; + // get doc, initialize if it does not exist yet + const doc = this.getYDoc(docName, true); + doc.conns.set(ws, new Set()); + + // listen and reply to events + ws.on('message', (message: ArrayBufferLike) => { + this.messageHandler(ws, doc, new Uint8Array(message)); + }); + + // Check if connection is still alive + let pongReceived = true; + const pingInterval = setInterval(() => { + const hasConn = doc.conns.has(ws); + + if (pongReceived) { + if (!hasConn) return; + pongReceived = false; + + try { + ws.ping(); + } catch (e) { + this.closeConn(doc, ws); + clearInterval(pingInterval); + } + return; + } + + if (hasConn) { + this.closeConn(doc, ws); + } + + clearInterval(pingInterval); + }, this.pingTimeout); + ws.on('close', () => { + this.closeConn(doc, ws); + clearInterval(pingInterval); + }); + ws.on('pong', () => { + pongReceived = true; + }); + { + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, WSMessageType.SYNC); + writeSyncStep1(encoder, doc); + this.send(doc, ws, encoding.toUint8Array(encoder)); + const awarenessStates = doc.awareness.getStates(); + if (awarenessStates.size > 0) { + encoding.writeVarUint(encoder, WSMessageType.AWARENESS); + encoding.writeVarUint8Array(encoder, encodeAwarenessUpdate(doc.awareness, Array.from(awarenessStates.keys()))); + this.send(doc, ws, encoding.toUint8Array(encoder)); + } + } + } + + public async updateDocument(docName: string, ydoc: WsSharedDocDo): Promise { + await this.tldrawBoardRepo.updateDocument(docName, ydoc); + } + + public async flushDocument(docName: string): Promise { + await this.tldrawBoardRepo.flushDocument(docName); + } +} diff --git a/apps/server/src/modules/tldraw/testing/test-connection.ts b/apps/server/src/modules/tldraw/testing/test-connection.ts new file mode 100644 index 00000000000..638c219ea18 --- /dev/null +++ b/apps/server/src/modules/tldraw/testing/test-connection.ts @@ -0,0 +1,22 @@ +import WebSocket from 'ws'; + +export class TestConnection { + public static getWsUrl = (gatewayPort: number): string => { + const wsUrl = `ws://localhost:${gatewayPort}`; + return wsUrl; + }; + + public static setupWs = async (wsUrl: string, docName?: string): Promise => { + let ws: WebSocket; + if (docName) { + ws = new WebSocket(`${wsUrl}/${docName}`); + } else { + ws = new WebSocket(`${wsUrl}`); + } + await new Promise((resolve) => { + ws.on('open', resolve); + }); + + return ws; + }; +} diff --git a/apps/server/src/modules/tldraw/tldraw-test.module.ts b/apps/server/src/modules/tldraw/tldraw-test.module.ts new file mode 100644 index 00000000000..c3118fa2cfe --- /dev/null +++ b/apps/server/src/modules/tldraw/tldraw-test.module.ts @@ -0,0 +1,25 @@ +import { DynamicModule, Module } from '@nestjs/common'; +import { MongoMemoryDatabaseModule, MongoDatabaseModuleOptions } from '@shared/infra/database'; +import { CoreModule } from '@src/core'; +import { ConfigModule } from '@nestjs/config'; +import { createConfigModuleOptions } from '@src/config'; +import { TldrawBoardRepo } from './repo'; +import { TldrawWsService } from './service'; +import { config } from './config'; +import { TldrawWs } from './controller'; + +const imports = [CoreModule, ConfigModule.forRoot(createConfigModuleOptions(config))]; +const providers = [TldrawWs, TldrawBoardRepo, TldrawWsService]; +@Module({ + imports, + providers, +}) +export class TldrawTestModule { + static forRoot(options?: MongoDatabaseModuleOptions): DynamicModule { + return { + module: TldrawTestModule, + imports: [...imports, MongoMemoryDatabaseModule.forRoot({ ...options })], + providers, + }; + } +} diff --git a/apps/server/src/modules/tldraw/tldraw.module.ts b/apps/server/src/modules/tldraw/tldraw.module.ts new file mode 100644 index 00000000000..2e3563f2ef1 --- /dev/null +++ b/apps/server/src/modules/tldraw/tldraw.module.ts @@ -0,0 +1,17 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { createConfigModuleOptions } from '@src/config'; +import { CoreModule } from '@src/core'; +import { Logger } from '@src/core/logger'; +import { TldrawBoardRepo } from './repo'; +import { TldrawWsService } from './service'; +import { TldrawWs } from './controller'; +import { config } from './config'; + +const imports = [CoreModule, ConfigModule.forRoot(createConfigModuleOptions(config))]; + +@Module({ + imports, + providers: [Logger, TldrawWs, TldrawWsService, TldrawBoardRepo], +}) +export class TldrawModule {} diff --git a/apps/server/src/modules/tldraw/types/connection-enum.ts b/apps/server/src/modules/tldraw/types/connection-enum.ts new file mode 100644 index 00000000000..6a9a4692e03 --- /dev/null +++ b/apps/server/src/modules/tldraw/types/connection-enum.ts @@ -0,0 +1,9 @@ +export enum WSConnectionState { + CONNECTING = 0, + OPEN = 1, +} + +export enum WSMessageType { + SYNC = 0, + AWARENESS = 1, +} diff --git a/apps/server/src/modules/tldraw/types/index.ts b/apps/server/src/modules/tldraw/types/index.ts new file mode 100644 index 00000000000..08399a7ad9d --- /dev/null +++ b/apps/server/src/modules/tldraw/types/index.ts @@ -0,0 +1,3 @@ +export * from './connection-enum'; +export * from './persistence-type'; +export * from '../domain/ws-shared-doc.do'; diff --git a/apps/server/src/modules/tldraw/types/persistence-type.ts b/apps/server/src/modules/tldraw/types/persistence-type.ts new file mode 100644 index 00000000000..edb0cde6a3e --- /dev/null +++ b/apps/server/src/modules/tldraw/types/persistence-type.ts @@ -0,0 +1,6 @@ +import { WsSharedDocDo } from '@src/modules/tldraw/domain/ws-shared-doc.do'; + +export type Persitence = { + bindState: (docName: string, ydoc: WsSharedDocDo) => Promise; + writeState: (docName: string, ydoc: WsSharedDocDo) => Promise; +}; diff --git a/apps/server/src/modules/tldraw/types/ws-close-code-enum.ts b/apps/server/src/modules/tldraw/types/ws-close-code-enum.ts new file mode 100644 index 00000000000..274fa99a6ae --- /dev/null +++ b/apps/server/src/modules/tldraw/types/ws-close-code-enum.ts @@ -0,0 +1,3 @@ +export enum WsCloseCodeEnum { + WS_CLIENT_BAD_REQUEST_CODE = 4400, +} diff --git a/apps/server/src/modules/tldraw/utils/index.ts b/apps/server/src/modules/tldraw/utils/index.ts new file mode 100644 index 00000000000..a51b9059bc1 --- /dev/null +++ b/apps/server/src/modules/tldraw/utils/index.ts @@ -0,0 +1 @@ +export * from './ydoc-utils'; diff --git a/apps/server/src/modules/tldraw/utils/ydoc-utils.ts b/apps/server/src/modules/tldraw/utils/ydoc-utils.ts new file mode 100644 index 00000000000..6d0817ecc9d --- /dev/null +++ b/apps/server/src/modules/tldraw/utils/ydoc-utils.ts @@ -0,0 +1,2 @@ +export const calculateDiff = (diff: Uint8Array): number => + diff.reduce((previousValue, currentValue) => previousValue + currentValue, 0); diff --git a/apps/server/src/shared/testing/factory/tldraw.ws.factory.ts b/apps/server/src/shared/testing/factory/tldraw.ws.factory.ts new file mode 100644 index 00000000000..88af733820b --- /dev/null +++ b/apps/server/src/shared/testing/factory/tldraw.ws.factory.ts @@ -0,0 +1,12 @@ +import { WsSharedDocDo } from '@src/modules/tldraw/domain/ws-shared-doc.do'; +import WebSocket from 'ws'; + +export class TldrawWsFactory { + public static createWsSharedDocDo(): WsSharedDocDo { + return { conns: new Map(), destroy() {} } as WsSharedDocDo; + } + + public static createWebsocket(readyState: number): WebSocket { + return { readyState, close: () => {} } as WebSocket; + } +} diff --git a/config/default.schema.json b/config/default.schema.json index bd637b719e9..780233bdc06 100644 --- a/config/default.schema.json +++ b/config/default.schema.json @@ -1343,6 +1343,55 @@ "description": "Number of simultaneously synchronized students, teachers and classes" } } + }, + "TLDRAW": { + "type": "object", + "description": "Tldraw managing variables.", + "required": ["PING_TIMEOUT", "SOCKET_PORT","GC_ENABLED", "DB_COLLECTION_NAME", "DB_FLUSH_SIZE", "DB_MULTIPLE_COLLECTIONS"], + "properties": { + "SOCKET_PORT": { + "type": "number", + "description": "Web socket port for tldraw" + }, + "PING_TIMEOUT": { + "type": "number", + "description": "Max time for waiting between calls for tldraw" + }, + "GC_ENABLED": { + "type": "boolean", + "description": "If tldraw garbage collector should be enabled" + }, + "DB_COLLECTION_NAME": { + "type": "string", + "description": "Collection name in which tldraw drawing are stored" + }, + "DB_FLUSH_SIZE": { + "type": "integer", + "description": "DB collection flushing size" + }, + "DB_MULTIPLE_COLLECTIONS": { + "type": "boolean", + "description": "DB collection allowing multiple collections for drawing" + } + }, + "default": { + "SOCKET_PORT": 3345, + "PING_TIMEOUT": 10000, + "GC_ENABLED": true, + "DB_COLLECTION_NAME": "drawing", + "DB_FLUSH_SIZE": 400, + "DB_MULTIPLE_COLLECTIONS": false + } + }, + "TLDRAW_DB_URL": { + "type": "string", + "default": "mongodb://127.0.0.1:27017/tldraw", + "description": "DB connection url" + }, + "FEATURE_TLDRAW_ENABLED": { + "type": "boolean", + "default": true, + "description": "Tldraw feature enabled" } }, "required": [], diff --git a/config/test.json b/config/test.json index c8b82b383ba..383d77e8648 100644 --- a/config/test.json +++ b/config/test.json @@ -68,5 +68,13 @@ }, "FEATURE_VIDEOCONFERENCE_ENABLED": true, "VIDEOCONFERENCE_HOST": "https://bigbluebutton.schul-cloud.org/bigbluebutton", - "VIDEOCONFERENCE_SALT": "ThisIsNOTaRealSaltThisIsNOTaRealSaltThisIsNOTaRealSalt1234567890" + "VIDEOCONFERENCE_SALT": "ThisIsNOTaRealSaltThisIsNOTaRealSaltThisIsNOTaRealSalt1234567890", + "TLDRAW": { + "SOCKET_PORT": 3346, + "PING_TIMEOUT": 1, + "GC_ENABLED": true, + "DB_COLLECTION_NAME": "drawing", + "DB_FLUSH_SIZE": 400, + "DB_MULTIPLE_COLLECTIONS": false + } } diff --git a/nest-cli.json b/nest-cli.json index 73dea03c093..6b03ac8672b 100644 --- a/nest-cli.json +++ b/nest-cli.json @@ -89,6 +89,15 @@ "compilerOptions": { "tsConfigPath": "apps/server/tsconfig.app.json" } + }, + "tldraw": { + "type": "application", + "root": "apps/server", + "entryFile": "apps/tldraw.app", + "sourceRoot": "apps/server/src", + "compilerOptions": { + "tsConfigPath": "apps/server/tsconfig.app.json" + } } } } diff --git a/package-lock.json b/package-lock.json index bf63523c91a..0df338ab2f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,9 @@ "@nestjs/microservices": "^10.2.4", "@nestjs/passport": "^10.0.1", "@nestjs/platform-express": "^10.2.4", + "@nestjs/platform-ws": "^10.2.4", "@nestjs/swagger": "^7.1.10", + "@nestjs/websockets": "^10.2.4", "@types/cache-manager-redis-store": "^2.0.1", "@types/connect-redis": "^0.0.19", "@types/gm": "^1.25.1", @@ -132,7 +134,10 @@ "universal-analytics": "^0.5.1", "urlsafe-base64": "^1.0.0", "uuid": "^8.3.0", - "winston": "^3.8.2" + "winston": "^3.8.2", + "y-mongodb-provider": "^0.1.8", + "y-protocols": "^1.0.5", + "yjs": "^13.6.7" }, "devDependencies": { "@aws-sdk/client-s3": "^3.352.0", @@ -581,7 +586,6 @@ "version": "3.352.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.352.0.tgz", "integrity": "sha512-qXqg7V/DpHu8oyEq22LMskCoHYZU6+ds9gaArwc3SjPwQN/UM6CpIUHtTtxevLEYr7nI5iMIPBBrEcoKOJefxg==", - "dev": true, "optional": true, "dependencies": { "@aws-crypto/sha256-browser": "3.0.0", @@ -798,7 +802,6 @@ "version": "3.352.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.352.0.tgz", "integrity": "sha512-395bdedGD0pangBT6dyyrTvtDRxr3lqbi8lfuJR/+7bpMIEJKVhF5D6IAgdjRDpASDRHUPhHuWzR3Qa9RHAcNA==", - "dev": true, "optional": true, "dependencies": { "@aws-sdk/client-cognito-identity": "3.352.0", @@ -982,7 +985,6 @@ "version": "3.352.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.352.0.tgz", "integrity": "sha512-hV6NO7+xzf3CPEsKZRsYflR05eNMvgVvOXFgQnOucUc85Kxt2XTSoH/HFtkolXDbxjA2Hku1pdaRG7qBzbiJHg==", - "dev": true, "optional": true, "dependencies": { "@aws-sdk/client-cognito-identity": "3.352.0", @@ -4462,6 +4464,14 @@ "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0" } }, + "node_modules/@mongodb-js/saslprep": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.0.tgz", + "integrity": "sha512-Xfijy7HvfzzqiOAhAepF4SGN5e9leLkMvg/OPOF97XemjfVCYN/oWa75wnkc6mltMSTwY+XlbhWgUOJmkFspSw==", + "dependencies": { + "sparse-bitfield": "^3.0.3" + } + }, "node_modules/@nestjs/axios": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-3.0.0.tgz", @@ -4941,6 +4951,44 @@ "@nestjs/core": "^10.0.0" } }, + "node_modules/@nestjs/platform-ws": { + "version": "10.2.7", + "resolved": "https://registry.npmjs.org/@nestjs/platform-ws/-/platform-ws-10.2.7.tgz", + "integrity": "sha512-4H4AeCQgM29Dju+zQb70Jt0JgWhQssOB8mh9n9icsSJ4B/joa+X7OiBBSjn72HZelj0tvX1gal6PaAhEaOdmGQ==", + "dependencies": { + "tslib": "2.6.2", + "ws": "8.14.2" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nest" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0", + "@nestjs/websockets": "^10.0.0", + "rxjs": "^7.1.0" + } + }, + "node_modules/@nestjs/platform-ws/node_modules/ws": { + "version": "8.14.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz", + "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/@nestjs/schematics": { "version": "10.0.2", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-10.0.2.tgz", @@ -5101,6 +5149,28 @@ } } }, + "node_modules/@nestjs/websockets": { + "version": "10.2.7", + "resolved": "https://registry.npmjs.org/@nestjs/websockets/-/websockets-10.2.7.tgz", + "integrity": "sha512-NKJMubkwpUBsudbiyjuLZDT/W68K+fS/pe3vG5Ur8QoPn+fkI9SFCiQw27Cv4K0qVX2eGJ41yNmVfu61zGa4CQ==", + "dependencies": { + "iterare": "1.2.1", + "object-hash": "3.0.0", + "tslib": "2.6.2" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0", + "@nestjs/core": "^10.0.0", + "@nestjs/platform-socket.io": "^10.0.0", + "reflect-metadata": "^0.1.12", + "rxjs": "^7.1.0" + }, + "peerDependenciesMeta": { + "@nestjs/platform-socket.io": { + "optional": true + } + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -13634,6 +13704,15 @@ "ws": "*" } }, + "node_modules/isomorphic.js": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", + "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==", + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", @@ -16159,6 +16238,25 @@ "node": ">= 0.8.0" } }, + "node_modules/lib0": { + "version": "0.2.87", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.87.tgz", + "integrity": "sha512-TbB63XJixvNToW2IHWAFsCJj9tVnajmwjE14p69i51Rx8byOQd2IP4ourE8v4d7vhyO++nVm1sQk3ePslfbucg==", + "dependencies": { + "isomorphic.js": "^0.2.4" + }, + "bin": { + "0gentesthtml": "bin/gentesthtml.js", + "0serve": "bin/0serve.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/libphonenumber-js": { "version": "1.10.24", "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.10.24.tgz", @@ -16704,8 +16802,7 @@ "node_modules/memory-pager": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", - "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", - "optional": true + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==" }, "node_modules/memory-stream": { "version": "0.0.3", @@ -18789,6 +18886,14 @@ "node": ">=0.10.0" } }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "engines": { + "node": ">= 6" + } + }, "node_modules/object-inspect": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", @@ -22338,7 +22443,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", - "optional": true, "dependencies": { "memory-pager": "^1.0.2" } @@ -25008,6 +25112,90 @@ "node": ">=0.4" } }, + "node_modules/y-mongodb-provider": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/y-mongodb-provider/-/y-mongodb-provider-0.1.8.tgz", + "integrity": "sha512-yV+rtS9nBEMqb6fG6sqyWNpMGzmTYe7hPiwWwWyrzK4frjMnkrQvJvyUiWjZI7eFFSKYzxYHucGEFA0j3QJEgA==", + "dependencies": { + "lib0": "^0.2.85", + "mongodb": "^6.1.0" + }, + "peerDependencies": { + "yjs": "^13.6.8" + } + }, + "node_modules/y-mongodb-provider/node_modules/bson": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.1.0.tgz", + "integrity": "sha512-yiQ3KxvpVoRpx1oD1uPz4Jit9tAVTJgjdmjDKtUErkOoL9VNoF8Dd58qtAOL5E40exx2jvAT9sqdRSK/r+SHlA==", + "engines": { + "node": ">=16.20.1" + } + }, + "node_modules/y-mongodb-provider/node_modules/mongodb": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.1.0.tgz", + "integrity": "sha512-AvzNY0zMkpothZ5mJAaIo2bGDjlJQqqAbn9fvtVgwIIUPEfdrqGxqNjjbuKyrgQxg2EvCmfWdjq+4uj96c0YPw==", + "dependencies": { + "@mongodb-js/saslprep": "^1.1.0", + "bson": "^6.1.0", + "mongodb-connection-string-url": "^2.6.0" + }, + "engines": { + "node": ">=16.20.1" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.188.0", + "@mongodb-js/zstd": "^1.1.0", + "gcp-metadata": "^5.2.0", + "kerberos": "^2.0.1", + "mongodb-client-encryption": ">=6.0.0 <7", + "snappy": "^7.2.2", + "socks": "^2.7.1" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/y-protocols": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.6.tgz", + "integrity": "sha512-vHRF2L6iT3rwj1jub/K5tYcTT/mEYDUppgNPXwp8fmLpui9f7Yeq3OEtTLVF012j39QnV+KEQpNqoN7CWU7Y9Q==", + "dependencies": { + "lib0": "^0.2.85" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "yjs": "^13.0.0" + } + }, "node_modules/y18n": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.2.tgz", @@ -25196,6 +25384,22 @@ "buffer-crc32": "~0.2.3" } }, + "node_modules/yjs": { + "version": "13.6.8", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.8.tgz", + "integrity": "sha512-ZPq0hpJQb6f59B++Ngg4cKexDJTvfOgeiv0sBc4sUm8CaBWH7OQC4kcCgrqbjJ/B2+6vO49exvTmYfdlPtcjbg==", + "dependencies": { + "lib0": "^0.2.74" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", @@ -25543,7 +25747,6 @@ "version": "3.352.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.352.0.tgz", "integrity": "sha512-qXqg7V/DpHu8oyEq22LMskCoHYZU6+ds9gaArwc3SjPwQN/UM6CpIUHtTtxevLEYr7nI5iMIPBBrEcoKOJefxg==", - "dev": true, "optional": true, "requires": { "@aws-crypto/sha256-browser": "3.0.0", @@ -25745,7 +25948,6 @@ "version": "3.352.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.352.0.tgz", "integrity": "sha512-395bdedGD0pangBT6dyyrTvtDRxr3lqbi8lfuJR/+7bpMIEJKVhF5D6IAgdjRDpASDRHUPhHuWzR3Qa9RHAcNA==", - "dev": true, "optional": true, "requires": { "@aws-sdk/client-cognito-identity": "3.352.0", @@ -25901,7 +26103,6 @@ "version": "3.352.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.352.0.tgz", "integrity": "sha512-hV6NO7+xzf3CPEsKZRsYflR05eNMvgVvOXFgQnOucUc85Kxt2XTSoH/HFtkolXDbxjA2Hku1pdaRG7qBzbiJHg==", - "dev": true, "optional": true, "requires": { "@aws-sdk/client-cognito-identity": "3.352.0", @@ -28429,6 +28630,14 @@ "integrity": "sha512-TrCdPsM7DApxrK3avBbijT6/6Er4TZhtiQ+qlMqtqva13vMCG4HiF2vIWGrKJbFukkLRuhOfZlES+KZ9Y1Lx2A==", "requires": {} }, + "@mongodb-js/saslprep": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.0.tgz", + "integrity": "sha512-Xfijy7HvfzzqiOAhAepF4SGN5e9leLkMvg/OPOF97XemjfVCYN/oWa75wnkc6mltMSTwY+XlbhWgUOJmkFspSw==", + "requires": { + "sparse-bitfield": "^3.0.3" + } + }, "@nestjs/axios": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-3.0.0.tgz", @@ -28695,6 +28904,23 @@ "tslib": "2.6.2" } }, + "@nestjs/platform-ws": { + "version": "10.2.7", + "resolved": "https://registry.npmjs.org/@nestjs/platform-ws/-/platform-ws-10.2.7.tgz", + "integrity": "sha512-4H4AeCQgM29Dju+zQb70Jt0JgWhQssOB8mh9n9icsSJ4B/joa+X7OiBBSjn72HZelj0tvX1gal6PaAhEaOdmGQ==", + "requires": { + "tslib": "2.6.2", + "ws": "8.14.2" + }, + "dependencies": { + "ws": { + "version": "8.14.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz", + "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==", + "requires": {} + } + } + }, "@nestjs/schematics": { "version": "10.0.2", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-10.0.2.tgz", @@ -28792,6 +29018,16 @@ "tslib": "2.6.2" } }, + "@nestjs/websockets": { + "version": "10.2.7", + "resolved": "https://registry.npmjs.org/@nestjs/websockets/-/websockets-10.2.7.tgz", + "integrity": "sha512-NKJMubkwpUBsudbiyjuLZDT/W68K+fS/pe3vG5Ur8QoPn+fkI9SFCiQw27Cv4K0qVX2eGJ41yNmVfu61zGa4CQ==", + "requires": { + "iterare": "1.2.1", + "object-hash": "3.0.0", + "tslib": "2.6.2" + } + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -35293,6 +35529,11 @@ "peer": true, "requires": {} }, + "isomorphic.js": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", + "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==" + }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", @@ -37247,6 +37488,14 @@ "type-check": "~0.4.0" } }, + "lib0": { + "version": "0.2.87", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.87.tgz", + "integrity": "sha512-TbB63XJixvNToW2IHWAFsCJj9tVnajmwjE14p69i51Rx8byOQd2IP4ourE8v4d7vhyO++nVm1sQk3ePslfbucg==", + "requires": { + "isomorphic.js": "^0.2.4" + } + }, "libphonenumber-js": { "version": "1.10.24", "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.10.24.tgz", @@ -37698,8 +37947,7 @@ "memory-pager": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", - "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", - "optional": true + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==" }, "memory-stream": { "version": "0.0.3", @@ -39345,6 +39593,11 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, + "object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==" + }, "object-inspect": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", @@ -42007,7 +42260,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", - "optional": true, "requires": { "memory-pager": "^1.0.2" } @@ -44048,6 +44300,40 @@ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, + "y-mongodb-provider": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/y-mongodb-provider/-/y-mongodb-provider-0.1.8.tgz", + "integrity": "sha512-yV+rtS9nBEMqb6fG6sqyWNpMGzmTYe7hPiwWwWyrzK4frjMnkrQvJvyUiWjZI7eFFSKYzxYHucGEFA0j3QJEgA==", + "requires": { + "lib0": "^0.2.85", + "mongodb": "^6.1.0" + }, + "dependencies": { + "bson": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.1.0.tgz", + "integrity": "sha512-yiQ3KxvpVoRpx1oD1uPz4Jit9tAVTJgjdmjDKtUErkOoL9VNoF8Dd58qtAOL5E40exx2jvAT9sqdRSK/r+SHlA==" + }, + "mongodb": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.1.0.tgz", + "integrity": "sha512-AvzNY0zMkpothZ5mJAaIo2bGDjlJQqqAbn9fvtVgwIIUPEfdrqGxqNjjbuKyrgQxg2EvCmfWdjq+4uj96c0YPw==", + "requires": { + "@mongodb-js/saslprep": "^1.1.0", + "bson": "^6.1.0", + "mongodb-connection-string-url": "^2.6.0" + } + } + } + }, + "y-protocols": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.6.tgz", + "integrity": "sha512-vHRF2L6iT3rwj1jub/K5tYcTT/mEYDUppgNPXwp8fmLpui9f7Yeq3OEtTLVF012j39QnV+KEQpNqoN7CWU7Y9Q==", + "requires": { + "lib0": "^0.2.85" + } + }, "y18n": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.2.tgz", @@ -44198,6 +44484,14 @@ "buffer-crc32": "~0.2.3" } }, + "yjs": { + "version": "13.6.8", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.8.tgz", + "integrity": "sha512-ZPq0hpJQb6f59B++Ngg4cKexDJTvfOgeiv0sBc4sUm8CaBWH7OQC4kcCgrqbjJ/B2+6vO49exvTmYfdlPtcjbg==", + "requires": { + "lib0": "^0.2.74" + } + }, "yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/package.json b/package.json index 53a0752ee10..ce313bc031f 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,9 @@ "nest:start:h5p": "nest start h5p-editor", "nest:start:h5p:dev": "nest start h5p-editor --debug --watch", "nest:start:h5p:prod": "node dist/apps/server/apps/h5p-editor.app", + "nest:start:tldraw": "nest start tldraw", + "nest:start:tldraw:dev": "nest start tldraw --debug --watch", + "nest:start:tldraw:prod": "node dist/apps/server/apps/tldraw.app", "nest:start:console": "nest start console --", "nest:start:console:dev": "nest start console --watch --", "nest:start:console:debug": "nest start console --debug --watch --", @@ -115,7 +118,9 @@ "@nestjs/microservices": "^10.2.4", "@nestjs/passport": "^10.0.1", "@nestjs/platform-express": "^10.2.4", + "@nestjs/platform-ws": "^10.2.4", "@nestjs/swagger": "^7.1.10", + "@nestjs/websockets": "^10.2.4", "@types/cache-manager-redis-store": "^2.0.1", "@types/connect-redis": "^0.0.19", "@types/gm": "^1.25.1", @@ -215,7 +220,10 @@ "universal-analytics": "^0.5.1", "urlsafe-base64": "^1.0.0", "uuid": "^8.3.0", - "winston": "^3.8.2" + "winston": "^3.8.2", + "y-mongodb-provider": "^0.1.8", + "y-protocols": "^1.0.5", + "yjs": "^13.6.7" }, "devDependencies": { "@aws-sdk/client-s3": "^3.352.0",