From fa96b3797175baa8b2afc7f5ec4b0eac044c2236 Mon Sep 17 00:00:00 2001 From: nael Date: Wed, 18 Sep 2024 22:20:45 +0200 Subject: [PATCH] :memo: Updated Docs --- .../open-source/self_hosting/envVariables.mdx | 19 +++++++++-- docs/rag/index.mdx | 8 +++-- docs/recipes/catch-connection-token.mdx | 4 +-- docs/webhooks/overview.mdx | 4 +-- .../panora-webhooks/dto/webhook.dto.ts | 29 +++++++++++++--- .../panora-webhooks/webhook.controller.ts | 3 +- .../panora-webhooks/webhook.service.ts | 7 ++-- packages/api/src/@core/rag/rag.dto.ts | 2 +- packages/api/swagger/swagger-spec.yaml | 34 ++++++++++++++----- 9 files changed, 82 insertions(+), 28 deletions(-) diff --git a/docs/open-source/self_hosting/envVariables.mdx b/docs/open-source/self_hosting/envVariables.mdx index 02d593f3e..71719ad2f 100644 --- a/docs/open-source/self_hosting/envVariables.mdx +++ b/docs/open-source/self_hosting/envVariables.mdx @@ -3,6 +3,7 @@ title: "Environment Variables" description: "" --- +If you selfhost, please make sure to create your own oAuth app and [report the credentials in the dashboard](/recipes/add-custom-provider-creds) or directly fill these env vars in your .env [here](/open-source/self_hosting/envVariables#providers-specific-credentials)! # Backend @@ -56,7 +57,7 @@ description: "" | Variable | Example | Purpose | | -------- | ------------------------------------- | ------------------------------------- | | REDIRECT_TUNNEL_INGRESS | https://MY-NGROK-DOMAIN.ngrok-free.app | Mandatory only when DISTRIBUTION=selfhost. Endpoint (an Ngrok tunnel domain) when you have to test your OAuth App and needs a HTTPS redirectUri that redirects to your localhost (useful for contributors that might need to test their oAuth flow) | -| WEBHOOK_INGRESS | https://MY-NGROK-DOMAIN.ngrok-free.app | Same job as REDIRECT_TUNNEL_INGRESS. You can use the same value for both if you prefer. | +| WEBHOOK_INGRESS | https://MY-NGROK-DOMAIN.ngrok-free.app | If you plan to use the Backend SDK to receive data, if your receiving server is running locally you must use a Ngrok redirect as well. (our main API is runnning within docker and doesnt have localhost context so requests must be https). Alternatively, you can just input it [using the dashboard](/webhooks/overview#:~:text=Register%20your%20endpoint%20within%20Panora%20using%20the%20Dashboard%20or%20the%20API.). | ## Providers Specific Credentials @@ -201,13 +202,24 @@ description: "" | SALESFORCE_CRM_CLOUD_CLIENT_SECRET | | | ## RAG + +### Embedding Models +Just use 1 model, it's not required of course to have them all. | Variable | Example | Purpose | | -------- | ------------------------------------- | ------------------------------------- | | OPENAI_API_KEY | | | | JINA_API_KEY | | | | COHERE_API_KEY | | | + +| Variable | Example | Purpose | +| -------- | ------------------------------------- | ------------------------------------- | | UNSTRUCTURED_API_KEY | | | | UNSTRUCTURED_API_URL | | | + +### Vector Database +Just use 1 Vector DB, it's not required of course to have them all. +| Variable | Example | Purpose | +| -------- | ------------------------------------- | ------------------------------------- | | PINECONE_API_KEY | | | | PINECONE_INDEX_NAME | | | | QDRANT_BASE_URL | | | @@ -239,7 +251,10 @@ description: "" ## Storage (Optional) -You can let them empty for now ! +If you selfhost, and you want the RAG feature you must fill these values.Just choose either AWS or Minio ! +We already put variables for Minio, so you dont have to do anything. +If you'd like to use AWS contact us for more info. +Minio bucket is available at http://localhost:9000 | Variable | Example | Purpose | | -------- | ------------------------------------- | ------------------------------------- | diff --git a/docs/rag/index.mdx b/docs/rag/index.mdx index b743ba738..b29c7908e 100644 --- a/docs/rag/index.mdx +++ b/docs/rag/index.mdx @@ -9,9 +9,9 @@ Once we've synced documents across File Storage systems, we embed and chunk them # Step 1: Import the code snippet - ```shell React - pnpm i @panora/sdk - ``` + ```shell React + pnpm i @panora/sdk + ``` #### Use the SDK @@ -43,6 +43,8 @@ Once we've synced documents across File Storage systems, we embed and chunk them Congrats ! You should be able to get back your embeddings and chunks for the query ! +If you selfhost, please make sure to do step 2 or directly fill these env vars in your .env [here](/open-source/self_hosting/envVariables#rag)! + By default, for embedding we use **OpenAI ADA-002** model and **Pinecone** managed vector database for storing the chunks. # Step 2 (Optional): Choose your own Vector DB + Embedding Model diff --git a/docs/recipes/catch-connection-token.mdx b/docs/recipes/catch-connection-token.mdx index 2dfc4ae76..810632d75 100644 --- a/docs/recipes/catch-connection-token.mdx +++ b/docs/recipes/catch-connection-token.mdx @@ -47,9 +47,7 @@ It is a field of the [Connection Object](/glossary/connection-object) that Panor } // Return a response to acknowledge receipt of the event - response.json({ - received: true - }); + response.json('Received !'); }); app.listen(8000, () => console.log('Running on port 8000')); diff --git a/docs/webhooks/overview.mdx b/docs/webhooks/overview.mdx index ee60b5be8..1147cbfa7 100644 --- a/docs/webhooks/overview.mdx +++ b/docs/webhooks/overview.mdx @@ -50,9 +50,7 @@ To enable webhook events, you need to register webhook endpoints. After you regi } // Return a response to acknowledge receipt of the event - response.json({ - received: true - }); + response.json('Received !'); }); app.listen(8000, () => console.log('Running on port 8000')); diff --git a/packages/api/src/@core/@core-services/webhooks/panora-webhooks/dto/webhook.dto.ts b/packages/api/src/@core/@core-services/webhooks/panora-webhooks/dto/webhook.dto.ts index e1f515416..42059c7d0 100644 --- a/packages/api/src/@core/@core-services/webhooks/panora-webhooks/dto/webhook.dto.ts +++ b/packages/api/src/@core/@core-services/webhooks/panora-webhooks/dto/webhook.dto.ts @@ -27,17 +27,38 @@ export class WebhookDto { } export class EventPayload { - [key: string]: any; -} + @ApiProperty({ + type: String, + example: '801f9ede-c698-4e66-a7fc-48d19eebaa4f', + nullable: true, + description: 'The id of the event.', + }) + id_event: string; + + @ApiProperty({ + type: String, + example: 'connection.created', + nullable: true, + description: 'The type of the event.', + }) + type: string; -export class SignatureVerificationDto { @ApiProperty({ type: Object, additionalProperties: true, nullable: true, + description: 'The data payload event of the webhook.', + }) + data: { [key: string]: any }; +} + +export class SignatureVerificationDto { + @ApiProperty({ + type: EventPayload, + nullable: true, description: 'The payload event of the webhook.', }) - payload: { [key: string]: any }; + payload: EventPayload; @ApiProperty({ type: String, diff --git a/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.controller.ts b/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.controller.ts index 1cabce443..80eb3beba 100644 --- a/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.controller.ts +++ b/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.controller.ts @@ -25,6 +25,7 @@ import { ApiParam, } from '@nestjs/swagger'; import { + EventPayload, SignatureVerificationDto, WebhookDto, WebhookResponse, @@ -171,7 +172,7 @@ export class WebhookController { summary: 'Verify payload signature of the webhook', }) @ApiBody({ type: SignatureVerificationDto }) - @ApiPostGenericJson('Dynamic event payload') + @ApiPostCustomResponse(EventPayload) @UseGuards(ApiKeyAuthGuard) @Post('verifyEvent') async verifyPayloadSignature(@Body() data: SignatureVerificationDto) { diff --git a/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.service.ts b/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.service.ts index e025ef298..45ac0ed33 100644 --- a/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.service.ts +++ b/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.service.ts @@ -3,7 +3,7 @@ import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; import { v4 as uuidv4 } from 'uuid'; import { LoggerService } from '@@core/@core-services/logger/logger.service'; import { WebhooksError } from '@@core/utils/errors'; -import { WebhookDto } from './dto/webhook.dto'; +import { EventPayload, WebhookDto } from './dto/webhook.dto'; import axios from 'axios'; import { createHmac } from 'crypto'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; @@ -220,6 +220,7 @@ export class WebhookService { { id_event: deliveryAttempt.id_event, data: deliveryAttempt.webhooks_payloads.data, + type: eventType, }, { headers: { @@ -289,10 +290,10 @@ export class WebhookService { } async verifyPayloadSignature( - payload: { [key: string]: any }, + payload: EventPayload, signature: string, secret: string, - ) { + ): Promise { try { const expected = this.generateSignature(payload.data, secret); if (expected !== signature) { diff --git a/packages/api/src/@core/rag/rag.dto.ts b/packages/api/src/@core/rag/rag.dto.ts index 79e1b8e72..4337f69ef 100644 --- a/packages/api/src/@core/rag/rag.dto.ts +++ b/packages/api/src/@core/rag/rag.dto.ts @@ -4,7 +4,7 @@ import { IsNumber, IsString } from 'class-validator'; export class RagQueryOutput { @ApiProperty({ type: String, - example: '\nDate : 06/07/2023', + example: 'Date : 06/07/2023', nullable: false, description: 'The chunk which matches the embed query', }) diff --git a/packages/api/swagger/swagger-spec.yaml b/packages/api/swagger/swagger-spec.yaml index 4a8bb822a..0293916cf 100644 --- a/packages/api/swagger/swagger-spec.yaml +++ b/packages/api/swagger/swagger-spec.yaml @@ -309,9 +309,7 @@ paths: content: application/json: schema: - type: object - additionalProperties: true - description: Dynamic event payload + $ref: '#/components/schemas/EventPayload' tags: *ref_1 x-speakeasy-group: webhooks /ticketing/tickets: @@ -9616,9 +9614,7 @@ components: properties: chunk: type: string - example: |- - - Date : 06/07/2023 + example: 'Date : 06/07/2023' nullable: false description: The chunk which matches the embed query metadata: @@ -9998,14 +9994,36 @@ components: required: - url - scope - SignatureVerificationDto: + EventPayload: type: object properties: - payload: + id_event: + type: string + example: 801f9ede-c698-4e66-a7fc-48d19eebaa4f + nullable: true + description: The id of the event. + type: + type: string + example: connection.created + nullable: true + description: The type of the event. + data: type: object additionalProperties: true + nullable: true + description: The data payload event of the webhook. + required: + - id_event + - type + - data + SignatureVerificationDto: + type: object + properties: + payload: nullable: true description: The payload event of the webhook. + allOf: + - $ref: '#/components/schemas/EventPayload' signature: type: string nullable: true