Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add twenty-tinybird package #8030

Merged
merged 11 commits into from
Oct 29, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/twenty-tinybird/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
.tinyb
33 changes: 33 additions & 0 deletions packages/twenty-tinybird/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@

## How to use
Create a virtual enviroment and install tinybird
```sh
python3 -m venv .venv
source .venv/bin/activate
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add .venv to .gitignore then :)

pip install tinybird-cli
```
Authenticate using your admin token from your workspace
```sh
tb auth -i

** List of available regions:
[1] us-east4 (gcp) (https://app.us-east.tinybird.co)
[2] europe-west3 (gcp) (https://app.tinybird.co/gcp/europe-west3)
[3] us-east-1 (aws) (https://app.tinybird.co/aws/us-east-1)
[4] us-west-2 (aws) (https://app.tinybird.co/aws/us-west-2)
[5] eu-central-1 (aws) (https://app.tinybird.co/aws/eu-central-1) <- this
[0] Cancel

Use region [1]:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Region 5 no?

Copy the "admin your@email" token from from https://app.tinybird.co/tokens and paste it here: <pasted Token>
** Auth successful!
** Configuration written to .tinyb file, consider adding it to .gitignore
```
To sync your changes to Tinybird use:
```sh
tb push --force --push-deps
```
To pull data from Tinybird use:
```sh
tb pull
```
12 changes: 12 additions & 0 deletions packages/twenty-tinybird/datasources/event.datasource
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@

SCHEMA >
`action` String `json:$.action`,
`timestamp` DateTime64(3) `json:$.timestamp`,
`version` String `json:$.version`,
`userId` String `json:$.userId` DEFAULT '',
`workspaceId` String `json:$.workspaceId` DEFAULT '',
`payload` String `json:$.payload`

ENGINE "MergeTree"
ENGINE_PARTITION_KEY "toYear(timestamp)"
ENGINE_SORTING_KEY "action, timestamp"
17 changes: 17 additions & 0 deletions packages/twenty-tinybird/datasources/pageview.datasource
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@

SCHEMA >
`href` String `json:$.href`,
`locale` String `json:$.locale`,
`pathname` String `json:$.pathname`,
`referrer` String `json:$.referrer`,
`sessionId` String `json:$.sessionId`,
`timeZone` String `json:$.timeZone`,
`timestamp` DateTime64(3) `json:$.timestamp`,
`userAgent` String `json:$.userAgent`,
`userId` String `json:$.userId`,
`version` String `json:$.version`,
`workspaceId` String `json:$.workspaceId`

ENGINE "MergeTree"
ENGINE_PARTITION_KEY "toYear(timestamp)"
ENGINE_SORTING_KEY "timestamp, userId, version, workspaceId"
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Data Source created from Pipe 'serverlessFunctionsEventMv'

SCHEMA >
`timestamp` DateTime64(3),
`workspaceId` String,
`functionId` String,
`durationInMs` Int64,
`success` Bool,
`errorType` String

ENGINE "MergeTree"
ENGINE_PARTITION_KEY "toYYYYMM(timestamp)"
ENGINE_SORTING_KEY "timestamp, functionId, success"
15 changes: 15 additions & 0 deletions packages/twenty-tinybird/datasources/webhooksEventMV.datasource
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Data Source created from Pipe 'webhooksEventMv'

SCHEMA >
`timestamp` DateTime64(3),
`workspaceId` String,
`webhookId` String,
`url` String,
`success` UInt8,
`status` Int64,
`eventName` String,
`version` String

ENGINE "MergeTree"
ENGINE_PARTITION_KEY "toYYYYMM(timestamp)"
ENGINE_SORTING_KEY "timestamp, workspaceId"
62 changes: 62 additions & 0 deletions packages/twenty-tinybird/pipes/getWebhooksAnalyticsV2.pipe
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
TOKEN "getWebhooksAnalyticsV2_endpoint_read_4255" READ

TAGS newVersion

NODE getWebhooksAnalyticsV2_0
SQL >

SELECT
timestamp,
workspaceId,
JSONExtractString(payload, 'webhookId') as webhookId,
JSONExtractString(payload, 'url') as url,
JSONExtractBool(payload, 'success') as success,
JSONExtractInt(payload, 'status') as status,
JSONExtractString(payload, 'eventName') as eventName,
version

FROM event
WHERE action = 'webhook.response'



NODE getWebhooksAnalyticsV3Jwt
SQL >

%
WITH
reference_time AS (
SELECT
toStartOfInterval(now(), INTERVAL {{ Int32(tickIntervalInMinutes, 420) }} MINUTE) AS ref_time
),
time_series AS (
SELECT
toDateTime(subtractHours(ref_time, {{ Int32(windowInHours, 168) }}))
+ INTERVAL number * {{ Int32(tickIntervalInMinutes, 420) }} MINUTE AS start_interval
FROM reference_time, numbers(0, 12 * 2 + 1)
WHERE start_interval <= now() -- Use now() instead of ref_time to include current interval
)
SELECT
ts.start_interval,
COALESCE(e.failure_count, 0) AS failure_count,
COALESCE(e.success_count, 0) AS success_count
FROM time_series ts
LEFT JOIN
(
SELECT
toStartOfInterval(
timestamp, INTERVAL {{ Int32(tickIntervalInMinutes, 420) }} MINUTE
) AS interval_start,
countIf(success = 0) AS failure_count,
countIf(success = 1) AS success_count
FROM getWebhooksAnalyticsV2_0
WHERE
timestamp >= subtractHours(now(), {{ Int32(windowInHours, 168) }})
AND webhookId = {{ String(webhookId, "c19ea782-417e-4d44-a486-fd794fc44a27") }}
AND workspaceId ={{ String(workspaceId,"20202020-1c25-4d02-bf25-6aeccf7ea419") }}
GROUP BY interval_start
) e
ON ts.start_interval = e.interval_start
ORDER BY ts.start_interval


18 changes: 18 additions & 0 deletions packages/twenty-tinybird/pipes/serverlessFunctionsEventMv.pipe
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
NODE serverlessFunctionsEvent_mv
SQL >

SELECT
timestamp,
workspaceId,
JSONExtractString(payload, 'functionId') as functionId,
JSONExtractInt(payload, 'duration') as durationInMs,
if(JSONExtractString(payload, 'status')='SUCCESS',TRUE, FALSE) as success,
JSONExtractString(payload, 'errorType') as errorType

FROM event
WHERE action = 'serverlessFunction.executed'

TYPE materialized
DATASOURCE serverlessFunctionsEventMV


Loading
Loading