diff --git a/go.mod b/go.mod index 63ede16be0e..734c5ea2722 100644 --- a/go.mod +++ b/go.mod @@ -128,6 +128,7 @@ require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect + github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 github.com/AthenZ/athenz v1.11.65 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect diff --git a/go.sum b/go.sum index 0f04d9c32ad..76ba63804dc 100644 --- a/go.sum +++ b/go.sum @@ -108,6 +108,8 @@ github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMb github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= github.com/99designs/keyring v1.2.2/go.mod h1:wes/FrByc8j7lFOAGLGSNEg8f/PaI3cgTBqhFkHUrPk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= github.com/AthenZ/athenz v1.11.65 h1:LV8zGlszam5Jccza/JbLgWsK+HwI2W836IyCTlfEKuw= diff --git a/processor/processor.go b/processor/processor.go index 08b43fcf6b4..c79b446f43d 100644 --- a/processor/processor.go +++ b/processor/processor.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "runtime/trace" "slices" "strconv" @@ -12,6 +13,8 @@ import ( "sync" "time" + obskit "github.com/rudderlabs/rudder-observability-kit/go/labels" + "github.com/rudderlabs/rudder-server/enterprise/trackedusers" "golang.org/x/sync/errgroup" @@ -55,6 +58,7 @@ import ( . "github.com/rudderlabs/rudder-server/utils/tx" //nolint:staticcheck "github.com/rudderlabs/rudder-server/utils/types" "github.com/rudderlabs/rudder-server/utils/workerpool" + wtrans "github.com/rudderlabs/rudder-server/warehouse/transformer" ) const ( @@ -84,10 +88,12 @@ type trackedUsersReporter interface { // Handle is a handle to the processor module type Handle struct { - conf *config.Config - tracer stats.Tracer - backendConfig backendconfig.BackendConfig - transformer transformer.Transformer + conf *config.Config + tracer stats.Tracer + backendConfig backendconfig.BackendConfig + transformer transformer.Transformer + warehouseTransformer transformer.DestinationTransformer + warehouseDebugLogger *wtrans.DebugLogger gatewayDB jobsdb.JobsDB routerDB jobsdb.JobsDB @@ -156,6 +162,7 @@ type Handle struct { eventAuditEnabled map[string]bool credentialsMap map[string][]transformer.Credential nonEventStreamSources map[string]bool + enableWarehouseTransformations config.ValueLoader[bool] } drainConfig struct { @@ -615,6 +622,9 @@ func (proc *Handle) Setup( "partition": partition, }) } + proc.warehouseTransformer = wtrans.New(proc.conf, proc.logger, proc.statsFactory) + proc.warehouseDebugLogger = wtrans.NewDebugLogger(proc.conf, proc.logger) + if proc.config.enableDedup { var err error proc.dedup, err = dedup.New(proc.conf, proc.statsFactory) @@ -815,6 +825,7 @@ func (proc *Handle) loadReloadableConfig(defaultPayloadLimit int64, defaultMaxEv proc.config.archivalEnabled = config.GetReloadableBoolVar(true, "archival.Enabled") // Capture event name as a tag in event level stats proc.config.captureEventNameStats = config.GetReloadableBoolVar(false, "Processor.Stats.captureEventName") + proc.config.enableWarehouseTransformations = config.GetReloadableBoolVar(false, "Processor.enableWarehouseTransformations") } type connection struct { @@ -2765,6 +2776,7 @@ func (proc *Handle) transformSrcDest( proc.logger.Debug("Dest Transform input size", len(eventsToTransform)) s := time.Now() response = proc.transformer.Transform(ctx, eventsToTransform, proc.config.transformBatchSize.Load()) + proc.handleResponseForWarehouseTransformation(ctx, eventsToTransform, response, commonMetaData, eventsByMessageID) destTransformationStat := proc.newDestinationTransformationStat(sourceID, workspaceID, transformAt, destination) destTransformationStat.transformTime.Since(s) @@ -2923,6 +2935,59 @@ func (proc *Handle) transformSrcDest( } } +func (proc *Handle) handleResponseForWarehouseTransformation( + ctx context.Context, + eventsToTransform []transformer.TransformerEvent, + pResponse transformer.Response, + commonMetaData *transformer.Metadata, + eventsByMessageID map[string]types.SingularEventWithReceivedAt, +) { + if len(eventsToTransform) == 0 || !proc.config.enableWarehouseTransformations.Load() { + return + } + defer proc.statsFactory.NewStat("proc_warehouse_transformations_time", stats.TimerType).RecordDuration()() + + wResponse := proc.warehouseTransformer.Transform(ctx, eventsToTransform, proc.config.transformBatchSize.Load()) + responsesDiffer := proc.responsesDiffer(eventsToTransform, pResponse, wResponse, eventsByMessageID) + if len(responsesDiffer) == 0 { + return + } + if err := proc.warehouseDebugLogger.LogEvents(responsesDiffer, commonMetaData); err != nil { + proc.logger.Warnn("Failed to log events for warehouse transformation debugging", obskit.Error(err)) + } +} + +func (proc *Handle) responsesDiffer( + eventsToTransform []transformer.TransformerEvent, + pResponse, wResponse transformer.Response, + eventsByMessageID map[string]types.SingularEventWithReceivedAt, +) (messages []types.SingularEventT) { + if len(pResponse.Events) != len(wResponse.Events) || len(pResponse.FailedEvents) != len(wResponse.FailedEvents) { + return lo.Map(eventsToTransform, func(e transformer.TransformerEvent, _ int) types.SingularEventT { + return eventsByMessageID[e.Metadata.MessageID].SingularEvent + }) + } + for i := range pResponse.Events { + if reflect.DeepEqual(pResponse.Events[i], wResponse.Events[i]) { + continue + } + messages = append(messages, lo.Map(pResponse.Events[i].Metadata.GetMessagesIDs(), func(msgID string, _ int) types.SingularEventT { + return eventsByMessageID[msgID].SingularEvent + })...) + } + for i := range pResponse.FailedEvents { + wResponse.FailedEvents[i].Error = pResponse.FailedEvents[i].Error + + if reflect.DeepEqual(pResponse.FailedEvents[i], wResponse.FailedEvents[i]) { + continue + } + messages = append(messages, lo.Map(pResponse.FailedEvents[i].Metadata.GetMessagesIDs(), func(msgID string, _ int) types.SingularEventT { + return eventsByMessageID[msgID].SingularEvent + })...) + } + return +} + func (proc *Handle) saveDroppedJobs(ctx context.Context, droppedJobs []*jobsdb.JobT, tx *Tx) error { if len(droppedJobs) > 0 { for i := range droppedJobs { // each dropped job should have a unique jobID in the scope of the batch diff --git a/warehouse/transformer/alias_test.go b/warehouse/transformer/alias_test.go index b530d99c321..6a8ace5d64f 100644 --- a/warehouse/transformer/alias_test.go +++ b/warehouse/transformer/alias_test.go @@ -7,13 +7,13 @@ import ( "github.com/ory/dockertest/v3" "github.com/stretchr/testify/require" - "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" transformertest "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/transformer" backendconfig "github.com/rudderlabs/rudder-server/backend-config" ptrans "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/warehouse/transformer/testhelper" ) func TestAlias(t *testing.T) { @@ -23,7 +23,7 @@ func TestAlias(t *testing.T) { transformerResource, err := transformertest.Setup(pool, t) require.NoError(t, err) - testsCases := []struct { + testCases := []struct { name string configOverride map[string]any eventPayload string @@ -34,84 +34,13 @@ func TestAlias(t *testing.T) { { name: "alias (Postgres)", eventPayload: `{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getAliasMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "previous_id": "previousId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "previous_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "aliases", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getAliasDefaultOutput(), + Metadata: getAliasMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -120,80 +49,15 @@ func TestAlias(t *testing.T) { { name: "alias (Postgres) without traits", eventPayload: `{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getAliasMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - "previous_id": "previousId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "previous_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "aliases", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getAliasDefaultOutput(). + RemoveDataFields("title", "url"). + RemoveColumnFields("title", "url"), + Metadata: getAliasMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -202,78 +66,16 @@ func TestAlias(t *testing.T) { { name: "alias (Postgres) without context", eventPayload: `{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"}}`, - metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getAliasMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "5.6.7.8", - "context_request_ip": "5.6.7.8", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "previous_id": "previousId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_request_ip": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "previous_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "aliases", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getAliasDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). // overriding the default value + RemoveDataFields("context_passed_ip", "context_traits_email", "context_traits_logins"). + RemoveColumnFields("context_passed_ip", "context_traits_email", "context_traits_logins"), + Metadata: getAliasMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -282,88 +84,17 @@ func TestAlias(t *testing.T) { { name: "alias (Postgres) store rudder event", eventPayload: `{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "storeFullEvent": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getAliasMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "storeFullEvent": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "previous_id": "previousId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "rudder_event": "{\"type\":\"alias\",\"anonymousId\":\"anonymousId\",\"channel\":\"web\",\"context\":{\"destinationId\":\"destinationID\",\"destinationType\":\"POSTGRES\",\"ip\":\"1.2.3.4\",\"sourceId\":\"sourceID\",\"sourceType\":\"sourceType\",\"traits\":{\"email\":\"rhedricks@example.com\",\"logins\":2}},\"messageId\":\"messageId\",\"originalTimestamp\":\"2021-09-01T00:00:00.000Z\",\"previousId\":\"previousId\",\"receivedAt\":\"2021-09-01T00:00:00.000Z\",\"request_ip\":\"5.6.7.8\",\"sentAt\":\"2021-09-01T00:00:00.000Z\",\"timestamp\":\"2021-09-01T00:00:00.000Z\",\"traits\":{\"title\":\"Home | RudderStack\",\"url\":\"https://www.rudderstack.com\"},\"userId\":\"userId\"}", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "previous_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "rudder_event": "json", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "aliases", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getAliasDefaultOutput(). + SetDataField("rudder_event", "{\"type\":\"alias\",\"anonymousId\":\"anonymousId\",\"channel\":\"web\",\"context\":{\"destinationId\":\"destinationID\",\"destinationType\":\"POSTGRES\",\"ip\":\"1.2.3.4\",\"sourceId\":\"sourceID\",\"sourceType\":\"sourceType\",\"traits\":{\"email\":\"rhedricks@example.com\",\"logins\":2}},\"messageId\":\"messageId\",\"originalTimestamp\":\"2021-09-01T00:00:00.000Z\",\"previousId\":\"previousId\",\"receivedAt\":\"2021-09-01T00:00:00.000Z\",\"request_ip\":\"5.6.7.8\",\"sentAt\":\"2021-09-01T00:00:00.000Z\",\"timestamp\":\"2021-09-01T00:00:00.000Z\",\"traits\":{\"title\":\"Home | RudderStack\",\"url\":\"https://www.rudderstack.com\"},\"userId\":\"userId\"}"). + SetColumnField("rudder_event", "json"), + Metadata: getAliasMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -372,78 +103,15 @@ func TestAlias(t *testing.T) { { name: "alias (Postgres) partial rules", eventPayload: `{"type":"alias","messageId":"messageId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getAliasMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "previous_id": "previousId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_ip": "string", - "context_passed_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "previous_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "aliases", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getAliasDefaultOutput(). + RemoveDataFields("anonymous_id", "channel", "context_request_ip"). + RemoveColumnFields("anonymous_id", "channel", "context_request_ip"), + Metadata: getAliasMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -455,141 +123,126 @@ func TestAlias(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getAliasMetadata("BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "previous_id": "previousId", - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "previous_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "loaded_at": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "aliases", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getAliasDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"), + Metadata: getAliasMetadata("BQ"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "merge_property_1_type": "user_id", - "merge_property_1_value": "userId", - "merge_property_2_type": "user_id", - "merge_property_2_value": "previousId", - }, - "metadata": map[string]any{ - "table": "rudder_identity_merge_rules", - "columns": map[string]any{"merge_property_1_type": "string", "merge_property_1_value": "string", "merge_property_2_type": "string", "merge_property_2_value": "string"}, - "isMergeRule": true, - "receivedAt": "2021-09-01T00:00:00.000Z", - "mergePropOne": "userId", - "mergePropTwo": "previousId", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getAliasDefaultMergeOutput(), + Metadata: getAliasMetadata("BQ"), StatusCode: http.StatusOK, }, }, }, }, } - - for _, tc := range testsCases { + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - c := config.New() - c.Set("DEST_TRANSFORM_URL", transformerResource.TransformerURL) - c.Set("USER_TRANSFORM_URL", transformerResource.TransformerURL) - - for k, v := range tc.configOverride { - c.Set(k, v) - } - - eventsInfos := []eventsInfo{ + c := setupConfig(transformerResource, tc.configOverride) + eventsInfos := []testhelper.EventInfo{ { - payload: []byte(tc.eventPayload), - metadata: tc.metadata, - destination: tc.destination, + Payload: []byte(tc.eventPayload), + Metadata: tc.metadata, + Destination: tc.destination, }, } destinationTransformer := ptrans.NewTransformer(c, logger.NOP, stats.Default) warehouseTransformer := New(c, logger.NOP, stats.NOP) - testEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) + testhelper.ValidateEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) }) } } + +func getAliasDefaultOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "anonymous_id": "anonymousId", + "channel": "web", + "context_ip": "1.2.3.4", + "context_passed_ip": "1.2.3.4", + "context_request_ip": "5.6.7.8", + "context_traits_email": "rhedricks@example.com", + "context_traits_logins": float64(2), + "id": "messageId", + "original_timestamp": "2021-09-01T00:00:00.000Z", + "received_at": "2021-09-01T00:00:00.000Z", + "sent_at": "2021-09-01T00:00:00.000Z", + "timestamp": "2021-09-01T00:00:00.000Z", + "title": "Home | RudderStack", + "url": "https://www.rudderstack.com", + "user_id": "userId", + "previous_id": "previousId", + "context_destination_id": "destinationID", + "context_destination_type": "POSTGRES", + "context_source_id": "sourceID", + "context_source_type": "sourceType", + }, + "metadata": map[string]any{ + "columns": map[string]any{ + "anonymous_id": "string", + "channel": "string", + "context_destination_id": "string", + "context_destination_type": "string", + "context_source_id": "string", + "context_source_type": "string", + "context_ip": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_logins": "int", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "title": "string", + "url": "string", + "user_id": "string", + "previous_id": "string", + "uuid_ts": "datetime", + }, + "receivedAt": "2021-09-01T00:00:00.000Z", + "table": "aliases", + }, + "userId": "", + } +} + +func getAliasDefaultMergeOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "merge_property_1_type": "user_id", + "merge_property_1_value": "userId", + "merge_property_2_type": "user_id", + "merge_property_2_value": "previousId", + }, + "metadata": map[string]any{ + "table": "rudder_identity_merge_rules", + "columns": map[string]any{"merge_property_1_type": "string", "merge_property_1_value": "string", "merge_property_2_type": "string", "merge_property_2_value": "string"}, + "isMergeRule": true, + "receivedAt": "2021-09-01T00:00:00.000Z", + "mergePropOne": "userId", + "mergePropTwo": "previousId", + }, + "userId": "", + } +} + +func getAliasMetadata(destinationType string) ptrans.Metadata { + return ptrans.Metadata{ + EventType: "alias", + DestinationType: destinationType, + ReceivedAt: "2021-09-01T00:00:00.000Z", + SourceID: "sourceID", + DestinationID: "destinationID", + SourceType: "sourceType", + } +} diff --git a/warehouse/transformer/debuglogger.go b/warehouse/transformer/debuglogger.go new file mode 100644 index 00000000000..dba14586f32 --- /dev/null +++ b/warehouse/transformer/debuglogger.go @@ -0,0 +1,77 @@ +package transformer + +import ( + "fmt" + "sync" + + "github.com/google/uuid" + "github.com/rudderlabs/rudder-go-kit/config" + "github.com/rudderlabs/rudder-go-kit/logger" + "github.com/rudderlabs/rudder-go-kit/stringify" + "github.com/samber/lo" + + ptrans "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/utils/misc" + "github.com/rudderlabs/rudder-server/utils/types" +) + +type DebugLogger struct { + logger logger.Logger + maxLoggedEvents config.ValueLoader[int] + eventLogMutex sync.Mutex + currentLogFileName string + loggedEvents int64 +} + +func NewDebugLogger(conf *config.Config, logger logger.Logger) *DebugLogger { + logFileName := generateLogFileName() + + return &DebugLogger{ + logger: logger.Child("debugLogger").With("currentLogFileName", logFileName), + maxLoggedEvents: conf.GetReloadableIntVar(10000, 1, "Processor.maxLoggedEvents"), + currentLogFileName: logFileName, + } +} + +func generateLogFileName() string { + return fmt.Sprintf("warehouse_transformations_debug_%s.log", uuid.NewString()) +} + +func (d *DebugLogger) LogEvents(events []types.SingularEventT, commonMedata *ptrans.Metadata) error { + d.eventLogMutex.Lock() + defer d.eventLogMutex.Unlock() + + if len(events) == 0 || d.loggedEvents >= int64(d.maxLoggedEvents.Load()) { + return nil + } + + logEntries := lo.Map(events, func(item types.SingularEventT, index int) string { + return stringify.Any(ptrans.TransformerEvent{ + Message: item, + Metadata: *commonMedata, + }) + }) + + if err := d.writeLogEntries(logEntries); err != nil { + return fmt.Errorf("logging events: %w", err) + } + + d.logger.Infon("Successfully logged events", logger.NewIntField("event_count", int64(len(logEntries)))) + d.loggedEvents += int64(len(logEntries)) + return nil +} + +func (d *DebugLogger) writeLogEntries(entries []string) error { + writer, err := misc.CreateBufferedWriter(d.currentLogFileName) + if err != nil { + return fmt.Errorf("creating buffered writer: %w", err) + } + defer func() { _ = writer.Close() }() + + for _, entry := range entries { + if _, err := writer.Write([]byte(entry + "\n")); err != nil { + return fmt.Errorf("writing log entry: %w", err) + } + } + return nil +} diff --git a/warehouse/transformer/extract.go b/warehouse/transformer/extract.go index a197c9baf3f..3a3f96e04af 100644 --- a/warehouse/transformer/extract.go +++ b/warehouse/transformer/extract.go @@ -1,11 +1,12 @@ package transformer import ( - "errors" "fmt" "github.com/rudderlabs/rudder-server/warehouse/transformer/internal/datatype" + "github.com/rudderlabs/rudder-server/warehouse/transformer/internal/response" "github.com/rudderlabs/rudder-server/warehouse/transformer/internal/rules" + "github.com/rudderlabs/rudder-server/warehouse/transformer/internal/utils" ) func (t *transformer) handleExtractEvent(pi *processingInfo) ([]map[string]any, error) { @@ -31,16 +32,9 @@ func (t *transformer) handleExtractEvent(pi *processingInfo) ([]map[string]any, return nil, fmt.Errorf("extract: safe column name: %w", err) } - d, ok := pi.event.Message[eventColName] - if !ok { - return nil, errors.New("extract: cannot create event table with empty event name, event name is missing in the payload") - } - eventName, ok := d.(string) - if !ok || len(eventName) == 0 { - return nil, errors.New("extract: cannot create event table with empty event name, event name is not a string") - } + eventName, _ := pi.event.Message[eventColName].(string) - event[eventColName] = TransformTableName(pi.event.Metadata.DestinationType, pi.itrOpts, pi.dstOpts, eventName) + event[eventColName] = TransformTableName(pi.itrOpts, pi.dstOpts, eventName) columnTypes[eventColName] = datatype.TypeString if err = t.setDataAndColumnTypeFromRules(pi, event, columnTypes, @@ -49,6 +43,10 @@ func (t *transformer) handleExtractEvent(pi *processingInfo) ([]map[string]any, return nil, fmt.Errorf("extract: setting data and column types from rules: %w", err) } + if val := event[eventColName]; val == nil || utils.IsBlank(val) { + return nil, response.ErrExtractEventNameEmpty + } + columnName := TransformColumnName(pi.event.Metadata.DestinationType, pi.itrOpts, pi.dstOpts, event[eventColName].(string)) tableName, err := SafeTableName(pi.event.Metadata.DestinationType, pi.itrOpts, columnName) if err != nil { diff --git a/warehouse/transformer/extract_test.go b/warehouse/transformer/extract_test.go index 5e2c109b384..23b41b40303 100644 --- a/warehouse/transformer/extract_test.go +++ b/warehouse/transformer/extract_test.go @@ -7,13 +7,13 @@ import ( "github.com/ory/dockertest/v3" "github.com/stretchr/testify/require" - "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" transformertest "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/transformer" backendconfig "github.com/rudderlabs/rudder-server/backend-config" ptrans "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/warehouse/transformer/testhelper" ) func TestExtract(t *testing.T) { @@ -23,7 +23,7 @@ func TestExtract(t *testing.T) { transformerResource, err := transformertest.Setup(pool, t) require.NoError(t, err) - testsCases := []struct { + testCases := []struct { name string eventPayload string metadata ptrans.Metadata @@ -33,74 +33,13 @@ func TestExtract(t *testing.T) { { name: "extract (Postgres)", eventPayload: `{"type":"extract","recordId":"recordID","event":"event","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "extract", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - RecordID: "recordID", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getExtractMetadata(), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "context_ip": "1.2.3.4", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "recordID", - "event": "event", - "received_at": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "event": "string", - "received_at": "datetime", - "title": "string", - "url": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "extract", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - RecordID: "recordID", - }, + Output: getExtractDefaultOutput(), + Metadata: getExtractMetadata(), StatusCode: http.StatusOK, }, }, @@ -109,68 +48,15 @@ func TestExtract(t *testing.T) { { name: "extract (Postgres) without properties", eventPayload: `{"type":"extract","recordId":"recordID","event":"event","receivedAt":"2021-09-01T00:00:00.000Z","context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "extract", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - RecordID: "recordID", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getExtractMetadata(), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "context_ip": "1.2.3.4", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "recordID", - "event": "event", - "received_at": "2021-09-01T00:00:00.000Z", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "event": "string", - "received_at": "datetime", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "extract", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - RecordID: "recordID", - }, + Output: getExtractDefaultOutput(). + RemoveDataFields("name", "title", "url"). + RemoveColumnFields("name", "title", "url"), + Metadata: getExtractMetadata(), StatusCode: http.StatusOK, }, }, @@ -179,66 +65,16 @@ func TestExtract(t *testing.T) { { name: "extract (Postgres) without context", eventPayload: `{"type":"extract","recordId":"recordID","event":"event","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"}}`, - metadata: ptrans.Metadata{ - EventType: "extract", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - RecordID: "recordID", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getExtractMetadata(), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "id": "recordID", - "event": "event", - "received_at": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "id": "string", - "event": "string", - "received_at": "datetime", - "title": "string", - "url": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "extract", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - RecordID: "recordID", - }, + Output: getExtractDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). // overriding the default value + RemoveDataFields("context_ip", "context_traits_email", "context_traits_logins", "context_traits_name"). + RemoveColumnFields("context_ip", "context_traits_email", "context_traits_logins", "context_traits_name"), + Metadata: getExtractMetadata(), StatusCode: http.StatusOK, }, }, @@ -247,76 +83,17 @@ func TestExtract(t *testing.T) { { name: "extract (Postgres) RudderCreatedTable", eventPayload: `{"type":"extract","recordId":"recordID","event":"accounts","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "extract", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - RecordID: "recordID", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "storeFullEvent": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getExtractMetadata(), + destination: getDestination("POSTGRES", map[string]any{ + "storeFullEvent": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "context_ip": "1.2.3.4", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "recordID", - "event": "accounts", - "received_at": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "event": "string", - "received_at": "datetime", - "title": "string", - "url": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "_accounts", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "extract", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - RecordID: "recordID", - }, + Output: getExtractDefaultOutput(). + SetDataField("event", "accounts"). + SetTableName("_accounts"), + Metadata: getExtractMetadata(), StatusCode: http.StatusOK, }, }, @@ -325,74 +102,15 @@ func TestExtract(t *testing.T) { { name: "extract (Postgres) RudderCreatedTable with skipReservedKeywordsEscaping", eventPayload: `{"type":"extract","recordId":"recordID","event":"accounts","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipReservedKeywordsEscaping":true}}}}`, - metadata: ptrans.Metadata{ - EventType: "extract", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - RecordID: "recordID", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getExtractMetadata(), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "context_ip": "1.2.3.4", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "recordID", - "event": "accounts", - "received_at": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "event": "string", - "received_at": "datetime", - "title": "string", - "url": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "accounts", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "extract", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - RecordID: "recordID", - }, + Output: getExtractDefaultOutput(). + SetDataField("event", "accounts"). + SetTableName("accounts"), + Metadata: getExtractMetadata(), StatusCode: http.StatusOK, }, }, @@ -401,98 +119,90 @@ func TestExtract(t *testing.T) { { name: "extract (Postgres) RudderIsolatedTable", eventPayload: `{"type":"extract","recordId":"recordID","event":"users","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "extract", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - RecordID: "recordID", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getExtractMetadata(), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "context_ip": "1.2.3.4", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "recordID", - "event": "users", - "received_at": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "event": "string", - "received_at": "datetime", - "title": "string", - "url": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "_users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "extract", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - RecordID: "recordID", - }, + Output: getExtractDefaultOutput(). + SetDataField("event", "users"). + SetTableName("_users"), + Metadata: getExtractMetadata(), StatusCode: http.StatusOK, }, }, }, }, } - - for _, tc := range testsCases { + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - c := config.New() - c.Set("DEST_TRANSFORM_URL", transformerResource.TransformerURL) - c.Set("USER_TRANSFORM_URL", transformerResource.TransformerURL) - - eventsInfos := []eventsInfo{ + c := setupConfig(transformerResource, map[string]any{}) + eventsInfos := []testhelper.EventInfo{ { - payload: []byte(tc.eventPayload), - metadata: tc.metadata, - destination: tc.destination, + Payload: []byte(tc.eventPayload), + Metadata: tc.metadata, + Destination: tc.destination, }, } destinationTransformer := ptrans.NewTransformer(c, logger.NOP, stats.Default) warehouseTransformer := New(c, logger.NOP, stats.NOP) - testEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) + testhelper.ValidateEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) }) } } + +func getExtractDefaultOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "name": "Home", + "context_ip": "1.2.3.4", + "context_traits_email": "rhedricks@example.com", + "context_traits_logins": float64(2), + "context_traits_name": "Richard Hendricks", + "id": "recordID", + "event": "event", + "received_at": "2021-09-01T00:00:00.000Z", + "title": "Home | RudderStack", + "url": "https://www.rudderstack.com", + "context_destination_id": "destinationID", + "context_destination_type": "POSTGRES", + "context_source_id": "sourceID", + "context_source_type": "sourceType", + }, + "metadata": map[string]any{ + "columns": map[string]any{ + "name": "string", + "context_destination_id": "string", + "context_destination_type": "string", + "context_source_id": "string", + "context_source_type": "string", + "context_ip": "string", + "context_traits_email": "string", + "context_traits_logins": "int", + "context_traits_name": "string", + "id": "string", + "event": "string", + "received_at": "datetime", + "title": "string", + "url": "string", + "uuid_ts": "datetime", + }, + "receivedAt": "2021-09-01T00:00:00.000Z", + "table": "event", + }, + "userId": "", + } +} + +func getExtractMetadata() ptrans.Metadata { + return ptrans.Metadata{ + EventType: "extract", + DestinationType: "POSTGRES", + ReceivedAt: "2021-09-01T00:00:00.000Z", + SourceID: "sourceID", + DestinationID: "destinationID", + SourceType: "sourceType", + RecordID: "recordID", + } +} diff --git a/warehouse/transformer/getcolumns_test.go b/warehouse/transformer/getcolumns_test.go new file mode 100644 index 00000000000..a147004a22e --- /dev/null +++ b/warehouse/transformer/getcolumns_test.go @@ -0,0 +1,106 @@ +package transformer + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/rudderlabs/rudder-go-kit/config" + + whutils "github.com/rudderlabs/rudder-server/warehouse/utils" +) + +func TestGetColumns(t *testing.T) { + testCases := []struct { + name string + destType string + data map[string]any + columnTypes map[string]string + maxColumns int32 + expected map[string]any + wantError bool + }{ + { + name: "Basic data types", + destType: whutils.POSTGRES, + data: map[string]any{ + "field1": "value1", "field2": 123, "field3": true, + }, + columnTypes: map[string]string{ + "field1": "string", "field2": "int", + }, + maxColumns: 10, + expected: map[string]any{ + "uuid_ts": "datetime", "field1": "string", "field2": "int", "field3": "boolean", + }, + }, + { + name: "Basic data types (BQ)", + destType: whutils.BQ, + data: map[string]any{ + "field1": "value1", "field2": 123, "field3": true, + }, + columnTypes: map[string]string{ + "field1": "string", "field2": "int", + }, + maxColumns: 10, + expected: map[string]any{ + "uuid_ts": "datetime", "field1": "string", "field2": "int", "field3": "boolean", "loaded_at": "datetime", + }, + }, + { + name: "Basic data types (SNOWFLAKE)", + destType: whutils.SNOWFLAKE, + data: map[string]any{ + "FIELD1": "value1", "FIELD2": 123, "FIELD3": true, + }, + columnTypes: map[string]string{ + "FIELD1": "string", "FIELD2": "int", + }, + maxColumns: 10, + expected: map[string]any{ + "UUID_TS": "datetime", "FIELD1": "string", "FIELD2": "int", "FIELD3": "boolean", + }, + }, + { + name: "Key not in columnTypes", + destType: whutils.POSTGRES, + data: map[string]any{ + "field1": "value1", "field2": 123, "field3": true, + }, + columnTypes: map[string]string{}, + maxColumns: 10, + expected: map[string]any{ + "uuid_ts": "datetime", "field1": "string", "field2": "int", "field3": "boolean", + }, + }, + { + name: "Too many columns", + destType: whutils.POSTGRES, + data: map[string]any{ + "field1": "value1", "field2": 123, "field3": true, "field4": "extra", + }, + columnTypes: map[string]string{ + "field1": "string", "field2": "int", + }, + maxColumns: 3, + expected: nil, + wantError: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + trans := &transformer{} + trans.config.maxColumnsInEvent = config.SingleValueLoader(int(tc.maxColumns)) + + columns, err := trans.getColumns(tc.destType, tc.data, tc.columnTypes) + if tc.wantError { + require.Error(t, err) + require.Nil(t, columns) + return + } + require.NoError(t, err) + require.Equal(t, tc.expected, columns) + }) + } +} diff --git a/warehouse/transformer/getdatatype_test.go b/warehouse/transformer/getdatatype_test.go index ae3255e3a67..2e17d49d267 100644 --- a/warehouse/transformer/getdatatype_test.go +++ b/warehouse/transformer/getdatatype_test.go @@ -34,11 +34,13 @@ func TestGetDataType(t *testing.T) { // Redshift with text and string types {"Redshift Text Type", whutils.RS, "someKey", string(make([]byte, 513)), false, false, "text"}, {"Redshift String Type", whutils.RS, "someKey", "shortValue", false, false, "string"}, + {"Redshift String Type", whutils.RS, "someKey", nil, false, false, "string"}, // ClickHouse - Array support enabled {"ClickHouse Array Type Int", whutils.CLICKHOUSE, "someKey", []any{1, 2, 3}, false, true, "array(int)"}, {"ClickHouse Array Type Mixed Int and Float", whutils.CLICKHOUSE, "someKey", []any{1, 2.5}, false, true, "array(float)"}, {"ClickHouse Array Type Mixed Int, Float, and String", whutils.CLICKHOUSE, "someKey", []any{1, 2.5, "text"}, false, true, "array(string)"}, + {"ClickHouse Array Type Mixed String, Int and Float", whutils.CLICKHOUSE, "someKey", []any{"text", 1, 2.5}, false, true, "array(string)"}, {"ClickHouse Array Type All Strings", whutils.CLICKHOUSE, "someKey", []any{"one", "two"}, false, true, "array(string)"}, {"ClickHouse Empty Array", whutils.CLICKHOUSE, "someKey", []any{}, false, true, "string"}, // Empty array should return "string" {"ClickHouse Array Type All Floats", whutils.CLICKHOUSE, "someKey", []any{1.1, 2.2, 3.3}, false, true, "array(float)"}, diff --git a/warehouse/transformer/group_test.go b/warehouse/transformer/group_test.go index aeb52103137..e24475e8d68 100644 --- a/warehouse/transformer/group_test.go +++ b/warehouse/transformer/group_test.go @@ -7,13 +7,13 @@ import ( "github.com/ory/dockertest/v3" "github.com/stretchr/testify/require" - "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" transformertest "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/transformer" backendconfig "github.com/rudderlabs/rudder-server/backend-config" ptrans "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/warehouse/transformer/testhelper" ) func TestGroup(t *testing.T) { @@ -23,7 +23,7 @@ func TestGroup(t *testing.T) { transformerResource, err := transformertest.Setup(pool, t) require.NoError(t, err) - testsCases := []struct { + testCases := []struct { name string configOverride map[string]any eventPayload string @@ -34,84 +34,13 @@ func TestGroup(t *testing.T) { { name: "group (Postgres)", eventPayload: `{"type":"group","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getGroupMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "group_id": "groupId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "group_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "groups", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getGroupDefaultOutput(), + Metadata: getGroupMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -120,80 +49,15 @@ func TestGroup(t *testing.T) { { name: "group (Postgres) without traits", eventPayload: `{"type":"group","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getGroupMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - "group_id": "groupId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "group_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "groups", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getGroupDefaultOutput(). + RemoveDataFields("title", "url"). + RemoveColumnFields("title", "url"), + Metadata: getGroupMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -202,78 +66,16 @@ func TestGroup(t *testing.T) { { name: "group (Postgres) without context", eventPayload: `{"type":"group","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"}}`, - metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getGroupMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "5.6.7.8", - "context_request_ip": "5.6.7.8", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "group_id": "groupId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_request_ip": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "group_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "groups", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getGroupDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). // overriding the default value + RemoveDataFields("context_passed_ip", "context_traits_email", "context_traits_logins"). + RemoveColumnFields("context_passed_ip", "context_traits_email", "context_traits_logins"), + Metadata: getGroupMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -282,88 +84,17 @@ func TestGroup(t *testing.T) { { name: "group (Postgres) store rudder event", eventPayload: `{"type":"group","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "storeFullEvent": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getGroupMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "storeFullEvent": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "group_id": "groupId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "rudder_event": "{\"type\":\"group\",\"anonymousId\":\"anonymousId\",\"channel\":\"web\",\"context\":{\"destinationId\":\"destinationID\",\"destinationType\":\"POSTGRES\",\"ip\":\"1.2.3.4\",\"sourceId\":\"sourceID\",\"sourceType\":\"sourceType\",\"traits\":{\"email\":\"rhedricks@example.com\",\"logins\":2}},\"groupId\":\"groupId\",\"messageId\":\"messageId\",\"originalTimestamp\":\"2021-09-01T00:00:00.000Z\",\"receivedAt\":\"2021-09-01T00:00:00.000Z\",\"request_ip\":\"5.6.7.8\",\"sentAt\":\"2021-09-01T00:00:00.000Z\",\"timestamp\":\"2021-09-01T00:00:00.000Z\",\"traits\":{\"title\":\"Home | RudderStack\",\"url\":\"https://www.rudderstack.com\"},\"userId\":\"userId\"}", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "group_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "rudder_event": "json", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "groups", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getGroupDefaultOutput(). + SetDataField("rudder_event", "{\"type\":\"group\",\"anonymousId\":\"anonymousId\",\"channel\":\"web\",\"context\":{\"destinationId\":\"destinationID\",\"destinationType\":\"POSTGRES\",\"ip\":\"1.2.3.4\",\"sourceId\":\"sourceID\",\"sourceType\":\"sourceType\",\"traits\":{\"email\":\"rhedricks@example.com\",\"logins\":2}},\"groupId\":\"groupId\",\"messageId\":\"messageId\",\"originalTimestamp\":\"2021-09-01T00:00:00.000Z\",\"receivedAt\":\"2021-09-01T00:00:00.000Z\",\"request_ip\":\"5.6.7.8\",\"sentAt\":\"2021-09-01T00:00:00.000Z\",\"timestamp\":\"2021-09-01T00:00:00.000Z\",\"traits\":{\"title\":\"Home | RudderStack\",\"url\":\"https://www.rudderstack.com\"},\"userId\":\"userId\"}"). + SetColumnField("rudder_event", "json"), + Metadata: getGroupMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -372,78 +103,15 @@ func TestGroup(t *testing.T) { { name: "group (Postgres) partial rules", eventPayload: `{"type":"group","messageId":"messageId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getGroupMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "group_id": "groupId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_ip": "string", - "context_passed_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "group_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "groups", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getGroupDefaultOutput(). + RemoveDataFields("anonymous_id", "channel", "context_request_ip"). + RemoveColumnFields("anonymous_id", "channel", "context_request_ip"), + Metadata: getGroupMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -455,141 +123,127 @@ func TestGroup(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"group","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getGroupMetadata("BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "group_id": "groupId", - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "group_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "loaded_at": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "_groups", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getGroupDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"). + SetTableName("_groups"), + Metadata: getGroupMetadata("BQ"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "merge_property_1_type": "anonymous_id", - "merge_property_1_value": "anonymousId", - "merge_property_2_type": "user_id", - "merge_property_2_value": "userId", - }, - "metadata": map[string]any{ - "table": "rudder_identity_merge_rules", - "columns": map[string]any{"merge_property_1_type": "string", "merge_property_1_value": "string", "merge_property_2_type": "string", "merge_property_2_value": "string"}, - "isMergeRule": true, - "receivedAt": "2021-09-01T00:00:00.000Z", - "mergePropOne": "anonymousId", - "mergePropTwo": "userId", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "group", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getGroupDefaultMergeOutput(), + Metadata: getGroupMetadata("BQ"), StatusCode: http.StatusOK, }, }, }, }, } - - for _, tc := range testsCases { + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - c := config.New() - c.Set("DEST_TRANSFORM_URL", transformerResource.TransformerURL) - c.Set("USER_TRANSFORM_URL", transformerResource.TransformerURL) - - for k, v := range tc.configOverride { - c.Set(k, v) - } - - eventsInfos := []eventsInfo{ + c := setupConfig(transformerResource, tc.configOverride) + eventsInfos := []testhelper.EventInfo{ { - payload: []byte(tc.eventPayload), - metadata: tc.metadata, - destination: tc.destination, + Payload: []byte(tc.eventPayload), + Metadata: tc.metadata, + Destination: tc.destination, }, } destinationTransformer := ptrans.NewTransformer(c, logger.NOP, stats.Default) warehouseTransformer := New(c, logger.NOP, stats.NOP) - testEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) + testhelper.ValidateEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) }) } } + +func getGroupDefaultOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "anonymous_id": "anonymousId", + "channel": "web", + "context_ip": "1.2.3.4", + "context_passed_ip": "1.2.3.4", + "context_request_ip": "5.6.7.8", + "context_traits_email": "rhedricks@example.com", + "context_traits_logins": float64(2), + "id": "messageId", + "original_timestamp": "2021-09-01T00:00:00.000Z", + "received_at": "2021-09-01T00:00:00.000Z", + "sent_at": "2021-09-01T00:00:00.000Z", + "timestamp": "2021-09-01T00:00:00.000Z", + "title": "Home | RudderStack", + "url": "https://www.rudderstack.com", + "user_id": "userId", + "group_id": "groupId", + "context_destination_id": "destinationID", + "context_destination_type": "POSTGRES", + "context_source_id": "sourceID", + "context_source_type": "sourceType", + }, + "metadata": map[string]any{ + "columns": map[string]any{ + "anonymous_id": "string", + "channel": "string", + "context_destination_id": "string", + "context_destination_type": "string", + "context_source_id": "string", + "context_source_type": "string", + "context_ip": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_logins": "int", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "title": "string", + "url": "string", + "user_id": "string", + "group_id": "string", + "uuid_ts": "datetime", + }, + "receivedAt": "2021-09-01T00:00:00.000Z", + "table": "groups", + }, + "userId": "", + } +} + +func getGroupDefaultMergeOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "merge_property_1_type": "anonymous_id", + "merge_property_1_value": "anonymousId", + "merge_property_2_type": "user_id", + "merge_property_2_value": "userId", + }, + "metadata": map[string]any{ + "table": "rudder_identity_merge_rules", + "columns": map[string]any{"merge_property_1_type": "string", "merge_property_1_value": "string", "merge_property_2_type": "string", "merge_property_2_value": "string"}, + "isMergeRule": true, + "receivedAt": "2021-09-01T00:00:00.000Z", + "mergePropOne": "anonymousId", + "mergePropTwo": "userId", + }, + "userId": "", + } +} + +func getGroupMetadata(destinationType string) ptrans.Metadata { + return ptrans.Metadata{ + EventType: "group", + DestinationType: destinationType, + ReceivedAt: "2021-09-01T00:00:00.000Z", + SourceID: "sourceID", + DestinationID: "destinationID", + SourceType: "sourceType", + } +} diff --git a/warehouse/transformer/identify_test.go b/warehouse/transformer/identify_test.go index 896109e62db..8a95125b893 100644 --- a/warehouse/transformer/identify_test.go +++ b/warehouse/transformer/identify_test.go @@ -7,13 +7,13 @@ import ( "github.com/ory/dockertest/v3" "github.com/stretchr/testify/require" - "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" transformertest "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/transformer" backendconfig "github.com/rudderlabs/rudder-server/backend-config" ptrans "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/warehouse/transformer/testhelper" ) func TestIdentify(t *testing.T) { @@ -23,7 +23,7 @@ func TestIdentify(t *testing.T) { transformerResource, err := transformertest.Setup(pool, t) require.NoError(t, err) - testsCases := []struct { + testCases := []struct { name string configOverride map[string]any eventPayload string @@ -34,166 +34,20 @@ func TestIdentify(t *testing.T) { { name: "identify (POSTGRES)", eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "allowUsersContextTraits": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getIdentifyMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "allowUsersContextTraits": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "messageId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "userId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getUserDefaultOutput(), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -202,160 +56,28 @@ func TestIdentify(t *testing.T) { { name: "identify (S3_DATALAKE)", eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "S3_DATALAKE", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "S3_DATALAKE", - Config: map[string]any{ - "allowUsersContextTraits": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "S3_DATALAKE", - }, - }, + metadata: getIdentifyMetadata("S3_DATALAKE"), + destination: getDestination("S3_DATALAKE", map[string]any{ + "allowUsersContextTraits": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "S3_DATALAKE", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "messageId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "_timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "_timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "S3_DATALAKE", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(). + SetDataField("_timestamp", "2021-09-01T00:00:00.000Z"). + SetColumnField("_timestamp", "datetime"). + RemoveDataFields("timestamp"). + RemoveColumnFields("timestamp"). + SetDataField("context_destination_type", "S3_DATALAKE"), + Metadata: getIdentifyMetadata("S3_DATALAKE"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "S3_DATALAKE", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "userId", - "logins": float64(2), - "name": "Richard Hendricks", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "S3_DATALAKE", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getUserDefaultOutput(). + RemoveDataFields("timestamp", "original_timestamp", "sent_at"). + RemoveColumnFields("timestamp", "original_timestamp", "sent_at"). + SetDataField("context_destination_type", "S3_DATALAKE"), + Metadata: getIdentifyMetadata("S3_DATALAKE"), StatusCode: http.StatusOK, }, }, @@ -364,158 +86,24 @@ func TestIdentify(t *testing.T) { { name: "identify (POSTGRES) without traits", eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "allowUsersContextTraits": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getIdentifyMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "allowUsersContextTraits": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "messageId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(). + RemoveDataFields("product_id", "review_id"). + RemoveColumnFields("product_id", "review_id"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "userId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getUserDefaultOutput(). + RemoveDataFields("product_id", "review_id"). + RemoveColumnFields("product_id", "review_id"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -524,158 +112,24 @@ func TestIdentify(t *testing.T) { { name: "identify (POSTGRES) without userProperties", eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "allowUsersContextTraits": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getIdentifyMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "allowUsersContextTraits": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "messageId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "received_at": "2021-09-01T00:00:00.000Z", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "received_at": "datetime", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(). + RemoveDataFields("rating", "review_body"). + RemoveColumnFields("rating", "review_body"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "userId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "received_at": "2021-09-01T00:00:00.000Z", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "received_at": "datetime", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getUserDefaultOutput(). + RemoveDataFields("rating", "review_body"). + RemoveColumnFields("rating", "review_body"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -684,142 +138,24 @@ func TestIdentify(t *testing.T) { { name: "identify (POSTGRES) without context.traits", eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "allowUsersContextTraits": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getIdentifyMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "allowUsersContextTraits": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(). + RemoveDataFields("context_traits_email", "context_traits_logins", "context_traits_name", "email", "logins", "name"). + RemoveColumnFields("context_traits_email", "context_traits_logins", "context_traits_name", "email", "logins", "name"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "id": "userId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getUserDefaultOutput(). + RemoveDataFields("context_traits_email", "context_traits_logins", "context_traits_name", "email", "logins", "name"). + RemoveColumnFields("context_traits_email", "context_traits_logins", "context_traits_name", "email", "logins", "name"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -828,138 +164,26 @@ func TestIdentify(t *testing.T) { { name: "identify (POSTGRES) without context", eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "allowUsersContextTraits": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getIdentifyMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "allowUsersContextTraits": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "5.6.7.8", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). // overriding the default value + RemoveDataFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name", "email", "logins", "name"). + RemoveColumnFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name", "email", "logins", "name"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "5.6.7.8", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "id": "userId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getUserDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). // overriding the default value + RemoveDataFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name", "email", "logins", "name"). + RemoveColumnFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name", "email", "logins", "name"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -968,298 +192,22 @@ func TestIdentify(t *testing.T) { { name: "identify (POSTGRES) not allowUsersContextTraits", eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, - expectedResponse: ptrans.Response{ - Events: []ptrans.TransformerResponse{ - { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - StatusCode: http.StatusOK, - }, - { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "userId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - StatusCode: http.StatusOK, - }, - }, - }, - }, - { - name: "identify (POSTGRES) context.traits not map", - eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":"traits","ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getIdentifyMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits": "traits", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(). + RemoveDataFields("email", "logins", "name"). + RemoveColumnFields("email", "logins", "name"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits": "traits", - "id": "userId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getUserDefaultOutput(). + RemoveDataFields("email", "logins", "name"). + RemoveColumnFields("email", "logins", "name"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -1268,166 +216,20 @@ func TestIdentify(t *testing.T) { { name: "identify (POSTGRES) user_id already exists", eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"user_id":"user_id","rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "allowUsersContextTraits": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getIdentifyMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "allowUsersContextTraits": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "messageId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "userId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getUserDefaultOutput(), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -1436,169 +238,23 @@ func TestIdentify(t *testing.T) { { name: "identify (POSTGRES) store rudder event", eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "storeFullEvent": true, - "allowUsersContextTraits": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getIdentifyMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "storeFullEvent": true, + "allowUsersContextTraits": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "messageId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - "rudder_event": "{\"type\":\"identify\",\"anonymousId\":\"anonymousId\",\"channel\":\"web\",\"context\":{\"destinationId\":\"destinationID\",\"destinationType\":\"POSTGRES\",\"ip\":\"1.2.3.4\",\"sourceId\":\"sourceID\",\"sourceType\":\"sourceType\",\"traits\":{\"email\":\"rhedricks@example.com\",\"logins\":2,\"name\":\"Richard Hendricks\"}},\"messageId\":\"messageId\",\"originalTimestamp\":\"2021-09-01T00:00:00.000Z\",\"receivedAt\":\"2021-09-01T00:00:00.000Z\",\"request_ip\":\"5.6.7.8\",\"sentAt\":\"2021-09-01T00:00:00.000Z\",\"timestamp\":\"2021-09-01T00:00:00.000Z\",\"traits\":{\"product_id\":\"9578257311\",\"review_id\":\"86ac1cd43\"},\"userId\":\"userId\",\"userProperties\":{\"rating\":3,\"review_body\":\"OK for the price. It works but the material feels flimsy.\"}}", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - "rudder_event": "json", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(). + SetDataField("rudder_event", "{\"anonymousId\":\"anonymousId\",\"channel\":\"web\",\"context\":{\"ip\":\"1.2.3.4\",\"traits\":{\"email\":\"rhedricks@example.com\",\"logins\":2,\"name\":\"Richard Hendricks\"},\"sourceId\":\"sourceID\",\"sourceType\":\"sourceType\",\"destinationId\":\"destinationID\",\"destinationType\":\"POSTGRES\"},\"messageId\":\"messageId\",\"originalTimestamp\":\"2021-09-01T00:00:00.000Z\",\"receivedAt\":\"2021-09-01T00:00:00.000Z\",\"request_ip\":\"5.6.7.8\",\"sentAt\":\"2021-09-01T00:00:00.000Z\",\"timestamp\":\"2021-09-01T00:00:00.000Z\",\"traits\":{\"product_id\":\"9578257311\",\"review_id\":\"86ac1cd43\"},\"type\":\"identify\",\"userId\":\"userId\",\"userProperties\":{\"rating\":3,\"review_body\":\"OK for the price. It works but the material feels flimsy.\"}}"). + SetColumnField("rudder_event", "json"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "userId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getUserDefaultOutput(), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -1607,158 +263,24 @@ func TestIdentify(t *testing.T) { { name: "identify (POSTGRES) partial rules", eventPayload: `{"type":"identify","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "allowUsersContextTraits": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getIdentifyMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "allowUsersContextTraits": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "messageId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(). + RemoveDataFields("anonymous_id", "channel", "context_request_ip"). + RemoveColumnFields("anonymous_id", "channel", "context_request_ip"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "userId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getUserDefaultOutput(). + RemoveDataFields("context_request_ip"). + RemoveColumnFields("context_request_ip"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -1767,96 +289,17 @@ func TestIdentify(t *testing.T) { { name: "identify (POSTGRES) no userID", eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "allowUsersContextTraits": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getIdentifyMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "allowUsersContextTraits": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "messageId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(). + RemoveDataFields("user_id"). + RemoveColumnFields("user_id"), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -1865,15 +308,7 @@ func TestIdentify(t *testing.T) { { name: "identify (POSTGRES) skipUsersTable (dstOpts)", eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + metadata: getIdentifyMetadata("POSTGRES"), destination: backendconfig.DestinationT{ Name: "POSTGRES", Config: map[string]any{ @@ -1887,77 +322,8 @@ func TestIdentify(t *testing.T) { expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "messageId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -1966,98 +332,15 @@ func TestIdentify(t *testing.T) { { name: "identify (POSTGRES) skipUsersTable (itrOpts)", eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipUsersTable":true}}}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "allowUsersContextTraits": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getIdentifyMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "allowUsersContextTraits": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "messageId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(), + Metadata: getIdentifyMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -2069,230 +352,210 @@ func TestIdentify(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipUsersTable":true}}}}`, - metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{ - "allowUsersContextTraits": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getIdentifyMetadata("BQ"), + destination: getDestination("BQ", map[string]any{ + "allowUsersContextTraits": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "messageId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - "loaded_at": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "identifies", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"), + Metadata: getIdentifyMetadata("BQ"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "merge_property_1_type": "anonymous_id", - "merge_property_1_value": "anonymousId", - "merge_property_2_type": "user_id", - "merge_property_2_value": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "merge_property_1_type": "string", - "merge_property_1_value": "string", - "merge_property_2_type": "string", - "merge_property_2_value": "string", - }, - "isMergeRule": true, - "mergePropOne": "anonymousId", - "mergePropTwo": "userId", - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "rudder_identity_merge_rules", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getIdentifyDefaultMergeOutput(), + Metadata: getIdentifyMetadata("BQ"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "email": "rhedricks@example.com", - "id": "userId", - "logins": float64(2), - "name": "Richard Hendricks", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "email": "string", - "id": "string", - "logins": "int", - "name": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "uuid_ts": "datetime", - "loaded_at": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "identify", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getUserDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"), + Metadata: getIdentifyMetadata("BQ"), StatusCode: http.StatusOK, }, }, }, }, } - - for _, tc := range testsCases { + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - c := config.New() - c.Set("DEST_TRANSFORM_URL", transformerResource.TransformerURL) - c.Set("USER_TRANSFORM_URL", transformerResource.TransformerURL) - - for k, v := range tc.configOverride { - c.Set(k, v) - } - - eventsInfos := []eventsInfo{ + c := setupConfig(transformerResource, tc.configOverride) + eventsInfos := []testhelper.EventInfo{ { - payload: []byte(tc.eventPayload), - metadata: tc.metadata, - destination: tc.destination, + Payload: []byte(tc.eventPayload), + Metadata: tc.metadata, + Destination: tc.destination, }, } destinationTransformer := ptrans.NewTransformer(c, logger.NOP, stats.Default) warehouseTransformer := New(c, logger.NOP, stats.NOP) - testEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) + testhelper.ValidateEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) }) } } + +func getIdentifyDefaultOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "anonymous_id": "anonymousId", + "channel": "web", + "context_destination_id": "destinationID", + "context_destination_type": "POSTGRES", + "context_ip": "1.2.3.4", + "context_passed_ip": "1.2.3.4", + "context_request_ip": "5.6.7.8", + "context_source_id": "sourceID", + "context_source_type": "sourceType", + "context_traits_email": "rhedricks@example.com", + "context_traits_logins": float64(2), + "context_traits_name": "Richard Hendricks", + "email": "rhedricks@example.com", + "id": "messageId", + "logins": float64(2), + "name": "Richard Hendricks", + "original_timestamp": "2021-09-01T00:00:00.000Z", + "product_id": "9578257311", + "rating": 3.0, + "received_at": "2021-09-01T00:00:00.000Z", + "review_body": "OK for the price. It works but the material feels flimsy.", + "review_id": "86ac1cd43", + "sent_at": "2021-09-01T00:00:00.000Z", + "timestamp": "2021-09-01T00:00:00.000Z", + "user_id": "userId", + }, + "metadata": map[string]any{ + "columns": map[string]any{ + "anonymous_id": "string", + "channel": "string", + "context_destination_id": "string", + "context_destination_type": "string", + "context_ip": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_source_id": "string", + "context_source_type": "string", + "context_traits_email": "string", + "context_traits_logins": "int", + "context_traits_name": "string", + "email": "string", + "id": "string", + "logins": "int", + "name": "string", + "original_timestamp": "datetime", + "product_id": "string", + "rating": "int", + "received_at": "datetime", + "review_body": "string", + "review_id": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime", + }, + "receivedAt": "2021-09-01T00:00:00.000Z", + "table": "identifies", + }, + "userId": "", + } +} + +func getUserDefaultOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "context_destination_id": "destinationID", + "context_destination_type": "POSTGRES", + "context_ip": "1.2.3.4", + "context_passed_ip": "1.2.3.4", + "context_request_ip": "5.6.7.8", + "context_source_id": "sourceID", + "context_source_type": "sourceType", + "context_traits_email": "rhedricks@example.com", + "context_traits_logins": float64(2), + "context_traits_name": "Richard Hendricks", + "email": "rhedricks@example.com", + "id": "userId", + "logins": float64(2), + "name": "Richard Hendricks", + "original_timestamp": "2021-09-01T00:00:00.000Z", + "product_id": "9578257311", + "rating": 3.0, + "received_at": "2021-09-01T00:00:00.000Z", + "review_body": "OK for the price. It works but the material feels flimsy.", + "review_id": "86ac1cd43", + "sent_at": "2021-09-01T00:00:00.000Z", + "timestamp": "2021-09-01T00:00:00.000Z", + }, + "metadata": map[string]any{ + "columns": map[string]any{ + "context_destination_id": "string", + "context_destination_type": "string", + "context_ip": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_source_id": "string", + "context_source_type": "string", + "context_traits_email": "string", + "context_traits_logins": "int", + "context_traits_name": "string", + "email": "string", + "id": "string", + "logins": "int", + "name": "string", + "original_timestamp": "datetime", + "product_id": "string", + "rating": "int", + "received_at": "datetime", + "review_body": "string", + "review_id": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "uuid_ts": "datetime", + }, + "receivedAt": "2021-09-01T00:00:00.000Z", + "table": "users", + }, + "userId": "", + } +} + +func getIdentifyDefaultMergeOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "merge_property_1_type": "anonymous_id", + "merge_property_1_value": "anonymousId", + "merge_property_2_type": "user_id", + "merge_property_2_value": "userId", + }, + "metadata": map[string]any{ + "columns": map[string]any{ + "merge_property_1_type": "string", + "merge_property_1_value": "string", + "merge_property_2_type": "string", + "merge_property_2_value": "string", + }, + "isMergeRule": true, + "mergePropOne": "anonymousId", + "mergePropTwo": "userId", + "receivedAt": "2021-09-01T00:00:00.000Z", + "table": "rudder_identity_merge_rules", + }, + "userId": "", + } +} + +func getIdentifyMetadata(destinationType string) ptrans.Metadata { + return ptrans.Metadata{ + EventType: "identify", + DestinationType: destinationType, + ReceivedAt: "2021-09-01T00:00:00.000Z", + SourceID: "sourceID", + DestinationID: "destinationID", + SourceType: "sourceType", + MessageID: "messageId", + } +} diff --git a/warehouse/transformer/internal/response/response.go b/warehouse/transformer/internal/response/response.go index 5ac1759eed3..92ef41337f4 100644 --- a/warehouse/transformer/internal/response/response.go +++ b/warehouse/transformer/internal/response/response.go @@ -25,6 +25,8 @@ var ( ErrEmptyTableName = NewTransformerError("Table name cannot be empty.", http.StatusBadRequest) ErrEmptyColumnName = NewTransformerError("Column name cannot be empty.", http.StatusBadRequest) ErrRecordIDEmpty = NewTransformerError("recordId cannot be empty for cloud sources events", http.StatusBadRequest) + ErrContextNotMap = NewTransformerError("context is not a map", http.StatusInternalServerError) + ErrExtractEventNameEmpty = NewTransformerError("cannot create event table with empty event name, event name is missing in the payload", http.StatusInternalServerError) ErrRecordIDObject = ErrRecordIDEmpty ) diff --git a/warehouse/transformer/internal/rules/rules.go b/warehouse/transformer/internal/rules/rules.go index 04c69bc784b..f550b75868e 100644 --- a/warehouse/transformer/internal/rules/rules.go +++ b/warehouse/transformer/internal/rules/rules.go @@ -39,10 +39,12 @@ func createReservedColumns(rules, functionRules []string) map[string]struct{} { } func IsRudderReservedColumn(eventType, columnName string) bool { - if _, ok := rudderReservedColumns[strings.ToLower(eventType)]; !ok { + lowerEventType := strings.ToLower(eventType) + if _, ok := rudderReservedColumns[lowerEventType]; !ok { return false } - if _, ok := rudderReservedColumns[strings.ToLower(eventType)][strings.ToLower(columnName)]; ok { + lowerColumnName := strings.ToLower(columnName) + if _, ok := rudderReservedColumns[lowerEventType][lowerColumnName]; ok { return true } return false diff --git a/warehouse/transformer/internal/rules/rules_test.go b/warehouse/transformer/internal/rules/rules_test.go index c18e1e3e536..d5f56c10da3 100644 --- a/warehouse/transformer/internal/rules/rules_test.go +++ b/warehouse/transformer/internal/rules/rules_test.go @@ -85,41 +85,45 @@ func TestExtractCloudRecordID(t *testing.T) { } func TestFunctionalRules(t *testing.T) { - t.Run("default", func(t *testing.T) { - testCases := []struct { - name string - functionalRule FunctionalRules - event ptrans.TransformerEvent - expected any - expectedError error - }{ - {name: "default (context.ip)", functionalRule: DefaultFunctionalRules["context_ip"], event: ptrans.TransformerEvent{Message: map[string]any{"context": map[string]any{"ip": "1.2.3.4"}}}, expected: "1.2.3.4", expectedError: nil}, - {name: "default (request_ip)", functionalRule: DefaultFunctionalRules["context_ip"], event: ptrans.TransformerEvent{Message: map[string]any{"request_ip": "1.2.3.4"}}, expected: "1.2.3.4", expectedError: nil}, - {name: "default (not available)", functionalRule: DefaultFunctionalRules["context_ip"], event: ptrans.TransformerEvent{Message: map[string]any{}}, expected: nil, expectedError: nil}, - {name: "extract (id)", functionalRule: ExtractFunctionalRules["id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{RecordID: "123"}}, expected: "123", expectedError: nil}, - {name: "extract (empty)", functionalRule: ExtractFunctionalRules["id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{RecordID: ""}}, expected: nil, expectedError: response.ErrRecordIDEmpty}, - {name: "identify (context.ip)", functionalRule: IdentifyFunctionalRules["context_ip"], event: ptrans.TransformerEvent{Message: map[string]any{"context": map[string]any{"ip": "1.2.3.4"}}}, expected: "1.2.3.4", expectedError: nil}, - {name: "identify (request_ip)", functionalRule: IdentifyFunctionalRules["context_ip"], event: ptrans.TransformerEvent{Message: map[string]any{"request_ip": "1.2.3.4"}}, expected: "1.2.3.4", expectedError: nil}, - {name: "identify (not available)", functionalRule: IdentifyFunctionalRules["context_ip"], event: ptrans.TransformerEvent{Message: map[string]any{}}, expected: nil, expectedError: nil}, - {name: "page (name)", functionalRule: PageFunctionalRules["name"], event: ptrans.TransformerEvent{Message: map[string]any{"name": "page name"}}, expected: "page name", expectedError: nil}, - {name: "page (properties.name)", functionalRule: PageFunctionalRules["name"], event: ptrans.TransformerEvent{Message: map[string]any{"properties": map[string]any{"name": "page name"}}}, expected: "page name", expectedError: nil}, - {name: "page (not available)", functionalRule: PageFunctionalRules["name"], event: ptrans.TransformerEvent{Message: map[string]any{}}, expected: nil, expectedError: nil}, - {name: "screen (name)", functionalRule: ScreenFunctionalRules["name"], event: ptrans.TransformerEvent{Message: map[string]any{"name": "screen name"}}, expected: "screen name", expectedError: nil}, - {name: "screen (properties.name)", functionalRule: ScreenFunctionalRules["name"], event: ptrans.TransformerEvent{Message: map[string]any{"properties": map[string]any{"name": "screen name"}}}, expected: "screen name", expectedError: nil}, - {name: "screen (not available)", functionalRule: ScreenFunctionalRules["name"], event: ptrans.TransformerEvent{Message: map[string]any{}}, expected: nil, expectedError: nil}, - {name: "track (record_id)", functionalRule: TrackTableFunctionalRules["record_id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "cloud", RecordID: "123"}, Message: types.SingularEventT{"context": map[string]any{"sources": map[string]any{"version": "1.0"}}}}, expected: "123", expectedError: nil}, - {name: "track (record_id) convert to string", functionalRule: TrackTableFunctionalRules["record_id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "cloud", RecordID: 123}, Message: types.SingularEventT{"context": map[string]any{"sources": map[string]any{"version": "1.0"}}}}, expected: "123", expectedError: nil}, - {name: "track (not cloud)", functionalRule: TrackTableFunctionalRules["record_id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "not cloud"}}, expected: nil, expectedError: nil}, - {name: "track (id)", functionalRule: TrackEventTableFunctionalRules["id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "cloud", RecordID: "123"}, Message: types.SingularEventT{"context": map[string]any{"sources": map[string]any{"version": "1.0"}}}}, expected: "123", expectedError: nil}, - {name: "track (id) don't convert to string", functionalRule: TrackEventTableFunctionalRules["id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "cloud", RecordID: 123}, Message: types.SingularEventT{"context": map[string]any{"sources": map[string]any{"version": "1.0"}}}}, expected: 123, expectedError: nil}, - {name: "track (not cloud)", functionalRule: TrackEventTableFunctionalRules["id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "not cloud", MessageID: "message-id"}}, expected: "message-id", expectedError: nil}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result, err := tc.functionalRule(tc.event) - require.Equal(t, tc.expectedError, err) - require.Equal(t, tc.expected, result) - }) - } - }) + testCases := []struct { + name string + functionalRule FunctionalRules + event ptrans.TransformerEvent + expected any + expectedError error + }{ + {name: "default (context.ip)", functionalRule: DefaultFunctionalRules["context_ip"], event: ptrans.TransformerEvent{Message: map[string]any{"context": map[string]any{"ip": "1.2.3.4"}}}, expected: "1.2.3.4", expectedError: nil}, + {name: "default (request_ip)", functionalRule: DefaultFunctionalRules["context_ip"], event: ptrans.TransformerEvent{Message: map[string]any{"request_ip": "1.2.3.4"}}, expected: "1.2.3.4", expectedError: nil}, + {name: "default (not available)", functionalRule: DefaultFunctionalRules["context_ip"], event: ptrans.TransformerEvent{Message: map[string]any{}}, expected: nil, expectedError: nil}, + {name: "extract (id)", functionalRule: ExtractFunctionalRules["id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{RecordID: "123"}}, expected: "123", expectedError: nil}, + {name: "extract (empty)", functionalRule: ExtractFunctionalRules["id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{RecordID: ""}}, expected: nil, expectedError: response.ErrRecordIDEmpty}, + {name: "identify (context.ip)", functionalRule: IdentifyFunctionalRules["context_ip"], event: ptrans.TransformerEvent{Message: map[string]any{"context": map[string]any{"ip": "1.2.3.4"}}}, expected: "1.2.3.4", expectedError: nil}, + {name: "identify (request_ip)", functionalRule: IdentifyFunctionalRules["context_ip"], event: ptrans.TransformerEvent{Message: map[string]any{"request_ip": "1.2.3.4"}}, expected: "1.2.3.4", expectedError: nil}, + {name: "identify (not available)", functionalRule: IdentifyFunctionalRules["context_ip"], event: ptrans.TransformerEvent{Message: map[string]any{}}, expected: nil, expectedError: nil}, + {name: "page (name)", functionalRule: PageFunctionalRules["name"], event: ptrans.TransformerEvent{Message: map[string]any{"name": "page name"}}, expected: "page name", expectedError: nil}, + {name: "page (properties.name)", functionalRule: PageFunctionalRules["name"], event: ptrans.TransformerEvent{Message: map[string]any{"properties": map[string]any{"name": "page name"}}}, expected: "page name", expectedError: nil}, + {name: "page (not available)", functionalRule: PageFunctionalRules["name"], event: ptrans.TransformerEvent{Message: map[string]any{}}, expected: nil, expectedError: nil}, + {name: "screen (name)", functionalRule: ScreenFunctionalRules["name"], event: ptrans.TransformerEvent{Message: map[string]any{"name": "screen name"}}, expected: "screen name", expectedError: nil}, + {name: "screen (properties.name)", functionalRule: ScreenFunctionalRules["name"], event: ptrans.TransformerEvent{Message: map[string]any{"properties": map[string]any{"name": "screen name"}}}, expected: "screen name", expectedError: nil}, + {name: "screen (not available)", functionalRule: ScreenFunctionalRules["name"], event: ptrans.TransformerEvent{Message: map[string]any{}}, expected: nil, expectedError: nil}, + {name: "track (record_id)", functionalRule: TrackTableFunctionalRules["record_id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "cloud", RecordID: "123"}, Message: types.SingularEventT{"context": map[string]any{"sources": map[string]any{"version": "1.0"}}}}, expected: "123", expectedError: nil}, + {name: "track (record_id) convert to string", functionalRule: TrackTableFunctionalRules["record_id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "cloud", RecordID: 123}, Message: types.SingularEventT{"context": map[string]any{"sources": map[string]any{"version": "1.0"}}}}, expected: "123", expectedError: nil}, + {name: "track (not cloud)", functionalRule: TrackTableFunctionalRules["record_id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "not cloud"}}, expected: nil, expectedError: nil}, + {name: "track (record_id) IsObject", functionalRule: TrackTableFunctionalRules["record_id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "cloud", RecordID: map[string]any{"a": "b"}}, Message: types.SingularEventT{"context": map[string]any{"sources": map[string]any{"version": "1.0"}}}}, expected: nil, expectedError: response.ErrRecordIDObject}, + {name: "track (id)", functionalRule: TrackEventTableFunctionalRules["id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "cloud", RecordID: "123"}, Message: types.SingularEventT{"context": map[string]any{"sources": map[string]any{"version": "1.0"}}}}, expected: "123", expectedError: nil}, + {name: "track (id) don't convert to string", functionalRule: TrackEventTableFunctionalRules["id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "cloud", RecordID: 123}, Message: types.SingularEventT{"context": map[string]any{"sources": map[string]any{"version": "1.0"}}}}, expected: 123, expectedError: nil}, + {name: "track (not cloud)", functionalRule: TrackEventTableFunctionalRules["id"], event: ptrans.TransformerEvent{Metadata: ptrans.Metadata{EventType: "track", SourceCategory: "not cloud", MessageID: "message-id"}}, expected: "message-id", expectedError: nil}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result, err := tc.functionalRule(tc.event) + if tc.expectedError != nil { + require.ErrorIs(t, err, tc.expectedError) + require.Nil(t, result) + return + } + require.NoError(t, err) + require.Equal(t, tc.expected, result) + }) + } } diff --git a/warehouse/transformer/internal/snakecase/snakecase_test.go b/warehouse/transformer/internal/snakecase/snakecase_test.go index 50e0b7094f3..6a8a18c3918 100644 --- a/warehouse/transformer/internal/snakecase/snakecase_test.go +++ b/warehouse/transformer/internal/snakecase/snakecase_test.go @@ -3,7 +3,6 @@ package snakecase import ( "strings" "testing" - "time" "github.com/samber/lo" "github.com/stretchr/testify/require" @@ -101,19 +100,133 @@ func TestToSnakeCase(t *testing.T) { require.Equal(t, expected, actual) } }) - t.Run("should prevent ReDoS", func(t *testing.T) { - largeWordLen := 50000 - largeWord := strings.Repeat("A", largeWordLen) - maxMs := 1000 - startTime := time.Now() - - expected := []string{largeWord, "Æiou", "Are", "Vowels"} - actual := extractWords(largeWord + "ÆiouAreVowels") - require.Equal(t, expected, actual) + }) + t.Run("extractWords", func(t *testing.T) { + t.Run("should match words containing Latin Unicode letters", func(t *testing.T) { + for _, letter := range burredLetters { + require.Equal(t, []string{string(letter)}, extractWordsWithNumbers(string(letter))) + } + }) + t.Run("should work with compound words", func(t *testing.T) { + require.Equal(t, []string{"12ft"}, extractWordsWithNumbers("12ft")) + require.Equal(t, []string{"aeiou", "Are", "Vowels"}, extractWordsWithNumbers("aeiouAreVowels")) + require.Equal(t, []string{"enable", "6h", "format"}, extractWordsWithNumbers("enable 6h format")) + require.Equal(t, []string{"enable", "24H", "format"}, extractWordsWithNumbers("enable 24H format")) + require.Equal(t, []string{"is", "ISO8601"}, extractWordsWithNumbers("isISO8601")) + require.Equal(t, []string{"LETTERS", "Aeiou", "Are", "Vowels"}, extractWordsWithNumbers("LETTERSAeiouAreVowels")) + require.Equal(t, []string{"too", "Legit2", "Quit"}, extractWordsWithNumbers("tooLegit2Quit")) + require.Equal(t, []string{"walk500", "Miles"}, extractWordsWithNumbers("walk500Miles")) + require.Equal(t, []string{"xhr2", "Request"}, extractWordsWithNumbers("xhr2Request")) + require.Equal(t, []string{"XML", "Http"}, extractWordsWithNumbers("XMLHttp")) + require.Equal(t, []string{"Xml", "HTTP"}, extractWordsWithNumbers("XmlHTTP")) + require.Equal(t, []string{"Xml", "Http"}, extractWordsWithNumbers("XmlHttp")) + }) + t.Run("should work with compound words containing diacritical marks", func(t *testing.T) { + require.Equal(t, []string{"LETTERS", "Æiou", "Are", "Vowels"}, extractWordsWithNumbers("LETTERSÆiouAreVowels")) + require.Equal(t, []string{"æiou", "Are", "Vowels"}, extractWordsWithNumbers("æiouAreVowels")) + require.Equal(t, []string{"æiou2", "Consonants"}, extractWordsWithNumbers("æiou2Consonants")) + }) + t.Run("should not treat contractions as separate words", func(t *testing.T) { + for _, apos := range []string{"'", string('\u2019')} { + t.Run("ToLower", func(t *testing.T) { + for _, postfix := range []string{"d", "ll", "m", "re", "s", "t", "ve"} { + input := "a b" + apos + postfix + " c" + actual := extractWordsWithNumbers(strings.ToLower(input)) + expected := lo.Map([]string{"a", "b" + apos + postfix, "c"}, func(item string, index int) string { + return strings.ToLower(item) + }) + require.Equal(t, expected, actual) + } + }) + t.Run("ToUpper", func(t *testing.T) { + for _, postfix := range []string{"d", "ll", "m", "re", "s", "t", "ve"} { + input := "a b" + apos + postfix + " c" + actual := extractWordsWithNumbers(strings.ToUpper(input)) + expected := lo.Map([]string{"a", "b" + apos + postfix, "c"}, func(item string, index int) string { + return strings.ToUpper(item) + }) + require.Equal(t, expected, actual) + } + }) + } + }) + t.Run("should not treat ordinal numbers as separate words", func(t *testing.T) { + ordinals := []string{"1st", "2nd", "3rd", "4th"} + for _, ordinal := range ordinals { + expected := []string{strings.ToLower(ordinal)} + actual := extractWordsWithNumbers(strings.ToLower(ordinal)) + require.Equal(t, expected, actual) - endTime := time.Now() - timeSpent := endTime.Sub(startTime) - require.Less(t, timeSpent.Milliseconds(), int64(maxMs)) + expected = []string{strings.ToUpper(ordinal)} + actual = extractWordsWithNumbers(strings.ToUpper(ordinal)) + require.Equal(t, expected, actual) + } + }) + }) + t.Run("ToSnakeCase", func(t *testing.T) { + t.Run("should remove Latin mathematical operators", func(t *testing.T) { + require.Equal(t, ToSnakeCase(string('\xd7')), "") + }) + t.Run("should coerce `string` to a string", func(t *testing.T) { + require.Equal(t, ToSnakeCase("foo bar"), "foo_bar") + }) + t.Run("should return an empty string for empty values", func(t *testing.T) { + require.Equal(t, ToSnakeCase(""), "") + }) + t.Run("should remove contraction apostrophes", func(t *testing.T) { + for _, apos := range []string{"'", string('\u2019')} { + for _, postfix := range []string{"d", "ll", "m", "re", "s", "t", "ve"} { + input := "a b" + apos + postfix + " c" + require.Equal(t, "a_b"+postfix+"_c", ToSnakeCase(input)) + } + } + }) + t.Run("should convert `string` to caseName case", func(t *testing.T) { + testCases := []string{"foo bar", "Foo bar", "foo Bar", "Foo Bar", "FOO BAR", "fooBar", "--foo-bar--", "__foo_bar__"} + expected := []string{"foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar"} + for i, input := range testCases { + require.Equal(t, expected[i], ToSnakeCase(input)) + } + }) + t.Run("should handle double-converting strings", func(t *testing.T) { + testCases := []string{"foo bar", "Foo bar", "foo Bar", "Foo Bar", "FOO BAR", "fooBar", "--foo-bar--", "__foo_bar__"} + expected := []string{"foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar"} + for i, input := range testCases { + require.Equal(t, expected[i], ToSnakeCase(ToSnakeCase(input))) + } + }) + }) + t.Run("ToSnakeCaseWithNumbers", func(t *testing.T) { + t.Run("should remove Latin mathematical operators", func(t *testing.T) { + require.Equal(t, ToSnakeCaseWithNumbers(string('\xd7')), "") + }) + t.Run("should coerce `string` to a string", func(t *testing.T) { + require.Equal(t, ToSnakeCaseWithNumbers("foo bar"), "foo_bar") + }) + t.Run("should return an empty string for empty values", func(t *testing.T) { + require.Equal(t, ToSnakeCaseWithNumbers(""), "") + }) + t.Run("should remove contraction apostrophes", func(t *testing.T) { + for _, apos := range []string{"'", string('\u2019')} { + for _, postfix := range []string{"d", "ll", "m", "re", "s", "t", "ve"} { + input := "a b" + apos + postfix + " c" + require.Equal(t, "a_b"+postfix+"_c", ToSnakeCaseWithNumbers(input)) + } + } + }) + t.Run("should convert `string` to caseName case", func(t *testing.T) { + testCases := []string{"foo bar", "Foo bar", "foo Bar", "Foo Bar", "FOO BAR", "fooBar", "--foo-bar--", "__foo_bar__"} + expected := []string{"foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar"} + for i, input := range testCases { + require.Equal(t, expected[i], ToSnakeCaseWithNumbers(input)) + } + }) + t.Run("should handle double-converting strings", func(t *testing.T) { + testCases := []string{"foo bar", "Foo bar", "foo Bar", "Foo Bar", "FOO BAR", "fooBar", "--foo-bar--", "__foo_bar__"} + expected := []string{"foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar", "foo_bar"} + for i, input := range testCases { + require.Equal(t, expected[i], ToSnakeCaseWithNumbers(ToSnakeCaseWithNumbers(input))) + } }) }) } diff --git a/warehouse/transformer/internal/utils/reservedcolumnstables.json b/warehouse/transformer/internal/utils/reservedcolumnstables.json new file mode 100644 index 00000000000..06bc6704701 --- /dev/null +++ b/warehouse/transformer/internal/utils/reservedcolumnstables.json @@ -0,0 +1,2241 @@ +{ + "AZURE_DATALAKE": [ + "NULL", + "PREORDER", + "BOTH", + "OVERLAY", + "REGR_SXX", + "ROW", + "START", + "ASSERTION", + "OLD", + "ON", + "PROC", + "RULE", + "ALTER", + "COMMIT", + "CONTINUE", + "NCLOB", + "EXCEPTION", + "HOST", + "READS", + "USE", + "VALUE", + "AFTER", + "EXTERNAL", + "FULL", + "PARAMETER", + "MATCH", + "SQLSTATE", + "TREAT", + "SQLCODE", + "YEAR", + "DEALLOCATE", + "ROLLBACK", + "STATISTICS", + "TABLE", + "DISTINCT", + "INNER", + "SENSITIVE", + "SEQUENCE", + "ABSOLUTE", + "CURRENT_TIMESTAMP", + "FOUND", + "ROLE", + "SMALLINT", + "VARCHAR", + "FROM", + "MONTH", + "OUTER", + "TIMEZONE_MINUTE", + "FILTER", + "GROUPING", + "NULLIF", + "OUT", + "PERCENTILE_CONT", + "WITHINGROUP", + "SELECT", + "XMLCONCAT", + "CONTAINS", + "EXECUTE", + "LATERAL", + "RETURNS", + "SAVE", + "SECURITYAUDIT", + "COVAR_POP", + "KILL", + "XMLITERATE", + "BROWSE", + "CORRESPONDING", + "TRY_CONVERT", + "CARDINALITY", + "DUMP", + "EXEC", + "DISCONNECT", + "NOCHECK", + "CURRENT_TRANSFORM_GROUP_FOR_TYPE", + "DOMAIN", + "ERRLVL", + "NEXT", + "DATE", + "LARGE", + "NORMALIZE", + "REGR_SYY", + "REVERT", + "TEMPORARY", + "PUBLIC", + "RELEASE", + "DROP", + "INITIALLY", + "CALL", + "CASCADE", + "TRAILING", + "CASCADED", + "HOUR", + "NATIONAL", + "SAVEPOINT", + "USER", + "INITIALIZE", + "NUMERIC", + "OPENROWSET", + "OPERATION", + "REGR_AVGY", + "ROWS", + "CONNECT", + "DEFERRED", + "IDENTITYCOL", + "VARYING", + "XMLCOMMENT", + "BULK", + "COMPUTE", + "FREE", + "FUSION", + "NONE", + "OBJECT", + "COLLECT", + "COLUMN", + "FORTRAN", + "CATALOG", + "DECIMAL", + "FLOAT", + "CONNECTION", + "CURSOR", + "MEMBER", + "TEXTSIZE", + "TRUE", + "CUME_DIST", + "PERCENTILE_DISC", + "CONDITION", + "DESC", + "FOREIGN", + "LEADING", + "LOCATOR", + "NO", + "DAY", + "MOD", + "PRECISION", + "REFERENCING", + "VAR_POP", + "ZONE", + "CHAR", + "INPUT", + "INTERSECTION", + "LOCALTIMESTAMP", + "NEW", + "SIZE", + "BY", + "INSERT", + "PREFIX", + "XMLCAST", + "DIAGNOSTICS", + "SYSTEM_USER", + "BIT", + "CROSS", + "ONLY", + "SYMMETRIC", + "DEREF", + "TRANSACTION", + "END-EXEC", + "MIN", + "UPPER", + "CURRENT_ROLE", + "SETS", + "TRANSLATE_REGEX", + "ELEMENT", + "LESS", + "PRINT", + "REGR_COUNT", + "ALLOCATE", + "CHAR_LENGTH", + "CYCLE", + "EXCEPT", + "NATURAL", + "OPTION", + "STRUCTURE", + "TRIGGER", + "DELETE", + "BINARY", + "DISK", + "SQLWARNING", + "CONSTRAINTS", + "FIRST", + "NONCLUSTERED", + "WHENEVER", + "CLUSTERED", + "INTEGER", + "SIMILAR", + "NAMES", + "TIME", + "TRIM", + "CLASS", + "EXIT", + "INDICATOR", + "MAP", + "ROLLUP", + "FREETEXTTABLE", + "MODIFY", + "ASC", + "CASE", + "GLOBAL", + "OPEN", + "REGR_SXY", + "ACTION", + "ASENSITIVE", + "LOWER", + "REGR_R2", + "WRITE", + "FALSE", + "REGR_AVGX", + "SCROLL", + "WHEN", + "POSTFIX", + "TRUNCATE", + "BREADTH", + "CURRENT_DATE", + "PRIOR", + "SEMANTICSIMILARITYDETAILSTABLE", + "IDENTITY_INSERT", + "CORR", + "FOR", + "OF", + "ALL", + "CLOB", + "MODIFIES", + "TRANSLATION", + "EACH", + "ESCAPE", + "PAD", + "REFERENCES", + "SPECIFIC", + "SUM", + "WRITETEXT", + "XMLPI", + "DISTRIBUTED", + "GROUP", + "IMMEDIATE", + "INCLUDE", + "POSITION", + "SUBSTRING", + "ASYMMETRIC", + "BEGIN", + "OCCURRENCES_REGEX", + "RESTRICT", + "SPECIFICTYPE", + "DBCC", + "DICTIONARY", + "KEY", + "NCHAR", + "RESULT", + "AS", + "COLLATE", + "CURRENT", + "OR", + "REGR_SLOPE", + "EXTRACT", + "ORDER", + "SEMANTICSIMILARITYTABLE", + "TSEQUAL", + "XMLDOCUMENT", + "LIMIT", + "TERMINATE", + "ADA", + "CLOSE", + "EQUALS", + "OFF", + "VARIABLE", + "END", + "PASCAL", + "TO", + "VAR_SAMP", + "ADD", + "COALESCE", + "RESTORE", + "USING", + "VIEW", + "ANY", + "CHARACTER", + "IDENTITY", + "ISOLATION", + "STATE", + "FILE", + "GOTO", + "MULTISET", + "WITH", + "AVG", + "STATIC", + "XMLQUERY", + "ARRAY", + "CURRENT_PATH", + "PARTIAL", + "VALUES", + "XMLAGG", + "WHILE", + "ARE", + "BETWEEN", + "CUBE", + "CURRENT_TIME", + "PREPARE", + "READ", + "EXISTS", + "GENERAL", + "JOIN", + "MAX", + "OPENXML", + "UNDER", + "SCHEMA", + "AT", + "FREETEXT", + "LOCAL", + "XMLVALIDATE", + "DYNAMIC", + "DEPTH", + "OPENDATASOURCE", + "POSITION_REGEX", + "SQLEXCEPTION", + "AND", + "OVER", + "OVERLAPS", + "PLAN", + "THEN", + "XMLEXISTS", + "CURRENT_SCHEMA", + "HAVING", + "RELATIVE", + "RIGHT", + "SQLCA", + "SYSTEM", + "CONVERT", + "PRIMARY", + "REF", + "SCOPE", + "UPDATE", + "BOOLEAN", + "DATABASE", + "XMLFOREST", + "EVERY", + "LEVEL", + "NOT", + "ROUTINE", + "GO", + "LOAD", + "SECTION", + "USAGE", + "CHECKPOINT", + "DECLARE", + "PRIVILEGES", + "INOUT", + "METHOD", + "XMLNAMESPACES", + "TOP", + "COVAR_SAMP", + "IF", + "LINENO", + "PRESERVE", + "PROCEDURE", + "STATEMENT", + "ATOMIC", + "BIT_LENGTH", + "HOLD", + "AUTHORIZATION", + "COLLATION", + "DETERMINISTIC", + "READTEXT", + "CONSTRAINT", + "LANGUAGE", + "THAN", + "WITHIN", + "CONTAINSTABLE", + "XMLSERIALIZE", + "FULLTEXTTABLE", + "INDEX", + "MINUTE", + "DENY", + "FUNCTION", + "RECONFIGURE", + "XMLATTRIBUTES", + "INTO", + "RETURN", + "SQLERROR", + "SUBMULTISET", + "XMLBINARY", + "CURRENT_CATALOG", + "LOCALTIME", + "STDDEV_SAMP", + "AGGREGATE", + "DESTRUCTOR", + "OCTET_LENGTH", + "PERCENT_RANK", + "SESSION", + "UPDATETEXT", + "DESCRIBE", + "RAISERROR", + "RANGE", + "ROWGUIDCOL", + "TRAN", + "CAST", + "DEC", + "DEFERRABLE", + "FETCH", + "OUTPUT", + "WIDTH_BUCKET", + "WITHOUT", + "BEFORE", + "ORDINALITY", + "SESSION_USER", + "WORK", + "CURRENT_USER", + "SQL", + "SUBSTRING_REGEX", + "XMLPARSE", + "DATA", + "INTERVAL", + "MODULE", + "SHUTDOWN", + "TIMEZONE_HOUR", + "UNIQUE", + "ADMIN", + "ALIAS", + "CALLED", + "CREATE", + "LAST", + "RECURSIVE", + "UNPIVOT", + "COUNT", + "INSENSITIVE", + "SECOND", + "SOME", + "STDDEV_POP", + "UNION", + "FILLFACTOR", + "PERCENT", + "REVOKE", + "BLOB", + "BREAK", + "DEFAULT", + "IN", + "LEFT", + "REPLICATION", + "DESCRIPTOR", + "ELSE", + "OPENQUERY", + "SET", + "GRANT", + "PARAMETERS", + "UESCAPE", + "WAITFOR", + "UNNEST", + "HOLDLOCK", + "LN", + "WINDOW", + "DOUBLE", + "LIKE", + "REAL", + "SEMANTICKEYPHRASETABLE", + "XMLELEMENT", + "IGNORE", + "ROWCOUNT", + "TRANSLATE", + "UNKNOWN", + "XMLTABLE", + "CURRENT_DEFAULT_TRANSFORM_GROUP", + "OFFSETS", + "SETUSER", + "COMPLETION", + "XMLTEXT", + "CHARACTER_LENGTH", + "GET", + "INT", + "MERGE", + "REGR_INTERCEPT", + "DESTROY", + "PARTITION", + "TABLESAMPLE", + "WHERE", + "INTERSECT", + "ITERATE", + "PIVOT", + "SEARCH", + "SPACE", + "BACKUP", + "CHECK", + "IS", + "LIKE_REGEX" + ], + "AZURE_SYNAPSE": [ + "BEGIN", + "LEVEL", + "LOCAL", + "PRINT", + "TRUNCATE", + "LINENO", + "NEW", + "OLD", + "RELEASE", + "ROW", + "CASCADE", + "FOR", + "INTERVAL", + "WRITETEXT", + "FILTER", + "SAVEPOINT", + "UNION", + "INTEGER", + "MIN", + "XMLBINARY", + "DEREF", + "RELATIVE", + "SQLWARNING", + "TEXTSIZE", + "CURRENT_TIMESTAMP", + "PROC", + "SPACE", + "START", + "REGR_R2", + "COLLECT", + "CROSS", + "FULLTEXTTABLE", + "OCTET_LENGTH", + "PRECISION", + "REPLICATION", + "ROWCOUNT", + "THEN", + "CURRENT_ROLE", + "INITIALIZE", + "NCLOB", + "PIVOT", + "CREATE", + "NULL", + "SIZE", + "SMALLINT", + "SPECIFICTYPE", + "SQLERROR", + "XMLTABLE", + "CORRESPONDING", + "DICTIONARY", + "MEMBER", + "ON", + "WITHINGROUP", + "AT", + "LEFT", + "PREFIX", + "VAR_SAMP", + "XMLCOMMENT", + "DENY", + "EXTERNAL", + "INTO", + "PREPARE", + "SQL", + "DESCRIPTOR", + "WHERE", + "XMLFOREST", + "XMLPI", + "COMMIT", + "FILE", + "LOAD", + "CONTAINSTABLE", + "CUBE", + "REF", + "VARCHAR", + "XMLELEMENT", + "DATE", + "LOWER", + "SYMMETRIC", + "VAR_POP", + "XMLAGG", + "INPUT", + "DECIMAL", + "DEPTH", + "GLOBAL", + "STRUCTURE", + "FIRST", + "STDDEV_SAMP", + "ADMIN", + "BOOLEAN", + "ELEMENT", + "HOUR", + "AFTER", + "BIT_LENGTH", + "PARTIAL", + "PERCENTILE_CONT", + "REGR_AVGX", + "ALLOCATE", + "HOLDLOCK", + "REGR_AVGY", + "CUME_DIST", + "INCLUDE", + "CONNECTION", + "OVERLAPS", + "DYNAMIC", + "RULE", + "TRAILING", + "COLUMN", + "LIMIT", + "PARTITION", + "SETS", + "VARYING", + "BEFORE", + "BY", + "IDENTITYCOL", + "NCHAR", + "OBJECT", + "ADD", + "RETURNS", + "SELECT", + "EQUALS", + "XMLDOCUMENT", + "CALL", + "EXECUTE", + "ORDER", + "VALUES", + "ADA", + "ROLLBACK", + "DISK", + "DISTRIBUTED", + "PERCENTILE_DISC", + "STATIC", + "UNNEST", + "XMLCAST", + "CONVERT", + "FOREIGN", + "HAVING", + "MERGE", + "STATISTICS", + "XMLEXISTS", + "ALIAS", + "MODIFY", + "RESTRICT", + "TRUE", + "OPENXML", + "BETWEEN", + "CHARACTER", + "CONNECT", + "LARGE", + "LATERAL", + "KILL", + "POSTFIX", + "READS", + "SECTION", + "CURSOR", + "IS", + "TRY_CONVERT", + "WHEN", + "COLLATE", + "LANGUAGE", + "SEMANTICSIMILARITYTABLE", + "UNIQUE", + "CONDITION", + "IN", + "REVERT", + "XMLCONCAT", + "AND", + "AS", + "CALLED", + "DOUBLE", + "HOLD", + "XMLSERIALIZE", + "AUTHORIZATION", + "CHAR", + "ELSE", + "END-EXEC", + "LOCALTIMESTAMP", + "BREAK", + "DIAGNOSTICS", + "SAVE", + "SESSION", + "DATA", + "PERCENT_RANK", + "USING", + "ROLE", + "ABSOLUTE", + "ANY", + "INSENSITIVE", + "ISOLATION", + "MATCH", + "ACTION", + "LOCATOR", + "REGR_COUNT", + "SEARCH", + "BOTH", + "NULLIF", + "REFERENCING", + "TABLESAMPLE", + "FILLFACTOR", + "OVER", + "COMPUTE", + "DROP", + "LEADING", + "NUMERIC", + "REGR_SXY", + "BIT", + "CLUSTERED", + "TOP", + "WITH", + "NATIONAL", + "NONCLUSTERED", + "SQLEXCEPTION", + "COLLATION", + "DOMAIN", + "ESCAPE", + "GROUPING", + "LIKE", + "TABLE", + "CLASS", + "INOUT", + "PARAMETER", + "SUM", + "WIDTH_BUCKET", + "WITHOUT", + "XMLITERATE", + "COMPLETION", + "GO", + "GRANT", + "NORMALIZE", + "REVOKE", + "DISCONNECT", + "NO", + "REAL", + "SQLSTATE", + "THAN", + "UNPIVOT", + "CARDINALITY", + "CYCLE", + "OPENQUERY", + "OUTER", + "SETUSER", + "UPDATETEXT", + "AGGREGATE", + "CONTINUE", + "POSITION", + "RANGE", + "SET", + "GROUP", + "LAST", + "PERCENT", + "ASC", + "EACH", + "FALSE", + "INSERT", + "OPTION", + "INDEX", + "MINUTE", + "OFFSETS", + "OUTPUT", + "WINDOW", + "OCCURRENCES_REGEX", + "TRIGGER", + "USE", + "EXISTS", + "RESTORE", + "FUSION", + "PARAMETERS", + "PLAN", + "FORTRAN", + "FUNCTION", + "CASCADED", + "COUNT", + "PASCAL", + "EXEC", + "XMLVALIDATE", + "CATALOG", + "RETURN", + "ROLLUP", + "SIMILAR", + "WORK", + "DECLARE", + "PAD", + "PRIMARY", + "REGR_SXX", + "REGR_SYY", + "SENSITIVE", + "TIMEZONE_MINUTE", + "TRANSACTION", + "VIEW", + "XMLPARSE", + "DETERMINISTIC", + "CHAR_LENGTH", + "IGNORE", + "JOIN", + "REGR_SLOPE", + "DEALLOCATE", + "FETCH", + "OR", + "RECURSIVE", + "CURRENT_CATALOG", + "READTEXT", + "UNDER", + "WHILE", + "INDICATOR", + "INT", + "MONTH", + "XMLQUERY", + "XMLTEXT", + "CAST", + "COVAR_SAMP", + "RIGHT", + "SECURITYAUDIT", + "UPPER", + "ASYMMETRIC", + "CURRENT_PATH", + "LOCALTIME", + "NEXT", + "REGR_INTERCEPT", + "TRIM", + "USER", + "CONSTRAINTS", + "DEFERRED", + "DESTROY", + "OPENROWSET", + "SCHEMA", + "CONSTRAINT", + "DESC", + "ONLY", + "SYSTEM_USER", + "TIME", + "DEC", + "DESTRUCTOR", + "RESULT", + "DAY", + "IMMEDIATE", + "OPERATION", + "PROCEDURE", + "SQLCA", + "UNKNOWN", + "WRITE", + "GENERAL", + "GET", + "METHOD", + "ORDINALITY", + "PRESERVE", + "CLOB", + "OPENDATASOURCE", + "WAITFOR", + "OUT", + "TEMPORARY", + "TRANSLATE", + "COVAR_POP", + "BLOB", + "OPEN", + "POSITION_REGEX", + "TRAN", + "YEAR", + "BULK", + "CASE", + "HOST", + "SECOND", + "CURRENT_DEFAULT_TRANSFORM_GROUP", + "NAMES", + "NOCHECK", + "OFF", + "ROWS", + "WHENEVER", + "BACKUP", + "IDENTITY", + "SEMANTICKEYPHRASETABLE", + "SESSION_USER", + "STATEMENT", + "STDDEV_POP", + "ALL", + "CHECK", + "CURRENT", + "DBCC", + "FROM", + "SYSTEM", + "ASSERTION", + "DUMP", + "LN", + "CHARACTER_LENGTH", + "EXCEPT", + "INITIALLY", + "NOT", + "ROWGUIDCOL", + "OF", + "ZONE", + "SUBSTRING_REGEX", + "DEFAULT", + "DEFERRABLE", + "EVERY", + "MODIFIES", + "SEQUENCE", + "ATOMIC", + "ROUTINE", + "ASENSITIVE", + "BROWSE", + "IDENTITY_INSERT", + "TREAT", + "ARRAY", + "FOUND", + "GOTO", + "MULTISET", + "SHUTDOWN", + "TO", + "CLOSE", + "ERRLVL", + "IF", + "XMLNAMESPACES", + "CHECKPOINT", + "CURRENT_USER", + "RECONFIGURE", + "TSEQUAL", + "SOME", + "AVG", + "CURRENT_TRANSFORM_GROUP_FOR_TYPE", + "FREE", + "ITERATE", + "MOD", + "CORR", + "CURRENT_DATE", + "INNER", + "SCOPE", + "TIMEZONE_HOUR", + "EXIT", + "NATURAL", + "KEY", + "UESCAPE", + "VARIABLE", + "XMLATTRIBUTES", + "BINARY", + "INTERSECT", + "PRIVILEGES", + "RAISERROR", + "TRANSLATION", + "END", + "FLOAT", + "FREETEXT", + "SQLCODE", + "DATABASE", + "TRANSLATE_REGEX", + "VALUE", + "SCROLL", + "SEMANTICSIMILARITYDETAILSTABLE", + "SPECIFIC", + "TERMINATE", + "ALTER", + "DISTINCT", + "MODULE", + "PRIOR", + "PUBLIC", + "CURRENT_SCHEMA", + "CURRENT_TIME", + "DESCRIBE", + "FREETEXTTABLE", + "LIKE_REGEX", + "COALESCE", + "MAP", + "READ", + "UPDATE", + "WITHIN", + "DELETE", + "REFERENCES", + "PREORDER", + "STATE", + "USAGE", + "CONTAINS", + "FULL", + "INTERSECTION", + "NONE", + "SUBSTRING", + "ARE", + "BREADTH", + "EXCEPTION", + "EXTRACT", + "LESS", + "MAX", + "OVERLAY", + "SUBMULTISET" + ], + "BQ": [ + "CUBE", + "RANGE", + "WHEN", + "RIGHT", + "SET", + "TREAT", + "ASC", + "CONTAINS", + "CREATE", + "ELSE", + "FETCH", + "FROM", + "HASH", + "LIKE", + "WINDOW", + "ROWS", + "TABLESAMPLE", + "TO", + "CURRENT", + "DEFINE", + "LATERAL", + "OF", + "ORDER", + "ESCAPE", + "TRUE", + "USING", + "ARRAY", + "BY", + "CROSS", + "RECURSIVE", + "SELECT", + "DEFAULT", + "OR", + "OVER", + "PARTITION", + "WHERE", + "UNION", + "FOR", + "INTO", + "JOIN", + "ROLLUP", + "SOME", + "LOOKUP", + "NULLS", + "ON", + "AND", + "ANY", + "EXCEPT", + "EXISTS", + "HAVING", + "WITHIN", + "COLLATE", + "DISTINCT", + "ENUM", + "INTERSECT", + "IS", + "NATURAL", + "PRECEDING", + "AT", + "EXTRACT", + "FALSE", + "IN", + "MERGE", + "STRUCT", + "CAST", + "IGNORE", + "INNER", + "NEW", + "OUTER", + "INTERVAL", + "LIMIT", + "RESPECT", + "UNNEST", + "WITH", + "DESC", + "EXCLUDE", + "GROUPS", + "PROTO", + "THEN", + "ASSERT_ROWS_MODIFIED", + "END", + "FULL", + "GROUPING", + "NOT", + "GROUP", + "IF", + "LEFT", + "ALL", + "AS", + "BETWEEN", + "CASE", + "FOLLOWING", + "NO", + "NULL", + "UNBOUNDED" + ], + "DELTALAKE": [ + "EXTERNAL", + "EXTRACT", + "PRIMARY", + "SEMI", + "UNIQUE", + "ALL", + "AT", + "JOIN", + "PARTITION", + "USING", + "ELSE", + "REFERENCES", + "SOME", + "BETWEEN", + "END", + "FOREIGN", + "GRANT", + "GROUPING", + "INNER", + "WINDOW", + "CURRENT_TIME", + "INSERT", + "LOCAL", + "ORDER", + "SELECT", + "COMMIT", + "EXISTS", + "OR", + "WHEN", + "AS", + "FROM", + "OF", + "ROLLUP", + "GLOBAL", + "NO", + "SET", + "START", + "TO", + "CAST", + "FALSE", + "OVERLAPS", + "ROW", + "ROWS", + "THEN", + "UPDATE", + "ANY", + "CREATE", + "DESCRIBE", + "REVOKE", + "TRAILING", + "TRUE", + "AUTHORIZATION", + "CASE", + "EVENT_DATE", + "HAVING", + "ALTER", + "FUNCTION", + "MINUS", + "OUT", + "RANGE", + "USER", + "WHERE", + "ANTI", + "LEADING", + "NATURAL", + "VALUES", + "CURRENT_USER", + "GROUP", + "OUTER", + "ROLLBACK", + "SESSION_USER", + "CHECK", + "LIKE", + "TABLE", + "DELETE", + "ESCAPE", + "COLLATE", + "COLUMN", + "POSITION", + "CONSTRAINT", + "EXCEPT", + "NOT", + "RIGHT", + "FULL", + "ONLY", + "TRUNCATE", + "CROSS", + "CURRENT", + "DISTINCT", + "FOR", + "INTERVAL", + "LATERAL", + "WITH", + "FILTER", + "IN", + "ON", + "UNKNOWN", + "FETCH", + "TIME", + "CUBE", + "UNION", + "CURRENT_DATE", + "INTERSECT", + "BOTH", + "NULL", + "AND", + "ARRAY", + "BY", + "CURRENT_TIMESTAMP", + "IS", + "TABLESAMPLE", + "DROP", + "INTO", + "LEFT" + ], + "GCS_DATALAKE": [ + "STRUCT", + "COLLATE", + "FOLLOWING", + "INTERSECT", + "NO", + "OF", + "RECURSIVE", + "TREAT", + "CONTAINS", + "ELSE", + "END", + "GROUPS", + "INNER", + "IS", + "ARRAY", + "AS", + "HAVING", + "NATURAL", + "ORDER", + "PRECEDING", + "PROTO", + "AND", + "BY", + "ESCAPE", + "IN", + "LEFT", + "OR", + "TABLESAMPLE", + "THEN", + "CREATE", + "FOR", + "LIKE", + "UNION", + "DEFINE", + "FROM", + "UNBOUNDED", + "DISTINCT", + "EXCLUDE", + "GROUPING", + "LOOKUP", + "OUTER", + "RIGHT", + "CASE", + "CURRENT", + "JOIN", + "NULLS", + "USING", + "WITHIN", + "AT", + "FALSE", + "INTERVAL", + "INTO", + "ALL", + "CROSS", + "ENUM", + "EXCEPT", + "EXTRACT", + "FETCH", + "UNNEST", + "WHERE", + "CUBE", + "LATERAL", + "PARTITION", + "ROLLUP", + "SELECT", + "SOME", + "EXISTS", + "NULL", + "OVER", + "WHEN", + "WINDOW", + "ASC", + "BETWEEN", + "DESC", + "HASH", + "ANY", + "CAST", + "IF", + "MERGE", + "SET", + "TRUE", + "ASSERT_ROWS_MODIFIED", + "FULL", + "LIMIT", + "NEW", + "ON", + "RANGE", + "TO", + "WITH", + "DEFAULT", + "GROUP", + "IGNORE", + "NOT", + "RESPECT", + "ROWS" + ], + "MSSQL": [ + "WHERE", + "ASYMMETRIC", + "DICTIONARY", + "ELSE", + "FREETEXT", + "NEW", + "PARAMETERS", + "USE", + "WHILE", + "REVOKE", + "COALESCE", + "CONSTRAINTS", + "CONTAINSTABLE", + "DUMP", + "LOCATOR", + "NAMES", + "PIVOT", + "UNPIVOT", + "CASE", + "CURSOR", + "DETERMINISTIC", + "EVERY", + "EXEC", + "HOUR", + "INCLUDE", + "DECIMAL", + "IDENTITYCOL", + "SYMMETRIC", + "AFTER", + "DEFERRABLE", + "FETCH", + "IMMEDIATE", + "OVERLAY", + "ASSERTION", + "CONVERT", + "LIMIT", + "MERGE", + "PRIVILEGES", + "BIT_LENGTH", + "BOTH", + "IF", + "LOWER", + "RIGHT", + "MEMBER", + "MODULE", + "RECURSIVE", + "ACTION", + "PROCEDURE", + "REVERT", + "UNNEST", + "AUTHORIZATION", + "CHAR_LENGTH", + "REFERENCES", + "SPECIFICTYPE", + "UNION", + "VARCHAR", + "CARDINALITY", + "CLOB", + "RANGE", + "SYSTEM_USER", + "THEN", + "CURRENT_TIMESTAMP", + "FUSION", + "INITIALIZE", + "LARGE", + "NULLIF", + "PREPARE", + "CONSTRAINT", + "CONTINUE", + "SIZE", + "TO", + "UPPER", + "ANY", + "ASC", + "DATE", + "INDEX", + "THAN", + "PROC", + "BROWSE", + "DEALLOCATE", + "EXECUTE", + "EXTERNAL", + "INDICATOR", + "INOUT", + "OVER", + "ESCAPE", + "MIN", + "NUMERIC", + "TRUNCATE", + "DATA", + "FILE", + "FOREIGN", + "CALLED", + "OPEN", + "XMLQUERY", + "CHECKPOINT", + "DBCC", + "DECLARE", + "FREETEXTTABLE", + "POSITION_REGEX", + "XMLCONCAT", + "CONNECT", + "IS", + "OPENDATASOURCE", + "ARE", + "NONE", + "ZONE", + "COMPUTE", + "SQLWARNING", + "VALUE", + "DELETE", + "TEMPORARY", + "TSEQUAL", + "WRITE", + "CALL", + "SUBMULTISET", + "BREAK", + "EACH", + "FUNCTION", + "RESTRICT", + "START", + "ERRLVL", + "IN", + "PRESERVE", + "DISTRIBUTED", + "MULTISET", + "OUT", + "ELEMENT", + "INTERVAL", + "OFF", + "POSTFIX", + "ROLLUP", + "STATIC", + "CHARACTER_LENGTH", + "MINUTE", + "XMLVALIDATE", + "DESCRIBE", + "OPENROWSET", + "OPENXML", + "REGR_SLOPE", + "REPLICATION", + "COVAR_SAMP", + "CORR", + "END-EXEC", + "TRAN", + "DESTROY", + "OFFSETS", + "WAITFOR", + "AND", + "CURRENT_ROLE", + "INTERSECTION", + "SCROLL", + "TIMEZONE_MINUTE", + "BULK", + "FALSE", + "FULLTEXTTABLE", + "INSENSITIVE", + "UPDATE", + "SHUTDOWN", + "AS", + "DESTRUCTOR", + "GENERAL", + "GO", + "ON", + "OPENQUERY", + "SCHEMA", + "STATE", + "GRANT", + "SEARCH", + "SELECT", + "XMLCOMMENT", + "DESCRIPTOR", + "SQL", + "COUNT", + "OCTET_LENGTH", + "SIMILAR", + "COLLATION", + "CREATE", + "INT", + "INTO", + "MAX", + "SETUSER", + "WINDOW", + "ATOMIC", + "EXCEPT", + "LOCALTIMESTAMP", + "SECURITYAUDIT", + "SEMANTICSIMILARITYDETAILSTABLE", + "CURRENT_DATE", + "SPACE", + "VAR_SAMP", + "CONDITION", + "DEREF", + "NCHAR", + "PUBLIC", + "ALLOCATE", + "SENSITIVE", + "RELATIVE", + "SECOND", + "XMLELEMENT", + "SCOPE", + "TABLESAMPLE", + "TRAILING", + "PERCENTILE_CONT", + "ROWGUIDCOL", + "TEXTSIZE", + "VALUES", + "BEGIN", + "ITERATE", + "OBJECT", + "ONLY", + "PARTITION", + "RELEASE", + "EXTRACT", + "NEXT", + "OLD", + "RECONFIGURE", + "TRIGGER", + "BETWEEN", + "INSERT", + "LEVEL", + "SYSTEM", + "DISCONNECT", + "NATURAL", + "SEQUENCE", + "CURRENT_TRANSFORM_GROUP_FOR_TYPE", + "DYNAMIC", + "GLOBAL", + "METHOD", + "UNDER", + "AGGREGATE", + "LIKE_REGEX", + "NOT", + "STRUCTURE", + "SUBSTRING", + "CURRENT_PATH", + "SEMANTICSIMILARITYTABLE", + "CURRENT_CATALOG", + "OCCURRENCES_REGEX", + "TRANSLATION", + "WITHIN", + "XMLAGG", + "REGR_R2", + "XMLFOREST", + "SMALLINT", + "ALL", + "CHARACTER", + "CLASS", + "COLLECT", + "CURRENT_USER", + "DOMAIN", + "FIRST", + "VARYING", + "ASENSITIVE", + "INTEGER", + "ORDER", + "SQLEXCEPTION", + "USING", + "FOUND", + "LINENO", + "DROP", + "FULL", + "PRECISION", + "SAVEPOINT", + "TRANSLATE_REGEX", + "VARIABLE", + "XMLCAST", + "DEC", + "FOR", + "HOLDLOCK", + "RESULT", + "OPTION", + "PASCAL", + "READTEXT", + "RESTORE", + "SAVE", + "XMLTEXT", + "BIT", + "MODIFIES", + "SQLERROR", + "KEY", + "NULL", + "SPECIFIC", + "SUM", + "NCLOB", + "NORMALIZE", + "REGR_AVGY", + "TRUE", + "WITHINGROUP", + "CONNECTION", + "CUBE", + "INITIALLY", + "POSITION", + "ROW", + "SESSION", + "XMLSERIALIZE", + "LATERAL", + "XMLEXISTS", + "IDENTITY", + "LESS", + "MATCH", + "PERCENTILE_DISC", + "STDDEV_SAMP", + "COMPLETION", + "DEFAULT", + "GOTO", + "INPUT", + "PRIMARY", + "UESCAPE", + "FLOAT", + "LEADING", + "REAL", + "USAGE", + "CORRESPONDING", + "DAY", + "RETURN", + "XMLNAMESPACES", + "XMLPI", + "BY", + "JOIN", + "TERMINATE", + "HOLD", + "ROWS", + "BLOB", + "CYCLE", + "DOUBLE", + "GROUP", + "LOCALTIME", + "PARTIAL", + "ROLE", + "CATALOG", + "LOAD", + "PARAMETER", + "XMLDOCUMENT", + "END", + "GROUPING", + "PREORDER", + "CAST", + "XMLATTRIBUTES", + "XMLBINARY", + "COLUMN", + "REGR_SYY", + "CONTAINS", + "CURRENT_SCHEMA", + "NO", + "ROWCOUNT", + "LIKE", + "OUTER", + "BEFORE", + "FROM", + "ORDINALITY", + "STATEMENT", + "BINARY", + "COVAR_POP", + "LAST", + "WORK", + "HAVING", + "KILL", + "SECTION", + "WITHOUT", + "GET", + "LEFT", + "TOP", + "TRANSLATE", + "TABLE", + "BOOLEAN", + "DISK", + "INNER", + "MONTH", + "NATIONAL", + "REGR_COUNT", + "ROLLBACK", + "VIEW", + "CHAR", + "MOD", + "PLAN", + "SETS", + "REF", + "TRANSACTION", + "YEAR", + "OVERLAPS", + "TRY_CONVERT", + "DATABASE", + "IDENTITY_INSERT", + "PERCENT", + "WIDTH_BUCKET", + "CURRENT", + "CURRENT_TIME", + "EQUALS", + "FILTER", + "INTERSECT", + "SOME", + "WHEN", + "XMLPARSE", + "CUME_DIST", + "LOCAL", + "XMLITERATE", + "DIAGNOSTICS", + "PERCENT_RANK", + "REGR_AVGX", + "REGR_INTERCEPT", + "RETURNS", + "PAD", + "OPERATION", + "ARRAY", + "AT", + "COLLATE", + "CROSS", + "DENY", + "EXIT", + "NONCLUSTERED", + "WHENEVER", + "CASCADE", + "FORTRAN", + "OR", + "UNKNOWN", + "CASCADED", + "HOST", + "PRIOR", + "NOCHECK", + "OF", + "READ", + "SQLCODE", + "TREAT", + "CURRENT_DEFAULT_TRANSFORM_GROUP", + "REFERENCING", + "TIMEZONE_HOUR", + "UPDATETEXT", + "USER", + "PRINT", + "REGR_SXY", + "SEMANTICKEYPHRASETABLE", + "SESSION_USER", + "VAR_POP", + "ADMIN", + "BACKUP", + "EXCEPTION", + "FILLFACTOR", + "LN", + "READS", + "ADA", + "ALIAS", + "CLOSE", + "ISOLATION", + "SQLCA", + "STDDEV_POP", + "WRITETEXT", + "DESC", + "SET", + "UNIQUE", + "ABSOLUTE", + "OUTPUT", + "RULE", + "ALTER", + "SQLSTATE", + "TRIM", + "BREADTH", + "MAP", + "COMMIT", + "FREE", + "RAISERROR", + "SUBSTRING_REGEX", + "WITH", + "XMLTABLE", + "AVG", + "DEFERRED", + "DISTINCT", + "REGR_SXX", + "MODIFY", + "STATISTICS", + "ADD", + "EXISTS", + "LANGUAGE", + "PREFIX", + "DEPTH", + "CHECK", + "CLUSTERED", + "IGNORE", + "ROUTINE", + "TIME" + ], + "POSTGRES": [ + "NATURAL", + "SIMILAR", + "ASC", + "BINARY", + "DEFERRABLE", + "EXCEPT", + "FREEZE", + "LEADING", + "LEFT", + "ANALYZE", + "CURRENT_ROLE", + "CURRENT_TIME", + "DO", + "IS", + "OVERLAPS", + "OUTER", + "UNIQUE", + "VERBOSE", + "CURRENT_DATE", + "CURRENT_USER", + "NOT", + "NOTNULL", + "PRIMARY", + "SESSION_USER", + "BETWEEN", + "CONSTRAINT", + "FOREIGN", + "ORDER", + "ONLY", + "OR", + "COLUMN", + "DEFAULT", + "INNER", + "OFFSET", + "GROUP", + "LOCALTIME", + "NEW", + "AND", + "AUTHORIZATION", + "CREATE", + "DISTINCT", + "CHECK", + "ILIKE", + "ASYMMETRIC", + "CURRENT_TIMESTAMP", + "INTERSECT", + "SYMMETRIC", + "TABLE", + "USER", + "ELSE", + "FULL", + "IN", + "SELECT", + "FROM", + "GRANT", + "INTO", + "LIKE", + "ALL", + "ANALYSE", + "END", + "FALSE", + "RIGHT", + "SOME", + "OLD", + "TRUE", + "WHEN", + "BOTH", + "HAVING", + "LOCALTIMESTAMP", + "OFF", + "CROSS", + "UNION", + "CASE", + "DESC", + "REFERENCES", + "THEN", + "ON", + "PLACING", + "USING", + "WHERE", + "ANY", + "ISNULL", + "JOIN", + "LIMIT", + "FOR", + "INITIALLY", + "NULL", + "TO", + "ARRAY", + "AS", + "CAST", + "COLLATE", + "TRAILING" + ], + "RS": [ + "TRUNCATECOLUMNS", + "UNIQUE", + "TEXT32K", + "COLLATE", + "IN", + "OLD", + "REFERENCES", + "AES256", + "LZOP", + "RESPECT", + "WALLET", + "AND", + "TOP", + "SYSDATE", + "IGNORE", + "LEADING", + "MOSTLY32", + "NOTNULL", + "OUTER", + "TRAILING", + "WHEN", + "BACKUP", + "GRANT", + "WITH", + "ANY", + "FOREIGN", + "GLOBALDICT256", + "ILIKE", + "RESORT", + "BETWEEN", + "DESC", + "LOCALTIMESTAMP", + "NULL", + "RECOVER", + "CURRENT_DATE", + "BOTH", + "CASE", + "CREDENTIALS", + "NEW", + "OVERLAPS", + "AZ64", + "ASC", + "DISABLE", + "EXPLICIT", + "ISNULL", + "AS", + "DELTA32K", + "FREEZE", + "INTO", + "LUN", + "MOSTLY8", + "BZIP2", + "ALL", + "CAST", + "DEFLATE", + "DELTA", + "END", + "PERMISSIONS", + "SESSION_USER", + "AES128", + "ENABLE", + "OPEN", + "RESTORE", + "ELSE", + "DEFAULT", + "ONLY", + "TEXT255", + "CURRENT_USER", + "EXCEPT", + "GROUP", + "IS", + "TDES", + "VERBOSE", + "ANALYZE", + "ENCODE", + "ENCRYPT ", + "FROM", + "SNAPSHOT ", + "UNION", + "ALLOWOVERWRITE", + "LZO", + "OR", + "TABLE", + "TAG", + "BLANKSASNULL", + "CURRENT_TIME", + "ENCRYPTION", + "FALSE", + "JOIN", + "BYTEDICT", + "NULLS", + "DEFRAG", + "LUNS", + "OFF", + "OFFSET", + "SOME", + "COLUMN", + "DO", + "FULL", + "BINARY", + "LIKE", + "OFFLINE", + "THEN", + "INNER", + "CHECK", + "NOT", + "ARRAY", + "DISTINCT", + "INITIALLY", + "ON", + "PERCENT", + "RAW", + "SYSTEM", + "DEFERRABLE", + "CURRENT_USER_ID", + "FOR", + "INTERSECT", + "LEFT", + "PLACING", + "USER", + "CROSS", + "LIMIT", + "PARALLEL", + "EMPTYASNULL", + "OID", + "REJECTLOG", + "CREATE", + "HAVING", + "MOSTLY13", + "PRIMARY", + "READRATIO", + "RIGHT", + "USING", + "WHERE", + "GZIP", + "GLOBALDICT64K", + "IDENTITY", + "LANGUAGE", + "MINUS", + "NATURAL", + "AUTHORIZATION", + "CONSTRAINT", + "LOCALTIME", + "PARTITION", + "TRUE", + "ANALYSE", + "TO", + "UUID", + "SELECT", + "ORDER", + "SIMILAR", + "WITHOUT", + "CURRENT_TIMESTAMP" + ], + "S3_DATALAKE": [ + "AS", + "CHAR", + "COMMIT", + "FOLLOWING", + "REDUCE", + "ROW", + "WHERE", + "BY", + "LOCAL", + "CURRENT_DATE", + "JOIN", + "OUT", + "PRESERVE", + "READS", + "ELSE", + "WITH", + "CROSS", + "EXISTS", + "GRANT", + "PRECEDING", + "TIMESTAMP", + "UNBOUNDED", + "DROP", + "START", + "TABLE", + "FLOOR", + "FULL", + "HAVING", + "IF", + "LESS", + "RIGHT", + "ROLLUP", + "UTC_TIMESTAMP", + "CURRENT_TIMESTAMP", + "OR", + "OVER", + "BETWEEN", + "DISTINCT", + "INTEGER", + "BIGINT", + "COLUMN", + "FETCH", + "MACRO", + "UNIQUEJOIN", + "ARRAY", + "CASHE", + "EXTRACT", + "REFERENCES", + "INSERT", + "LIKE", + "VALUES", + "TRUE", + "BOOLEAN", + "EXCHANGE", + "IN", + "INNER", + "LATERAL", + "PRECISION", + "TRIGGER", + "BINARY", + "CAST", + "DOUBLE", + "END", + "FALSE", + "ON", + "PRIMARY", + "CREATE", + "DECIMAL", + "FLOAT", + "NULL", + "TABLESAMPLE", + "THEN", + "REVOKE", + "AND", + "DELETE", + "LEFT", + "NONE", + "PERCENT", + "PROCEDURE", + "REGEXP", + "TRUNCATE", + "UNION", + "BOTH", + "MORE", + "VARCHAR", + "ALL", + "DATE", + "SELECT", + "CURSOR", + "FUNCTION", + "RLIKE", + "NUMERIC", + "OF", + "AUTHORIZATION", + "CASE", + "CONF", + "FOR", + "GROUP", + "INTERSECT", + "ROLLBACK", + "DATABASE", + "INT", + "NOT", + "SMALLINT", + "WHEN", + "CONSTRAINT", + "IMPORT", + "OUTER", + "RANGE", + "TIME", + "EXTENDED", + "INTO", + "CUBE", + "FOREIGN", + "INTERVAL", + "PARTIALSCAN", + "SET", + "TO", + "CURRENT", + "FROM", + "VIEWS", + "ALTER", + "DESCRIBE", + "EXTERNAL", + "GROUPING", + "IS", + "PARTITION", + "USING", + "DAYOFWEEK", + "MAP", + "TRANSFORM", + "USER", + "ONLY", + "ROWS", + "ORDER", + "UPDATE", + "WINDOW" + ], + "SNOWFLAKE": [ + "SET", + "TABLE", + "GRANT", + "INNER", + "MINUS", + "REGEXP", + "CONNECTION", + "CROSS", + "START", + "IN", + "DELETE", + "DISTINCT", + "EXISTS", + "FROM", + "UPDATE", + "DATABASE", + "FULL", + "GROUP", + "INTO", + "FALSE", + "INSERT", + "NOT", + "ORDER", + "ACCOUNT", + "ANY", + "BETWEEN", + "CAST", + "TRIGGER", + "USING", + "ILIKE", + "NULL", + "ON", + "OR", + "ALL", + "CONNECT", + "REVOKE", + "WHERE", + "SAMPLE", + "SELECT", + "CHECK", + "JOIN", + "RLIKE", + "ROW", + "CURRENT", + "LATERAL", + "NATURAL", + "WHENEVER", + "LIKE", + "SOME", + "TRUE", + "VIEW", + "CASE", + "CURRENT_TIMESTAMP", + "INCREMENT", + "IS", + "ELSE", + "FOLLOWING", + "UNION", + "AND", + "BY", + "CREATE", + "CURRENT_TIME", + "TO", + "CURRENT_DATE", + "DROP", + "ISSUE", + "SCHEMA", + "QUALIFY", + "ROWS", + "TABLESAMPLE", + "UNIQUE", + "CONSTRAINT", + "GSCLUSTER", + "HAVING", + "ORGANIZATION", + "COLUMN", + "FOR", + "INTERSECT", + "OF", + "THEN", + "VALUES", + "WITH", + "ALTER", + "CURRENT_USER", + "LOCALTIME", + "RIGHT", + "WHEN", + "AS", + "LEFT", + "LOCALTIMESTAMP", + "TRY_CAST" + ], + "CLICKHOUSE": [] +} diff --git a/warehouse/transformer/internal/utils/reservedkeywords.go b/warehouse/transformer/internal/utils/reservedkeywords.go index d454d7d93d9..0c5187e2e64 100644 --- a/warehouse/transformer/internal/utils/reservedkeywords.go +++ b/warehouse/transformer/internal/utils/reservedkeywords.go @@ -10,25 +10,30 @@ import ( ) var ( - //go:embed reservedkeywords.json - reservedKeywordsFile embed.FS + //go:embed reservedcolumnstables.json + reservedKeywordsForColumnsTablesFile embed.FS - reservedKeywords map[string]map[string]struct{} + //go:embed reservednamespaces.json + reservedKeywordsForNamespacesFile embed.FS + + reservedKeywordsForColumnsTables map[string]map[string]struct{} + reservedKeywordsForNamespaces map[string]map[string]struct{} ) func init() { - reservedKeywords = loadReservedKeywords() + reservedKeywordsForColumnsTables = loadReservedKeywords(reservedKeywordsForColumnsTablesFile, "reservedcolumnstables.json") + reservedKeywordsForNamespaces = loadReservedKeywords(reservedKeywordsForNamespacesFile, "reservednamespaces.json") } -func loadReservedKeywords() map[string]map[string]struct{} { - data, err := reservedKeywordsFile.ReadFile("reservedkeywords.json") +func loadReservedKeywords(reservedKeywordsFile embed.FS, name string) map[string]map[string]struct{} { + data, err := reservedKeywordsFile.ReadFile(name) if err != nil { - log.Fatalf("failed to load reserved keywords: %v", err) + log.Fatalf("failed to load reserved keywords for %s: %v", name, err) } var tempKeywords map[string][]string if err := json.Unmarshal(data, &tempKeywords); err != nil { - log.Fatalf("failed to parse reserved keywords: %v", err) + log.Fatalf("failed to parse reserved keywords for %s: %v", name, err) } return lo.MapValues(tempKeywords, func(keywords []string, _ string) map[string]struct{} { @@ -38,7 +43,12 @@ func loadReservedKeywords() map[string]map[string]struct{} { }) } -func IsReservedKeyword(destType, keyword string) bool { - _, exists := reservedKeywords[destType][strings.ToUpper(keyword)] +func IsReservedKeywordForColumnsTables(destType, keyword string) bool { + _, exists := reservedKeywordsForColumnsTables[destType][strings.ToUpper(keyword)] + return exists +} + +func IsReservedKeywordForNamespaces(destType, keyword string) bool { + _, exists := reservedKeywordsForNamespaces[destType][strings.ToUpper(keyword)] return exists } diff --git a/warehouse/transformer/internal/utils/reservedkeywords_test.go b/warehouse/transformer/internal/utils/reservedkeywords_test.go index 7a0d535cd3e..10832e41ec1 100644 --- a/warehouse/transformer/internal/utils/reservedkeywords_test.go +++ b/warehouse/transformer/internal/utils/reservedkeywords_test.go @@ -10,6 +10,7 @@ import ( func TestReservedKeywordsMapping(t *testing.T) { for _, destType := range whutils.WarehouseDestinations { - require.NotNilf(t, reservedKeywords[destType], "Reserved keywords not found for destination type %s", destType) + require.NotNilf(t, reservedKeywordsForColumnsTables[destType], "Reserved keywords not found for destination type %s", destType) + require.NotNilf(t, reservedKeywordsForNamespaces[destType], "Reserved keywords not found for destination type %s", destType) } } diff --git a/warehouse/transformer/internal/utils/reservedkeywords.json b/warehouse/transformer/internal/utils/reservednamespaces.json similarity index 100% rename from warehouse/transformer/internal/utils/reservedkeywords.json rename to warehouse/transformer/internal/utils/reservednamespaces.json diff --git a/warehouse/transformer/internal/utils/stringlikeobject.go b/warehouse/transformer/internal/utils/stringlikeobject.go index 1d1805c78a3..49b505facb2 100644 --- a/warehouse/transformer/internal/utils/stringlikeobject.go +++ b/warehouse/transformer/internal/utils/stringlikeobject.go @@ -3,6 +3,9 @@ package utils import ( "sort" "strconv" + "strings" + + "github.com/samber/lo" ) func IsStringLikeObject(obj map[string]any) bool { @@ -40,29 +43,20 @@ func IsStringLikeObject(obj map[string]any) bool { } func isNonNegativeInteger(str string) bool { - if len(str) == 0 { - return false - } - for _, c := range str { - if c < '0' || c > '9' { - return false - } - } - return true + return lo.EveryBy([]rune(str), func(c rune) bool { + return c >= '0' && c <= '9' + }) } -func StringLikeObjectToString(obj map[string]any) any { - keys := make([]int, 0, len(obj)) - for key := range obj { +func StringLikeObjectToString(obj map[string]any) string { + keys := lo.Map(lo.Keys(obj), func(key string, _ int) int { numKey, _ := strconv.Atoi(key) - keys = append(keys, numKey) - } - + return numKey + }) sort.Ints(keys) - result := "" - for _, key := range keys { - result += ToString(obj[strconv.Itoa(key)]) - } - return result + values := lo.Map(keys, func(key, _ int) string { + return ToString(obj[strconv.Itoa(key)]) + }) + return strings.Join(values, "") } diff --git a/warehouse/transformer/internal/utils/stringlikeobject_test.go b/warehouse/transformer/internal/utils/stringlikeobject_test.go index 2989f87f99d..1fb30444eba 100644 --- a/warehouse/transformer/internal/utils/stringlikeobject_test.go +++ b/warehouse/transformer/internal/utils/stringlikeobject_test.go @@ -123,11 +123,18 @@ func TestStringLikeObjectToString(t *testing.T) { }, expected: "abc", }, + { + name: "valid string-like object with 1 and 2", + input: map[string]any{ + "1": "x", + "2": "y", + }, + expected: "xy", + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - require.True(t, IsStringLikeObject(tc.input)) require.Equal(t, tc.expected, StringLikeObjectToString(tc.input)) }) } diff --git a/warehouse/transformer/internal/utils/utils.go b/warehouse/transformer/internal/utils/utils.go index 20b1c093835..ae23bbbae32 100644 --- a/warehouse/transformer/internal/utils/utils.go +++ b/warehouse/transformer/internal/utils/utils.go @@ -13,10 +13,11 @@ import ( ) var ( - rudderCreatedTables = sliceToMap([]string{"tracks", "pages", "screens", "aliases", "groups", "accounts"}) - rudderIsolatedTables = sliceToMap([]string{"users", "identifies"}) - sourceCategoriesToUseRecordID = sliceToMap([]string{"cloud", "singer-protocol"}) - identityEnabledWarehouses = sliceToMap([]string{whutils.SNOWFLAKE, whutils.BQ}) + rudderCreatedTables = sliceToMap([]string{"tracks", "pages", "screens", "aliases", "groups", "accounts"}) + rudderIsolatedTables = sliceToMap([]string{"users", "identifies"}) + sourceCategoriesToUseRecordID = sliceToMap([]string{"cloud", "singer-protocol"}) + identityEnabledWarehouses = sliceToMap([]string{whutils.SNOWFLAKE, whutils.BQ}) + destinationSupportJSONPathAsPartOfConfig = sliceToMap([]string{whutils.POSTGRES, whutils.RS, whutils.SNOWFLAKE, whutils.BQ}) supportedJSONPathPrefixes = []string{"track.", "identify.", "page.", "screen.", "alias.", "group.", "extract."} fullEventColumnTypeByDestType = map[string]string{ @@ -87,8 +88,9 @@ func CanUseRecordID(sourceCategory string) bool { } func HasJSONPathPrefix(jsonPath string) bool { + lowerJSONPath := strings.ToLower(jsonPath) for _, prefix := range supportedJSONPathPrefixes { - if strings.HasPrefix(strings.ToLower(jsonPath), prefix) { + if strings.HasPrefix(lowerJSONPath, prefix) { return true } } @@ -134,3 +136,8 @@ func ToString(value interface{}) string { func IsBlank(value interface{}) bool { return len(ToString(value)) == 0 } + +func IsJSONPathSupportedAsPartOfConfig(destType string) bool { + _, ok := destinationSupportJSONPathAsPartOfConfig[destType] + return ok +} diff --git a/warehouse/transformer/merge_test.go b/warehouse/transformer/merge_test.go index 78bfa96a4e2..2e388878157 100644 --- a/warehouse/transformer/merge_test.go +++ b/warehouse/transformer/merge_test.go @@ -7,7 +7,6 @@ import ( "github.com/ory/dockertest/v3" "github.com/stretchr/testify/require" - "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" transformertest "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/transformer" @@ -15,6 +14,7 @@ import ( backendconfig "github.com/rudderlabs/rudder-server/backend-config" ptrans "github.com/rudderlabs/rudder-server/processor/transformer" "github.com/rudderlabs/rudder-server/warehouse/transformer/internal/response" + "github.com/rudderlabs/rudder-server/warehouse/transformer/testhelper" ) func TestMerge(t *testing.T) { @@ -24,7 +24,7 @@ func TestMerge(t *testing.T) { transformerResource, err := transformertest.Setup(pool, t) require.NoError(t, err) - testsCases := []struct { + testCases := []struct { name string configOverride map[string]any eventPayload string @@ -37,22 +37,9 @@ func TestMerge(t *testing.T) { configOverride: map[string]any{ "Warehouse.enableIDResolution": true, }, - eventPayload: `{"type":"merge"}`, - metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + eventPayload: `{"type":"merge"}`, + metadata: getMergeMetadata("merge", "POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{}, }, { @@ -61,21 +48,8 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"merge","mergeProperties":[{"type":"email","value":"alex@example.com"},{"type":"mobile","value":"+1-202-555-0146"}]}`, - metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("merge", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { @@ -96,14 +70,7 @@ func TestMerge(t *testing.T) { }, "userId": "", }, - Metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Metadata: getMergeMetadata("merge", "BQ"), StatusCode: http.StatusOK, }, }, @@ -114,22 +81,9 @@ func TestMerge(t *testing.T) { configOverride: map[string]any{ "Warehouse.enableIDResolution": false, }, - eventPayload: `{"type":"merge"}`, - metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + eventPayload: `{"type":"merge"}`, + metadata: getMergeMetadata("merge", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{}, }, { @@ -138,34 +92,14 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"merge"}`, - metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("merge", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ FailedEvents: []ptrans.TransformerResponse{ { Error: response.ErrMergePropertiesMissing.Error(), StatusCode: response.ErrMergePropertiesMissing.StatusCode(), - Metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Metadata: getMergeMetadata("merge", "BQ"), }, }, }, @@ -176,34 +110,14 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"merge", "mergeProperties": "invalid"}`, - metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("merge", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ FailedEvents: []ptrans.TransformerResponse{ { Error: response.ErrMergePropertiesNotArray.Error(), StatusCode: response.ErrMergePropertiesNotArray.StatusCode(), - Metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Metadata: getMergeMetadata("merge", "BQ"), }, }, }, @@ -214,34 +128,14 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"merge", "mergeProperties": []}`, - metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("merge", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ FailedEvents: []ptrans.TransformerResponse{ { Error: response.ErrMergePropertiesNotSufficient.Error(), StatusCode: response.ErrMergePropertiesNotSufficient.StatusCode(), - Metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Metadata: getMergeMetadata("merge", "BQ"), }, }, }, @@ -252,34 +146,14 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"merge","mergeProperties":[{"type":"email","value":"alex@example.com"}]}`, - metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("merge", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ FailedEvents: []ptrans.TransformerResponse{ { Error: response.ErrMergePropertiesNotSufficient.Error(), StatusCode: response.ErrMergePropertiesNotSufficient.StatusCode(), - Metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Metadata: getMergeMetadata("merge", "BQ"), }, }, }, @@ -290,34 +164,14 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"merge","mergeProperties":["invalid",{"type":"email","value":"alex@example.com"}]}`, - metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("merge", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ FailedEvents: []ptrans.TransformerResponse{ { Error: response.ErrMergePropertyOneInvalid.Error(), StatusCode: response.ErrMergePropertyOneInvalid.StatusCode(), - Metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Metadata: getMergeMetadata("merge", "BQ"), }, }, }, @@ -328,34 +182,14 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"merge","mergeProperties":[{"type":"email","value":"alex@example.com"},"invalid"]}`, - metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("merge", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ FailedEvents: []ptrans.TransformerResponse{ { Error: response.ErrMergePropertyTwoInvalid.Error(), StatusCode: response.ErrMergePropertyTwoInvalid.StatusCode(), - Metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Metadata: getMergeMetadata("merge", "BQ"), }, }, }, @@ -366,34 +200,14 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"merge","mergeProperties":[{"type1":"email","value1":"alex@example.com"},{"type1":"mobile","value1":"+1-202-555-0146"}]}`, - metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("merge", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ FailedEvents: []ptrans.TransformerResponse{ { Error: response.ErrMergePropertyEmpty.Error(), StatusCode: response.ErrMergePropertyEmpty.StatusCode(), - Metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Metadata: getMergeMetadata("merge", "BQ"), }, }, }, @@ -404,21 +218,8 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"merge","mergeProperties":[{"type":"email","value":"alex@example.com"},{"type":"mobile","value":"+1-202-555-0146"}]}`, - metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "SNOWFLAKE", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "SNOWFLAKE", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "SNOWFLAKE", - }, - }, + metadata: getMergeMetadata("merge", "SNOWFLAKE"), + destination: getDestination("SNOWFLAKE", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { @@ -439,14 +240,7 @@ func TestMerge(t *testing.T) { }, "userId": "", }, - Metadata: ptrans.Metadata{ - EventType: "merge", - DestinationType: "SNOWFLAKE", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Metadata: getMergeMetadata("merge", "SNOWFLAKE"), StatusCode: http.StatusOK, }, }, @@ -458,113 +252,20 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("alias", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "previous_id": "previousId", - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "previous_id": "string", - "uuid_ts": "datetime", - "loaded_at": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "aliases", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getAliasDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"), + Metadata: getMergeMetadata("alias", "BQ"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "merge_property_1_type": "user_id", - "merge_property_1_value": "userId", - "merge_property_2_type": "user_id", - "merge_property_2_value": "previousId", - }, - "metadata": map[string]any{ - "table": "rudder_identity_merge_rules", - "columns": map[string]any{"merge_property_1_type": "string", "merge_property_1_value": "string", "merge_property_2_type": "string", "merge_property_2_value": "string"}, - "isMergeRule": true, - "receivedAt": "2021-09-01T00:00:00.000Z", - "mergePropOne": "userId", - "mergePropTwo": "previousId", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getAliasDefaultMergeOutput(), + Metadata: getMergeMetadata("alias", "BQ"), StatusCode: http.StatusOK, }, }, @@ -576,81 +277,17 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("alias", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "uuid_ts": "datetime", - "loaded_at": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "aliases", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getAliasDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"). + RemoveDataFields("user_id", "previous_id"). + RemoveColumnFields("user_id", "previous_id"), + Metadata: getMergeMetadata("alias", "BQ"), StatusCode: http.StatusOK, }, }, @@ -662,81 +299,17 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"","previousId":"","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("alias", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "uuid_ts": "datetime", - "loaded_at": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "aliases", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "alias", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getAliasDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"). + RemoveDataFields("user_id", "previous_id"). + RemoveColumnFields("user_id", "previous_id"), + Metadata: getMergeMetadata("alias", "BQ"), StatusCode: http.StatusOK, }, }, @@ -748,115 +321,20 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"page","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("page", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "uuid_ts": "datetime", - "loaded_at": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "pages", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getPageDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"), + Metadata: getMergeMetadata("page", "BQ"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "merge_property_1_type": "anonymous_id", - "merge_property_1_value": "anonymousId", - "merge_property_2_type": "user_id", - "merge_property_2_value": "userId", - }, - "metadata": map[string]any{ - "table": "rudder_identity_merge_rules", - "columns": map[string]any{"merge_property_1_type": "string", "merge_property_1_value": "string", "merge_property_2_type": "string", "merge_property_2_value": "string"}, - "isMergeRule": true, - "receivedAt": "2021-09-01T00:00:00.000Z", - "mergePropOne": "anonymousId", - "mergePropTwo": "userId", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getPageDefaultMergeOutput(), + Metadata: getMergeMetadata("page", "BQ"), StatusCode: http.StatusOK, }, }, @@ -868,85 +346,17 @@ func TestMerge(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"page","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getMergeMetadata("page", "BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "uuid_ts": "datetime", - "loaded_at": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "pages", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getPageDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"). + RemoveDataFields("anonymous_id"). + RemoveColumnFields("anonymous_id"), + Metadata: getMergeMetadata("page", "BQ"), StatusCode: http.StatusOK, }, { @@ -964,42 +374,39 @@ func TestMerge(t *testing.T) { }, "userId": "", }, - Metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Metadata: getMergeMetadata("page", "BQ"), StatusCode: http.StatusOK, }, }, }, }, } - - for _, tc := range testsCases { + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - c := config.New() - c.Set("DEST_TRANSFORM_URL", transformerResource.TransformerURL) - c.Set("USER_TRANSFORM_URL", transformerResource.TransformerURL) - - for k, v := range tc.configOverride { - c.Set(k, v) - } - - eventsInfos := []eventsInfo{ + c := setupConfig(transformerResource, tc.configOverride) + eventsInfos := []testhelper.EventInfo{ { - payload: []byte(tc.eventPayload), - metadata: tc.metadata, - destination: tc.destination, + Payload: []byte(tc.eventPayload), + Metadata: tc.metadata, + Destination: tc.destination, }, } destinationTransformer := ptrans.NewTransformer(c, logger.NOP, stats.Default) warehouseTransformer := New(c, logger.NOP, stats.NOP) - testEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) + testhelper.ValidateEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) }) } } + +func getMergeMetadata(eventType, destinationType string) ptrans.Metadata { + return ptrans.Metadata{ + EventType: eventType, + DestinationType: destinationType, + ReceivedAt: "2021-09-01T00:00:00.000Z", + SourceID: "sourceID", + DestinationID: "destinationID", + SourceType: "sourceType", + MessageID: "messageId", + } +} diff --git a/warehouse/transformer/options.go b/warehouse/transformer/options.go index 4e41b5062d9..77298b11fd8 100644 --- a/warehouse/transformer/options.go +++ b/warehouse/transformer/options.go @@ -49,6 +49,7 @@ func prepareIntegrationOptions(event ptrans.TransformerEvent) (opts integrations if src == nil || !utils.IsObject(src) { return } + var jsonPaths []any srcMap := src.(map[string]any) @@ -56,11 +57,17 @@ func prepareIntegrationOptions(event ptrans.TransformerEvent) (opts integrations setOption(srcMap, "useBlendoCasing", &opts.useBlendoCasing) setOption(srcMap, "skipTracksTable", &opts.skipTracksTable) setOption(srcMap, "skipUsersTable", &opts.skipUsersTable) - setOption(srcMap, "jsonPaths", &opts.jsonPaths) + setOption(srcMap, "jsonPaths", &jsonPaths) + + for _, jp := range jsonPaths { + if jpStr, ok := jp.(string); ok { + opts.jsonPaths = append(opts.jsonPaths, jpStr) + } + } return } -func prepareDestinationOptions(destConfig map[string]any) (opts destConfigOptions) { +func prepareDestinationOptions(destType string, destConfig map[string]any) (opts destConfigOptions) { var jsonPaths string setOption(destConfig, "skipTracksTable", &opts.skipTracksTable) @@ -70,7 +77,7 @@ func prepareDestinationOptions(destConfig map[string]any) (opts destConfigOption setOption(destConfig, "storeFullEvent", &opts.storeFullEvent) setOption(destConfig, "jsonPaths", &jsonPaths) - if len(jsonPaths) > 0 { + if len(jsonPaths) > 0 && utils.IsJSONPathSupportedAsPartOfConfig(destType) { opts.jsonPaths = strings.Split(jsonPaths, ",") } return diff --git a/warehouse/transformer/options_test.go b/warehouse/transformer/options_test.go index fd00e004d1e..c105ed5be01 100644 --- a/warehouse/transformer/options_test.go +++ b/warehouse/transformer/options_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" ptrans "github.com/rudderlabs/rudder-server/processor/transformer" + whutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) func TestIntegrationOptions(t *testing.T) { @@ -19,7 +20,7 @@ func TestIntegrationOptions(t *testing.T) { "useBlendoCasing": false, "skipTracksTable": true, "skipUsersTable": false, - "jsonPaths": []string{"path1", "path2", "path3"}, + "jsonPaths": []any{"path1", "path2", "path3"}, }, }, }, @@ -86,7 +87,7 @@ func TestIntegrationOptions(t *testing.T) { "destinationType": map[string]any{ "options": map[string]any{ "skipUsersTable": true, - "jsonPaths": []string{"path1"}, + "jsonPaths": []any{"path1"}, }, }, }, @@ -117,7 +118,7 @@ func TestDestinationOptions(t *testing.T) { "jsonPaths": "path1,path2", } - opts := prepareDestinationOptions(destConfig) + opts := prepareDestinationOptions(whutils.POSTGRES, destConfig) require.True(t, opts.skipTracksTable) require.False(t, opts.skipUsersTable) @@ -129,7 +130,7 @@ func TestDestinationOptions(t *testing.T) { t.Run("MissingOptions", func(t *testing.T) { destConfig := map[string]any{} - opts := prepareDestinationOptions(destConfig) + opts := prepareDestinationOptions(whutils.POSTGRES, destConfig) require.False(t, opts.skipTracksTable) require.False(t, opts.skipUsersTable) @@ -139,7 +140,7 @@ func TestDestinationOptions(t *testing.T) { require.Empty(t, opts.jsonPaths) }) t.Run("NilDestinationConfig", func(t *testing.T) { - opts := prepareDestinationOptions(nil) + opts := prepareDestinationOptions(whutils.POSTGRES, nil) require.False(t, opts.skipTracksTable) require.False(t, opts.skipUsersTable) @@ -155,7 +156,7 @@ func TestDestinationOptions(t *testing.T) { "allowUsersContextTraits": true, } - opts := prepareDestinationOptions(destConfig) + opts := prepareDestinationOptions(whutils.POSTGRES, destConfig) require.True(t, opts.skipTracksTable) require.False(t, opts.skipUsersTable) @@ -164,4 +165,12 @@ func TestDestinationOptions(t *testing.T) { require.False(t, opts.storeFullEvent) require.Equal(t, []string{"path1", "path2"}, opts.jsonPaths) }) + t.Run("JSONPathSupported", func(t *testing.T) { + destConfig := map[string]any{ + "jsonPaths": "path1,path2", + } + + require.Equal(t, []string{"path1", "path2"}, prepareDestinationOptions(whutils.POSTGRES, destConfig).jsonPaths) + require.Empty(t, prepareDestinationOptions(whutils.CLICKHOUSE, destConfig).jsonPaths) + }) } diff --git a/warehouse/transformer/page_test.go b/warehouse/transformer/page_test.go index 2e4b1c5f151..e63b4d0ad3a 100644 --- a/warehouse/transformer/page_test.go +++ b/warehouse/transformer/page_test.go @@ -7,13 +7,13 @@ import ( "github.com/ory/dockertest/v3" "github.com/stretchr/testify/require" - "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" transformertest "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/transformer" backendconfig "github.com/rudderlabs/rudder-server/backend-config" ptrans "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/warehouse/transformer/testhelper" ) func TestPageEvents(t *testing.T) { @@ -23,7 +23,7 @@ func TestPageEvents(t *testing.T) { transformerResource, err := transformertest.Setup(pool, t) require.NoError(t, err) - testsCases := []struct { + testCases := []struct { name string configOverride map[string]any eventPayload string @@ -34,86 +34,13 @@ func TestPageEvents(t *testing.T) { { name: "page (Postgres)", eventPayload: `{"type":"page","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getPageMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "pages", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getPageDefaultOutput(), + Metadata: getPageMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -122,82 +49,15 @@ func TestPageEvents(t *testing.T) { { name: "page (Postgres) without properties", eventPayload: `{"type":"page","name":"Home","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getPageMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "pages", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getPageDefaultOutput(). + RemoveDataFields("title", "url"). + RemoveColumnFields("title", "url"), + Metadata: getPageMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -206,78 +66,16 @@ func TestPageEvents(t *testing.T) { { name: "page (Postgres) without context", eventPayload: `{"type":"page","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"}}`, - metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getPageMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "5.6.7.8", - "context_request_ip": "5.6.7.8", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_request_ip": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "pages", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getPageDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). // overriding the default value + RemoveDataFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name"). + RemoveColumnFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name"), + Metadata: getPageMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -286,90 +84,17 @@ func TestPageEvents(t *testing.T) { { name: "page (Postgres) store rudder event", eventPayload: `{"type":"page","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "storeFullEvent": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getPageMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "storeFullEvent": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "rudder_event": "{\"anonymousId\":\"anonymousId\",\"channel\":\"web\",\"context\":{\"destinationId\":\"destinationID\",\"destinationType\":\"POSTGRES\",\"ip\":\"1.2.3.4\",\"sourceId\":\"sourceID\",\"sourceType\":\"sourceType\",\"traits\":{\"email\":\"rhedricks@example.com\",\"logins\":2,\"name\":\"Richard Hendricks\"}},\"messageId\":\"messageId\",\"originalTimestamp\":\"2021-09-01T00:00:00.000Z\",\"properties\":{\"name\":\"Home\",\"title\":\"Home | RudderStack\",\"url\":\"https://www.rudderstack.com\"},\"receivedAt\":\"2021-09-01T00:00:00.000Z\",\"request_ip\":\"5.6.7.8\",\"sentAt\":\"2021-09-01T00:00:00.000Z\",\"timestamp\":\"2021-09-01T00:00:00.000Z\",\"type\":\"page\",\"userId\":\"userId\"}", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "rudder_event": "json", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "pages", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getPageDefaultOutput(). + SetDataField("rudder_event", "{\"anonymousId\":\"anonymousId\",\"channel\":\"web\",\"context\":{\"ip\":\"1.2.3.4\",\"traits\":{\"email\":\"rhedricks@example.com\",\"logins\":2,\"name\":\"Richard Hendricks\"},\"sourceId\":\"sourceID\",\"sourceType\":\"sourceType\",\"destinationId\":\"destinationID\",\"destinationType\":\"POSTGRES\"},\"messageId\":\"messageId\",\"originalTimestamp\":\"2021-09-01T00:00:00.000Z\",\"properties\":{\"name\":\"Home\",\"title\":\"Home | RudderStack\",\"url\":\"https://www.rudderstack.com\"},\"receivedAt\":\"2021-09-01T00:00:00.000Z\",\"request_ip\":\"5.6.7.8\",\"sentAt\":\"2021-09-01T00:00:00.000Z\",\"timestamp\":\"2021-09-01T00:00:00.000Z\",\"type\":\"page\",\"userId\":\"userId\"}"). + SetColumnField("rudder_event", "json"), + Metadata: getPageMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -378,80 +103,15 @@ func TestPageEvents(t *testing.T) { { name: "page (Postgres) partial rules", eventPayload: `{"type":"page","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getPageMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "pages", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getPageDefaultOutput(). + RemoveDataFields("anonymous_id", "channel", "context_request_ip"). + RemoveColumnFields("anonymous_id", "channel", "context_request_ip"), + Metadata: getPageMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -463,143 +123,128 @@ func TestPageEvents(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"page","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getPageMetadata("BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Home", - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "loaded_at": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "pages", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getPageDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"), + Metadata: getPageMetadata("BQ"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "merge_property_1_type": "anonymous_id", - "merge_property_1_value": "anonymousId", - "merge_property_2_type": "user_id", - "merge_property_2_value": "userId", - }, - "metadata": map[string]any{ - "table": "rudder_identity_merge_rules", - "columns": map[string]any{"merge_property_1_type": "string", "merge_property_1_value": "string", "merge_property_2_type": "string", "merge_property_2_value": "string"}, - "isMergeRule": true, - "receivedAt": "2021-09-01T00:00:00.000Z", - "mergePropOne": "anonymousId", - "mergePropTwo": "userId", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "page", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getPageDefaultMergeOutput(), + Metadata: getPageMetadata("BQ"), StatusCode: http.StatusOK, }, }, }, }, } - - for _, tc := range testsCases { + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - c := config.New() - c.Set("DEST_TRANSFORM_URL", transformerResource.TransformerURL) - c.Set("USER_TRANSFORM_URL", transformerResource.TransformerURL) - - for k, v := range tc.configOverride { - c.Set(k, v) - } - - eventsInfos := []eventsInfo{ + c := setupConfig(transformerResource, tc.configOverride) + eventsInfos := []testhelper.EventInfo{ { - payload: []byte(tc.eventPayload), - metadata: tc.metadata, - destination: tc.destination, + Payload: []byte(tc.eventPayload), + Metadata: tc.metadata, + Destination: tc.destination, }, } destinationTransformer := ptrans.NewTransformer(c, logger.NOP, stats.Default) warehouseTransformer := New(c, logger.NOP, stats.NOP) - testEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) + testhelper.ValidateEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) }) } } + +func getPageDefaultOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "name": "Home", + "anonymous_id": "anonymousId", + "channel": "web", + "context_ip": "1.2.3.4", + "context_passed_ip": "1.2.3.4", + "context_request_ip": "5.6.7.8", + "context_traits_email": "rhedricks@example.com", + "context_traits_logins": float64(2), + "context_traits_name": "Richard Hendricks", + "id": "messageId", + "original_timestamp": "2021-09-01T00:00:00.000Z", + "received_at": "2021-09-01T00:00:00.000Z", + "sent_at": "2021-09-01T00:00:00.000Z", + "timestamp": "2021-09-01T00:00:00.000Z", + "title": "Home | RudderStack", + "url": "https://www.rudderstack.com", + "user_id": "userId", + "context_destination_id": "destinationID", + "context_destination_type": "POSTGRES", + "context_source_id": "sourceID", + "context_source_type": "sourceType", + }, + "metadata": map[string]any{ + "columns": map[string]any{ + "name": "string", + "anonymous_id": "string", + "channel": "string", + "context_destination_id": "string", + "context_destination_type": "string", + "context_source_id": "string", + "context_source_type": "string", + "context_ip": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_logins": "int", + "context_traits_name": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "title": "string", + "url": "string", + "user_id": "string", + "uuid_ts": "datetime", + }, + "receivedAt": "2021-09-01T00:00:00.000Z", + "table": "pages", + }, + "userId": "", + } +} + +func getPageDefaultMergeOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "merge_property_1_type": "anonymous_id", + "merge_property_1_value": "anonymousId", + "merge_property_2_type": "user_id", + "merge_property_2_value": "userId", + }, + "metadata": map[string]any{ + "table": "rudder_identity_merge_rules", + "columns": map[string]any{"merge_property_1_type": "string", "merge_property_1_value": "string", "merge_property_2_type": "string", "merge_property_2_value": "string"}, + "isMergeRule": true, + "receivedAt": "2021-09-01T00:00:00.000Z", + "mergePropOne": "anonymousId", + "mergePropTwo": "userId", + }, + "userId": "", + } +} + +func getPageMetadata(destinationType string) ptrans.Metadata { + return ptrans.Metadata{ + EventType: "page", + DestinationType: destinationType, + ReceivedAt: "2021-09-01T00:00:00.000Z", + SourceID: "sourceID", + DestinationID: "destinationID", + SourceType: "sourceType", + } +} diff --git a/warehouse/transformer/safe.go b/warehouse/transformer/safe.go index ddf4726a6ae..a20a7b34ea2 100644 --- a/warehouse/transformer/safe.go +++ b/warehouse/transformer/safe.go @@ -37,7 +37,7 @@ func SafeNamespace(conf *config.Config, destType, input string) string { if namespace == "" { namespace = "stringempty" } - if utils.IsReservedKeyword(destType, namespace) { + if utils.IsReservedKeywordForNamespaces(destType, namespace) { namespace = "_" + namespace } return misc.TruncateStr(namespace, 127) @@ -79,7 +79,7 @@ func SafeTableName(destType string, options integrationsOptions, tableName strin if len(tableName) == 0 { return "", response.ErrEmptyTableName } - return safeName(destType, options, tableName) + return safeName(destType, options, tableName), nil } // SafeColumnName processes the input column name based on the destination type and integration options. @@ -89,10 +89,10 @@ func SafeColumnName(destType string, options integrationsOptions, columnName str if len(columnName) == 0 { return "", response.ErrEmptyColumnName } - return safeName(destType, options, columnName) + return safeName(destType, options, columnName), nil } -func safeName(destType string, options integrationsOptions, name string) (string, error) { +func safeName(destType string, options integrationsOptions, name string) string { switch destType { case whutils.SNOWFLAKE: name = strings.ToUpper(name) @@ -103,23 +103,39 @@ func safeName(destType string, options integrationsOptions, name string) (string name = strings.ToLower(name) } - if !options.skipReservedKeywordsEscaping && utils.IsReservedKeyword(destType, name) { + if !options.skipReservedKeywordsEscaping && utils.IsReservedKeywordForColumnsTables(destType, name) { name = "_" + name } if utils.IsDataLake(destType) { - return name, nil + return name } - return misc.TruncateStr(name, 127), nil + return misc.TruncateStr(name, 127) } // TransformTableName applies transformation to the input table name based on the destination type and configuration options. // If `useBlendoCasing` is enabled, it converts the table name to lowercase and trims spaces. // Otherwise, it applies a more general transformation using the `transformName` function. -func TransformTableName(destType string, integrationsOptions integrationsOptions, destConfigOptions destConfigOptions, tableName string) string { +func TransformTableName(integrationsOptions integrationsOptions, destConfigOptions destConfigOptions, tableName string) string { if integrationsOptions.useBlendoCasing { return strings.TrimSpace(strings.ToLower(tableName)) } - return transformName(destType, destConfigOptions, tableName) + name := strings.Join(extractAlphanumericValues(tableName), "_") + + var snakeCaseFn func(s string) string + if destConfigOptions.underscoreDivideNumbers { + snakeCaseFn = snakecase.ToSnakeCase + } else { + snakeCaseFn = snakecase.ToSnakeCaseWithNumbers + } + if strings.HasPrefix(tableName, "_") { + name = reLeadingUnderscores.FindString(tableName) + snakeCaseFn(reLeadingUnderscores.ReplaceAllString(name, "")) + } else { + name = snakeCaseFn(name) + } + if startsWithDigit(name) { + name = "_" + name + } + return name } // TransformColumnName applies transformation to the input column name based on the destination type and configuration options. @@ -129,22 +145,17 @@ func TransformColumnName(destType string, integrationsOptions integrationsOption if integrationsOptions.useBlendoCasing { return transformNameToBlendoCase(destType, columnName) } - return transformName(destType, destConfigOptions, columnName) -} -// transformName normalizes the input string by extracting alphanumeric values and converting it into snake case based on the configuration options. -// It handles leading underscores, adds a leading underscore if the first character is a digit, and truncates the name if necessary (e.g., for Postgres). -func transformName(destType string, options destConfigOptions, input string) string { - name := strings.Join(extractAlphanumericValues(input), "_") + name := strings.Join(extractAlphanumericValues(columnName), "_") var snakeCaseFn func(s string) string - if options.underscoreDivideNumbers { + if destConfigOptions.underscoreDivideNumbers { snakeCaseFn = snakecase.ToSnakeCase } else { snakeCaseFn = snakecase.ToSnakeCaseWithNumbers } - if strings.HasPrefix(input, "_") { - name = reLeadingUnderscores.FindString(input) + snakeCaseFn(reLeadingUnderscores.ReplaceAllString(name, "")) + if strings.HasPrefix(columnName, "_") { + name = reLeadingUnderscores.FindString(columnName) + snakeCaseFn(reLeadingUnderscores.ReplaceAllString(name, "")) } else { name = snakeCaseFn(name) } diff --git a/warehouse/transformer/safe_test.go b/warehouse/transformer/safe_test.go index 1cbcc6efa7d..a316ed6c063 100644 --- a/warehouse/transformer/safe_test.go +++ b/warehouse/transformer/safe_test.go @@ -187,7 +187,6 @@ func TestSafeColumnName(t *testing.T) { func TestTransformTableName(t *testing.T) { testCases := []struct { name string - destType string integrationsOptions integrationsOptions destConfigOptions destConfigOptions tableName string @@ -195,7 +194,6 @@ func TestTransformTableName(t *testing.T) { }{ { name: "Blendo casing - table name trimmed and lowercased", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: true}, destConfigOptions: destConfigOptions{}, tableName: " TableName ", @@ -203,7 +201,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Blendo casing - mixedcased to lowercased", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: true}, destConfigOptions: destConfigOptions{}, tableName: "CaMeLcAsE", @@ -211,7 +208,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Blendo casing - mixedcased to lowercased", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: true}, destConfigOptions: destConfigOptions{}, tableName: "Table@Name!", @@ -219,7 +215,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Blendo casing - alphanumeric", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: true}, destConfigOptions: destConfigOptions{}, tableName: "TableName123", @@ -228,7 +223,6 @@ func TestTransformTableName(t *testing.T) { { name: "Standard casing - underscoreDivideNumbers(true) - remove symbols and join continuous letters and numbers with a single underscore", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "&4yasdfa(84224_fs9##_____*3q", @@ -236,7 +230,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - omega to omega", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "omega", @@ -244,7 +237,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - omega v2 to omega_v_2", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "omega v2", @@ -252,7 +244,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - prepend underscore if name starts with a number", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "9mega", @@ -260,7 +251,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - remove trailing special characters", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "mega&", @@ -268,7 +258,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - replace special character in the middle with underscore", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "ome$ga", @@ -276,7 +265,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - remove trailing $ character", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "omega$", @@ -284,7 +272,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - spaces and special characters by converting to underscores", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "ome_ ga", @@ -292,7 +279,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - multiple underscores and hyphens by reducing to single underscores", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "9mega________-________90", @@ -300,7 +286,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - non-ASCII characters by converting them to underscores", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "Cízǔ", @@ -308,7 +293,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - CamelCase123Key to camel_case_123_key", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "CamelCase123Key", @@ -316,7 +300,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - numbers and commas", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "path to $1,00,000", @@ -324,7 +307,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - no valid characters", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "@#$%", @@ -332,7 +314,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - underscores between letters and numbers", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "test123", @@ -340,7 +321,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - multiple underscore-number sequences", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "abc123def456", @@ -348,7 +328,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - multiple underscore-number sequences", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "abc_123_def_456", @@ -356,7 +335,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - single underscore", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "__abc_123_def_456", @@ -364,7 +342,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - multiple underscore", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: true}, tableName: "_abc_123_def_456", @@ -373,7 +350,6 @@ func TestTransformTableName(t *testing.T) { { name: "Standard casing - underscoreDivideNumbers(false) - remove symbols and join continuous letters and numbers with a single underscore", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "&4yasdfa(84224_fs9##_____*3q", @@ -381,7 +357,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - omega to omega", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "omega", @@ -389,7 +364,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - omega v2 to omega_v_2", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "omega v2", @@ -397,7 +371,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - prepend underscore if name starts with a number", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "9mega", @@ -405,7 +378,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - remove trailing special characters", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "mega&", @@ -413,7 +385,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - replace special character in the middle with underscore", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "ome$ga", @@ -421,7 +392,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - remove trailing $ character", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "omega$", @@ -429,7 +399,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - spaces and special characters by converting to underscores", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "ome_ ga", @@ -437,7 +406,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - multiple underscores and hyphens by reducing to single underscores", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "9mega________-________90", @@ -445,7 +413,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - non-ASCII characters by converting them to underscores", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "Cízǔ", @@ -453,7 +420,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - CamelCase123Key to camel_case_123_key", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "CamelCase123Key", @@ -461,7 +427,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - numbers and commas", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "path to $1,00,000", @@ -469,7 +434,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - no valid characters", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "@#$%", @@ -477,7 +441,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - underscores between letters and numbers", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "test123", @@ -485,7 +448,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(true) - multiple underscore-number sequences", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "abc123def456", @@ -493,7 +455,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - multiple underscore-number sequences", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "abc_123_def_456", @@ -501,7 +462,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - single underscore", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "__abc_123_def_456", @@ -509,7 +469,6 @@ func TestTransformTableName(t *testing.T) { }, { name: "Standard casing - underscoreDivideNumbers(false) - multiple underscore", - destType: whutils.SNOWFLAKE, integrationsOptions: integrationsOptions{useBlendoCasing: false}, destConfigOptions: destConfigOptions{underscoreDivideNumbers: false}, tableName: "_abc_123_def_456", @@ -519,7 +478,7 @@ func TestTransformTableName(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - tableName := TransformTableName(tc.destType, tc.integrationsOptions, tc.destConfigOptions, tc.tableName) + tableName := TransformTableName(tc.integrationsOptions, tc.destConfigOptions, tc.tableName) require.Equal(t, tc.expected, tableName) }) } diff --git a/warehouse/transformer/screen_test.go b/warehouse/transformer/screen_test.go index 97d2fb6121f..ccbb9b18b1d 100644 --- a/warehouse/transformer/screen_test.go +++ b/warehouse/transformer/screen_test.go @@ -7,13 +7,13 @@ import ( "github.com/ory/dockertest/v3" "github.com/stretchr/testify/require" - "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" transformertest "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/transformer" backendconfig "github.com/rudderlabs/rudder-server/backend-config" ptrans "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/warehouse/transformer/testhelper" ) func TestScreen(t *testing.T) { @@ -23,7 +23,7 @@ func TestScreen(t *testing.T) { transformerResource, err := transformertest.Setup(pool, t) require.NoError(t, err) - testsCases := []struct { + testCases := []struct { name string configOverride map[string]any eventPayload string @@ -34,86 +34,13 @@ func TestScreen(t *testing.T) { { name: "screen (Postgres)", eventPayload: `{"type":"screen","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getScreenMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Main", - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "screens", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getScreenDefaultOutput(), + Metadata: getScreenMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -122,82 +49,15 @@ func TestScreen(t *testing.T) { { name: "screen (Postgres) without properties", eventPayload: `{"type":"screen","name":"Main","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getScreenMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Main", - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "screens", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getScreenDefaultOutput(). + RemoveDataFields("title", "url"). + RemoveColumnFields("title", "url"), + Metadata: getScreenMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -206,78 +66,16 @@ func TestScreen(t *testing.T) { { name: "screen (Postgres) without context", eventPayload: `{"type":"screen","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"}}`, - metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getScreenMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Main", - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "5.6.7.8", - "context_request_ip": "5.6.7.8", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_request_ip": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "screens", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getScreenDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). // overriding the default value + RemoveDataFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name"). + RemoveColumnFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name"), + Metadata: getScreenMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -286,90 +84,17 @@ func TestScreen(t *testing.T) { { name: "screen (Postgres) store rudder event", eventPayload: `{"type":"screen","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "storeFullEvent": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getScreenMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{ + "storeFullEvent": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Main", - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "rudder_event": "{\"type\":\"screen\",\"anonymousId\":\"anonymousId\",\"channel\":\"web\",\"context\":{\"destinationId\":\"destinationID\",\"destinationType\":\"POSTGRES\",\"ip\":\"1.2.3.4\",\"sourceId\":\"sourceID\",\"sourceType\":\"sourceType\",\"traits\":{\"email\":\"rhedricks@example.com\",\"logins\":2,\"name\":\"Richard Hendricks\"}},\"messageId\":\"messageId\",\"originalTimestamp\":\"2021-09-01T00:00:00.000Z\",\"properties\":{\"name\":\"Main\",\"title\":\"Home | RudderStack\",\"url\":\"https://www.rudderstack.com\"},\"receivedAt\":\"2021-09-01T00:00:00.000Z\",\"request_ip\":\"5.6.7.8\",\"sentAt\":\"2021-09-01T00:00:00.000Z\",\"timestamp\":\"2021-09-01T00:00:00.000Z\",\"userId\":\"userId\"}", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "rudder_event": "json", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "screens", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getScreenDefaultOutput(). + SetDataField("rudder_event", "{\"anonymousId\":\"anonymousId\",\"channel\":\"web\",\"context\":{\"ip\":\"1.2.3.4\",\"traits\":{\"email\":\"rhedricks@example.com\",\"logins\":2,\"name\":\"Richard Hendricks\"},\"sourceId\":\"sourceID\",\"sourceType\":\"sourceType\",\"destinationId\":\"destinationID\",\"destinationType\":\"POSTGRES\"},\"messageId\":\"messageId\",\"originalTimestamp\":\"2021-09-01T00:00:00.000Z\",\"properties\":{\"name\":\"Main\",\"title\":\"Home | RudderStack\",\"url\":\"https://www.rudderstack.com\"},\"receivedAt\":\"2021-09-01T00:00:00.000Z\",\"request_ip\":\"5.6.7.8\",\"sentAt\":\"2021-09-01T00:00:00.000Z\",\"timestamp\":\"2021-09-01T00:00:00.000Z\",\"type\":\"screen\",\"userId\":\"userId\"}"). + SetColumnField("rudder_event", "json"), + Metadata: getScreenMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -378,80 +103,15 @@ func TestScreen(t *testing.T) { { name: "screen (Postgres) partial rules", eventPayload: `{"type":"screen","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getScreenMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Main", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "screens", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getScreenDefaultOutput(). + RemoveDataFields("anonymous_id", "channel", "context_request_ip"). + RemoveColumnFields("anonymous_id", "channel", "context_request_ip"), + Metadata: getScreenMetadata("POSTGRES"), StatusCode: http.StatusOK, }, }, @@ -463,143 +123,128 @@ func TestScreen(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"screen","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getScreenMetadata("BQ"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "name": "Main", - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "title": "Home | RudderStack", - "url": "https://www.rudderstack.com", - "user_id": "userId", - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "name": "string", - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "title": "string", - "url": "string", - "user_id": "string", - "uuid_ts": "datetime", - "context_destination_id": "string", - "context_destination_type": "string", - "context_source_id": "string", - "context_source_type": "string", - "loaded_at": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "screens", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getScreenDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"), + Metadata: getScreenMetadata("BQ"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "merge_property_1_type": "anonymous_id", - "merge_property_1_value": "anonymousId", - "merge_property_2_type": "user_id", - "merge_property_2_value": "userId", - }, - "metadata": map[string]any{ - "table": "rudder_identity_merge_rules", - "columns": map[string]any{"merge_property_1_type": "string", "merge_property_1_value": "string", "merge_property_2_type": "string", "merge_property_2_value": "string"}, - "isMergeRule": true, - "receivedAt": "2021-09-01T00:00:00.000Z", - "mergePropOne": "anonymousId", - "mergePropTwo": "userId", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "screen", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - }, + Output: getScreenDefaultMergeOutput(), + Metadata: getScreenMetadata("BQ"), StatusCode: http.StatusOK, }, }, }, }, } - - for _, tc := range testsCases { + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - c := config.New() - c.Set("DEST_TRANSFORM_URL", transformerResource.TransformerURL) - c.Set("USER_TRANSFORM_URL", transformerResource.TransformerURL) - - for k, v := range tc.configOverride { - c.Set(k, v) - } - - eventsInfos := []eventsInfo{ + c := setupConfig(transformerResource, tc.configOverride) + eventsInfos := []testhelper.EventInfo{ { - payload: []byte(tc.eventPayload), - metadata: tc.metadata, - destination: tc.destination, + Payload: []byte(tc.eventPayload), + Metadata: tc.metadata, + Destination: tc.destination, }, } destinationTransformer := ptrans.NewTransformer(c, logger.NOP, stats.Default) warehouseTransformer := New(c, logger.NOP, stats.NOP) - testEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) + testhelper.ValidateEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) }) } } + +func getScreenDefaultOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "name": "Main", + "anonymous_id": "anonymousId", + "channel": "web", + "context_ip": "1.2.3.4", + "context_passed_ip": "1.2.3.4", + "context_request_ip": "5.6.7.8", + "context_traits_email": "rhedricks@example.com", + "context_traits_logins": float64(2), + "context_traits_name": "Richard Hendricks", + "id": "messageId", + "original_timestamp": "2021-09-01T00:00:00.000Z", + "received_at": "2021-09-01T00:00:00.000Z", + "sent_at": "2021-09-01T00:00:00.000Z", + "timestamp": "2021-09-01T00:00:00.000Z", + "title": "Home | RudderStack", + "url": "https://www.rudderstack.com", + "user_id": "userId", + "context_destination_id": "destinationID", + "context_destination_type": "POSTGRES", + "context_source_id": "sourceID", + "context_source_type": "sourceType", + }, + "metadata": map[string]any{ + "columns": map[string]any{ + "name": "string", + "anonymous_id": "string", + "channel": "string", + "context_destination_id": "string", + "context_destination_type": "string", + "context_source_id": "string", + "context_source_type": "string", + "context_ip": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_traits_email": "string", + "context_traits_logins": "int", + "context_traits_name": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "title": "string", + "url": "string", + "user_id": "string", + "uuid_ts": "datetime", + }, + "receivedAt": "2021-09-01T00:00:00.000Z", + "table": "screens", + }, + "userId": "", + } +} + +func getScreenDefaultMergeOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "merge_property_1_type": "anonymous_id", + "merge_property_1_value": "anonymousId", + "merge_property_2_type": "user_id", + "merge_property_2_value": "userId", + }, + "metadata": map[string]any{ + "table": "rudder_identity_merge_rules", + "columns": map[string]any{"merge_property_1_type": "string", "merge_property_1_value": "string", "merge_property_2_type": "string", "merge_property_2_value": "string"}, + "isMergeRule": true, + "receivedAt": "2021-09-01T00:00:00.000Z", + "mergePropOne": "anonymousId", + "mergePropTwo": "userId", + }, + "userId": "", + } +} + +func getScreenMetadata(destinationType string) ptrans.Metadata { + return ptrans.Metadata{ + EventType: "screen", + DestinationType: destinationType, + ReceivedAt: "2021-09-01T00:00:00.000Z", + SourceID: "sourceID", + DestinationID: "destinationID", + SourceType: "sourceType", + } +} diff --git a/warehouse/transformer/setdata.go b/warehouse/transformer/setdata.go index b6733d6f980..81510faa765 100644 --- a/warehouse/transformer/setdata.go +++ b/warehouse/transformer/setdata.go @@ -22,7 +22,7 @@ func (t *transformer) setDataAndColumnTypeFromInput( data map[string]any, columnType map[string]string, prefixDetails *prefixInfo, ) error { - if !utils.IsObject(input) { + if input == nil || !utils.IsObject(input) { return nil } @@ -82,7 +82,7 @@ func (t *transformer) handleValidJSONPath(processInfo *processingInfo, prefixDet if err != nil { return fmt.Errorf("marshalling value: %w", err) } - return t.addColumnTypeAndValue(processInfo, prefixDetails.prefix+key, valJSON, true, data, columnType) + return t.addColumnTypeAndValue(processInfo, prefixDetails.prefix+key, string(valJSON), true, data, columnType) } func (t *transformer) shouldProcessNestedObject(processInfo *processingInfo, prefixDetails *prefixInfo, val any) bool { @@ -104,15 +104,15 @@ func (t *transformer) processNestedObject( } func (t *transformer) processNonNestedObject(processInfo *processingInfo, prefixDetails *prefixInfo, key string, val any, data map[string]any, columnType map[string]string) error { - tempData := val + finalValue := val if processInfo.event.Metadata.SourceCategory == "cloud" && prefixDetails.level >= 3 && utils.IsObject(val) { - var err error - tempData, err = json.Marshal(val) + jsonData, err := json.Marshal(val) if err != nil { return fmt.Errorf("marshalling value: %w", err) } + finalValue = string(jsonData) } - return t.addColumnTypeAndValue(processInfo, prefixDetails.prefix+key, tempData, false, data, columnType) + return t.addColumnTypeAndValue(processInfo, prefixDetails.prefix+key, finalValue, false, data, columnType) } func (t *transformer) addColumnTypeAndValue(pi *processingInfo, key string, val any, isJSONKey bool, data map[string]any, columnType map[string]string) error { diff --git a/warehouse/transformer/testhelper/outputbuilder.go b/warehouse/transformer/testhelper/outputbuilder.go new file mode 100644 index 00000000000..672bb8aecee --- /dev/null +++ b/warehouse/transformer/testhelper/outputbuilder.go @@ -0,0 +1,67 @@ +package testhelper + +type OutputBuilder map[string]any + +func (ob OutputBuilder) SetDataField(key string, value any) OutputBuilder { + if _, ok := ob["data"]; !ok { + ob["data"] = make(map[string]any) + } + if dataMap, ok := ob["data"].(map[string]any); ok { + dataMap[key] = value + } + return ob +} + +func (ob OutputBuilder) SetColumnField(key string, value any) OutputBuilder { + if _, ok := ob["metadata"]; !ok { + ob["metadata"] = make(map[string]any) + } + if metadataMap, ok := ob["metadata"].(map[string]any); ok { + if _, ok := metadataMap["columns"]; !ok { + metadataMap["columns"] = make(map[string]any) + } + if columnsMap, ok := metadataMap["columns"].(map[string]any); ok { + columnsMap[key] = value + } + } + return ob +} + +func (ob OutputBuilder) SetTableName(tableName string) OutputBuilder { + if _, ok := ob["metadata"]; !ok { + ob["metadata"] = make(map[string]any) + } + if metadataMap, ok := ob["metadata"].(map[string]any); ok { + metadataMap["table"] = tableName + } + return ob +} + +func (ob OutputBuilder) RemoveDataFields(fields ...string) OutputBuilder { + if dataMap, ok := ob["data"].(map[string]any); ok { + for _, key := range fields { + delete(dataMap, key) + } + } + return ob +} + +func (ob OutputBuilder) RemoveColumnFields(fields ...string) OutputBuilder { + if metadataMap, ok := ob["metadata"].(map[string]any); ok { + if columnsMap, ok := metadataMap["columns"].(map[string]any); ok { + for _, key := range fields { + delete(columnsMap, key) + } + } + } + return ob +} + +func (ob OutputBuilder) AddRandomEntries(count int, predicate func(index int) (dataKey, dataValue, columnKey, columnValue string)) OutputBuilder { + for i := 0; i < count; i++ { + dataKey, dataValue, columnKey, columnValue := predicate(i) + ob.SetDataField(dataKey, dataValue) + ob.SetColumnField(columnKey, columnValue) + } + return ob +} diff --git a/warehouse/transformer/testhelper/testhelper.go b/warehouse/transformer/testhelper/testhelper.go new file mode 100644 index 00000000000..5aa3b822a86 --- /dev/null +++ b/warehouse/transformer/testhelper/testhelper.go @@ -0,0 +1,42 @@ +package testhelper + +import ( + "fmt" + "strings" + + "github.com/samber/lo" +) + +func AddRandomColumns(eventPayload string, numColumns int) string { + return fmt.Sprintf(eventPayload, strings.Join( + lo.RepeatBy(numColumns, func(index int) string { + return fmt.Sprintf(`"random_column_%d": "random_value_%d"`, index, index) + }), ",", + )) +} + +func AddLargeColumns(eventPayload string, numColumns int) string { + return fmt.Sprintf(eventPayload, strings.Join( + lo.RepeatBy(numColumns, func(index int) string { + return fmt.Sprintf(`"large_column_`+strings.Repeat("a", 1000)+`": "large_value_%d"`, index) + }), ",", + )) +} + +func AddNestedLevels(eventPayload string, numLevels int) string { + var nestedBuilder strings.Builder + + for i := numLevels; i > 0; i-- { + if i < numLevels { + nestedBuilder.WriteString(", ") + } + nestedBuilder.WriteString(fmt.Sprintf(`"nested_level_%d": {`, i)) + } + for i := 0; i < numLevels; i++ { + nestedBuilder.WriteString("}") + if i < numLevels-1 { + nestedBuilder.WriteString(", ") + } + } + return strings.Replace(eventPayload, "{}", "{"+nestedBuilder.String()+"}", 1) +} diff --git a/warehouse/transformer/testhelper/validate.go b/warehouse/transformer/testhelper/validate.go new file mode 100644 index 00000000000..22575722a40 --- /dev/null +++ b/warehouse/transformer/testhelper/validate.go @@ -0,0 +1,99 @@ +package testhelper + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/rudderlabs/rudder-server/backend-config" + ptrans "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/utils/types" +) + +type EventInfo struct { + Payload []byte + Metadata ptrans.Metadata + Destination backendconfig.DestinationT +} + +func ValidateEvents(t *testing.T, infos []EventInfo, pTransformer, dTransformer ptrans.DestinationTransformer, expectedResponse ptrans.Response) { + t.Helper() + + events := prepareEvents(t, infos) + + ctx := context.Background() + batchSize := 100 + + pResponse := pTransformer.Transform(ctx, events, batchSize) + wResponse := dTransformer.Transform(ctx, events, batchSize) + + validateResponseLengths(t, expectedResponse, pResponse, wResponse) + validateRudderEventIfExists(t, expectedResponse, pResponse, wResponse) + validateEventEquality(t, expectedResponse, pResponse, wResponse) + validateFailedEventEquality(t, expectedResponse, pResponse, wResponse) +} + +func prepareEvents(t *testing.T, infos []EventInfo) []ptrans.TransformerEvent { + var events []ptrans.TransformerEvent + for _, info := range infos { + var singularEvent types.SingularEventT + err := json.Unmarshal(info.Payload, &singularEvent) + require.NoError(t, err) + + events = append(events, ptrans.TransformerEvent{ + Message: singularEvent, + Metadata: info.Metadata, + Destination: info.Destination, + }) + } + return events +} + +func validateResponseLengths(t *testing.T, expectedResponse, pResponse, wResponse ptrans.Response) { + require.Equal(t, len(expectedResponse.Events), len(pResponse.Events)) + require.Equal(t, len(expectedResponse.Events), len(wResponse.Events)) + require.Equal(t, len(expectedResponse.FailedEvents), len(pResponse.FailedEvents)) + require.Equal(t, len(expectedResponse.FailedEvents), len(wResponse.FailedEvents)) +} + +func validateRudderEventIfExists(t *testing.T, expectedResponse, pResponse, wResponse ptrans.Response) { + for i := range pResponse.Events { + data := expectedResponse.Events[i].Output["data"] + if data != nil && data.(map[string]any)["rudder_event"] != nil { + expectedRudderEvent := expectedResponse.Events[i].Output["data"].(map[string]any)["rudder_event"].(string) + require.JSONEq(t, expectedRudderEvent, pResponse.Events[i].Output["data"].(map[string]any)["rudder_event"].(string)) + require.JSONEq(t, expectedRudderEvent, wResponse.Events[i].Output["data"].(map[string]any)["rudder_event"].(string)) + require.JSONEq(t, wResponse.Events[i].Output["data"].(map[string]any)["rudder_event"].(string), pResponse.Events[i].Output["data"].(map[string]any)["rudder_event"].(string)) + + // Clean up rudder_event key after comparison + delete(pResponse.Events[i].Output["data"].(map[string]any), "rudder_event") + delete(wResponse.Events[i].Output["data"].(map[string]any), "rudder_event") + delete(expectedResponse.Events[i].Output["data"].(map[string]any), "rudder_event") + } + } +} + +func validateEventEquality(t *testing.T, expectedResponse, pResponse, wResponse ptrans.Response) { + for i := range pResponse.Events { + require.EqualValues(t, expectedResponse.Events[i], pResponse.Events[i]) + require.EqualValues(t, expectedResponse.Events[i], wResponse.Events[i]) + require.EqualValues(t, wResponse.Events[i], pResponse.Events[i]) + } +} + +func validateFailedEventEquality(t *testing.T, expectedResponse, pResponse, wResponse ptrans.Response) { + for i := range pResponse.FailedEvents { + require.NotEmpty(t, pResponse.FailedEvents[i].Error) + require.NotEmpty(t, wResponse.FailedEvents[i].Error) + require.NotEmpty(t, expectedResponse.FailedEvents[i].Error) + + expectedResponse.FailedEvents[i].Error = pResponse.FailedEvents[i].Error + wResponse.FailedEvents[i].Error = pResponse.FailedEvents[i].Error + + require.EqualValues(t, expectedResponse.FailedEvents[i], pResponse.FailedEvents[i]) + require.EqualValues(t, expectedResponse.FailedEvents[i], wResponse.FailedEvents[i]) + require.EqualValues(t, wResponse.FailedEvents[i], pResponse.FailedEvents[i]) + } +} diff --git a/warehouse/transformer/track.go b/warehouse/transformer/track.go index a0142aa68ce..dc03987064b 100644 --- a/warehouse/transformer/track.go +++ b/warehouse/transformer/track.go @@ -57,7 +57,7 @@ func (t *transformer) trackCommonProps(pi *processingInfo) (map[string]any, map[ if d, dok := commonProps[eventTextColName]; dok { eventName, _ = d.(string) } - transformerEventName = TransformTableName(pi.event.Metadata.DestinationType, pi.itrOpts, pi.dstOpts, eventName) + transformerEventName = TransformTableName(pi.itrOpts, pi.dstOpts, eventName) commonProps[eventColName] = transformerEventName commonColumnTypes[eventColName] = "string" diff --git a/warehouse/transformer/track_test.go b/warehouse/transformer/track_test.go index 0871a0f3cdd..0e09d27ebe8 100644 --- a/warehouse/transformer/track_test.go +++ b/warehouse/transformer/track_test.go @@ -7,13 +7,13 @@ import ( "github.com/ory/dockertest/v3" "github.com/stretchr/testify/require" - "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" transformertest "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/transformer" backendconfig "github.com/rudderlabs/rudder-server/backend-config" ptrans "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/warehouse/transformer/testhelper" ) func TestTrack(t *testing.T) { @@ -23,7 +23,7 @@ func TestTrack(t *testing.T) { transformerResource, err := transformertest.Setup(pool, t) require.NoError(t, err) - testsCases := []struct { + testCases := []struct { name string configOverride map[string]any eventPayload string @@ -34,158 +34,18 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES)", eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -194,154 +54,20 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES) without properties", eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(). + RemoveDataFields("product_id", "review_id"). + RemoveColumnFields("product_id", "review_id"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -350,154 +76,20 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES) without userProperties", eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "received_at": "2021-09-01T00:00:00.000Z", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "received_at": "datetime", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(). + RemoveDataFields("rating", "review_body"). + RemoveColumnFields("rating", "review_body"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -506,142 +98,24 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES) without context", eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "5.6.7.8", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). // overriding the default value + RemoveDataFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name"). + RemoveColumnFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "5.6.7.8", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). // overriding the default value + RemoveDataFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name"). + RemoveColumnFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -650,158 +124,23 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES) RudderCreatedTable", eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"accounts","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "accounts", - "event_text": "accounts", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(). + SetDataField("event", "accounts"). + SetDataField("event_text", "accounts"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "accounts", - "event_text": "accounts", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "_accounts", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(). + SetDataField("event", "accounts"). + SetDataField("event_text", "accounts"). + SetTableName("_accounts"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -810,158 +149,23 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES) RudderCreatedTable with skipReservedKeywordsEscaping", eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"accounts","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipReservedKeywordsEscaping":true}}}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "accounts", - "event_text": "accounts", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(). + SetDataField("event", "accounts"). + SetDataField("event_text", "accounts"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "accounts", - "event_text": "accounts", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "accounts", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(). + SetDataField("event", "accounts"). + SetDataField("event_text", "accounts"). + SetTableName("accounts"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -970,158 +174,23 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES) RudderIsolatedTable", eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"users","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "users", - "event_text": "users", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(). + SetDataField("event", "users"). + SetDataField("event_text", "users"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "users", - "event_text": "users", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "_users", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(). + SetDataField("event", "users"). + SetDataField("event_text", "users"). + SetTableName("_users"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -1130,84 +199,16 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES) empty event", eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "event": "", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "event": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(). + SetDataField("event", ""). + RemoveDataFields("event_text"). + RemoveColumnFields("event_text"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -1216,84 +217,16 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES) no event", eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "id": "messageId", - "event": "", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "id": "string", - "event": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(). + SetDataField("event", ""). + RemoveDataFields("event_text"). + RemoveColumnFields("event_text"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -1302,162 +235,22 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES) store rudder event", eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "storeFullEvent": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{ + "storeFullEvent": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - "rudder_event": "{\"type\":\"track\",\"anonymousId\":\"anonymousId\",\"channel\":\"web\",\"context\":{\"destinationId\":\"destinationID\",\"destinationType\":\"POSTGRES\",\"ip\":\"1.2.3.4\",\"sourceId\":\"sourceID\",\"sourceType\":\"sourceType\",\"traits\":{\"email\":\"rhedricks@example.com\",\"logins\":2,\"name\":\"Richard Hendricks\"}},\"event\":\"event\",\"messageId\":\"messageId\",\"originalTimestamp\":\"2021-09-01T00:00:00.000Z\",\"properties\":{\"product_id\":\"9578257311\",\"review_id\":\"86ac1cd43\"},\"receivedAt\":\"2021-09-01T00:00:00.000Z\",\"request_ip\":\"5.6.7.8\",\"sentAt\":\"2021-09-01T00:00:00.000Z\",\"timestamp\":\"2021-09-01T00:00:00.000Z\",\"userId\":\"userId\",\"userProperties\":{\"rating\":3,\"review_body\":\"OK for the price. It works but the material feels flimsy.\"}}", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "rudder_event": "json", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(). + SetDataField("rudder_event", "{\"type\":\"track\",\"anonymousId\":\"anonymousId\",\"channel\":\"web\",\"context\":{\"destinationId\":\"destinationID\",\"destinationType\":\"POSTGRES\",\"ip\":\"1.2.3.4\",\"sourceId\":\"sourceID\",\"sourceType\":\"sourceType\",\"traits\":{\"email\":\"rhedricks@example.com\",\"logins\":2,\"name\":\"Richard Hendricks\"}},\"event\":\"event\",\"messageId\":\"messageId\",\"originalTimestamp\":\"2021-09-01T00:00:00.000Z\",\"properties\":{\"product_id\":\"9578257311\",\"review_id\":\"86ac1cd43\"},\"receivedAt\":\"2021-09-01T00:00:00.000Z\",\"request_ip\":\"5.6.7.8\",\"sentAt\":\"2021-09-01T00:00:00.000Z\",\"timestamp\":\"2021-09-01T00:00:00.000Z\",\"userId\":\"userId\",\"userProperties\":{\"rating\":3,\"review_body\":\"OK for the price. It works but the material feels flimsy.\"}}"). + SetColumnField("rudder_event", "json"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -1466,146 +259,22 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES) partial rules", eventPayload: `{"type":"track","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","event":"event","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(). + RemoveDataFields("anonymous_id", "channel", "context_request_ip"). + RemoveColumnFields("anonymous_id", "channel", "context_request_ip"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(). + RemoveDataFields("anonymous_id", "channel", "context_request_ip"). + RemoveColumnFields("anonymous_id", "channel", "context_request_ip"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -1614,96 +283,15 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES) skipTracksTable (dstOpts)", eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{ - "skipTracksTable": true, - }, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{ + "skipTracksTable": true, + }), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -1712,94 +300,13 @@ func TestTrack(t *testing.T) { { name: "track (POSTGRES) skipTracksTable (itrOpts)", eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipTracksTable":true}}}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, @@ -1811,222 +318,203 @@ func TestTrack(t *testing.T) { "Warehouse.enableIDResolution": true, }, eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "BQ", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "BQ", - }, - }, + metadata: getTrackMetadata("BQ", "webhook"), + destination: getDestination("BQ", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "loaded_at": "datetime", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"), + Metadata: getTrackMetadata("BQ", "webhook"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "BQ", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "loaded_at": "datetime", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(). + SetDataField("context_destination_type", "BQ"). + SetColumnField("loaded_at", "datetime"), + Metadata: getTrackMetadata("BQ", "webhook"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "merge_property_1_type": "anonymous_id", - "merge_property_1_value": "anonymousId", - "merge_property_2_type": "user_id", - "merge_property_2_value": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "merge_property_1_type": "string", - "merge_property_1_value": "string", - "merge_property_2_type": "string", - "merge_property_2_value": "string", - }, - "isMergeRule": true, - "mergePropOne": "anonymousId", - "mergePropTwo": "userId", - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "rudder_identity_merge_rules", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "BQ", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultMergeOutput(), + Metadata: getTrackMetadata("BQ", "webhook"), StatusCode: http.StatusOK, }, }, }, }, } - - for _, tc := range testsCases { + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - c := config.New() - c.Set("DEST_TRANSFORM_URL", transformerResource.TransformerURL) - c.Set("USER_TRANSFORM_URL", transformerResource.TransformerURL) - - for k, v := range tc.configOverride { - c.Set(k, v) - } - - eventsInfos := []eventsInfo{ + c := setupConfig(transformerResource, tc.configOverride) + eventsInfos := []testhelper.EventInfo{ { - payload: []byte(tc.eventPayload), - metadata: tc.metadata, - destination: tc.destination, + Payload: []byte(tc.eventPayload), + Metadata: tc.metadata, + Destination: tc.destination, }, } destinationTransformer := ptrans.NewTransformer(c, logger.NOP, stats.Default) warehouseTransformer := New(c, logger.NOP, stats.NOP) - testEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) + testhelper.ValidateEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) }) } } + +func getTrackDefaultOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "anonymous_id": "anonymousId", + "channel": "web", + "context_destination_id": "destinationID", + "context_destination_type": "POSTGRES", + "context_ip": "1.2.3.4", + "context_passed_ip": "1.2.3.4", + "context_request_ip": "5.6.7.8", + "context_source_id": "sourceID", + "context_source_type": "sourceType", + "context_traits_email": "rhedricks@example.com", + "context_traits_logins": float64(2), + "context_traits_name": "Richard Hendricks", + "event": "event", + "event_text": "event", + "id": "messageId", + "original_timestamp": "2021-09-01T00:00:00.000Z", + "received_at": "2021-09-01T00:00:00.000Z", + "sent_at": "2021-09-01T00:00:00.000Z", + "timestamp": "2021-09-01T00:00:00.000Z", + "user_id": "userId", + }, + "metadata": map[string]any{ + "columns": map[string]any{ + "anonymous_id": "string", + "channel": "string", + "context_destination_id": "string", + "context_destination_type": "string", + "context_ip": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_source_id": "string", + "context_source_type": "string", + "context_traits_email": "string", + "context_traits_logins": "int", + "context_traits_name": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime", + }, + "receivedAt": "2021-09-01T00:00:00.000Z", + "table": "tracks", + }, + "userId": "", + } +} + +func getEventDefaultOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "anonymous_id": "anonymousId", + "channel": "web", + "context_destination_id": "destinationID", + "context_destination_type": "POSTGRES", + "context_ip": "1.2.3.4", + "context_passed_ip": "1.2.3.4", + "context_request_ip": "5.6.7.8", + "context_source_id": "sourceID", + "context_source_type": "sourceType", + "context_traits_email": "rhedricks@example.com", + "context_traits_logins": float64(2), + "context_traits_name": "Richard Hendricks", + "event": "event", + "event_text": "event", + "id": "messageId", + "original_timestamp": "2021-09-01T00:00:00.000Z", + "product_id": "9578257311", + "rating": 3.0, + "received_at": "2021-09-01T00:00:00.000Z", + "review_body": "OK for the price. It works but the material feels flimsy.", + "review_id": "86ac1cd43", + "sent_at": "2021-09-01T00:00:00.000Z", + "timestamp": "2021-09-01T00:00:00.000Z", + "user_id": "userId", + }, + "metadata": map[string]any{ + "columns": map[string]any{ + "anonymous_id": "string", + "channel": "string", + "context_destination_id": "string", + "context_destination_type": "string", + "context_ip": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_source_id": "string", + "context_source_type": "string", + "context_traits_email": "string", + "context_traits_logins": "int", + "context_traits_name": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "product_id": "string", + "rating": "int", + "received_at": "datetime", + "review_body": "string", + "review_id": "string", + "sent_at": "datetime", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime", + }, + "receivedAt": "2021-09-01T00:00:00.000Z", + "table": "event", + }, + "userId": "", + } +} + +func getTrackDefaultMergeOutput() testhelper.OutputBuilder { + return testhelper.OutputBuilder{ + "data": map[string]any{ + "merge_property_1_type": "anonymous_id", + "merge_property_1_value": "anonymousId", + "merge_property_2_type": "user_id", + "merge_property_2_value": "userId", + }, + "metadata": map[string]any{ + "columns": map[string]any{ + "merge_property_1_type": "string", + "merge_property_1_value": "string", + "merge_property_2_type": "string", + "merge_property_2_value": "string", + }, + "isMergeRule": true, + "mergePropOne": "anonymousId", + "mergePropTwo": "userId", + "receivedAt": "2021-09-01T00:00:00.000Z", + "table": "rudder_identity_merge_rules", + }, + "userId": "", + } +} + +func getTrackMetadata(destinationType, sourceCategory string) ptrans.Metadata { + return ptrans.Metadata{ + EventType: "track", + DestinationType: destinationType, + ReceivedAt: "2021-09-01T00:00:00.000Z", + SourceID: "sourceID", + DestinationID: "destinationID", + SourceType: "sourceType", + SourceCategory: sourceCategory, + MessageID: "messageId", + } +} diff --git a/warehouse/transformer/transformer.go b/warehouse/transformer/transformer.go index a74e3168efb..d8cbb536fce 100644 --- a/warehouse/transformer/transformer.go +++ b/warehouse/transformer/transformer.go @@ -47,7 +47,7 @@ type ( func New(conf *config.Config, logger logger.Logger, statsFactory stats.Stats) ptrans.DestinationTransformer { t := &transformer{ conf: conf, - logger: logger, + logger: logger.Child("warehouse-transformer"), statsFactory: statsFactory, now: time.Now, } @@ -102,30 +102,38 @@ func (t *transformer) Transform(_ context.Context, clientEvents []ptrans.Transfo } func (t *transformer) processWarehouseMessage(event ptrans.TransformerEvent) ([]map[string]any, error) { - t.enhanceContextWithSourceDestInfo(event) + if err := t.enhanceContextWithSourceDestInfo(event); err != nil { + return nil, fmt.Errorf("enhancing context with source and destination info: %w", err) + } return t.handleEvent(event) } -func (t *transformer) enhanceContextWithSourceDestInfo(event ptrans.TransformerEvent) { +func (t *transformer) enhanceContextWithSourceDestInfo(event ptrans.TransformerEvent) error { if !t.config.populateSrcDestInfoInContext.Load() { - return + return nil } - messageContext, ok := event.Message["context"].(map[string]any) + messageContext, ok := event.Message["context"] if !ok || messageContext == nil { messageContext = map[string]any{} + event.Message["context"] = messageContext } - messageContext["sourceId"] = event.Metadata.SourceID - messageContext["sourceType"] = event.Metadata.SourceType - messageContext["destinationId"] = event.Metadata.DestinationID - messageContext["destinationType"] = event.Metadata.DestinationType + messageContextMap, ok := messageContext.(map[string]any) + if !ok { + return response.ErrContextNotMap + } + messageContextMap["sourceId"] = event.Metadata.SourceID + messageContextMap["sourceType"] = event.Metadata.SourceType + messageContextMap["destinationId"] = event.Metadata.DestinationID + messageContextMap["destinationType"] = event.Metadata.DestinationType - event.Message["context"] = messageContext + event.Message["context"] = messageContextMap + return nil } func (t *transformer) handleEvent(event ptrans.TransformerEvent) ([]map[string]any, error) { itrOpts := prepareIntegrationOptions(event) - dstOpts := prepareDestinationOptions(event.Destination.Config) + dstOpts := prepareDestinationOptions(event.Metadata.DestinationType, event.Destination.Config) jsonPathsInfo := extractJSONPathInfo(append(itrOpts.jsonPaths, dstOpts.jsonPaths...)) eventType := strings.ToLower(event.Metadata.EventType) diff --git a/warehouse/transformer/transformer_fuzz_test.go b/warehouse/transformer/transformer_fuzz_test.go new file mode 100644 index 00000000000..47caadb66ab --- /dev/null +++ b/warehouse/transformer/transformer_fuzz_test.go @@ -0,0 +1,560 @@ +package transformer + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/ory/dockertest/v3" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + + "github.com/rudderlabs/rudder-go-kit/logger" + "github.com/rudderlabs/rudder-go-kit/stats" + transformertest "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/transformer" + + backendconfig "github.com/rudderlabs/rudder-server/backend-config" + ptrans "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/utils/misc" + "github.com/rudderlabs/rudder-server/utils/types" + "github.com/rudderlabs/rudder-server/warehouse/transformer/testhelper" + whutils "github.com/rudderlabs/rudder-server/warehouse/utils" +) + +func FuzzTransformer(f *testing.F) { + pool, err := dockertest.NewPool("") + require.NoError(f, err) + + transformerResource, err := transformertest.Setup(pool, f) + require.NoError(f, err) + + f.Log("Providing seed corpus for event types") + f.Add(`{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"}}`) + f.Add(`{"type":"alias","messageId":"messageId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"page","name":"Home","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"page","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"}}`) + f.Add(`{"type":"page","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"screen","name":"Main","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"screen","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"}}`) + f.Add(`{"type":"screen","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"group","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"group","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"}}`) + f.Add(`{"type":"group","messageId":"messageId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"merge","mergeProperties":[{"type":"email","value":"alex@example.com"},{"type":"mobile","value":"+1-202-555-0146"}]}`) + f.Add(`{"type":"merge"}`) + f.Add(`{"type":"merge", "mergeProperties": "invalid"}`) + f.Add(`{"type":"merge", "mergeProperties": []}`) + f.Add(`{"type":"merge","mergeProperties":[{"type":"email","value":"alex@example.com"}]}`) + f.Add(`{"type":"merge","mergeProperties":["invalid",{"type":"email","value":"alex@example.com"}]}`) + f.Add(`{"type":"merge","mergeProperties":[{"type":"email","value":"alex@example.com"},"invalid"]}`) + f.Add(`{"type":"merge","mergeProperties":[{"type1":"email","value1":"alex@example.com"},{"type1":"mobile","value1":"+1-202-555-0146"}]}`) + f.Add(`{"type":"merge","mergeProperties":[{"type1":"email","value1":"alex@example.com"},{"type1":"mobile","value1":"+1-202-555-0146"}]}`) + f.Add(`{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"","previousId":"","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"page","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"page","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."}}`) + f.Add(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"accounts","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"accounts","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipReservedKeywordsEscaping":true}}}}`) + f.Add(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"users","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"track","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","event":"event","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipTracksTable":true}}}}`) + f.Add(`{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"ip":"1.2.3.4"}}`) + f.Add(`{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."}}`) + f.Add(`{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"user_id":"user_id","rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"identify","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipUsersTable":true}}}}`) + f.Add(`{"type":"extract","recordId":"recordID","event":"event","receivedAt":"2021-09-01T00:00:00.000Z","context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"extract","recordId":"recordID","event":"event","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"}}`) + f.Add(`{"type":"extract","recordId":"recordID","event":"accounts","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"extract","recordId":"recordID","event":"accounts","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipReservedKeywordsEscaping":true}}}}`) + f.Add(`{"type":"extract","recordId":"recordID","event":"users","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + + f.Log("Providing seed corpus for column names") + columnNames := []string{ + // SQL keywords and reserved words + "select", "from", "where", "and", "or", "not", "insert", "update", "delete", + "create", "alter", "drop", "table", "index", "view", "primary", "foreign", + "key", "constraint", "default", "null", "unique", "check", "references", + "join", "inner", "outer", "left", "right", "full", "on", "group", "by", + "having", "order", "asc", "desc", "limit", "offset", "union", "all", + "distinct", "as", "in", "between", "like", "is", "null", "true", "false", + + // Data types (which can vary by database system) + "int", "integer", "bigint", "smallint", "tinyint", "decimal", "numeric", + "float", "real", "double", "precision", "char", "varchar", "text", "date", + "time", "timestamp", "datetime", "boolean", "blob", "clob", "binary", + + // Names starting with numbers or special characters + "1column", "2_column", "@column", "#column", "$column", + + // Names with spaces or special characters + "column name", "column-name", "column.name", "column@name", "column#name", + "column$name", "column%name", "column&name", "column*name", "column+name", + "column/name", "column\\name", "column'name", "column\"name", "column`name", + + // Names with non-ASCII characters + "columnñame", "colûmnname", "columnнаме", "列名", "カラム名", + + // Very long names (may exceed maximum length in some databases) + "this_is_a_very_long_column_name_that_exceeds_the_maximum_allowed_length_in_many_database_systems", + + // Names that could be confused with functions + "count", "sum", "avg", "max", "min", "first", "last", "now", "current_timestamp", + + // Names with potential encoding issues + "column\u0000name", "column\ufffdname", + + // Names that might conflict with ORM conventions + "id", "_id", "created_at", "updated_at", "deleted_at", + + // Names that might conflict with common programming conventions + "class", "interface", "enum", "struct", "function", "var", "let", "const", + + // Names with emoji or other Unicode symbols + "column😀name", "column→name", "column★name", + + // Names with mathematical symbols + "column+name", "column-name", "column*name", "column/name", "column^name", + "column=name", "columnname", "column≠name", "column≈name", + + // Names with comment-like syntax + "column--name", "column/*name*/", + + // Names that might be interpreted as operators + "column||name", "column&&name", "column!name", "column?name", + + // Names with control characters + "column\tname", "column\nname", "column\rname", + + // Names that might conflict with schema notation + "schema.column", "database.schema.column", + + // Names with brackets or parentheses + "column(name)", "column[name]", "column{name}", + + // Names with quotes + "'column'", "\"column\"", "`column`", + + // Names that might be interpreted as aliases + "column as alias", + + // Names that might conflict with database-specific features + "rowid", "oid", "xmin", "ctid", // These are specific to certain databases + + // Names that might conflict with common column naming conventions + "fk_", "idx_", "pk_", "ck_", "uq_", + + // Names with invisible characters + "column\u200bname", // Zero-width space + "column\u00A0name", // Non-breaking space + + // Names with combining characters + "columǹ", // 'n' with combining grave accent + + // Names with bi-directional text + "column\u202Ename\u202C", // Using LTR and RTL markers + + // Names with unusual capitalization + "COLUMN", "Column", "cOlUmN", + + // Names that are empty or only whitespace + "", " ", "\t", "\n", + + // Names with currency symbols + "column¢name", "column£name", "column€name", "column¥name", + + // Names with less common punctuation + "column·name", "column…name", "column•name", "column‽name", + + // Names with fractions or other numeric forms + "column½name", "column²name", "columnⅣname", + } + for _, columnName := range columnNames { + f.Add(`{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","context":{"traits":{"` + columnName + `":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"page","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","` + columnName + `":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"screen","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","` + columnName + `":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"group","messageId":"messageId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"` + columnName + `":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","` + columnName + `":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipTracksTable":true}}}}`) + f.Add(`{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","` + columnName + `":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipUsersTable":true}}}}`) + f.Add(`{"type":"extract","recordId":"recordID","event":"users","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","` + columnName + `":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + } + + f.Log("Providing seed corpus for event names") + eventNames := []string{ + "omega", + "omega v2 ", + "9mega", + "mega&", + "ome$ga", + "omega$", + "ome_ ga", + "9mega________-________90", + "Cízǔ", + "Rudderstack", + "___", + "group", + "k3_namespace", + "k3_namespace", + "select", + "drop", + "create", + "alter", + "index", + "table", + "from", + "where", + "join", + "union", + "insert", + "update", + "delete", + "truncate", + "1invalid", + "invalid-name", + "invalid.name", + "name with spaces", + "name@with@special@chars", + "verylongnamethatiswaytoolongforadatabasetablenameandexceedsthemaximumlengthallowed", + "ñáme_wíth_áccents", + "schema.tablename", + "'quoted_name'", + "name--with--comments", + "name/*with*/comments", + "name;with;semicolons", + "name,with,commas", + "name(with)parentheses", + "name[with]brackets", + "name{with}braces", + "name+with+plus", + "name=with=equals", + "nameangle_brackets", + "name|with|pipes", + "name\\with\\backslashes", + "name/with/slashes", + "name\"with\"quotes", + "name'with'single_quotes", + "name`with`backticks", + "name!with!exclamation", + "name?with?question", + "name#with#hash", + "name%with%percent", + "name^with^caret", + "name~with~tilde", + "primary", + "foreign", + "key", + "constraint", + "default", + "null", + "not null", + "auto_increment", + "identity", + "unique", + "check", + "references", + "on delete", + "on update", + "cascade", + "restrict", + "set null", + "set default", + "temporary", + "temp", + "view", + "function", + "procedure", + "trigger", + "序列化", // Chinese for "serialization" + "テーブル", // Japanese for "table" + "таблица", // Russian for "table" + "0day", + "_system", + "__hidden__", + "name:with:colons", + "name★with★stars", + "name→with→arrows", + "name•with•bullets", + "name‼with‼double_exclamation", + "name⁉with⁉interrobang", + "name‽with‽interrobang", + "name⚠with⚠warning", + "name☢with☢radiation", + "name❗with❗exclamation", + "name❓with❓question", + "name⏎with⏎return", + "name⌘with⌘command", + "name⌥with⌥option", + "name⇧with⇧shift", + "name⌃with⌃control", + "name⎋with⎋escape", + "name␣with␣space", + "name⍽with⍽space", + "name¶with¶pilcrow", + "name§with§section", + "name‖with‖double_vertical_bar", + "name¦with¦broken_bar", + "name¬with¬negation", + "name¤with¤currency", + "name‰with‰permille", + "name‱with‱permyriad", + "name∞with∞infinity", + "name≠with≠not_equal", + "name≈with≈approximately_equal", + "name≡with≡identical", + "name√with√square_root", + "name∛with∛cube_root", + "name∜with∜fourth_root", + "name∫with∫integral", + "name∑with∑sum", + "name∏with∏product", + "name∀with∀for_all", + "name∃with∃exists", + "name∄with∄does_not_exist", + "name∅with∅empty_set", + "name∈with∈element_of", + "name∉with∉not_element_of", + "name∋with∋contains", + "name∌with∌does_not_contain", + "name∩with∩intersection", + "name∪with∪union", + "name⊂with⊂subset", + "name⊃with⊃superset", + "name⊄with⊄not_subset", + "name⊅with⊅not_superset", + } + for _, eventName := range eventNames { + f.Add(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"` + eventName + `","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipTracksTable":true}}}}`) + f.Add(`{"type":"extract","recordId":"recordID","event":"` + eventName + `","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + } + + f.Log("Providing seed corpus for random columns") + f.Add(testhelper.AddRandomColumns(`{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 500)) + f.Add(testhelper.AddRandomColumns(`{"type":"page","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 500)) + f.Add(testhelper.AddRandomColumns(`{"type":"screen","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 500)) + f.Add(testhelper.AddRandomColumns(`{"type":"group","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 500)) + f.Add(testhelper.AddRandomColumns(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 500)) + f.Add(testhelper.AddRandomColumns(`{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 500)) + f.Add(testhelper.AddRandomColumns(`{"type":"extract","recordId":"recordID","event":"event","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 500)) + + f.Log("Providing seed corpus for big columns") + f.Add(testhelper.AddLargeColumns(`{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + f.Add(testhelper.AddLargeColumns(`{"type":"page","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + f.Add(testhelper.AddLargeColumns(`{"type":"screen","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + f.Add(testhelper.AddLargeColumns(`{"type":"group","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + f.Add(testhelper.AddLargeColumns(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + f.Add(testhelper.AddLargeColumns(`{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + f.Add(testhelper.AddLargeColumns(`{"type":"extract","recordId":"recordID","event":"event","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + + f.Log("Providing seed corpus for nested levels") + f.Add(testhelper.AddNestedLevels(`{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + f.Add(testhelper.AddNestedLevels(`{"type":"page","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + f.Add(testhelper.AddNestedLevels(`{"type":"screen","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + f.Add(testhelper.AddNestedLevels(`{"type":"group","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + f.Add(testhelper.AddNestedLevels(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + f.Add(testhelper.AddNestedLevels(`{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + f.Add(testhelper.AddNestedLevels(`{"type":"extract","recordId":"recordID","event":"event","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 10)) + + f.Log("Providing seed corpus for channel") + for _, channel := range []string{"web", "sources", "android", "ios", "server", "backend", "frontend", "mobile", "desktop", "webapp", "mobileapp", "desktopapp", "website"} { + f.Add(`{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"` + channel + `","request_ip":"5.6.7.8","context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"page","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"` + channel + `","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"screen","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"` + channel + `","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"group","messageId":"messageId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"` + channel + `","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"` + channel + `","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipTracksTable":true}}}}`) + f.Add(`{"type":"identify","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"` + channel + `","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipUsersTable":true}}}}`) + f.Add(`{"type":"extract","recordId":"recordID","event":"users","receivedAt":"2021-09-01T00:00:00.000Z","channel":"` + channel + `","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + } + + f.Log("Providing seed corpus for caps key names") + f.Add(`{"TYPE":"alias","MESSAGEID":"messageId","ANONYMOUSID":"anonymousId","USERID":"userId","PREVIOUSID":"previousId","SENTAT":"2021-09-01T00:00:00.000Z","TIMESTAMP":"2021-09-01T00:00:00.000Z","RECEIVEDAT":"2021-09-01T00:00:00.000Z","ORIGINALTIMESTAMP":"2021-09-01T00:00:00.000Z","CHANNEL":"web","REQUEST_IP":"5.6.7.8","CONTEXT":{"TRAITS":{"EMAIL":"rhedrICKS@example.com","LOGINS":2},"IP":"1.2.3.4"}}`) + f.Add(`{"TYPE":"page","MESSAGEID":"messageId","USERID":"userId","SENTAT":"2021-09-01T00:00:00.000Z","TIMESTAMP":"2021-09-01T00:00:00.000Z","RECEIVEDAT":"2021-09-01T00:00:00.000Z","ORIGINALTIMESTAMP":"2021-09-01T00:00:00.000Z","PROPERTIES":{"NAME":"Home","TITLE":"Home | RudderStack","URL":"https://www.rudderstack.com"},"CONTEXT":{"TRAITS":{"NAME":"Richard Hendricks","EMAIL":"rhedrICKS@example.com","LOGINS":2},"IP":"1.2.3.4"}}`) + f.Add(`{"TYPE":"screen","MESSAGEID":"messageId","USERID":"userId","SENTAT":"2021-09-01T00:00:00.000Z","TIMESTAMP":"2021-09-01T00:00:00.000Z","RECEIVEDAT":"2021-09-01T00:00:00.000Z","ORIGINALTIMESTAMP":"2021-09-01T00:00:00.000Z","PROPERTIES":{"NAME":"Main","TITLE":"Home | RudderStack","URL":"https://www.rudderstack.com"},"CONTEXT":{"TRAITS":{"NAME":"Richard Hendricks","EMAIL":"rhedrICKS@example.com","LOGINS":2},"IP":"1.2.3.4"}}`) + f.Add(`{"TYPE":"group","MESSAGEID":"messageId","USERID":"userId","GROUPID":"groupId","SENTAT":"2021-09-01T00:00:00.000Z","TIMESTAMP":"2021-09-01T00:00:00.000Z","RECEIVEDAT":"2021-09-01T00:00:00.000Z","ORIGINALTIMESTAMP":"2021-09-01T00:00:00.000Z","TRAITS":{"TITLE":"Home | RudderStack","URL":"https://www.rudderstack.com"},"CONTEXT":{"TRAITS":{"EMAIL":"rhedrICKS@example.com","LOGINS":2},"IP":"1.2.3.4"}}`) + f.Add(`{"TYPE":"track","MESSAGEID":"messageId","ANONYMOUSID":"anonymousId","USERID":"userId","SENTAT":"2021-09-01T00:00:00.000Z","TIMESTAMP":"2021-09-01T00:00:00.000Z","RECEIVEDAT":"2021-09-01T00:00:00.000Z","ORIGINALTIMESTAMP":"2021-09-01T00:00:00.000Z","CHANNEL":"web","EVENT":"event","REQUEST_IP":"5.6.7.8","PROPERTIES":{"REVIEW_ID":"86ac1cd43","PRODUCT_ID":"9578257311"},"USERPROPERTIES":{"RATING":3.0,"REVIEW_BODY":"OK for the price. It works but the material feels flimsy."},"CONTEXT":{"TRAITS":{"NAME":"Richard Hendricks","EMAIL":"rhedrICKS@example.com","LOGINS":2},"IP":"1.2.3.4"},"INTEGRATIONS":{"POSTGRES":{"OPTIONS":{"SKIPTRACKSTABLE":true}}}}`) + f.Add(`{"TYPE":"identify","MESSAGEID":"messageId","ANONYMOUSID":"anonymousId","USERID":"userId","SENTAT":"2021-09-01T00:00:00.000Z","TIMESTAMP":"2021-09-01T00:00:00.000Z","RECEIVEDAT":"2021-09-01T00:00:00.000Z","ORIGINALTIMESTAMP":"2021-09-01T00:00:00.000Z","CHANNEL":"web","REQUEST_IP":"5.6.7.8","TRAITS":{"REVIEW_ID":"86ac1cd43","PRODUCT_ID":"9578257311"},"USERPROPERTIES":{"RATING":3.0,"REVIEW_BODY":"OK for the price. It works but the material feels flimsy."},"CONTEXT":{"TRAITS":{"NAME":"Richard Hendricks","EMAIL":"rhedrICKS@example.com","LOGINS":2},"IP":"1.2.3.4"},"INTEGRATIONS":{"POSTGRES":{"OPTIONS":{"SKIPUSERSTABLE":true}}}}`) + f.Add(`{"TYPE":"extract","RECORDID":"recordID","EVENT":"users","RECEIVEDAT":"2021-09-01T00:00:00.000Z","PROPERTIES":{"NAME":"Home","TITLE":"Home | RudderStack","URL":"https://www.rudderstack.com"},"CONTEXT":{"TRAITS":{"NAME":"Richard Hendricks","EMAIL":"rhedrICKS@example.com","LOGINS":2},"IP":"1.2.3.4"}}`) + + f.Log("Providing seed corpus for caps event type") + f.Add(`{"type":"ALIAS","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"PAGE","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"SCREEN","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"GROUP","messageId":"messageId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + f.Add(`{"type":"TRACK","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipTracksTable":true}}}}`) + f.Add(`{"type":"IDENTIFY","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},"integrations":{"POSTGRES":{"options":{"skipUsersTable":true}}}}`) + f.Add(`{"type":"EXTRACT","recordId":"recordID","event":"users","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`) + + f.Log("Providing seed corpus for integrations options") + integrationOpts := []string{ + `{"options":{"skipReservedKeywordsEscaping":true}}`, + `{"options":{"skipReservedKeywordsEscaping":false}}`, + `{"options":{"useBlendoCasing":false}}`, + `{"options":{"useBlendoCasing":true}}`, + `{"options":{"skipTracksTable":true}}`, + `{"options":{"skipTracksTable":false}}`, + `{"options":{"skipUsersTable":true}}`, + `{"options":{"skipUsersTable":false}}`, + `{"options":{"jsonPaths":["context", "properties", "userProperties", "context.traits"]}}`, + `{"options":{"jsonPaths":["track.context", "track.properties", "track.userProperties", "track.context.traits"]}}`, + `{"options":{"jsonPaths":["properties", "context.traits"]}}`, + `{"options":{"jsonPaths":["page.properties", "page.context.traits"]}}`, + `{"options":{"jsonPaths":["screen.properties", "screen.context.traits"]}}`, + `{"options":{"jsonPaths":["alias.properties", "alias.context.traits"]}}`, + `{"options":{"jsonPaths":["group.properties", "group.context.traits"]}}`, + `{"options":{"jsonPaths":["extract.properties", "extract.context.traits"]}}`, + `{"options":{"jsonPaths":["identify.traits", "identify.context.traits", "identify.userProperties"]}}`, + } + for _, destType := range whutils.WarehouseDestinations { + for _, opt := range integrationOpts { + itrOptsPayload := fmt.Sprintf(`"integrations":{"%s":%s}`, destType, opt) + f.Add(fmt.Sprintf(`{"type":"alias","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","previousId":"previousId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},%s}`, itrOptsPayload)) + f.Add(fmt.Sprintf(`{"type":"page","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},%s}`, itrOptsPayload)) + f.Add(fmt.Sprintf(`{"type":"screen","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","properties":{"name":"Main","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},%s}`, itrOptsPayload)) + f.Add(fmt.Sprintf(`{"type":"group","messageId":"messageId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},%s}`, itrOptsPayload)) + f.Add(fmt.Sprintf(`{"type":"page","messageId":"messageId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},%s}`, itrOptsPayload)) + f.Add(fmt.Sprintf(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},%s}`, itrOptsPayload)) + f.Add(fmt.Sprintf(`{"type":"IDENTIFY","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},%s}`, itrOptsPayload)) + f.Add(fmt.Sprintf(`{"type":"EXTRACT","recordId":"recordID","event":"users","receivedAt":"2021-09-01T00:00:00.000Z","properties":{"name":"Home","title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"},%s}`, itrOptsPayload)) + } + } + + f.Log("Providing seed corpus for destination configurations options") + destConfigOpts := []map[string]any{ + {"skipTracksTable": true}, + {"skipTracksTable": false}, + {"skipUsersTable": true}, + {"skipUsersTable": false}, + {"underscoreDivideNumbers": true}, + {"underscoreDivideNumbers": false}, + {"allowUsersContextTraits": true}, + {"allowUsersContextTraits": false}, + {"jsonPaths": strings.Join([]string{"context", "properties", "userProperties", "context.traits"}, ",")}, + {"jsonPaths": strings.Join([]string{"track.context", "track.properties", "track.userProperties", "track.context.traits"}, ",")}, + {"jsonPaths": strings.Join([]string{"properties", "context.traits"}, ",")}, + {"jsonPaths": strings.Join([]string{"page.properties", "page.context.traits"}, ",")}, + {"jsonPaths": strings.Join([]string{"screen.properties", "screen.context.traits"}, ",")}, + {"jsonPaths": strings.Join([]string{"alias.properties", "alias.context.traits"}, ",")}, + {"jsonPaths": strings.Join([]string{"group.properties", "group.context.traits"}, ",")}, + {"jsonPaths": strings.Join([]string{"extract.properties", "extract.context.traits"}, ",")}, + {"jsonPaths": strings.Join([]string{"identify.traits", "identify.context.traits", "identify.userProperties"}, ",")}, + } + + f.Fuzz(func(t *testing.T, payload string) { + payload, err := sanitizeJSON(payload) + if err != nil { + return + } + + eventType := gjson.Get(payload, "type").String() + eventName := gjson.Get(payload, "event").String() + messageID := gjson.Get(payload, "messageId").String() + receivedAt := gjson.Get(payload, "receivedAt").Time() + recordID := gjson.Get(payload, "recordId").Value() + + c := setupConfig(transformerResource, map[string]any{}) + + destinationTransformer := ptrans.NewTransformer(c, logger.NOP, stats.Default) + warehouseTransformer := New(c, logger.NOP, stats.NOP) + + for _, destType := range whutils.WarehouseDestinations { + destConfig := map[string]any{} + for k, v := range destConfigOpts[len(payload)%len(destConfigOpts)] { + destConfig[k] = v + } + + eventsInfos := []testhelper.EventInfo{ + { + Payload: []byte(payload), + Metadata: ptrans.Metadata{ + EventType: eventType, + EventName: eventName, + DestinationType: destType, + ReceivedAt: receivedAt.Format(misc.RFC3339Milli), + SourceID: "sourceID", + DestinationID: "destinationID", + SourceType: "sourceType", + MessageID: messageID, + RecordID: recordID, + }, + Destination: backendconfig.DestinationT{ + Name: destType, + Config: destConfig, + DestinationDefinition: backendconfig.DestinationDefinitionT{ + Name: destType, + }, + }, + }, + } + + cmpEvents(t, eventsInfos, destinationTransformer, warehouseTransformer) + } + }) +} + +func cmpEvents(t *testing.T, infos []testhelper.EventInfo, pTransformer, dTransformer ptrans.DestinationTransformer) { + t.Helper() + + var events []ptrans.TransformerEvent + for _, info := range infos { + var singularEvent types.SingularEventT + err := json.Unmarshal(info.Payload, &singularEvent) + require.NoError(t, err) + + events = append(events, ptrans.TransformerEvent{ + Message: singularEvent, + Metadata: info.Metadata, + Destination: info.Destination, + }) + } + + ctx := context.Background() + batchSize := 100 + + pResponse := pTransformer.Transform(ctx, events, batchSize) + wResponse := dTransformer.Transform(ctx, events, batchSize) + + require.Equal(t, len(pResponse.Events), len(wResponse.Events)) + require.Equal(t, len(pResponse.FailedEvents), len(wResponse.FailedEvents)) + + for i := range pResponse.Events { + require.EqualValues(t, wResponse.Events[i], pResponse.Events[i]) + } + for i := range pResponse.FailedEvents { + require.NotEmpty(t, pResponse.FailedEvents[i].Error) + require.NotEmpty(t, wResponse.FailedEvents[i].Error) + + wResponse.FailedEvents[i].Error = pResponse.FailedEvents[i].Error + + require.EqualValues(t, wResponse.FailedEvents[i], pResponse.FailedEvents[i]) + } +} + +func sanitizeJSON(input string) (string, error) { + sanitized := strings.ReplaceAll(input, `\u0000`, "") + + if len(strings.TrimSpace(sanitized)) == 0 { + return "{}", nil + } + + var result any + if err := json.Unmarshal([]byte(sanitized), &result); err != nil { + return "", errors.New("invalid JSON format") + } + + output, err := json.Marshal(result) + if err != nil { + return "", err + } + return string(output), nil +} diff --git a/warehouse/transformer/transformer_test.go b/warehouse/transformer/transformer_test.go index 26b67d4d688..251088f7de9 100644 --- a/warehouse/transformer/transformer_test.go +++ b/warehouse/transformer/transformer_test.go @@ -1,16 +1,11 @@ package transformer import ( - "context" - "encoding/json" "fmt" "net/http" - "strconv" - "strings" "testing" "github.com/ory/dockertest/v3" - "github.com/samber/lo" "github.com/stretchr/testify/require" "github.com/rudderlabs/rudder-go-kit/config" @@ -20,92 +15,12 @@ import ( backendconfig "github.com/rudderlabs/rudder-server/backend-config" ptrans "github.com/rudderlabs/rudder-server/processor/transformer" - "github.com/rudderlabs/rudder-server/utils/types" + "github.com/rudderlabs/rudder-server/warehouse/transformer/internal/response" + "github.com/rudderlabs/rudder-server/warehouse/transformer/testhelper" ) -type eventsInfo struct { - payload []byte - metadata ptrans.Metadata - destination backendconfig.DestinationT -} - -func testEvents(t *testing.T, infos []eventsInfo, pTransformer, dTransformer ptrans.DestinationTransformer, expectedResponse ptrans.Response) { - t.Helper() - - events := prepareEvents(t, infos) - - ctx := context.Background() - batchSize := 100 - - pResponse := pTransformer.Transform(ctx, events, batchSize) - wResponse := dTransformer.Transform(ctx, events, batchSize) - - validateResponseLengths(t, expectedResponse, pResponse, wResponse) - validateEventData(t, expectedResponse, pResponse, wResponse) - validateEventEquality(t, expectedResponse, pResponse, wResponse) - validateFailedEventEquality(t, expectedResponse, pResponse, wResponse) -} - -func prepareEvents(t *testing.T, infos []eventsInfo) []ptrans.TransformerEvent { - var events []ptrans.TransformerEvent - for _, info := range infos { - var singularEvent types.SingularEventT - err := json.Unmarshal(info.payload, &singularEvent) - require.NoError(t, err) - - events = append(events, ptrans.TransformerEvent{ - Message: singularEvent, - Metadata: info.metadata, - Destination: info.destination, - }) - } - return events -} - -func validateResponseLengths(t *testing.T, expectedResponse, pResponse, wResponse ptrans.Response) { - require.Equal(t, len(expectedResponse.Events), len(pResponse.Events)) - require.Equal(t, len(expectedResponse.Events), len(wResponse.Events)) - require.Equal(t, len(expectedResponse.FailedEvents), len(pResponse.FailedEvents)) - require.Equal(t, len(expectedResponse.FailedEvents), len(wResponse.FailedEvents)) -} - -func validateEventData(t *testing.T, expectedResponse, pResponse, wResponse ptrans.Response) { - for i := range pResponse.Events { - data := expectedResponse.Events[i].Output["data"] - if data != nil && data.(map[string]any)["rudder_event"] != nil { - expectedRudderEvent := expectedResponse.Events[i].Output["data"].(map[string]any)["rudder_event"].(string) - require.JSONEq(t, expectedRudderEvent, pResponse.Events[i].Output["data"].(map[string]any)["rudder_event"].(string)) - require.JSONEq(t, expectedRudderEvent, wResponse.Events[i].Output["data"].(map[string]any)["rudder_event"].(string)) - require.JSONEq(t, wResponse.Events[i].Output["data"].(map[string]any)["rudder_event"].(string), pResponse.Events[i].Output["data"].(map[string]any)["rudder_event"].(string)) - - // Clean up rudder_event key after comparison - delete(pResponse.Events[i].Output["data"].(map[string]any), "rudder_event") - delete(wResponse.Events[i].Output["data"].(map[string]any), "rudder_event") - delete(expectedResponse.Events[i].Output["data"].(map[string]any), "rudder_event") - } - } -} - -func validateEventEquality(t *testing.T, expectedResponse, pResponse, wResponse ptrans.Response) { - for i := range pResponse.Events { - require.EqualValues(t, expectedResponse.Events[i], pResponse.Events[i]) - require.EqualValues(t, expectedResponse.Events[i], wResponse.Events[i]) - require.EqualValues(t, wResponse.Events[i], pResponse.Events[i]) - } -} - -func validateFailedEventEquality(t *testing.T, expectedResponse, pResponse, wResponse ptrans.Response) { - for i := range pResponse.FailedEvents { - require.EqualValues(t, expectedResponse.FailedEvents[i], pResponse.FailedEvents[i]) - require.EqualValues(t, expectedResponse.FailedEvents[i], wResponse.FailedEvents[i]) - require.EqualValues(t, wResponse.FailedEvents[i], pResponse.FailedEvents[i]) - } -} - func TestTransformer(t *testing.T) { - t.Skip() - - testsCases := []struct { + testCases := []struct { name string configOverride map[string]any envOverride []string @@ -115,664 +30,411 @@ func TestTransformer(t *testing.T) { expectedResponse ptrans.Response }{ { - name: "unknown event (POSTGRES)", + name: "Unknown event", eventPayload: `{"type":"unknown"}`, - metadata: ptrans.Metadata{ - EventType: "unknown", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", - }, - }, + metadata: getMetadata("unknown", "POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ FailedEvents: []ptrans.TransformerResponse{ { Error: "Unknown event type: \"unknown\"", + Metadata: getMetadata("unknown", "POSTGRES"), StatusCode: http.StatusBadRequest, - Metadata: ptrans.Metadata{ - EventType: "unknown", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, }, }, }, }, + // TODO: Enable this once we have the https://github.com/rudderlabs/rudder-transformer/pull/3806 changes in latest + //{ + // name: "Not populateSrcDestInfoInContext", + // configOverride: map[string]any{ + // "Warehouse.populateSrcDestInfoInContext": false, + // }, + // envOverride: []string{"WH_POPULATE_SRC_DEST_INFO_IN_CONTEXT=false"}, + // eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, + // metadata: getMetadata("track", "POSTGRES"), + // destination: getDestination("POSTGRES", map[string]any{}), + // expectedResponse: ptrans.Response{ + // Events: []ptrans.TransformerResponse{ + // { + // Output: getTrackDefaultOutput(). + // RemoveDataFields("context_destination_id", "context_destination_type", "context_source_id", "context_source_type"). + // RemoveColumnFields("context_destination_id", "context_destination_type", "context_source_id", "context_source_type"), + // Metadata: getMetadata("track", "POSTGRES"), + // StatusCode: http.StatusOK, + // }, + // { + // Output: getEventDefaultOutput(). + // RemoveDataFields("context_destination_id", "context_destination_type", "context_source_id", "context_source_type"). + // RemoveColumnFields("context_destination_id", "context_destination_type", "context_source_id", "context_source_type"), + // Metadata: getMetadata("track", "POSTGRES"), + // StatusCode: http.StatusOK, + // }, + // }, + // }, + //}, { - name: "track (POSTGRES) not populateSrcDestInfoInContext", - configOverride: map[string]any{ - "Warehouse.populateSrcDestInfoInContext": false, - }, - envOverride: []string{"WH_POPULATE_SRC_DEST_INFO_IN_CONTEXT=false"}, - eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", + name: "Too many columns", + eventPayload: testhelper.AddRandomColumns(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","context":{%s},"ip":"1.2.3.4"}`, 500), + metadata: getMetadata("track", "POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), + expectedResponse: ptrans.Response{ + FailedEvents: []ptrans.TransformerResponse{ + { + Error: "postgres transformer: Too many columns outputted from the event", + Metadata: getMetadata("track", "POSTGRES"), + StatusCode: http.StatusBadRequest, + }, + }, }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", + }, + { + name: "Too many columns (DataLake)", + eventPayload: testhelper.AddRandomColumns(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 500), + metadata: getMetadata("track", "GCS_DATALAKE"), + destination: getDestination("GCS_DATALAKE", map[string]any{}), + expectedResponse: ptrans.Response{ + Events: []ptrans.TransformerResponse{ + { + Output: getTrackDefaultOutput().SetDataField("context_destination_type", "GCS_DATALAKE").AddRandomEntries(500, func(index int) (string, string, string, string) { + return fmt.Sprintf("context_random_column_%d", index), fmt.Sprintf("random_value_%d", index), fmt.Sprintf("context_random_column_%d", index), "string" + }), + Metadata: getMetadata("track", "GCS_DATALAKE"), + StatusCode: http.StatusOK, + }, + { + Output: getEventDefaultOutput().SetDataField("context_destination_type", "GCS_DATALAKE").AddRandomEntries(500, func(index int) (string, string, string, string) { + return fmt.Sprintf("context_random_column_%d", index), fmt.Sprintf("random_value_%d", index), fmt.Sprintf("context_random_column_%d", index), "string" + }), + Metadata: getMetadata("track", "GCS_DATALAKE"), + StatusCode: http.StatusOK, + }, }, }, + }, + { + name: "Too many columns channel as sources", + eventPayload: testhelper.AddRandomColumns(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"sources","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, 500), + metadata: getMetadata("track", "POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput().SetDataField("channel", "sources").AddRandomEntries(500, func(index int) (string, string, string, string) { + return fmt.Sprintf("context_random_column_%d", index), fmt.Sprintf("random_value_%d", index), fmt.Sprintf("context_random_column_%d", index), "string" + }), + Metadata: getMetadata("track", "POSTGRES"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - "metadata": map[string]any{ - "columns": map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput().SetDataField("channel", "sources").AddRandomEntries(500, func(index int) (string, string, string, string) { + return fmt.Sprintf("context_random_column_%d", index), fmt.Sprintf("random_value_%d", index), fmt.Sprintf("context_random_column_%d", index), "string" + }), + Metadata: getMetadata("track", "POSTGRES"), StatusCode: http.StatusOK, }, }, }, }, { - name: "track (POSTGRES) too many columns", - eventPayload: fmt.Sprintf(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","context":{%s},"ip":"1.2.3.4"}`, strings.Join( - lo.RepeatBy(500, func(index int) string { - return fmt.Sprintf(`"column_%d": "value_%d"`, index, index) - }), ",", - )), - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", + name: "StringLikeObject for context traits", + eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"0":"a","1":"b","2":"c"},"ip":"1.2.3.4"}}`, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), + expectedResponse: ptrans.Response{ + Events: []ptrans.TransformerResponse{ + { + Output: getTrackDefaultOutput(). + RemoveDataFields("context_traits_email", "context_traits_logins", "context_traits_name"). + RemoveColumnFields("context_traits_email", "context_traits_logins", "context_traits_name"). + SetDataField("context_traits", "abc"). + SetColumnField("context_traits", "string"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), + StatusCode: http.StatusOK, + }, + { + Output: getEventDefaultOutput(). + RemoveDataFields("context_traits_email", "context_traits_logins", "context_traits_name"). + RemoveColumnFields("context_traits_email", "context_traits_logins", "context_traits_name"). + SetDataField("context_traits", "abc"). + SetColumnField("context_traits", "string"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), + StatusCode: http.StatusOK, + }, + }, }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", + }, + { + name: "StringLikeObject for group traits", + eventPayload: `{"type":"group","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","groupId":"groupId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","request_ip":"5.6.7.8","traits":{"title":"Home | RudderStack","url":"https://www.rudderstack.com"},"context":{"traits":{"0":"a","1":"b","2":"c"},"ip":"1.2.3.4"}}`, + metadata: getGroupMetadata("POSTGRES"), + destination: getDestination("POSTGRES", map[string]any{}), + expectedResponse: ptrans.Response{ + Events: []ptrans.TransformerResponse{ + { + Output: getGroupDefaultOutput(). + RemoveDataFields("context_traits_email", "context_traits_logins", "context_traits_name"). + RemoveColumnFields("context_traits_email", "context_traits_logins", "context_traits_name"). + SetDataField("context_traits", "abc"). + SetColumnField("context_traits", "string"), + Metadata: getGroupMetadata("POSTGRES"), + StatusCode: http.StatusOK, + }, }, }, + }, + { + name: "Not StringLikeObject for context properties", + eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"0":"a","1":"b","2":"c"},"userProperties":{"rating":3,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},"ip":"1.2.3.4"}}`, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ - FailedEvents: []ptrans.TransformerResponse{ + Events: []ptrans.TransformerResponse{ { - Error: "postgres transformer: Too many columns outputted from the event", - StatusCode: http.StatusBadRequest, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(). + RemoveDataFields("product_id", "review_id"). + RemoveColumnFields("product_id", "review_id"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), + StatusCode: http.StatusOK, + }, + { + Output: getEventDefaultOutput(). + RemoveDataFields("product_id", "review_id"). + RemoveColumnFields("product_id", "review_id"). + SetDataField("_0", "a").SetColumnField("_0", "string"). + SetDataField("_1", "b").SetColumnField("_1", "string"). + SetDataField("_2", "c").SetColumnField("_2", "string"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), + StatusCode: http.StatusOK, }, }, }, }, { - name: "track (GCS_DATALAKE) too many columns", - eventPayload: fmt.Sprintf(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, strings.Join( - lo.RepeatBy(500, func(index int) string { - return fmt.Sprintf(`"column_%d": "value_%d"`, index, index) - }), ",", - )), - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "GCS_DATALAKE", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", + name: "context, properties and userProperties as null", + eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":null,"userProperties":null,"context":null}`, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), + expectedResponse: ptrans.Response{ + Events: []ptrans.TransformerResponse{ + { + Output: getTrackDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). + RemoveDataFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name"). + RemoveColumnFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), + StatusCode: http.StatusOK, + }, + { + Output: getEventDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). + RemoveDataFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name", "product_id", "rating", "review_body", "review_id"). + RemoveColumnFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name", "product_id", "rating", "review_body", "review_id"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), + StatusCode: http.StatusOK, + }, + }, }, - destination: backendconfig.DestinationT{ - Name: "GCS_DATALAKE", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "GCS_DATALAKE", + }, + { + name: "context, properties and userProperties as not a object", + eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":"properties","userProperties":"userProperties","context":"context"}`, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), + expectedResponse: ptrans.Response{ + FailedEvents: []ptrans.TransformerResponse{ + { + Error: response.ErrContextNotMap.Error(), + StatusCode: response.ErrContextNotMap.StatusCode(), + Metadata: getTrackMetadata("POSTGRES", "webhook"), + }, }, }, + }, + { + name: "context, properties and userProperties as empty map", + eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{},"userProperties":{},"context":{}}`, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": lo.Assign( - map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "GCS_DATALAKE", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - lo.SliceToMap( - lo.RepeatBy(500, func(index int) string { - return strconv.Itoa(index) - }), func(item string) (string, any) { - return fmt.Sprintf(`context_column_%s`, item), fmt.Sprintf(`value_%s`, item) - }, - ), - ), - "metadata": map[string]any{ - "columns": lo.Assign( - map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - lo.SliceToMap( - lo.RepeatBy(500, func(index int) string { - return strconv.Itoa(index) - }), func(item string) (string, any) { - return fmt.Sprintf(`context_column_%s`, item), "string" - }, - ), - ), - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "GCS_DATALAKE", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). + RemoveDataFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name"). + RemoveColumnFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": lo.Assign( - map[string]any{ - "anonymous_id": "anonymousId", - "channel": "web", - "context_destination_id": "destinationID", - "context_destination_type": "GCS_DATALAKE", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - lo.SliceToMap( - lo.RepeatBy(500, func(index int) string { - return strconv.Itoa(index) - }), func(item string) (string, any) { - return fmt.Sprintf(`context_column_%s`, item), fmt.Sprintf(`value_%s`, item) - }, - ), - ), - "metadata": map[string]any{ - "columns": lo.Assign( - map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - lo.SliceToMap( - lo.RepeatBy(500, func(index int) string { - return strconv.Itoa(index) - }), func(item string) (string, any) { - return fmt.Sprintf(`context_column_%s`, item), "string" - }, - ), - ), - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "GCS_DATALAKE", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(). + SetDataField("context_ip", "5.6.7.8"). + RemoveDataFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name", "product_id", "rating", "review_body", "review_id"). + RemoveColumnFields("context_passed_ip", "context_traits_email", "context_traits_logins", "context_traits_name", "product_id", "rating", "review_body", "review_id"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), StatusCode: http.StatusOK, }, }, }, }, { - name: "track (POSTGRES) with sources channel too many columns", - eventPayload: fmt.Sprintf(`{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"sources","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3.0,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2},%s,"ip":"1.2.3.4"}}`, strings.Join( - lo.RepeatBy(500, func(index int) string { - return fmt.Sprintf(`"column_%d": "value_%d"`, index, index) - }), ",", - )), - metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, - destination: backendconfig.DestinationT{ - Name: "POSTGRES", - Config: map[string]any{}, - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "POSTGRES", + name: "Nested object level with no limit when source category is not cloud", + eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2,"location":{"city":"Palo Alto","state":"California","country":"USA","coordinates":{"latitude":37.4419,"longitude":-122.143,"geo":{"altitude":30.5,"accuracy":5,"details":{"altitudeUnits":"meters","accuracyUnits":"meters"}}}}},"ip":"1.2.3.4"}}`, + metadata: getTrackMetadata("POSTGRES", "webhook"), + destination: getDestination("POSTGRES", map[string]any{}), + expectedResponse: ptrans.Response{ + Events: []ptrans.TransformerResponse{ + { + Output: getTrackDefaultOutput(). + SetDataField("context_traits_location_city", "Palo Alto"). + SetDataField("context_traits_location_state", "California"). + SetDataField("context_traits_location_country", "USA"). + SetDataField("context_traits_location_coordinates_latitude", 37.4419). + SetDataField("context_traits_location_coordinates_longitude", -122.143). + SetDataField("context_traits_location_coordinates_geo_altitude", 30.5). + SetDataField("context_traits_location_coordinates_geo_accuracy", 5.0). + SetDataField("context_traits_location_coordinates_geo_details_altitude_units", "meters"). + SetDataField("context_traits_location_coordinates_geo_details_accuracy_units", "meters"). + SetColumnField("context_traits_location_city", "string"). + SetColumnField("context_traits_location_state", "string"). + SetColumnField("context_traits_location_country", "string"). + SetColumnField("context_traits_location_coordinates_latitude", "float"). + SetColumnField("context_traits_location_coordinates_longitude", "float"). + SetColumnField("context_traits_location_coordinates_geo_altitude", "float"). + SetColumnField("context_traits_location_coordinates_geo_accuracy", "int"). + SetColumnField("context_traits_location_coordinates_geo_details_altitude_units", "string"). + SetColumnField("context_traits_location_coordinates_geo_details_accuracy_units", "string"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), + StatusCode: http.StatusOK, + }, + { + Output: getEventDefaultOutput(). + SetDataField("context_traits_location_city", "Palo Alto"). + SetDataField("context_traits_location_state", "California"). + SetDataField("context_traits_location_country", "USA"). + SetDataField("context_traits_location_coordinates_latitude", 37.4419). + SetDataField("context_traits_location_coordinates_longitude", -122.143). + SetDataField("context_traits_location_coordinates_geo_altitude", 30.5). + SetDataField("context_traits_location_coordinates_geo_accuracy", 5.0). + SetDataField("context_traits_location_coordinates_geo_details_altitude_units", "meters"). + SetDataField("context_traits_location_coordinates_geo_details_accuracy_units", "meters"). + SetColumnField("context_traits_location_city", "string"). + SetColumnField("context_traits_location_state", "string"). + SetColumnField("context_traits_location_country", "string"). + SetColumnField("context_traits_location_coordinates_latitude", "float"). + SetColumnField("context_traits_location_coordinates_longitude", "float"). + SetColumnField("context_traits_location_coordinates_geo_altitude", "float"). + SetColumnField("context_traits_location_coordinates_geo_accuracy", "int"). + SetColumnField("context_traits_location_coordinates_geo_details_altitude_units", "string"). + SetColumnField("context_traits_location_coordinates_geo_details_accuracy_units", "string"), + Metadata: getTrackMetadata("POSTGRES", "webhook"), + StatusCode: http.StatusOK, + }, }, }, + }, + { + name: "Nested object level limits to 3 when source category is cloud", + eventPayload: `{"type":"track","messageId":"messageId","anonymousId":"anonymousId","userId":"userId","sentAt":"2021-09-01T00:00:00.000Z","timestamp":"2021-09-01T00:00:00.000Z","receivedAt":"2021-09-01T00:00:00.000Z","originalTimestamp":"2021-09-01T00:00:00.000Z","channel":"web","event":"event","request_ip":"5.6.7.8","properties":{"review_id":"86ac1cd43","product_id":"9578257311"},"userProperties":{"rating":3,"review_body":"OK for the price. It works but the material feels flimsy."},"context":{"traits":{"name":"Richard Hendricks","email":"rhedricks@example.com","logins":2,"location":{"city":"Palo Alto","state":"California","country":"USA","coordinates":{"latitude":37.4419,"longitude":-122.143,"geo":{"altitude":30.5,"accuracy":5,"details":{"altitudeUnits":"meters","accuracyUnits":"meters"}}}}},"ip":"1.2.3.4"}}`, + metadata: getTrackMetadata("POSTGRES", "cloud"), + destination: getDestination("POSTGRES", map[string]any{}), expectedResponse: ptrans.Response{ Events: []ptrans.TransformerResponse{ { - Output: map[string]any{ - "data": lo.Assign( - map[string]any{ - "anonymous_id": "anonymousId", - "channel": "sources", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "received_at": "2021-09-01T00:00:00.000Z", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - lo.SliceToMap( - lo.RepeatBy(500, func(index int) string { - return strconv.Itoa(index) - }), func(item string) (string, any) { - return fmt.Sprintf(`context_column_%s`, item), fmt.Sprintf(`value_%s`, item) - }, - ), - ), - "metadata": map[string]any{ - "columns": lo.Assign( - map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "received_at": "datetime", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - lo.SliceToMap( - lo.RepeatBy(500, func(index int) string { - return strconv.Itoa(index) - }), func(item string) (string, any) { - return fmt.Sprintf(`context_column_%s`, item), "string" - }, - ), - ), - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "tracks", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getTrackDefaultOutput(). + SetDataField("context_traits_location_city", "Palo Alto"). + SetDataField("context_traits_location_state", "California"). + SetDataField("context_traits_location_country", "USA"). + SetDataField("context_traits_location_coordinates_latitude", 37.4419). + SetDataField("context_traits_location_coordinates_longitude", -122.143). + SetDataField("context_traits_location_coordinates_geo", `{"accuracy":5,"altitude":30.5,"details":{"accuracyUnits":"meters","altitudeUnits":"meters"}}`). + SetColumnField("context_traits_location_city", "string"). + SetColumnField("context_traits_location_state", "string"). + SetColumnField("context_traits_location_country", "string"). + SetColumnField("context_traits_location_coordinates_latitude", "float"). + SetColumnField("context_traits_location_coordinates_geo", "string"). + SetColumnField("context_traits_location_coordinates_longitude", "float"), + Metadata: getTrackMetadata("POSTGRES", "cloud"), StatusCode: http.StatusOK, }, { - Output: map[string]any{ - "data": lo.Assign( - map[string]any{ - "anonymous_id": "anonymousId", - "channel": "sources", - "context_destination_id": "destinationID", - "context_destination_type": "POSTGRES", - "context_ip": "1.2.3.4", - "context_passed_ip": "1.2.3.4", - "context_request_ip": "5.6.7.8", - "context_source_id": "sourceID", - "context_source_type": "sourceType", - "context_traits_email": "rhedricks@example.com", - "context_traits_logins": float64(2), - "context_traits_name": "Richard Hendricks", - "event": "event", - "event_text": "event", - "id": "messageId", - "original_timestamp": "2021-09-01T00:00:00.000Z", - "product_id": "9578257311", - "rating": 3.0, - "received_at": "2021-09-01T00:00:00.000Z", - "review_body": "OK for the price. It works but the material feels flimsy.", - "review_id": "86ac1cd43", - "sent_at": "2021-09-01T00:00:00.000Z", - "timestamp": "2021-09-01T00:00:00.000Z", - "user_id": "userId", - }, - lo.SliceToMap( - lo.RepeatBy(500, func(index int) string { - return strconv.Itoa(index) - }), func(item string) (string, any) { - return fmt.Sprintf(`context_column_%s`, item), fmt.Sprintf(`value_%s`, item) - }, - ), - ), - "metadata": map[string]any{ - "columns": lo.Assign( - map[string]any{ - "anonymous_id": "string", - "channel": "string", - "context_destination_id": "string", - "context_destination_type": "string", - "context_ip": "string", - "context_passed_ip": "string", - "context_request_ip": "string", - "context_source_id": "string", - "context_source_type": "string", - "context_traits_email": "string", - "context_traits_logins": "int", - "context_traits_name": "string", - "event": "string", - "event_text": "string", - "id": "string", - "original_timestamp": "datetime", - "product_id": "string", - "rating": "int", - "received_at": "datetime", - "review_body": "string", - "review_id": "string", - "sent_at": "datetime", - "timestamp": "datetime", - "user_id": "string", - "uuid_ts": "datetime", - }, - lo.SliceToMap( - lo.RepeatBy(500, func(index int) string { - return strconv.Itoa(index) - }), func(item string) (string, any) { - return fmt.Sprintf(`context_column_%s`, item), "string" - }, - ), - ), - "receivedAt": "2021-09-01T00:00:00.000Z", - "table": "event", - }, - "userId": "", - }, - Metadata: ptrans.Metadata{ - EventType: "track", - DestinationType: "POSTGRES", - ReceivedAt: "2021-09-01T00:00:00.000Z", - SourceID: "sourceID", - DestinationID: "destinationID", - SourceType: "sourceType", - MessageID: "messageId", - }, + Output: getEventDefaultOutput(). + SetDataField("context_traits_location_city", "Palo Alto"). + SetDataField("context_traits_location_state", "California"). + SetDataField("context_traits_location_country", "USA"). + SetDataField("context_traits_location_coordinates_latitude", 37.4419). + SetDataField("context_traits_location_coordinates_longitude", -122.143). + SetDataField("context_traits_location_coordinates_geo", `{"accuracy":5,"altitude":30.5,"details":{"accuracyUnits":"meters","altitudeUnits":"meters"}}`). + SetColumnField("context_traits_location_city", "string"). + SetColumnField("context_traits_location_state", "string"). + SetColumnField("context_traits_location_country", "string"). + SetColumnField("context_traits_location_coordinates_latitude", "float"). + SetColumnField("context_traits_location_coordinates_geo", "string"). + SetColumnField("context_traits_location_coordinates_longitude", "float"), + Metadata: getTrackMetadata("POSTGRES", "cloud"), StatusCode: http.StatusOK, }, }, }, }, } - - for _, tc := range testsCases { + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { pool, err := dockertest.NewPool("") require.NoError(t, err) - //opts := lo.Map(tc.envOverride, func(item string, index int) transformertest.Option { - // return transformertest.WithEnv(item) - //}) - //opts = append(opts, transformertest.WithRepository("rudderstack/develop-rudder-transformer")) - transformerResource, err := transformertest.Setup(pool, t) - require.NoError(t, err) - - c := config.New() - c.Set("DEST_TRANSFORM_URL", transformerResource.TransformerURL) - c.Set("USER_TRANSFORM_URL", transformerResource.TransformerURL) - - for k, v := range tc.configOverride { - c.Set(k, v) + var opts []transformertest.Option + for _, envOverride := range tc.envOverride { + opts = append(opts, transformertest.WithEnv(envOverride)) } + transformerResource, err := transformertest.Setup(pool, t, opts...) + require.NoError(t, err) - eventsInfos := []eventsInfo{ + c := setupConfig(transformerResource, tc.configOverride) + eventsInfos := []testhelper.EventInfo{ { - payload: []byte(tc.eventPayload), - metadata: tc.metadata, - destination: tc.destination, + Payload: []byte(tc.eventPayload), + Metadata: tc.metadata, + Destination: tc.destination, }, } destinationTransformer := ptrans.NewTransformer(c, logger.NOP, stats.Default) warehouseTransformer := New(c, logger.NOP, stats.NOP) - testEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) + testhelper.ValidateEvents(t, eventsInfos, destinationTransformer, warehouseTransformer, tc.expectedResponse) }) } } + +func setupConfig(resource *transformertest.Resource, configOverride map[string]any) *config.Config { + c := config.New() + c.Set("DEST_TRANSFORM_URL", resource.TransformerURL) + c.Set("USER_TRANSFORM_URL", resource.TransformerURL) + + for k, v := range configOverride { + c.Set(k, v) + } + return c +} + +func getDestination(destinationType string, config map[string]any) backendconfig.DestinationT { + return backendconfig.DestinationT{ + Name: destinationType, + Config: config, + DestinationDefinition: backendconfig.DestinationDefinitionT{ + Name: destinationType, + }, + } +} + +func getMetadata(eventType, destinationType string) ptrans.Metadata { + return ptrans.Metadata{ + EventType: eventType, + DestinationType: destinationType, + ReceivedAt: "2021-09-01T00:00:00.000Z", + SourceID: "sourceID", + DestinationID: "destinationID", + SourceType: "sourceType", + MessageID: "messageId", + } +}