Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: logging implementation via metadata #3424

Closed
wants to merge 3 commits into from
Closed

chore: logging implementation via metadata #3424

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d06cf2f
chore: logging capability through metadata
May 28, 2024
50cc903
chore: metadata enabled logging for gaec
May 28, 2024
7253061
chore: refactor log implementation using injection into metadata
May 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 30 additions & 12 deletions src/controllers/delivery.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import { FixMe } from '../util/types';
import tags from '../v0/util/tags';
import { ControllerUtility } from './util';
import logger from '../logger';
import MetaBinder from '../helpers/metadata/binder';

const NON_DETERMINABLE = 'Non-determinable';

Expand All @@ -28,6 +29,17 @@ export class DeliveryController {
const deliveryRequest = ctx.request.body as ProxyV0Request;
const { destination }: { destination: string } = ctx.params;
const integrationService = ServiceSelector.getNativeDestinationService();
const metaTO = integrationService.getTags(
destination,
deliveryRequest.metadata?.destinationId,
deliveryRequest.metadata?.workspaceId,
tags.FEATURES.PROCESSOR,
);
MetaBinder.bindTransformMetaToDeliveryV0(deliveryRequest, {
module: metaTO.errorDetails.module,
implementation: metaTO.errorDetails.implementation,
feature: metaTO.errorDetails.feature,
});
try {
deliveryResponse = (await integrationService.deliver(
deliveryRequest,
Expand All @@ -37,12 +49,12 @@ export class DeliveryController {
)) as DeliveryV0Response;
} catch (error: any) {
const { metadata } = deliveryRequest;
const metaTO = integrationService.getTags(
destination,
metadata?.destinationId || NON_DETERMINABLE,
metadata?.workspaceId || NON_DETERMINABLE,
tags.FEATURES.DATA_DELIVERY,
);
// const metaTO = integrationService.getTags(
// destination,
// metadata?.destinationId || NON_DETERMINABLE,
// metadata?.workspaceId || NON_DETERMINABLE,
// tags.FEATURES.DATA_DELIVERY,
// );
metaTO.metadata = metadata;
deliveryResponse = DestinationPostTransformationService.handleDeliveryFailureEvents(
error,
Expand All @@ -63,7 +75,18 @@ export class DeliveryController {
const deliveryRequest = ctx.request.body as ProxyV1Request;
const { destination }: { destination: string } = ctx.params;
const integrationService = ServiceSelector.getNativeDestinationService();
const metaTO = integrationService.getTags(
destination,
deliveryRequest.metadata[0].destinationId || NON_DETERMINABLE,
deliveryRequest.metadata[0].workspaceId || NON_DETERMINABLE,
tags.FEATURES.DATA_DELIVERY,
);
try {
MetaBinder.bindTransformMetaToDeliveryV1(deliveryRequest.metadata, {
module: metaTO.errorDetails.module,
implementation: metaTO.errorDetails.implementation,
feature: metaTO.errorDetails.feature,
});
deliveryResponse = (await integrationService.deliver(
deliveryRequest,
destination,
Expand All @@ -72,12 +95,7 @@ export class DeliveryController {
)) as DeliveryV1Response;
} catch (error: any) {
const { metadata } = deliveryRequest;
const metaTO = integrationService.getTags(
destination,
metadata[0].destinationId || NON_DETERMINABLE,
metadata[0].workspaceId || NON_DETERMINABLE,
tags.FEATURES.DATA_DELIVERY,
);

metaTO.metadatas = metadata;
deliveryResponse = DestinationPostTransformationService.handlevV1DeliveriesFailureEvents(
error,
Expand Down
46 changes: 34 additions & 12 deletions src/controllers/destination.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { checkInvalidRtTfEvents } from '../v0/util';
import tags from '../v0/util/tags';
import { ControllerUtility } from './util';
import logger from '../logger';
import MetaBinder from '../helpers/metadata/binder';

export class DestinationController {
public static async destinationTransformAtProcessor(ctx: Context) {
Expand All @@ -34,6 +35,17 @@ export class DestinationController {
const integrationService = ServiceSelector.getDestinationService(events);
try {
integrationService.init();
const metaTO = integrationService.getTags(
destination,
events[0].metadata?.destinationId,
events[0].metadata?.workspaceId,
tags.FEATURES.PROCESSOR,
);
MetaBinder.bindTransformMetaToTransformation(events, {
module: metaTO.errorDetails.module,
implementation: metaTO.errorDetails.implementation,
feature: metaTO.errorDetails.feature,
});
events = DestinationPreTransformationService.preProcess(
events,
ctx,
Expand Down Expand Up @@ -114,7 +126,18 @@ export class DestinationController {
});
const integrationService = ServiceSelector.getDestinationService(events);
let resplist: RouterTransformationResponse[];
const metaTO = integrationService.getTags(
destination,
events[0].metadata?.destinationId,
events[0].metadata?.workspaceId,
tags.FEATURES.ROUTER,
);
try {
MetaBinder.bindTransformMetaToTransformation(events, {
module: metaTO.errorDetails.module,
implementation: metaTO.errorDetails.implementation,
feature: metaTO.errorDetails.feature,
});
events = DestinationPreTransformationService.preProcess(events, ctx);
const timestampCorrectEvents = ControllerUtility.handleTimestampInEvents(events);
events = DynamicConfigParser.process(timestampCorrectEvents);
Expand All @@ -125,12 +148,6 @@ export class DestinationController {
requestMetadata,
);
} catch (error: any) {
const metaTO = integrationService.getTags(
destination,
events[0].metadata?.destinationId,
events[0].metadata?.workspaceId,
tags.FEATURES.ROUTER,
);
metaTO.metadatas = events.map((ev) => ev.metadata);
const errResp = DestinationPostTransformationService.handleRouterTransformFailureEvents(
error,
Expand Down Expand Up @@ -162,7 +179,18 @@ export class DestinationController {
const destination = routerRequest.destType;
let events = routerRequest.input;
const integrationService = ServiceSelector.getDestinationService(events);
const metaTO = integrationService.getTags(
destination,
routerRequest.input[0].metadata.destinationId,
routerRequest.input[0].metadata.workspaceId,
tags.FEATURES.BATCH,
);
try {
MetaBinder.bindTransformMetaToTransformation(events, {
module: metaTO.errorDetails.module,
implementation: metaTO.errorDetails.implementation,
feature: metaTO.errorDetails.feature,
});
events = DestinationPreTransformationService.preProcess(events, ctx);
const timestampCorrectEvents = ControllerUtility.handleTimestampInEvents(events);
const resplist = integrationService.doBatchTransformation(
Expand All @@ -173,12 +201,6 @@ export class DestinationController {
);
ctx.body = resplist;
} catch (error: any) {
const metaTO = integrationService.getTags(
destination,
events[0].metadata.destinationId,
events[0].metadata.workspaceId,
tags.FEATURES.BATCH,
);
metaTO.metadatas = events.map((ev) => ev.metadata);
const errResp = DestinationPostTransformationService.handleBatchTransformFailureEvents(
error,
Expand Down
39 changes: 39 additions & 0 deletions src/helpers/metadata/binder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/* eslint-disable no-param-reassign */
import { ProxyV0Request } from '../../types';
import { requestLog, responseLog } from './metadata';
import { Metadata, ProxyMetdata, TransformationKindInfo } from './types';

export default class MetaBinder {
// TODO: Update implementation for `implementation` so that it can be taken from input args
public static bindTransformMetaToDeliveryV0(
event: ProxyV0Request,
kindInfo: TransformationKindInfo,
) {
// @ts-expect-error this assignment is necessary
event.metadata.requestLog = requestLog(event.metadata, kindInfo);
// @ts-expect-error this assignment is necessary
event.metadata.responseLog = responseLog(event.metadata, kindInfo);
}

public static bindTransformMetaToDeliveryV1(
metadatas: ProxyMetdata[],
kindInfo: TransformationKindInfo,
) {
// @ts-expect-error this assignment is necessary
metadatas.requestLog = requestLog(metadatas, kindInfo);
// @ts-expect-error this assignment is necessary
metadatas.responseLog = responseLog(metadatas, kindInfo);
}

public static bindTransformMetaToTransformation(
events: { metadata: Metadata }[],
kindInfo: TransformationKindInfo,
) {
events.forEach((ev) => {
// @ts-expect-error this assignment is necessary
ev.metadata.requestLog = requestLog(ev.metadata, kindInfo);
// @ts-expect-error this assignment is necessary
ev.metadata.responseLog = responseLog(ev.metadata, kindInfo);
});
}
}
64 changes: 64 additions & 0 deletions src/helpers/metadata/metadata.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/* eslint-disable @typescript-eslint/ban-ts-comment */
// @ts-nocheck

import logger from '../../logger';
import {
AllMetadata,
Metadata,
ProxyMetdata,
RequestInfo,
ResponseInfo,
TransformationKindInfo,
} from './types';

const pickMetaInfo = (m: Metadata | ProxyMetdata, tfKind: TransformationKindInfo) => {
const { sourceId, destinationId, workspaceId, jobId } = m;
const { module, feature, implementation } = tfKind;
return {
sourceId,
destinationId,
jobId,
// destinationType,
workspaceId,
module,
feature,
implementation,
};
};
export function requestLog(this: AllMetadata, kindInfo: TransformationKindInfo) {
return function internalReqLog(identifierMsg: string, args: RequestInfo) {
// do something before if necessary
if (Array.isArray(this)) {
this.forEach((m) => {
logger.debug(identifierMsg, {
...pickMetaInfo(m, kindInfo),
...args,
});
});
return;
}
logger.debug(identifierMsg, {
...pickMetaInfo(this, kindInfo),
...args,
});
};
}

export function responseLog(this: AllMetadata, kindInfo: TransformationKindInfo) {
return function internalResLog(identifierMsg: string, args: ResponseInfo) {
// do something before if necessary
if (Array.isArray(this)) {
this.forEach((m) => {
logger.debug(identifierMsg, {
...pickMetaInfo(m, kindInfo),
...args,
});
});
return;
}
logger.debug(identifierMsg, {
...pickMetaInfo(this, kindInfo),
...args,
});
};
}
87 changes: 87 additions & 0 deletions src/helpers/metadata/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import { FixMe } from '../../util/types';

export type AllMetadata = Metadata | ProxyMetdata | Metadata[] | ProxyMetdata[];

export type MetadataT = {
requestLog(
m: AllMetadata,
kindInfo: TransformationKindInfo,
): (msg: string, args: Record<string, FixMe>) => void;
responseLog(
m: AllMetadata,
kindInfo: TransformationKindInfo,
): (msg: string, args: Record<string, FixMe>) => void;
} & (Metadata | ProxyMetdata | Metadata[] | ProxyMetdata[]);

export interface MetadataI {
requestLog(msg: string, args: Record<string, FixMe>): void;
responseLog(msg: string, args: Record<string, FixMe>): void;
}

export type CommonMetadata = {
sourceId: string;
destinationId: string;
workspaceId: string;
jobId: number;
};

export type Metadata = CommonMetadata & {
namespace: string;
instanceId: string;
sourceType: string;
sourceCategory: string;
trackingPlanId: string;
trackingPlanVersion: number;
sourceTpConfig: object;
mergedTpConfig: object;
jobRunId: string;
jobId: number;
sourceBatchId: string;
sourceJobId: string;
sourceJobRunId: string;
sourceTaskId: string;
sourceTaskRunId: string;
recordId: object;
destinationType: string;
messageId: string;
oauthAccessToken: string;
messageIds: string[];
rudderId: string;
receivedAt: string;
eventName: string;
eventType: string;
sourceDefinitionId: string;
destinationDefinitionId: string;
transformationId: string;
dontBatch?: boolean;
};

export type ProxyMetdata = CommonMetadata & {
jobId: number;
attemptNum: number;
userId: string;
secret: Record<string, unknown>;
destInfo?: Record<string, unknown>;
omitempty?: Record<string, unknown>;
dontBatch: boolean;
};

export type TransformationKindInfo = {
module: string;
implementation: string;
feature: string;
};

export type TransformationMeta = CommonMetadata & TransformationKindInfo;

export type RequestInfo = {
url: string;
body: FixMe;
method: string;
};

export type ResponseInfo = {
responseBody: FixMe;
status: number;
responseHeaders: Record<string, FixMe>;
};
Loading
Loading