Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf: remove unnecessary base64 encode+decode from OTLP export #4343

Merged
merged 10 commits into from
Dec 19, 2023
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ For experimental package changes, see the [experimental CHANGELOG](experimental/

### :rocket: (Enhancement)

* perf(otlp-transformer): skip unnecessary base64 encode of span contexts [#4343](https://github.com/open-telemetry/opentelemetry-js/pull/4343) @seemk

### :books: (Refine Doc)

### :house: (Internal)
Expand Down
20 changes: 11 additions & 9 deletions experimental/packages/otlp-transformer/src/common/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import type { OtlpEncodingOptions, Fixed64, LongBits } from './types';
import { HrTime } from '@opentelemetry/api';
import { hexToBase64, hrTimeToNanoseconds } from '@opentelemetry/core';
import { hexToBinary, hrTimeToNanoseconds } from '@opentelemetry/core';

const NANOSECONDS = BigInt(1_000_000_000);

Expand Down Expand Up @@ -44,10 +44,12 @@ const encodeTimestamp =
typeof BigInt !== 'undefined' ? encodeAsString : hrTimeToNanoseconds;

export type HrTimeEncodeFunction = (hrTime: HrTime) => Fixed64;
export type SpanContextEncodeFunction = (spanContext: string) => string;
export type SpanContextEncodeFunction = (
spanContext: string
) => string | Uint8Array;
export type OptionalSpanContextEncodeFunction = (
spanContext: string | undefined
) => string | undefined;
) => string | Uint8Array | undefined;

export interface Encoder {
encodeHrTime: HrTimeEncodeFunction;
Expand All @@ -59,15 +61,15 @@ function identity<T>(value: T): T {
return value;
}

function optionalHexToBase64(str: string | undefined): string | undefined {
function optionalHexToBinary(str: string | undefined): Uint8Array | undefined {
if (str === undefined) return undefined;
return hexToBase64(str);
return hexToBinary(str);
}

const DEFAULT_ENCODER: Encoder = {
encodeHrTime: encodeAsLongBits,
encodeSpanContext: hexToBase64,
encodeOptionalSpanContext: optionalHexToBase64,
encodeSpanContext: hexToBinary,
encodeOptionalSpanContext: optionalHexToBinary,
};

export function getOtlpEncoder(options?: OtlpEncodingOptions): Encoder {
Expand All @@ -79,7 +81,7 @@ export function getOtlpEncoder(options?: OtlpEncodingOptions): Encoder {
const useHex = options.useHex ?? false;
return {
encodeHrTime: useLongBits ? encodeAsLongBits : encodeTimestamp,
encodeSpanContext: useHex ? identity : hexToBase64,
encodeOptionalSpanContext: useHex ? identity : optionalHexToBase64,
encodeSpanContext: useHex ? identity : hexToBinary,
encodeOptionalSpanContext: useHex ? identity : optionalHexToBinary,
};
}
4 changes: 2 additions & 2 deletions experimental/packages/otlp-transformer/src/logs/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,10 @@ export interface ILogRecord {
flags?: number;

/** LogRecord traceId */
traceId?: string;
traceId?: string | Uint8Array;

/** LogRecord spanId */
spanId?: string;
spanId?: string | Uint8Array;
}

/**
Expand Down
4 changes: 2 additions & 2 deletions experimental/packages/otlp-transformer/src/metrics/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -287,10 +287,10 @@ export interface IExemplar {
asInt?: number;

/** Exemplar spanId */
spanId?: string;
spanId?: string | Uint8Array;

/** Exemplar traceId */
traceId?: string;
traceId?: string | Uint8Array;
}

/**
Expand Down
10 changes: 5 additions & 5 deletions experimental/packages/otlp-transformer/src/trace/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,16 +63,16 @@ export interface IScopeSpans {
/** Properties of a Span. */
export interface ISpan {
/** Span traceId */
traceId: string;
traceId: string | Uint8Array;

/** Span spanId */
spanId: string;
spanId: string | Uint8Array;

/** Span traceState */
traceState?: string | null;

/** Span parentSpanId */
parentSpanId?: string;
parentSpanId?: string | Uint8Array;

/** Span name */
name: string;
Expand Down Expand Up @@ -181,10 +181,10 @@ export interface IEvent {
/** Properties of a Link. */
export interface ILink {
/** Link traceId */
traceId: string;
traceId: string | Uint8Array;

/** Link spanId */
spanId: string;
spanId: string | Uint8Array;

/** Link traceState */
traceState?: string;
Expand Down
12 changes: 6 additions & 6 deletions experimental/packages/otlp-transformer/test/common.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* limitations under the License.
*/

import { hexToBase64 } from '@opentelemetry/core';
import { hexToBinary } from '@opentelemetry/core';
import { getOtlpEncoder } from '../src';
import { toAnyValue } from '../src/common/internal';
import * as assert from 'assert';
Expand Down Expand Up @@ -70,19 +70,19 @@ describe('common', () => {
});

describe('otlp encoder', () => {
it('defaults to long timestamps and base64 encoding given no options', () => {
it('defaults to long timestamps and binary encoding given no options', () => {
const encoder = getOtlpEncoder();
assert.deepStrictEqual(encoder.encodeHrTime([1697978649, 99870675]), {
low: 3352011219,
high: 395341461,
});
assert.deepStrictEqual(
encoder.encodeSpanContext(traceId),
hexToBase64(traceId)
hexToBinary(traceId)
);
assert.deepStrictEqual(
encoder.encodeOptionalSpanContext(spanId),
hexToBase64(spanId)
hexToBinary(spanId)
);
assert.deepStrictEqual(
encoder.encodeOptionalSpanContext(undefined),
Expand All @@ -98,11 +98,11 @@ describe('common', () => {
});
assert.deepStrictEqual(
encoder.encodeSpanContext(traceId),
hexToBase64(traceId)
hexToBinary(traceId)
);
assert.deepStrictEqual(
encoder.encodeOptionalSpanContext(spanId),
hexToBase64(spanId)
hexToBinary(spanId)
);
assert.deepStrictEqual(
encoder.encodeOptionalSpanContext(undefined),
Expand Down
6 changes: 3 additions & 3 deletions experimental/packages/otlp-transformer/test/logs.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* limitations under the License.
*/
import { HrTime, TraceFlags } from '@opentelemetry/api';
import { InstrumentationScope, hexToBase64 } from '@opentelemetry/core';
import { InstrumentationScope, hexToBinary } from '@opentelemetry/core';
import { Resource } from '@opentelemetry/resources';
import * as assert from 'assert';
import {
Expand All @@ -28,8 +28,8 @@ import { SeverityNumber } from '@opentelemetry/api-logs';
function createExpectedLogJson(useHex: boolean): IExportLogsServiceRequest {
const traceId = useHex
? '00000000000000000000000000000001'
: hexToBase64('00000000000000000000000000000001');
const spanId = useHex ? '0000000000000002' : hexToBase64('0000000000000002');
: hexToBinary('00000000000000000000000000000001');
const spanId = useHex ? '0000000000000002' : hexToBinary('0000000000000002');

return {
resourceLogs: [
Expand Down
12 changes: 6 additions & 6 deletions experimental/packages/otlp-transformer/test/trace.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* limitations under the License.
*/
import { SpanKind, SpanStatusCode, TraceFlags } from '@opentelemetry/api';
import { TraceState, hexToBase64 } from '@opentelemetry/core';
import { TraceState, hexToBinary } from '@opentelemetry/core';
import { Resource } from '@opentelemetry/resources';
import { ReadableSpan } from '@opentelemetry/sdk-trace-base';
import * as assert from 'assert';
Expand All @@ -41,17 +41,17 @@ function createExpectedSpanJson(options: OtlpEncodingOptions) {

const traceId = useHex
? '00000000000000000000000000000001'
: hexToBase64('00000000000000000000000000000001');
const spanId = useHex ? '0000000000000002' : hexToBase64('0000000000000002');
: hexToBinary('00000000000000000000000000000001');
const spanId = useHex ? '0000000000000002' : hexToBinary('0000000000000002');
const parentSpanId = useHex
? '0000000000000001'
: hexToBase64('0000000000000001');
: hexToBinary('0000000000000001');
const linkSpanId = useHex
? '0000000000000003'
: hexToBase64('0000000000000003');
: hexToBinary('0000000000000003');
const linkTraceId = useHex
? '00000000000000000000000000000002'
: hexToBase64('00000000000000000000000000000002');
: hexToBinary('00000000000000000000000000000002');

return {
resourceSpans: [
Expand Down
43 changes: 43 additions & 0 deletions packages/opentelemetry-core/src/common/hex-to-binary.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

function intValue(charCode: number): number {
// 0-9
if (charCode >= 48 && charCode <= 57) {
return charCode - 48;
}

// a-f
if (charCode >= 97 && charCode <= 102) {
return charCode - 87;
}

// A-F
return charCode - 55;
}

export function hexToBinary(hexStr: string): Uint8Array {
const buf = new Uint8Array(hexStr.length / 2);
let offset = 0;

for (let i = 0; i < hexStr.length; i += 2) {
const hi = intValue(hexStr.charCodeAt(i));
const lo = intValue(hexStr.charCodeAt(i + 1));
buf[offset++] = (hi << 4) | lo;
}

return buf;
}
1 change: 1 addition & 0 deletions packages/opentelemetry-core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ export * from './common/global-error-handler';
export * from './common/logging-error-handler';
export * from './common/time';
export * from './common/types';
export * from './common/hex-to-binary';
export * from './ExportResult';
export * as baggageUtils from './baggage/utils';
export * from './platform';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import { hexToBinary } from '../../common/hex-to-binary';

export function hexToBase64(hexStr: string): string {
const hexStrLen = hexStr.length;
let hexAsciiCharsStr = '';
for (let i = 0; i < hexStrLen; i += 2) {
const hexPair = hexStr.substring(i, i + 2);
const hexVal = parseInt(hexPair, 16);
hexAsciiCharsStr += String.fromCharCode(hexVal);
}
return btoa(hexAsciiCharsStr);
return btoa(String.fromCharCode(...hexToBinary(hexStr)));
}
36 changes: 2 additions & 34 deletions packages/opentelemetry-core/src/platform/node/hex-to-base64.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,40 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
function intValue(charCode: number): number {
// 0-9
if (charCode >= 48 && charCode <= 57) {
return charCode - 48;
}

// a-f
if (charCode >= 97 && charCode <= 102) {
return charCode - 87;
}

// A-F
return charCode - 55;
}

const buf8 = Buffer.alloc(8);
const buf16 = Buffer.alloc(16);
import { hexToBinary } from '../../common/hex-to-binary';

export function hexToBase64(hexStr: string): string {
let buf;
if (hexStr.length === 16) {
buf = buf8;
} else if (hexStr.length === 32) {
buf = buf16;
} else {
buf = Buffer.alloc(hexStr.length / 2);
}
let offset = 0;

for (let i = 0; i < hexStr.length; i += 2) {
const hi = intValue(hexStr.charCodeAt(i));
const lo = intValue(hexStr.charCodeAt(i + 1));
buf.writeUInt8((hi << 4) | lo, offset++);
}

return buf.toString('base64');
return Buffer.from(hexToBinary(hexStr)).toString('base64');
}