Skip to content

Commit

Permalink
feat: collect logs
Browse files Browse the repository at this point in the history
  • Loading branch information
basti1302 committed Jun 13, 2024
1 parent 269c94e commit f246405
Show file tree
Hide file tree
Showing 12 changed files with 270 additions and 83 deletions.
40 changes: 37 additions & 3 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,12 @@
"dependencies": {
"@opentelemetry/api": "^1.8.0",
"@opentelemetry/auto-instrumentations-node": "^0.44.0",
"@opentelemetry/exporter-logs-otlp-proto": "^0.51.0",
"@opentelemetry/exporter-metrics-otlp-proto": "^0.51.0",
"@opentelemetry/exporter-trace-otlp-proto": "^0.51.0",
"@opentelemetry/resource-detector-container": "^0.3.11",
"@opentelemetry/resources": "^1.24.0",
"@opentelemetry/sdk-logs": "^0.51.0",
"@opentelemetry/sdk-metrics": "^1.24.0",
"@opentelemetry/sdk-node": "^0.51.0",
"@opentelemetry/sdk-trace-base": "^1.24.0",
Expand Down
11 changes: 11 additions & 0 deletions src/init.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@

import { SpanKind, trace } from '@opentelemetry/api';
import { getNodeAutoInstrumentations, getResourceDetectors } from '@opentelemetry/auto-instrumentations-node';
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
import { containerDetector } from '@opentelemetry/resource-detector-container';
import { Detector, DetectorSync, envDetector, hostDetector, processDetector, Resource } from '@opentelemetry/resources';
import { BatchLogRecordProcessor } from '@opentelemetry/sdk-logs';
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
import { NodeSDK, NodeSDKConfiguration } from '@opentelemetry/sdk-node';
import { BatchSpanProcessor, SpanProcessor } from '@opentelemetry/sdk-trace-base';
Expand Down Expand Up @@ -47,6 +49,12 @@ const spanProcessors: SpanProcessor[] = [
),
];

const logRecordProcessor = new BatchLogRecordProcessor(
new OTLPLogExporter({
url: `${baseUrl}/v1/logs`,
}),
);

if (process.env.DASH0_DEBUG_PRINT_SPANS != null) {
if (process.env.DASH0_DEBUG_PRINT_SPANS.toLocaleLowerCase() === 'true') {
spanProcessors.push(new BatchSpanProcessor(new ConsoleSpanExporter()));
Expand All @@ -57,12 +65,15 @@ if (process.env.DASH0_DEBUG_PRINT_SPANS != null) {

const configuration: Partial<NodeSDKConfiguration> = {
spanProcessors: spanProcessors,

metricReader: new PeriodicExportingMetricReader({
exporter: new OTLPMetricExporter({
url: `${baseUrl}/v1/metrics`,
}),
}),

logRecordProcessor,

instrumentations: [getNodeAutoInstrumentations(instrumentationConfig)],

resource: new Resource({
Expand Down
9 changes: 9 additions & 0 deletions test/apps/express-typescript/app.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,23 @@
// SPDX-FileCopyrightText: Copyright 2024 Dash0 Inc.
// SPDX-License-Identifier: Apache-2.0

import * as logsApi from '@opentelemetry/api-logs';
import express, { Express } from 'express';

import { sendReadyToParentProcess } from '../../util/sendToParentProcess';

const port: number = parseInt(process.env.PORT || '1302');
const app: Express = express();

const logger = logsApi.logs.getLoggerProvider().getLogger('default');

app.get('/ohai', (req, res) => {
logger.emit({
severityNumber: logsApi.SeverityNumber.INFO,
severityText: 'INFO',
body: 'log body',
attributes: { 'log.type': 'LogRecord' },
});
res.json({ message: 'We make Observability easy for every developer.' });
});

Expand Down
10 changes: 10 additions & 0 deletions test/collector/CollectorChildProcessWrapper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,16 @@ export default class CollectorChildProcessWrapper extends ChildProcessWrapper {
return stats.traces >= 1;
}

async hasMetrics() {
const stats = await collector().fetchStats();
return stats.metrics >= 1;
}

async hasLogs() {
const stats = await collector().fetchStats();
return stats.logs >= 1;
}

async hasTelemetry() {
const stats = await collector().fetchStats();
return stats.traces >= 1 || stats.metrics >= 1 || stats.logs >= 1;
Expand Down
1 change: 1 addition & 0 deletions test/integration/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
spans.json
6 changes: 4 additions & 2 deletions test/integration/ChildProcessWrapper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,10 @@ export function defaultAppConfiguration(appPort: number): ChildProcessWrapperOpt
env: {
...process.env,
PORT: appPort.toString(),
// have the Node.js SDK send spans every 100 ms instead of every 5 seconds to speed up tests
OTEL_BSP_SCHEDULE_DELAY: '100',
// have the Node.js SDK send spans every 20 ms instead of every 5 seconds to speed up tests
OTEL_BSP_SCHEDULE_DELAY: '20',
// have the Node.js SDK send logs every 20 ms instead of every 5 seconds to speed up tests
OTEL_BLRP_SCHEDULE_DELAY: '20',
DASH0_OTEL_COLLECTOR_BASE_URL: 'http://localhost:4318',
},
};
Expand Down
74 changes: 56 additions & 18 deletions test/integration/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ import { FileHandle, open, readFile, unlink } from 'node:fs/promises';
import { join } from 'node:path';
import semver from 'semver';

import { SeverityNumber } from '../collector/types/opentelemetry/proto/logs/v1/logs';
import delay from '../util/delay';

import { expectResourceAttribute, expectSpanAttribute } from '../util/expectAttribute';
import { expectLogRecordAttribute, expectResourceAttribute, expectSpanAttribute } from '../util/expectAttribute';
import { expectMatchingLogRecord } from '../util/expectMatchingLogRecord';
import { expectMatchingSpan, expectMatchingSpanInFileDump } from '../util/expectMatchingSpan';
import runCommand from '../util/runCommand';
import waitUntil from '../util/waitUntil';
Expand Down Expand Up @@ -38,7 +39,7 @@ describe('attach', () => {
collector().clear();
});

describe('basic tracing', () => {
describe('basic signals', () => {
let appUnderTest: ChildProcessWrapper;

before(async () => {
Expand All @@ -53,9 +54,9 @@ describe('attach', () => {

it('should attach via --require and capture spans', async () => {
await waitUntil(async () => {
const telemetry = await sendRequestAndWaitForTraceData();
const traces = await sendRequestAndWaitForTraceData();
expectMatchingSpan(
telemetry.traces,
traces,
[
resource => expectResourceAttribute(resource, 'telemetry.sdk.name', 'opentelemetry'),
resource => expectResourceAttribute(resource, 'telemetry.sdk.language', 'nodejs'),
Expand All @@ -69,6 +70,31 @@ describe('attach', () => {
);
});
});

it('should attach via --require and capture logs', async () => {
await waitUntil(async () => {
const logs = await sendRequestAndWaitForLogRecords();
expectMatchingLogRecord(
logs,
[
resource => expectResourceAttribute(resource, 'telemetry.sdk.name', 'opentelemetry'),
resource => expectResourceAttribute(resource, 'telemetry.sdk.language', 'nodejs'),
resource => expectResourceAttribute(resource, 'telemetry.distro.name', 'dash0-nodejs'),
resource => expectResourceAttribute(resource, 'telemetry.distro.version', expectedDistroVersion),
],
[
logRecord => expect(logRecord.body).to.deep.equal({ string_value: 'log body' }),
logRecord =>
expect(logRecord.severity_number).to.equal(
SeverityNumber.SEVERITY_NUMBER_INFO,
'severity number should be info',
),
logRecord => expect(logRecord.severity_text).to.equal('INFO'),
logRecord => expectLogRecordAttribute(logRecord, 'log.type', 'LogRecord'),
],
);
});
});
});

describe('pod uid detection', () => {
Expand All @@ -87,9 +113,9 @@ describe('attach', () => {

it('should attach via --require and detect the pod uid', async () => {
await waitUntil(async () => {
const telemetry = await sendRequestAndWaitForTraceData();
const traces = await sendRequestAndWaitForTraceData();
expectMatchingSpan(
telemetry.traces,
traces,
[resource => expectResourceAttribute(resource, 'k8s.pod.uid', 'f57400dc-94ce-4806-a52e-d2726f448f15')],
[
span => expect(span.kind).to.equal(SpanKind.SERVER, 'span kind should be server'),
Expand All @@ -115,9 +141,9 @@ describe('attach', () => {

it('should attach via --require and derive a service name from the package.json file', async () => {
await waitUntil(async () => {
const telemetry = await sendRequestAndWaitForTraceData();
const traces = await sendRequestAndWaitForTraceData();
expectMatchingSpan(
telemetry.traces,
traces,
[
resource =>
expectResourceAttribute(resource, 'service.name', '[email protected]'),
Expand Down Expand Up @@ -150,9 +176,9 @@ describe('attach', () => {
// (because the top level beforeHook is executed after this suite's before hook).
await appUnderTest.start();
await waitUntil(async () => {
const telemetry = await waitForTraceData();
const traces = await waitForTraceData();
expectMatchingSpan(
telemetry.traces,
traces,
[
resource => expectResourceAttribute(resource, 'telemetry.sdk.name', 'opentelemetry'),
resource => expectResourceAttribute(resource, 'telemetry.sdk.language', 'nodejs'),
Expand Down Expand Up @@ -187,9 +213,9 @@ describe('attach', () => {
await appUnderTest.start();
await appUnderTest.stop();
await waitUntil(async () => {
const telemetry = await waitForTraceData();
const traces = await waitForTraceData();
expectMatchingSpan(
telemetry.traces,
traces,
[
resource => expectResourceAttribute(resource, 'telemetry.sdk.name', 'opentelemetry'),
resource => expectResourceAttribute(resource, 'telemetry.sdk.language', 'nodejs'),
Expand All @@ -205,9 +231,9 @@ describe('attach', () => {
await appUnderTest.start();
await appUnderTest.stop('SIGINT');
await waitUntil(async () => {
const telemetry = await waitForTraceData();
const traces = await waitForTraceData();
expectMatchingSpan(
telemetry.traces,
traces,
[
resource => expectResourceAttribute(resource, 'telemetry.sdk.name', 'opentelemetry'),
resource => expectResourceAttribute(resource, 'telemetry.sdk.language', 'nodejs'),
Expand Down Expand Up @@ -241,9 +267,9 @@ describe('attach', () => {
it('should flush telemetry before process exit due to empty event loop', async () => {
await appUnderTest.start();
await waitUntil(async () => {
const telemetry = await waitForTraceData();
const traces = await waitForTraceData();
expectMatchingSpan(
telemetry.traces,
traces,
[
resource => expectResourceAttribute(resource, 'telemetry.sdk.name', 'opentelemetry'),
resource => expectResourceAttribute(resource, 'telemetry.sdk.language', 'nodejs'),
Expand Down Expand Up @@ -351,6 +377,11 @@ describe('attach', () => {
return waitForTraceData();
}

async function sendRequestAndWaitForLogRecords() {
await sendRequestAndVerifyResponse();
return waitForLogRecords();
}

async function sendRequestAndVerifyResponse() {
const response = await fetch(`http://localhost:${appPort}/ohai`);
expect(response.status).to.equal(200);
Expand All @@ -362,7 +393,14 @@ describe('attach', () => {
if (!(await collector().hasTraces())) {
throw new Error('The collector never received any spans.');
}
return await collector().fetchTelemetry();
return (await collector().fetchTelemetry()).traces;
}

async function waitForLogRecords() {
if (!(await collector().hasLogs())) {
throw new Error('The collector never received any log records.');
}
return (await collector().fetchTelemetry()).logs;
}

async function verifyFileHasBeenCreated(filename: string): Promise<FileHandle> {
Expand Down
5 changes: 5 additions & 0 deletions test/util/expectAttribute.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { expect } from 'chai';
import { KeyValue } from '../collector/types/opentelemetry/proto/common/v1/common';
import { Resource } from '../collector/types/opentelemetry/proto/resource/v1/resource';
import { Span } from '../collector/types/opentelemetry/proto/trace/v1/trace';
import { LogRecord } from '../collector/types/opentelemetry/proto/logs/v1/logs';

const { fail } = expect;

Expand Down Expand Up @@ -41,6 +42,10 @@ export function expectSpanAttribute(span: Span, key: string, expectedValue: any)
expectAttribute(span, key, expectedValue, 'span');
}

export function expectLogRecordAttribute(logRecord: LogRecord, key: string, expectedValue: any) {
expectAttribute(logRecord, key, expectedValue, 'log record');
}

function getValue(attribute: KeyValue) {
const v = attribute.value;
if (v == null) {
Expand Down
Loading

0 comments on commit f246405

Please sign in to comment.