structure changes, add web fetch api support

packages/node/src/lib/ReadMe.ts

@@ -1,14 +1,14 @@
-import type { Options } from './log';
+import type { Options } from './shared/options';
 import type { NextFunction, Request, Response } from 'express';
 
 import pkg from '../../package.json';
 import config from '../config';
 
 import findAPIKey from './find-api-key';
 import { getGroupByApiKey } from './get-group-id';
-import { log } from './log';
-import { logger } from './logger';
+import { log } from './metrics-node/log';
 import { buildSetupView } from './setup-readme-view';
+import { logger } from './shared/logger';
 import { testVerifyWebhook } from './test-verify-webhook';
 import verifyWebhook from './verify-webhook';
 

packages/node/src/lib/metrics-fetch/construct-payload.ts

@@ -0,0 +1,64 @@
+import type { OutgoingLogBody } from '../shared/metrics-log';
+import type { PayloadData, LogOptions } from '../shared/options';
+
+import { randomUUID } from 'node:crypto';
+import os from 'os';
+
+import { version } from '../../../package.json';
+import { mask } from '../shared/mask';
+
+import { processRequest } from './process-request';
+import { processResponse } from './process-response';
+
+export function getProto(req: Request): 'http' | 'https' {
+  return req.url.startsWith('https://') ? 'https' : 'http';
+}
+
+export function constructPayload(
+  req: Request,
+  res: Response,
+  payloadData: PayloadData,
+  logOptions: LogOptions,
+): OutgoingLogBody {
+  const serverTime = payloadData.responseEndDateTime.getTime() - payloadData.startedDateTime.getTime();
+
+  return {
+    _id: payloadData.logId || randomUUID(),
+    _version: 3,
+    group: {
+      id: mask(payloadData.apiKey),
+      label: payloadData.label,
+      email: payloadData.email,
+    },
+    clientIPAddress: req.headers.get('x-forwarded-for') || '',
+    development: !!logOptions?.development,
+    request: {
+      log: {
+        version: '1.2',
+        creator: {
+          name: 'readme-metrics (node)',
+          version,
+          // x64-darwin21.3.0/14.19.3
+          comment: `${os.arch()}-${os.platform()}${os.release()}/${process.versions.node}`,
+        },
+        entries: [
+          {
+            pageref: payloadData.routePath
+              ? payloadData.routePath
+              : new URL(req.url || '', `${getProto(req)}://${req.headers.get('host')}`).toString(),
+            startedDateTime: payloadData.startedDateTime.toISOString(),
+            time: serverTime,
+            request: processRequest(req, payloadData.requestBody, logOptions),
+            response: processResponse(res, payloadData.responseBody, logOptions),
+            cache: {},
+            timings: {
+              // This requires us to know the time the request was sent to the server, so we're skipping it for now
+              wait: 0,
+              receive: serverTime,
+            },
+          },
+        ],
+      },
+    },
+  };
+}

packages/node/src/lib/metrics-fetch/extract-body.ts

@@ -0,0 +1,16 @@
+export default function extractBody(target: Request | Response) {
+  const contentType = target.headers.get('content-type');
+  let body;
+
+  if (contentType?.includes('json')) {
+    target.json().then(data => {
+      body = data;
+    });
+  } else {
+    target.text().then(data => {
+      body = data;
+    });
+  }
+
+  return body;
+}

packages/node/src/lib/metrics-fetch/index.ts

@@ -0,0 +1,5 @@
+import { getProjectBaseUrl } from '../shared/get-project-base-url';
+
+import { log } from './log';
+
+export { getProjectBaseUrl, log };

packages/node/src/lib/metrics-fetch/log.ts

@@ -0,0 +1,111 @@
+import type { GroupingObject, OutgoingLogBody } from '../shared/metrics-log';
+import type { Options } from '../shared/options';
+
+import { randomUUID } from 'node:crypto';
+
+import clamp from 'lodash/clamp';
+
+import config from '../../config';
+import { getProjectBaseUrl } from '../shared/get-project-base-url';
+import { logger } from '../shared/logger';
+import { metricsAPICall } from '../shared/metrics-log';
+
+import { constructPayload } from './construct-payload';
+import extractBody from './extract-body';
+
+let queue: OutgoingLogBody[] = [];
+
+export function doSend(readmeApiKey: string, options: Options) {
+  // Copy the queue so we can send all the requests in one batch
+  const json = [...queue];
+  // Clear out the queue so we don't resend any data in the future
+  queue = [];
+
+  // Make the log call
+  metricsAPICall(readmeApiKey, json, options).catch(err => {
+    // Silently discard errors and timeouts.
+    if (options.development) {
+      logger.error({ message: 'Failed to capture API request.', err });
+    }
+  });
+
+  logger.debug({ message: 'Queue flushed.', args: { queue } });
+}
+// Make sure we flush the queue if the process is exited
+process.on('exit', doSend);
+
+function setDocumentationHeader(res: Response, baseLogUrl: string, logId: string) {
+  // This is to catch the potential race condition where `getProjectBaseUrl()`
+  // takes longer to respond than the original req/res to finish. Without this
+  // we would get an error that would be very difficult to trace. This could
+  // do with a test, but it's a little difficult to test. Maybe with a nock()
+  // delay timeout.
+  const documentationUrl = `${baseLogUrl}/logs/${logId}`;
+  logger.verbose({
+    message: 'Created URL to your API request log.',
+    args: { 'x-documentation-url': documentationUrl },
+  });
+  res.headers.set('x-documentation-url', documentationUrl);
+}
+/**
+ * This method will send supplied API requests to ReadMe Metrics.
+ *
+ * @see {@link https://readme.com/metrics}
+ * @see {@link https://docs.readme.com/docs/sending-logs-to-readme-with-nodejs}
+ * @param readmeApiKey The API key for your ReadMe project. This ensures your requests end up in
+ *    your dashboard. You can read more about the API key in
+ *    [our docs](https://docs.readme.com/reference/authentication).
+ * @param req This is your incoming request object from your HTTP server and/or framework.
+ * @param res This is your outgoing response object for your HTTP server and/or framework.
+ * @param group A function that helps translate incoming request data to our metrics grouping data.
+ * @param options Additional options. See the documentation for more details.
+ */
+export function log(readmeApiKey: string, req: Request, res: Response, group: GroupingObject, options: Options = {}) {
+  if (req.method === 'OPTIONS') return undefined;
+  if (!readmeApiKey) throw new Error('You must provide your ReadMe API key');
+  if (!group) throw new Error('You must provide a group');
+  if (options.logger) {
+    if (typeof options.logger === 'boolean') logger.configure({ isLoggingEnabled: true });
+    else logger.configure({ isLoggingEnabled: true, strategy: options.logger });
+  }
+
+  // Ensures the buffer length is between 1 and 30
+  const bufferLength = clamp(options.bufferLength || config.bufferLength, 1, 30);
+
+  const startedDateTime = new Date();
+  const logId = randomUUID();
+
+  // baseLogUrl can be provided, but if it isn't then we
+  // attempt to fetch it from the ReadMe API
+  if (typeof options.baseLogUrl === 'string') {
+    setDocumentationHeader(res, options.baseLogUrl, logId);
+  } else {
+    getProjectBaseUrl(readmeApiKey).then(baseLogUrl => {
+      setDocumentationHeader(res, baseLogUrl, logId);
+    });
+  }
+
+  const requestBody = extractBody(req);
+  const responseBody = extractBody(res);
+
+  const payload = constructPayload(
+    req,
+    res,
+    {
+      ...group,
+      logId,
+      startedDateTime,
+      responseEndDateTime: new Date(),
+      routePath: '',
+      responseBody,
+      requestBody,
+    },
+    options,
+  );
+
+  queue.push(payload);
+  logger.debug({ message: 'Request enqueued.', args: { queue } });
+  if (queue.length >= bufferLength) doSend(readmeApiKey, options);
+
+  return logId;
+}

packages/node/src/lib/metrics-fetch/process-request.ts

@@ -0,0 +1,118 @@
+import type { LogOptions } from '../shared/options';
+import type { Cookie, Param, PostData, Request as HarRequest } from 'har-format';
+
+import * as qs from 'querystring';
+import url, { URL } from 'url';
+
+import * as contentType from 'content-type';
+
+import { mask } from '../shared/mask';
+import { objectToArray, searchToArray } from '../shared/object-to-array';
+import {
+  fixHeader,
+  redactOtherProperties,
+  redactProperties,
+  isApplicationJson,
+  parseRequestBody,
+} from '../shared/processing-helpers';
+
+import { getProto } from './construct-payload';
+import { headersToObject } from './process-response';
+
+export function processRequest(
+  req: Request,
+  requestBody?: Record<string, unknown> | string,
+  options?: LogOptions,
+): HarRequest {
+  const protocol = fixHeader(req.headers.get('x-forwarded-proto') || '')?.toLowerCase() || getProto(req);
+  const host = fixHeader(req.headers.get('x-forwarded-host') || '') || req.headers.get('host');
+
+  const denylist = options?.denylist || options?.blacklist;
+  const allowlist = options?.allowlist || options?.whitelist;
+
+  let mimeType = '';
+  try {
+    mimeType = contentType.parse(req.headers.get('content-type') || '').type;
+  } catch (e) {} // eslint-disable-line no-empty
+
+  let reqBody = typeof requestBody === 'string' ? parseRequestBody(requestBody, mimeType) : requestBody;
+  let postData: PostData | undefined;
+
+  let headers = headersToObject(req.headers);
+
+  if (denylist) {
+    reqBody = typeof reqBody === 'object' ? redactProperties(reqBody, denylist) : reqBody;
+    headers = redactProperties(headers, denylist);
+  }
+
+  if (allowlist && !denylist) {
+    reqBody = typeof reqBody === 'object' ? redactOtherProperties(reqBody, allowlist) : reqBody;
+    headers = redactOtherProperties(headers, allowlist);
+  }
+
+  if (mimeType === 'application/x-www-form-urlencoded') {
+    postData = {
+      mimeType,
+      // `reqBody` is likely to be an object, but can be empty if no HTTP body sent
+      params: objectToArray((reqBody || {}) as Record<string, unknown>) as Param[],
+    };
+  } else if (isApplicationJson(mimeType)) {
+    postData = {
+      mimeType,
+      text: typeof reqBody === 'object' || Array.isArray(reqBody) ? JSON.stringify(reqBody) : reqBody || '',
+    };
+  } else if (mimeType) {
+    let stringBody = '';
+
+    try {
+      stringBody = typeof reqBody === 'string' ? reqBody : JSON.stringify(reqBody);
+    } catch (e) {
+      stringBody = '[ReadMe is unable to handle circular JSON. Please contact support if you have any questions.]';
+    }
+
+    postData = {
+      mimeType,
+      // Do our best to record *some sort of body* even if it's not 100% accurate.
+      text: stringBody,
+    };
+  }
+
+  // We use a fake host here because we rely on the host header which could be redacted.
+  // We only ever use this reqUrl with the fake hostname for the pathname and querystring.
+  // req.originalUrl is express specific, req.url is node.js
+  const reqUrl = new URL(req.url || '', 'https://readme.io');
+
+  if (headers.authorization) {
+    req.headers.set('authorization', mask(headers.authorization as string));
+  }
+
+  const requestData: HarRequest = {
+    method: req.method || '',
+    url: url.format({
+      // Handle cases where some reverse proxies put two protocols into x-forwarded-proto
+      // This line does the following: "https,http" -> "https"
+      // https://github.com/readmeio/metrics-sdks/issues/378
+      protocol: protocol.split(',')[0],
+      host,
+      pathname: reqUrl.pathname,
+      // Search includes the leading questionmark, format assumes there isn't one, so we trim that off.
+      query: qs.parse(reqUrl.search.substring(1)),
+    }),
+    httpVersion: `${getProto(req).toUpperCase()}/5`, // todo: figure out what we can do with this, there is no analogue in fetch api
+    headers: objectToArray(headers, { castToString: true }),
+    queryString: searchToArray(reqUrl.searchParams),
+    postData,
+    // TODO: When readme starts accepting these, send the correct values
+    cookies: [] satisfies Cookie[],
+    headersSize: -1,
+    bodySize: -1,
+  } as const;
+
+  if (typeof requestData.postData === 'undefined') {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const { postData: postDataToBeOmitted, ...remainingRequestData } = requestData;
+    return remainingRequestData;
+  }
+
+  return requestData;
+}