From e416938950109178462ac92ec18ef4bc2daa6fab Mon Sep 17 00:00:00 2001 From: Sander Bruens Date: Fri, 18 Oct 2024 18:08:34 -0400 Subject: [PATCH] feat(server): add tunnel time metric to opt-in server usage report (#1551) * feat(server): add ASN metric to opt-in server usage report * feat(server): add tunnel time metric to opt-in server usage report * Rename variable. * Let Prometheus join the metrics. * Use a Map. * Revert changes to `prometheus_scraper.ts`. * Rename `LocationUsage` to `ReportedUsage`. * Add test cases for different ASN+country combinations. * Use a different approach where we don't let Prometheus combine the series. --- src/shadowbox/server/mocks/mocks.ts | 2 +- src/shadowbox/server/shared_metrics.spec.ts | 70 ++++++++++----------- src/shadowbox/server/shared_metrics.ts | 57 +++++++++++++---- 3 files changed, 80 insertions(+), 49 deletions(-) diff --git a/src/shadowbox/server/mocks/mocks.ts b/src/shadowbox/server/mocks/mocks.ts index 248cd55b0..58f6a6458 100644 --- a/src/shadowbox/server/mocks/mocks.ts +++ b/src/shadowbox/server/mocks/mocks.ts @@ -59,7 +59,7 @@ export class FakePrometheusClient extends PrometheusClient { const bytesTransferred = this.bytesTransferredById[accessKeyId] || 0; queryResultData.result.push({ metric: {access_key: accessKeyId}, - value: [bytesTransferred, `${bytesTransferred}`], + value: [Date.now() / 1000, `${bytesTransferred}`], }); } return queryResultData; diff --git a/src/shadowbox/server/shared_metrics.spec.ts b/src/shadowbox/server/shared_metrics.spec.ts index 5e21b1ace..ebe570258 100644 --- a/src/shadowbox/server/shared_metrics.spec.ts +++ b/src/shadowbox/server/shared_metrics.spec.ts @@ -79,11 +79,11 @@ describe('OutlineSharedMetricsPublisher', () => { publisher.startSharing(); usageMetrics.reportedUsage = [ - {country: 'AA', inboundBytes: 11}, - {country: 'BB', inboundBytes: 11}, - {country: 'CC', inboundBytes: 22}, - {country: 'AA', inboundBytes: 33}, - {country: 'DD', inboundBytes: 33}, + {country: 'AA', inboundBytes: 11, tunnelTimeSec: 99}, + {country: 'BB', inboundBytes: 11, tunnelTimeSec: 88}, + {country: 'CC', inboundBytes: 22, tunnelTimeSec: 77}, + {country: 'AA', inboundBytes: 33, tunnelTimeSec: 66}, + {country: 'DD', inboundBytes: 33, tunnelTimeSec: 55}, ]; clock.nowMs += 60 * 60 * 1000; @@ -93,18 +93,18 @@ describe('OutlineSharedMetricsPublisher', () => { startUtcMs: startTime, endUtcMs: clock.nowMs, userReports: [ - {bytesTransferred: 11, countries: ['AA']}, - {bytesTransferred: 11, countries: ['BB']}, - {bytesTransferred: 22, countries: ['CC']}, - {bytesTransferred: 33, countries: ['AA']}, - {bytesTransferred: 33, countries: ['DD']}, + {bytesTransferred: 11, countries: ['AA'], tunnelTimeSec: 99}, + {bytesTransferred: 11, countries: ['BB'], tunnelTimeSec: 88}, + {bytesTransferred: 22, countries: ['CC'], tunnelTimeSec: 77}, + {bytesTransferred: 33, countries: ['AA'], tunnelTimeSec: 66}, + {bytesTransferred: 33, countries: ['DD'], tunnelTimeSec: 55}, ], }); startTime = clock.nowMs; usageMetrics.reportedUsage = [ - {country: 'EE', inboundBytes: 44}, - {country: 'FF', inboundBytes: 55}, + {country: 'EE', inboundBytes: 44, tunnelTimeSec: 11}, + {country: 'FF', inboundBytes: 55, tunnelTimeSec: 22}, ]; clock.nowMs += 60 * 60 * 1000; @@ -114,8 +114,8 @@ describe('OutlineSharedMetricsPublisher', () => { startUtcMs: startTime, endUtcMs: clock.nowMs, userReports: [ - {bytesTransferred: 44, countries: ['EE']}, - {bytesTransferred: 55, countries: ['FF']}, + {bytesTransferred: 44, countries: ['EE'], tunnelTimeSec: 11}, + {bytesTransferred: 55, countries: ['FF'], tunnelTimeSec: 22}, ], }); @@ -137,15 +137,15 @@ describe('OutlineSharedMetricsPublisher', () => { publisher.startSharing(); usageMetrics.reportedUsage = [ - {country: 'DD', asn: 999, inboundBytes: 44}, - {country: 'EE', inboundBytes: 55}, + {country: 'DD', inboundBytes: 44, tunnelTimeSec: 11, asn: 999}, + {country: 'EE', inboundBytes: 55, tunnelTimeSec: 22}, ]; clock.nowMs += 60 * 60 * 1000; await clock.runCallbacks(); expect(metricsCollector.collectedServerUsageReport.userReports).toEqual([ - {bytesTransferred: 44, countries: ['DD'], asn: 999}, - {bytesTransferred: 55, countries: ['EE']}, + {bytesTransferred: 44, tunnelTimeSec: 11, countries: ['DD'], asn: 999}, + {bytesTransferred: 55, tunnelTimeSec: 22, countries: ['EE']}, ]); publisher.stopSharing(); }); @@ -165,15 +165,15 @@ describe('OutlineSharedMetricsPublisher', () => { publisher.startSharing(); usageMetrics.reportedUsage = [ - {country: 'DD', asn: 999, inboundBytes: 44}, - {country: 'DD', asn: 888, inboundBytes: 55}, + {country: 'DD', asn: 999, tunnelTimeSec: 11, inboundBytes: 44}, + {country: 'DD', asn: 888, tunnelTimeSec: 22, inboundBytes: 55}, ]; clock.nowMs += 60 * 60 * 1000; await clock.runCallbacks(); expect(metricsCollector.collectedServerUsageReport.userReports).toEqual([ - {bytesTransferred: 44, countries: ['DD'], asn: 999}, - {bytesTransferred: 55, countries: ['DD'], asn: 888}, + {bytesTransferred: 44, tunnelTimeSec: 11, countries: ['DD'], asn: 999}, + {bytesTransferred: 55, tunnelTimeSec: 22, countries: ['DD'], asn: 888}, ]); publisher.stopSharing(); }); @@ -193,15 +193,15 @@ describe('OutlineSharedMetricsPublisher', () => { publisher.startSharing(); usageMetrics.reportedUsage = [ - {country: 'DD', asn: 999, inboundBytes: 44}, - {country: 'EE', asn: 999, inboundBytes: 66}, + {country: 'DD', asn: 999, tunnelTimeSec: 11, inboundBytes: 44}, + {country: 'EE', asn: 999, tunnelTimeSec: 22, inboundBytes: 55}, ]; clock.nowMs += 60 * 60 * 1000; await clock.runCallbacks(); expect(metricsCollector.collectedServerUsageReport.userReports).toEqual([ - {bytesTransferred: 44, countries: ['DD'], asn: 999}, - {bytesTransferred: 66, countries: ['EE'], asn: 999}, + {bytesTransferred: 44, tunnelTimeSec: 11, countries: ['DD'], asn: 999}, + {bytesTransferred: 55, tunnelTimeSec: 22, countries: ['EE'], asn: 999}, ]); publisher.stopSharing(); }); @@ -222,11 +222,11 @@ describe('OutlineSharedMetricsPublisher', () => { publisher.startSharing(); usageMetrics.reportedUsage = [ - {country: 'AA', inboundBytes: 11}, - {country: 'SY', inboundBytes: 11}, - {country: 'CC', inboundBytes: 22}, - {country: 'AA', inboundBytes: 33}, - {country: 'DD', inboundBytes: 33}, + {country: 'AA', tunnelTimeSec: 99, inboundBytes: 11}, + {country: 'SY', tunnelTimeSec: 88, inboundBytes: 11}, + {country: 'CC', tunnelTimeSec: 77, inboundBytes: 22}, + {country: 'AA', tunnelTimeSec: 66, inboundBytes: 33}, + {country: 'DD', tunnelTimeSec: 55, inboundBytes: 33}, ]; clock.nowMs += 60 * 60 * 1000; @@ -236,10 +236,10 @@ describe('OutlineSharedMetricsPublisher', () => { startUtcMs: startTime, endUtcMs: clock.nowMs, userReports: [ - {bytesTransferred: 11, countries: ['AA']}, - {bytesTransferred: 22, countries: ['CC']}, - {bytesTransferred: 33, countries: ['AA']}, - {bytesTransferred: 33, countries: ['DD']}, + {bytesTransferred: 11, tunnelTimeSec: 99, countries: ['AA']}, + {bytesTransferred: 22, tunnelTimeSec: 77, countries: ['CC']}, + {bytesTransferred: 33, tunnelTimeSec: 66, countries: ['AA']}, + {bytesTransferred: 33, tunnelTimeSec: 55, countries: ['DD']}, ], }); publisher.stopSharing(); diff --git a/src/shadowbox/server/shared_metrics.ts b/src/shadowbox/server/shared_metrics.ts index c0076e35d..8cb4b9492 100644 --- a/src/shadowbox/server/shared_metrics.ts +++ b/src/shadowbox/server/shared_metrics.ts @@ -16,7 +16,7 @@ import {Clock} from '../infrastructure/clock'; import * as follow_redirects from '../infrastructure/follow_redirects'; import {JsonConfig} from '../infrastructure/json_config'; import * as logging from '../infrastructure/logging'; -import {PrometheusClient} from '../infrastructure/prometheus_scraper'; +import {PrometheusClient, QueryResultData} from '../infrastructure/prometheus_scraper'; import * as version from './version'; import {AccessKeyConfigJson} from './server_access_key'; @@ -30,6 +30,7 @@ export interface ReportedUsage { country: string; asn?: number; inboundBytes: number; + tunnelTimeSec: number; } // JSON format for the published report. @@ -47,6 +48,7 @@ export interface HourlyUserMetricsReportJson { countries: string[]; asn?: number; bytesTransferred: number; + tunnelTimeSec: number; } // JSON format for the feature metrics report. @@ -84,18 +86,46 @@ export class PrometheusUsageMetrics implements UsageMetrics { async getReportedUsage(): Promise { const timeDeltaSecs = Math.round((Date.now() - this.resetTimeMs) / 1000); - // We measure the traffic to and from the target, since that's what we are protecting. - const result = await this.prometheusClient.query( + + const usage = new Map(); + const processResults = ( + data: QueryResultData, + setValue: (entry: ReportedUsage, value: string) => void + ) => { + for (const result of data.result) { + const country = result.metric['location'] || ''; + const asn = result.metric['asn'] ? Number(result.metric['asn']) : undefined; + const key = `${country}-${asn}`; + const entry = usage.get(key) || { + country, + asn, + inboundBytes: 0, + tunnelTimeSec: 0, + }; + setValue(entry, result.value[1]); + if (!usage.has(key)) { + usage.set(key, entry); + } + } + }; + + // Query and process inbound data bytes by country+ASN. + const dataBytesQueryResponse = await this.prometheusClient.query( `sum(increase(shadowsocks_data_bytes_per_location{dir=~"p>t|p { + entry.inboundBytes = Math.round(parseFloat(value)); + }); + + // Query and process tunneltime by country+ASN. + const tunnelTimeQueryResponse = await this.prometheusClient.query( + `sum(increase(shadowsocks_tunnel_time_seconds_per_location[${timeDeltaSecs}s])) by (location, asn)` + ); + processResults(tunnelTimeQueryResponse, (entry, value) => { + entry.tunnelTimeSec = Math.round(parseFloat(value)); + }); + + return Array.from(usage.values()); } reset() { @@ -205,7 +235,7 @@ export class OutlineSharedMetricsPublisher implements SharedMetricsPublisher { const userReports: HourlyUserMetricsReportJson[] = []; for (const locationUsage of locationUsageMetrics) { - if (locationUsage.inboundBytes === 0) { + if (locationUsage.inboundBytes === 0 && locationUsage.tunnelTimeSec === 0) { continue; } if (isSanctionedCountry(locationUsage.country)) { @@ -215,8 +245,9 @@ export class OutlineSharedMetricsPublisher implements SharedMetricsPublisher { // It's used to differentiate the row from the legacy key usage rows. const country = locationUsage.country || 'ZZ'; const report: HourlyUserMetricsReportJson = { - bytesTransferred: locationUsage.inboundBytes, countries: [country], + bytesTransferred: locationUsage.inboundBytes, + tunnelTimeSec: locationUsage.tunnelTimeSec, }; if (locationUsage.asn) { report.asn = locationUsage.asn;