From d745cc77e17404281307612077178613366ba493 Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Mon, 28 Oct 2024 16:45:01 -0700 Subject: [PATCH 01/36] Change optout loading exception to a warning temporarily --- .../java/com/uid2/operator/store/CloudSyncOptOutStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java index 49f7d0385..1bc882c73 100644 --- a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java @@ -545,8 +545,8 @@ else if (OptOutUtils.isPartitionFile(f)) if (tsOld != Instant.EPOCH && tsNew != Instant.EPOCH && !tsOld.isBefore(tsNew)) { final String errorMsg = "Last partition timestamp of indexed files " + tsOld.getEpochSecond() + " is after last partition of non-indexed files " + tsNew.getEpochSecond(); - LOGGER.error(errorMsg); - throw new IllegalStateException(errorMsg); + // Leaving this as a warning until issue is fixed permanently + LOGGER.warn(errorMsg); } // if there are new partitions in this update, let index delete some in-mem delta caches that is old if (tsNew != Instant.EPOCH) { From bfde65ca8da3f7302fc1f719551a0e954dbc26dc Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Tue, 5 Nov 2024 14:21:18 -0700 Subject: [PATCH 02/36] add back use metrics for the operator-served SDKs --- .../vertx/ClientVersionCapturingHandler.java | 52 +++++++++++++++++++ .../operator/vertx/UIDOperatorVerticle.java | 1 + 2 files changed, 53 insertions(+) create mode 100644 src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java diff --git a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java new file mode 100644 index 000000000..d63626952 --- /dev/null +++ b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java @@ -0,0 +1,52 @@ +package com.uid2.operator.vertx; + +import com.uid2.shared.Const; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Metrics; +import io.vertx.core.Handler; +import io.vertx.ext.web.RoutingContext; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +public class ClientVersionCapturingHandler implements Handler { + private final Map _clientVersionCounters = new HashMap<>(); + + public ClientVersionCapturingHandler(String dir, String whitelistGlob) throws IOException { + try (DirectoryStream dirStream = Files.newDirectoryStream(Paths.get(dir), whitelistGlob)) { + dirStream.forEach(path -> { + final String version = getFileNameWithoutExtension(path); + final Counter counter = Counter + .builder("uid2.client_sdk_versions") + .description("counter for how many http requests are processed per each client sdk version") + .tags("client_version", version) + .register(Metrics.globalRegistry); + _clientVersionCounters.put(version, counter); + }); + } + } + @Override + public void handle(RoutingContext context) { + String clientVersion = context.request().headers().get(Const.Http.ClientVersionHeader); + if (clientVersion == null) { + clientVersion = !context.queryParam("client").isEmpty() ? context.queryParam("client").get(0) : null; + } + if (clientVersion != null) { + final Counter counter = _clientVersionCounters.get(clientVersion); + if (counter != null) { + counter.increment(); + } + } + context.next(); + } + + private static String getFileNameWithoutExtension(Path path) { + final String fileName = path.getFileName().toString(); + return fileName.indexOf(".") > 0 ? fileName.substring(0, fileName.lastIndexOf(".")) : fileName; + } +} \ No newline at end of file diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 210b08571..245aae445 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -218,6 +218,7 @@ private Router createRoutesSetup() throws IOException { router.allowForward(AllowForwardHeaders.X_FORWARD); router.route().handler(new RequestCapturingHandler()); + router.route().handler(new ClientVersionCapturingHandler("static/js", "*.js")); router.route().handler(CorsHandler.create() .addRelativeOrigin(".*.") .allowedMethod(io.vertx.core.http.HttpMethod.GET) From 7149e815b59b74d6a19055b67d594a3dcfc420c0 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 5 Nov 2024 22:35:29 +0000 Subject: [PATCH 03/36] [CI Pipeline] Released Patch version: 5.41.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4dddd15aa..66d95d376 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.41.0 + 5.41.6 UTF-8 From b880e15ac78afe6af84565f9507340c87a144af5 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 6 Nov 2024 16:51:24 +0000 Subject: [PATCH 04/36] [CI Pipeline] Released Patch version: 5.41.8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 66d95d376..f92477342 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.41.6 + 5.41.8 UTF-8 From 5c81dc86f2a722fd94d7ac00e53a2ff4af527489 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 6 Nov 2024 17:42:51 +0000 Subject: [PATCH 05/36] [CI Pipeline] Released Patch version: 5.41.8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 66d95d376..f92477342 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.41.6 + 5.41.8 UTF-8 From facd5a609f4708a00e2d27e6ff438518f41d3b99 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Wed, 6 Nov 2024 14:18:15 -0700 Subject: [PATCH 06/36] Added counter for tcf --- conf/local-config.json | 2 +- .../uid2/operator/vertx/UIDOperatorVerticle.java | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/conf/local-config.json b/conf/local-config.json index f19a4357d..f4687b6c1 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -18,7 +18,7 @@ "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", - "enable_v2_encryption": false, + "enable_v2_encryption": true, "sharing_token_expiry_seconds": 2592000, "cloud_download_threads": 8, "cloud_upload_threads": 2, diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 4f6fd97db..6433268bd 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -99,6 +99,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final Map, DistributionSummary> _refreshDurationMetricSummaries = new HashMap<>(); private final Map, Counter> _advertisingTokenExpiryStatus = new HashMap<>(); private final Map, Counter> _tokenGeneratePolicyCounters = new HashMap<>(); + private final Map _tokenGenerateTCFUsage = new HashMap<>(); private final Map> _identityMapUnmappedIdentifiers = new HashMap<>(); private final Map _identityMapRequestWithUnmapped = new HashMap<>(); @@ -962,7 +963,7 @@ private void handleTokenGenerateV2(RoutingContext rc) { if (isTokenInputValid(input, rc)) { final String apiContact = getApiContact(rc); - switch (validateUserConsent(req)) { + switch (validateUserConsent(req, apiContact)) { case INVALID: { SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, "User consent is invalid", siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.InvalidUserConsentString, siteProvider, platformType); return; @@ -1891,9 +1892,10 @@ private InputUtil.InputVal[] createInputListV1(JsonArray a, IdentityType identit return resp; } - private UserConsentStatus validateUserConsent(JsonObject req) { - // TCF string is an optional parameter and we should only check tcf if in EUID and the string is present + private UserConsentStatus validateUserConsent(JsonObject req, String apiContact) { + // TCF string is an optional parameter, and we should only check tcf if in EUID and the string is present if (identityScope.equals(IdentityScope.EUID) && req.containsKey("tcf_consent_string")) { + recordTokenGenerateTCFUsage(apiContact); TransparentConsentParseResult tcResult = this.getUserConsentV2(req); if (!tcResult.isSuccess()) { return UserConsentStatus.INVALID; @@ -1960,6 +1962,13 @@ private void recordTokenGeneratePolicy(String apiContact, OptoutCheckPolicy poli .register(Metrics.globalRegistry)).increment(); } + private void recordTokenGenerateTCFUsage(String apiContact) { + _tokenGenerateTCFUsage.computeIfAbsent(apiContact, contact -> Counter + .builder("uid2.token_generate_tcf_usage") + .description("Counter for token generate tcf usage") + .tags("api_contact", contact, "policy") + .register(Metrics.globalRegistry)).increment(); + } private TransparentConsentParseResult getUserConsentV2(JsonObject req) { final String rawTcString = req.getString("tcf_consent_string"); From de1569a25e006eab52d652fc15e0545f541b8770 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Wed, 6 Nov 2024 14:19:18 -0700 Subject: [PATCH 07/36] fixed bad config value: --- conf/local-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/local-config.json b/conf/local-config.json index f4687b6c1..f19a4357d 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -18,7 +18,7 @@ "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", - "enable_v2_encryption": true, + "enable_v2_encryption": false, "sharing_token_expiry_seconds": 2592000, "cloud_download_threads": 8, "cloud_upload_threads": 2, From d58eb09b7ba5dfeb30a2ecd39a51b2a8bf9bcf1e Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Thu, 7 Nov 2024 10:32:59 -0700 Subject: [PATCH 08/36] find participants on old sdks --- .../vertx/ClientVersionCapturingHandler.java | 63 +++++++++++++++---- .../operator/vertx/UIDOperatorVerticle.java | 2 +- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java index d63626952..1626bb397 100644 --- a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java +++ b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java @@ -1,10 +1,16 @@ package com.uid2.operator.vertx; +import com.uid2.operator.util.Tuple; import com.uid2.shared.Const; +import com.uid2.shared.auth.IAuthorizable; +import com.uid2.shared.auth.IAuthorizableProvider; +import com.uid2.shared.middleware.AuthMiddleware; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; import io.vertx.core.Handler; import io.vertx.ext.web.RoutingContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.DirectoryStream; @@ -12,21 +18,23 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; public class ClientVersionCapturingHandler implements Handler { - private final Map _clientVersionCounters = new HashMap<>(); + private static final Logger LOGGER = LoggerFactory.getLogger(ClientVersionCapturingHandler.class); + private static final String BEARER_TOKEN_PREFIX = "bearer "; + private final Map, Counter> _clientVersionCounters = new HashMap<>(); + private IAuthorizableProvider authKeyStore; + private final Set versions = new HashSet<>(); - public ClientVersionCapturingHandler(String dir, String whitelistGlob) throws IOException { + public ClientVersionCapturingHandler(String dir, String whitelistGlob, IAuthorizableProvider authKeyStore) throws IOException { + this.authKeyStore = authKeyStore; try (DirectoryStream dirStream = Files.newDirectoryStream(Paths.get(dir), whitelistGlob)) { dirStream.forEach(path -> { final String version = getFileNameWithoutExtension(path); - final Counter counter = Counter - .builder("uid2.client_sdk_versions") - .description("counter for how many http requests are processed per each client sdk version") - .tags("client_version", version) - .register(Metrics.globalRegistry); - _clientVersionCounters.put(version, counter); + versions.add(version); }); } } @@ -36,11 +44,22 @@ public void handle(RoutingContext context) { if (clientVersion == null) { clientVersion = !context.queryParam("client").isEmpty() ? context.queryParam("client").get(0) : null; } - if (clientVersion != null) { - final Counter counter = _clientVersionCounters.get(clientVersion); - if (counter != null) { - counter.increment(); - } + String apiContact; + try { + final String authHeaderValue = context.request().getHeader("Authorization"); + final String authKey = extractBearerToken(authHeaderValue); + final IAuthorizable profile = this.authKeyStore.get(authKey); + apiContact = profile.getContact(); + apiContact = apiContact == null ? "unknown" : apiContact; + } catch (Exception ex) { + apiContact = "unknown"; + } + if (clientVersion != null && versions.contains(clientVersion)) { + _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(apiContact, clientVersion), tuple -> Counter + .builder("uid2.client_sdk_versions") + .description("counter for how many http requests are processed per each client sdk version") + .tags("api_contact", tuple.getItem1(), "client_version", tuple.getItem2()) + .register(Metrics.globalRegistry)).increment();; } context.next(); } @@ -49,4 +68,22 @@ private static String getFileNameWithoutExtension(Path path) { final String fileName = path.getFileName().toString(); return fileName.indexOf(".") > 0 ? fileName.substring(0, fileName.lastIndexOf(".")) : fileName; } + + private static String extractBearerToken(final String headerValue) { + if (headerValue == null) { + return null; + } + + final String v = headerValue.trim(); + if (v.length() < BEARER_TOKEN_PREFIX.length()) { + return null; + } + + final String givenPrefix = v.substring(0, BEARER_TOKEN_PREFIX.length()); + + if (!BEARER_TOKEN_PREFIX.equals(givenPrefix.toLowerCase())) { + return null; + } + return v.substring(BEARER_TOKEN_PREFIX.length()); + } } \ No newline at end of file diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 4f6fd97db..8c92da12e 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -221,7 +221,7 @@ private Router createRoutesSetup() throws IOException { router.allowForward(AllowForwardHeaders.X_FORWARD); router.route().handler(new RequestCapturingHandler()); - router.route().handler(new ClientVersionCapturingHandler("static/js", "*.js")); + router.route().handler(new ClientVersionCapturingHandler("static/js", "*.js", clientKeyProvider)); router.route().handler(CorsHandler.create() .addRelativeOrigin(".*.") .allowedMethod(io.vertx.core.http.HttpMethod.GET) From 015091b0296a32bf423f6adccf640971927dc87c Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Thu, 7 Nov 2024 10:42:55 -0700 Subject: [PATCH 09/36] update workflow refs --- .github/workflows/build-and-test.yaml | 2 +- .github/workflows/publish-all-operators.yaml | 2 +- .github/workflows/validate-image.yaml | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 00695f1db..aa13387c6 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -3,7 +3,7 @@ on: [pull_request, push, workflow_dispatch] jobs: build: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-build-and-test.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-build-and-test.yaml@v3 with: java_version: 21 secrets: inherit \ No newline at end of file diff --git a/.github/workflows/publish-all-operators.yaml b/.github/workflows/publish-all-operators.yaml index c5db3a3b0..5e5bf559b 100644 --- a/.github/workflows/publish-all-operators.yaml +++ b/.github/workflows/publish-all-operators.yaml @@ -55,7 +55,7 @@ jobs: fetch-depth: 0 - name: Scan vulnerabilities - uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan_filesystem@v2 + uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan_filesystem@v3 with: scan_severity: HIGH,CRITICAL failure_severity: CRITICAL diff --git a/.github/workflows/validate-image.yaml b/.github/workflows/validate-image.yaml index 524f19102..37b4bf912 100644 --- a/.github/workflows/validate-image.yaml +++ b/.github/workflows/validate-image.yaml @@ -19,7 +19,7 @@ on: jobs: build-publish-docker-default: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3 with: failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }} fail_on_error: ${{ inputs.fail_on_error || true }} @@ -27,7 +27,7 @@ jobs: java_version: 21 secrets: inherit build-publish-docker-aws: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3 with: failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }} fail_on_error: ${{ inputs.fail_on_error || true }} @@ -36,7 +36,7 @@ jobs: secrets: inherit needs: [build-publish-docker-default] build-publish-docker-gcp: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3 with: failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }} fail_on_error: ${{ inputs.fail_on_error || true }} @@ -45,7 +45,7 @@ jobs: secrets: inherit needs: [build-publish-docker-aws] build-publish-docker-azure: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3 with: failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }} fail_on_error: ${{ inputs.fail_on_error || true }} From 13e033ee1bb666a177565f9d897391efb6cca42d Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Thu, 7 Nov 2024 13:24:34 -0700 Subject: [PATCH 10/36] Fixed extra tag --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 6433268bd..cc1639b22 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -1966,7 +1966,7 @@ private void recordTokenGenerateTCFUsage(String apiContact) { _tokenGenerateTCFUsage.computeIfAbsent(apiContact, contact -> Counter .builder("uid2.token_generate_tcf_usage") .description("Counter for token generate tcf usage") - .tags("api_contact", contact, "policy") + .tags("api_contact", contact) .register(Metrics.globalRegistry)).increment(); } From 81260018c5fa6cb693c751a8432e99dab1495831 Mon Sep 17 00:00:00 2001 From: Scott Sundahl Date: Thu, 7 Nov 2024 14:29:45 -0700 Subject: [PATCH 11/36] support phone number --- scripts/aws/conf/prod-euid-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/aws/conf/prod-euid-config.json b/scripts/aws/conf/prod-euid-config.json index 104f3f4d4..a8befc45f 100644 --- a/scripts/aws/conf/prod-euid-config.json +++ b/scripts/aws/conf/prod-euid-config.json @@ -26,7 +26,7 @@ "identity_scope": "euid", "advertising_token_v3": true, "refresh_token_v3": true, - "enable_phone_support": false, + "enable_phone_support": true, "enable_v1_phone_support": false, "enable_v2_encryption": true } From aae3f7365de11f4428d9cdf9bae9b42ce2464206 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Thu, 7 Nov 2024 14:55:32 -0700 Subject: [PATCH 12/36] update shared version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f92477342..af18e37d4 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 7.20.0 + 7.20.4 ${project.version} 21 21 From 49761070ad9887995fb58296888831b571f0a225 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 7 Nov 2024 22:12:11 +0000 Subject: [PATCH 13/36] [CI Pipeline] Released Patch version: 5.41.15 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index af18e37d4..38e26b327 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.41.8 + 5.41.15 UTF-8 From 8b86c2c397a26bbdc110f7c7af2fcf9394e0afb4 Mon Sep 17 00:00:00 2001 From: Andrei Tarassov Date: Mon, 28 Oct 2024 13:14:44 +1100 Subject: [PATCH 14/36] UID2-4429 More efficient disk space utilisation when building operator enclaves - Clean up unneeded files/resources after they no longer needed to reduce disk space usage and reducing the risk of github runner running out of disk space --- Makefile.eif | 4 ++-- scripts/aws/pipeline/amazonlinux2023.Dockerfile | 11 +++++++---- scripts/aws/pipeline/aws_nitro_eif.sh | 1 + 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Makefile.eif b/Makefile.eif index 9b5b391ac..c50b59b57 100644 --- a/Makefile.eif +++ b/Makefile.eif @@ -14,11 +14,11 @@ all: build_eif build_eif: uid2operator.eif euidoperator.eif uid2operator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile build/load_config.py build/make_config.py - cd build; docker build -t uid2operator . --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./uid2operator.tar uid2operator; docker cp ./uid2operator.tar amazonlinux:/uid2operator.tar + cd build; docker build -t uid2operator . --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./uid2operator.tar uid2operator; docker cp ./uid2operator.tar amazonlinux:/uid2operator.tar; rm -f ./uid2operator.tar docker exec amazonlinux bash aws_nitro_eif.sh uid2operator euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile build/load_config.py build/make_config.py - cd build; docker build -t euidoperator . --build-arg IDENTITY_SCOPE='EUID' --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./euidoperator.tar euidoperator; docker cp ./euidoperator.tar amazonlinux:/euidoperator.tar + cd build; docker build -t euidoperator . --build-arg IDENTITY_SCOPE='EUID' --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./euidoperator.tar euidoperator; docker cp ./euidoperator.tar amazonlinux:/euidoperator.tar; rm -f ./euidoperator.tar docker exec amazonlinux bash aws_nitro_eif.sh euidoperator ################################################################################################################################################################## diff --git a/scripts/aws/pipeline/amazonlinux2023.Dockerfile b/scripts/aws/pipeline/amazonlinux2023.Dockerfile index 2914c9ee3..79bcd66df 100644 --- a/scripts/aws/pipeline/amazonlinux2023.Dockerfile +++ b/scripts/aws/pipeline/amazonlinux2023.Dockerfile @@ -4,8 +4,9 @@ FROM amazonlinux:2023 RUN dnf update -y # systemd is not a hard requirement for Amazon ECS Anywhere, but the installation script currently only supports systemd to run. # Amazon ECS Anywhere can be used without systemd, if you set up your nodes and register them into your ECS cluster **without** the installation script. -RUN dnf -y groupinstall "Development Tools" -RUN dnf -y install systemd vim-common wget git tar libstdc++-static.x86_64 cmake cmake3 aws-nitro-enclaves-cli aws-nitro-enclaves-cli-devel +RUN dnf -y groupinstall "Development Tools" \ + && dnf -y install systemd vim-common wget git tar libstdc++-static.x86_64 cmake cmake3 aws-nitro-enclaves-cli aws-nitro-enclaves-cli-devel \ + && dnf clean all RUN systemctl enable docker @@ -14,12 +15,14 @@ RUN wget https://www.inet.no/dante/files/dante-1.4.3.tar.gz \ && sha256sum --check dante_checksum \ && tar -xf dante-1.4.3.tar.gz \ && cd dante-1.4.3; ./configure; make; cd .. \ - && cp dante-1.4.3/sockd/sockd ./ + && cp dante-1.4.3/sockd/sockd ./ \ + && rm -rf dante-1.4.3 dante-1.4.3.tar.gz RUN git clone https://github.com/IABTechLab/uid2-aws-enclave-vsockproxy.git \ && mkdir uid2-aws-enclave-vsockproxy/build \ && cd uid2-aws-enclave-vsockproxy/build; cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo; make; cd ../.. \ - && cp uid2-aws-enclave-vsockproxy/build/vsock-bridge/src/vsock-bridge ./vsockpx + && cp uid2-aws-enclave-vsockproxy/build/vsock-bridge/src/vsock-bridge ./vsockpx \ + && rm -rf uid2-aws-enclave-vsockproxy COPY ./scripts/aws/pipeline/aws_nitro_eif.sh /aws_nitro_eif.sh diff --git a/scripts/aws/pipeline/aws_nitro_eif.sh b/scripts/aws/pipeline/aws_nitro_eif.sh index 2d8f0216b..904d3f3ea 100644 --- a/scripts/aws/pipeline/aws_nitro_eif.sh +++ b/scripts/aws/pipeline/aws_nitro_eif.sh @@ -10,5 +10,6 @@ while (! docker stats --no-stream >/dev/null 2>&1); do sleep 1 done docker load -i $1.tar +rm -f $1.tar nitro-cli build-enclave --docker-uri $1 --output-file $1.eif nitro-cli describe-eif --eif-path $1.eif | jq -r '.Measurements.PCR0' | xxd -r -p | base64 > pcr0.txt From 62421ce7d26371c8ada45d103736129c31a6d496 Mon Sep 17 00:00:00 2001 From: Andrei Tarassov Date: Wed, 30 Oct 2024 10:51:43 +1100 Subject: [PATCH 15/36] UID2-4430 Automatic allocation of cores to vsockpx and operator vertx - Avoid oversubscribing host and enclave - Host: half cores to vsockpx - Enclave: 2/3 cores to operator reactor vertices, 1/4 cores to vsockpx --- scripts/aws/eks-pod/entrypoint.sh | 2 +- scripts/aws/entrypoint.sh | 2 +- scripts/aws/make_config.py | 2 +- scripts/aws/start.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index adc8a4634..2dc0483e2 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -27,7 +27,7 @@ function setup_vsockproxy() { echo "setup_vsockproxy" VSOCK_PROXY=${VSOCK_PROXY:-/home/vsockpx} VSOCK_CONFIG=${VSOCK_CONFIG:-/home/proxies.host.yaml} - VSOCK_THREADS=${VSOCK_THREADS:-$(( $(nproc) * 2 )) } + VSOCK_THREADS=${VSOCK_THREADS:-$(( ( $(nproc) + 1 ) / 2 )) } VSOCK_LOG_LEVEL=${VSOCK_LOG_LEVEL:-3} echo "starting vsock proxy at $VSOCK_PROXY with $VSOCK_THREADS worker threads..." $VSOCK_PROXY -c $VSOCK_CONFIG --workers $VSOCK_THREADS --log-level $VSOCK_LOG_LEVEL --daemon diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index ef622ff80..37214388b 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -16,7 +16,7 @@ ifconfig lo 127.0.0.1 # -- start vsock proxy echo "Starting vsock proxy..." -/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 3 +/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3 # -- load config from identity service echo "Loading config from identity service via proxy..." diff --git a/scripts/aws/make_config.py b/scripts/aws/make_config.py index 5777dce61..93439ef28 100644 --- a/scripts/aws/make_config.py +++ b/scripts/aws/make_config.py @@ -26,7 +26,7 @@ def apply_override(config, overrides, key, type): config['optout_api_token'] = overrides['api_token'] # number of threads -config['service_instances'] = thread_count +config['service_instances'] = int((thread_count + 1) * 2 / 3) # environment if overrides.get('environment') == 'integ': diff --git a/scripts/aws/start.sh b/scripts/aws/start.sh index 440ae58d7..429826928 100644 --- a/scripts/aws/start.sh +++ b/scripts/aws/start.sh @@ -81,7 +81,7 @@ function update_allocation() { function setup_vsockproxy() { VSOCK_PROXY=${VSOCK_PROXY:-/usr/bin/vsockpx} VSOCK_CONFIG=${VSOCK_CONFIG:-/etc/uid2operator/proxy.yaml} - VSOCK_THREADS=${VSOCK_THREADS:-$(( $(nproc) * 2 )) } + VSOCK_THREADS=${VSOCK_THREADS:-$(( ( $(nproc) + 1 ) / 2 )) } VSOCK_LOG_LEVEL=${VSOCK_LOG_LEVEL:-3} echo "starting vsock proxy at $VSOCK_PROXY with $VSOCK_THREADS worker threads..." $VSOCK_PROXY -c $VSOCK_CONFIG --workers $VSOCK_THREADS --log-level $VSOCK_LOG_LEVEL --daemon From 3dd9ba5e1246c78e3b1b6add06215328f90927fa Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 8 Nov 2024 20:11:30 +0000 Subject: [PATCH 16/36] [CI Pipeline] Released Minor version: 5.42.0 --- pom.xml | 2 +- version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 38e26b327..ac7ec96b4 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.41.15 + 5.42.0 UTF-8 diff --git a/version.json b/version.json index a43cbb357..ad32c1ceb 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.41", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.42", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } From 564b274549509a2e9a58b20cadd752e4440f71a5 Mon Sep 17 00:00:00 2001 From: thomasm-ttd <117058351+thomasm-ttd@users.noreply.github.com> Date: Fri, 15 Nov 2024 18:05:45 +1100 Subject: [PATCH 17/36] Ignore CVE-2024-47535 and CVE-2024-7254 (#1153) --- .trivyignore | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.trivyignore b/.trivyignore index 3aa85f54a..f3b81e9c6 100644 --- a/.trivyignore +++ b/.trivyignore @@ -2,4 +2,9 @@ # See https://aquasecurity.github.io/trivy/v0.35/docs/vulnerability/examples/filter/ # for more details # e.g. -# CVE-2022-3996 + +# https://thetradedesk.atlassian.net/browse/UID2-4460 +CVE-2024-47535 + +# https://thetradedesk.atlassian.net/browse/UID2-4461 +CVE-2024-7254 From 75dfd6d3416e8f9d8d5c2e7ea29315b62b9fbb79 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 18 Nov 2024 01:36:07 +0000 Subject: [PATCH 18/36] [CI Pipeline] Released Patch version: 5.42.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ac7ec96b4..6965c297b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.0 + 5.42.3 UTF-8 From b83fa903016dc98e8ede2bf4e9de234ef953febf Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 18 Nov 2024 01:43:44 +0000 Subject: [PATCH 19/36] [CI Pipeline] Released Patch version: 5.42.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6965c297b..fbce103bd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.3 + 5.42.4 UTF-8 From 0a51642d52cb243d6ecf77df3cdc58b5c31f6f22 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 18 Nov 2024 02:58:28 +0000 Subject: [PATCH 20/36] [CI Pipeline] Released Patch version: 5.42.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fbce103bd..28eab1fc8 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.4 + 5.42.5 UTF-8 From 1385f2feb81fd121508687e068eb24193973d045 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 18 Nov 2024 03:05:59 +0000 Subject: [PATCH 21/36] [CI Pipeline] Released Patch version: 5.42.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 28eab1fc8..a1311d45d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.5 + 5.42.6 UTF-8 From 75a9c29ca63dc23d1a5a54c8d90b0661e0f2f063 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Mon, 18 Nov 2024 13:37:18 -0700 Subject: [PATCH 22/36] Added site name as contact for CSTG --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index f64405588..febbbd931 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -357,6 +357,7 @@ private void handleClientSideTokenGenerateImpl(RoutingContext rc) throws NoSuchA null, TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.BadSubscriptionId, siteProvider, platformType); return; } + rc.data().put("api-contact", siteProvider.getSite(clientSideKeypair.getSiteId()).getName()); if(clientSideKeypair.isDisabled()) { SendClientErrorResponseAndRecordStats(ResponseStatus.Unauthorized, 401, rc, "Unauthorized", From c20cc48601fca945a2ffde48ab68129e23b8a0ec Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Mon, 18 Nov 2024 14:03:34 -0700 Subject: [PATCH 23/36] Added site ID --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index febbbd931..7cba5a866 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -357,7 +357,8 @@ private void handleClientSideTokenGenerateImpl(RoutingContext rc) throws NoSuchA null, TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.BadSubscriptionId, siteProvider, platformType); return; } - rc.data().put("api-contact", siteProvider.getSite(clientSideKeypair.getSiteId()).getName()); + rc.data().put(AuthMiddleware.API_CONTACT_PROP, siteProvider.getSite(clientSideKeypair.getSiteId()).getName()); + rc.put(com.uid2.shared.Const.RoutingContextData.SiteId, clientSideKeypair.getSiteId()); if(clientSideKeypair.isDisabled()) { SendClientErrorResponseAndRecordStats(ResponseStatus.Unauthorized, 401, rc, "Unauthorized", From b2e30d957015b5ff639dc836ec93dcf88e949d84 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Tue, 19 Nov 2024 10:13:54 -0700 Subject: [PATCH 24/36] Removing API contact --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 7cba5a866..c3784ae38 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -357,7 +357,6 @@ private void handleClientSideTokenGenerateImpl(RoutingContext rc) throws NoSuchA null, TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.BadSubscriptionId, siteProvider, platformType); return; } - rc.data().put(AuthMiddleware.API_CONTACT_PROP, siteProvider.getSite(clientSideKeypair.getSiteId()).getName()); rc.put(com.uid2.shared.Const.RoutingContextData.SiteId, clientSideKeypair.getSiteId()); if(clientSideKeypair.isDisabled()) { From 4359b3dfeb672f2c9e66dc4e597180e6decef820 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Fri, 22 Nov 2024 10:44:23 -0700 Subject: [PATCH 25/36] Upgrading vertx version --- .trivyignore | 3 --- pom.xml | 4 ++-- src/main/java/com/uid2/operator/Main.java | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.trivyignore b/.trivyignore index f3b81e9c6..be4a48a9d 100644 --- a/.trivyignore +++ b/.trivyignore @@ -5,6 +5,3 @@ # https://thetradedesk.atlassian.net/browse/UID2-4460 CVE-2024-47535 - -# https://thetradedesk.atlassian.net/browse/UID2-4461 -CVE-2024-7254 diff --git a/pom.xml b/pom.xml index a1311d45d..99584430b 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ UTF-8 - 4.5.3 + 4.5.11 1.0.22 5.11.2 5.11.2 @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 7.20.4 + 8.0.1-alpha-167-SNAPSHOT ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 6aa069604..6b88d715b 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -414,7 +414,7 @@ private static Vertx createVertx() { } private static void setupMetrics(MicrometerMetricsOptions metricOptions) { - BackendRegistries.setupBackend(metricOptions); + BackendRegistries.setupBackend(metricOptions, null); MeterRegistry backendRegistry = BackendRegistries.getDefaultNow(); if (backendRegistry instanceof PrometheusMeterRegistry) { From b23dcdc835a0fbdda462c6c8f1d6535bcb71bf31 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 22 Nov 2024 17:47:44 +0000 Subject: [PATCH 26/36] [CI Pipeline] Released Snapshot version: 5.42.7-alpha-139-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 99584430b..d64fb772a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.6 + 5.42.7-alpha-139-SNAPSHOT UTF-8 From 3768f821402215f6497da1e28984538af8157a7e Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Fri, 22 Nov 2024 11:06:42 -0700 Subject: [PATCH 27/36] Adding new shared version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 99584430b..f81efef11 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.0.1-alpha-167-SNAPSHOT + 8.0.6 ${project.version} 21 21 From 39540466f3fc87980ac211e1bc9ef0de547243c9 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Tue, 26 Nov 2024 14:24:24 +1100 Subject: [PATCH 28/36] Remove siteIdsUsingV4Tokens (#1178) * Remove siteIdsUsingV4Tokens * Remove site_ids_using_v4_tokens * Remove v4 variable * Remove test case which generate v2 token * Set advertising_token_v4_percentage to be 100 * Remove siteId param --- conf/local-config.json | 1 - scripts/aws/conf/default-config.json | 1 - scripts/azure-cc/conf/default-config.json | 1 - scripts/gcp-oidc/conf/default-config.json | 1 - .../com/uid2/operator/service/TokenUtils.java | 17 ------- .../operator/service/UIDOperatorService.java | 21 +++------ .../uid2/operator/UIDOperatorServiceTest.java | 12 ++--- .../operator/UIDOperatorVerticleTest.java | 1 - .../uid2/operator/service/TokenUtilsTest.java | 46 ------------------- 9 files changed, 11 insertions(+), 90 deletions(-) delete mode 100644 src/test/java/com/uid2/operator/service/TokenUtilsTest.java diff --git a/conf/local-config.json b/conf/local-config.json index f19a4357d..7a3200f12 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -14,7 +14,6 @@ "refresh_identity_token_after_seconds": 900, "advertising_token_v3": false, "advertising_token_v4_percentage": 0, - "site_ids_using_v4_tokens": "", "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", diff --git a/scripts/aws/conf/default-config.json b/scripts/aws/conf/default-config.json index 6db89fd29..1836f1aa7 100644 --- a/scripts/aws/conf/default-config.json +++ b/scripts/aws/conf/default-config.json @@ -35,6 +35,5 @@ "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, "advertising_token_v4_percentage": 100, - "site_ids_using_v4_tokens": "", "operator_type": "private" } diff --git a/scripts/azure-cc/conf/default-config.json b/scripts/azure-cc/conf/default-config.json index fbe3e7184..c47eef8f6 100644 --- a/scripts/azure-cc/conf/default-config.json +++ b/scripts/azure-cc/conf/default-config.json @@ -39,6 +39,5 @@ "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, "advertising_token_v4_percentage": 100, - "site_ids_using_v4_tokens": "", "operator_type": "private" } diff --git a/scripts/gcp-oidc/conf/default-config.json b/scripts/gcp-oidc/conf/default-config.json index 302a8c3c3..aaea43364 100644 --- a/scripts/gcp-oidc/conf/default-config.json +++ b/scripts/gcp-oidc/conf/default-config.json @@ -39,6 +39,5 @@ "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, "advertising_token_v4_percentage": 100, - "site_ids_using_v4_tokens": "", "operator_type": "private" } diff --git a/src/main/java/com/uid2/operator/service/TokenUtils.java b/src/main/java/com/uid2/operator/service/TokenUtils.java index 4a13145c6..2cabc641b 100644 --- a/src/main/java/com/uid2/operator/service/TokenUtils.java +++ b/src/main/java/com/uid2/operator/service/TokenUtils.java @@ -62,21 +62,4 @@ public static byte encodeIdentityScope(IdentityScope identityScope) { public static byte encodeIdentityType(IdentityType identityType) { return (byte) (identityType.value << 2); } - - public static Set getSiteIdsUsingV4Tokens(String siteIdsUsingV4TokensInString) { - String[] siteIdsV4TokensList = siteIdsUsingV4TokensInString.split(","); - - Set siteIdsV4TokensSet = new HashSet<>(); - try { - for (String siteId : siteIdsV4TokensList) { - String siteIdTrimmed = siteId.trim(); - if (!siteIdTrimmed.isEmpty()) { - siteIdsV4TokensSet.add(Integer.parseInt(siteIdTrimmed)); - } - } - } catch (NumberFormatException ex) { - throw new IllegalArgumentException(String.format("Invalid integer format found in site_ids_using_v4_tokens: %s", siteIdsUsingV4TokensInString)); - } - return siteIdsV4TokensSet; - } } diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 672cec238..7e6450829 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -22,7 +22,6 @@ import java.util.*; import static com.uid2.operator.IdentityConst.*; -import static com.uid2.operator.service.TokenUtils.getSiteIdsUsingV4Tokens; public class UIDOperatorService implements IUIDOperatorService { public static final String IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = "identity_token_expires_after_seconds"; @@ -49,7 +48,6 @@ public class UIDOperatorService implements IUIDOperatorService { private final OperatorIdentity operatorIdentity; protected final TokenVersion tokenVersionToUseIfNotV4; protected final int advertisingTokenV4Percentage; - protected final Set siteIdsUsingV4Tokens; private final TokenVersion refreshTokenVersion; private final boolean identityV3Enabled; @@ -94,7 +92,6 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv } this.advertisingTokenV4Percentage = config.getInteger("advertising_token_v4_percentage", 0); //0 indicates token v4 will not be used - this.siteIdsUsingV4Tokens = getSiteIdsUsingV4Tokens(config.getString("site_ids_using_v4_tokens", "")); this.tokenVersionToUseIfNotV4 = config.getBoolean("advertising_token_v3", false) ? TokenVersion.V3 : TokenVersion.V2; this.refreshTokenVersion = TokenVersion.V3; @@ -271,18 +268,14 @@ private RefreshToken createRefreshToken(PublisherIdentity publisherIdentity, Use private AdvertisingToken createAdvertisingToken(PublisherIdentity publisherIdentity, UserIdentity userIdentity, Instant now) { TokenVersion tokenVersion; - if (siteIdsUsingV4Tokens.contains(publisherIdentity.siteId)) { - tokenVersion = TokenVersion.V4; - } else { - int pseudoRandomNumber = 1; - final var rawUid = userIdentity.id; - if (rawUid.length > 2) - { - int hash = ((rawUid[0] & 0xFF) << 12) | ((rawUid[1] & 0xFF) << 4) | ((rawUid[2] & 0xFF) & 0xF); //using same logic as ModBasedSaltEntryIndexer.getIndex() in uid2-shared - pseudoRandomNumber = (hash % 100) + 1; //1 to 100 - } - tokenVersion = (pseudoRandomNumber <= this.advertisingTokenV4Percentage) ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; + int pseudoRandomNumber = 1; + final var rawUid = userIdentity.id; + if (rawUid.length > 2) + { + int hash = ((rawUid[0] & 0xFF) << 12) | ((rawUid[1] & 0xFF) << 4) | ((rawUid[2] & 0xFF) & 0xF); //using same logic as ModBasedSaltEntryIndexer.getIndex() in uid2-shared + pseudoRandomNumber = (hash % 100) + 1; //1 to 100 } + tokenVersion = (pseudoRandomNumber <= this.advertisingTokenV4Percentage) ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; return new AdvertisingToken(tokenVersion, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, publisherIdentity, userIdentity); } diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index bc1d49bb2..4a8aef9d1 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -58,11 +58,8 @@ public ExtendedUIDOperatorService(JsonObject config, IOptOutStore optOutStore, I super(config, optOutStore, saltProvider, encoder, clock, identityScope, saltRetrievalResponseHandler); } - public TokenVersion getAdvertisingTokenVersionForTests(int siteId) { + public TokenVersion getAdvertisingTokenVersionForTests() { assert this.advertisingTokenV4Percentage == 0 || this.advertisingTokenV4Percentage == 100; //we want tests to be deterministic - if (this.siteIdsUsingV4Tokens.contains(siteId)) { - return TokenVersion.V4; - } return this.advertisingTokenV4Percentage == 100 ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; } } @@ -96,8 +93,7 @@ void setup() throws Exception { uid2Config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - uid2Config.put("advertising_token_v4_percentage", 0); - uid2Config.put("site_ids_using_v4_tokens", "127,128"); + uid2Config.put("advertising_token_v4_percentage", 100); uid2Config.put("advertising_token_v3", false); // prod is using v2 token version for now uid2Config.put("identity_v3", false); @@ -152,7 +148,7 @@ private UserIdentity createUserIdentity(String rawIdentityHash, IdentityScope sc } private AdvertisingToken validateAndGetToken(EncryptedTokenEncoder tokenEncoder, String advertisingTokenString, IdentityScope scope, IdentityType type, int siteId) { - TokenVersion tokenVersion = (scope == IdentityScope.UID2) ? uid2Service.getAdvertisingTokenVersionForTests(siteId) : euidService.getAdvertisingTokenVersionForTests(siteId); + TokenVersion tokenVersion = (scope == IdentityScope.UID2) ? uid2Service.getAdvertisingTokenVersionForTests() : euidService.getAdvertisingTokenVersionForTests(); UIDOperatorVerticleTest.validateAdvertisingToken(advertisingTokenString, tokenVersion, scope, type); return tokenEncoder.decodeAdvertisingToken(advertisingTokenString); } @@ -164,7 +160,7 @@ private void assertIdentityScopeIdentityTypeAndEstablishedAt(UserIdentity expcte } @ParameterizedTest - @CsvSource({"123, V2","127, V4","128, V4"}) //site id 127 and 128 is for testing "site_ids_using_v4_tokens" + @CsvSource({"123, V4","127, V4","128, V4"}) public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { final IdentityRequest identityRequest = new IdentityRequest( new PublisherIdentity(siteId, 124, 125), diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index eafa14f9a..6ef9ba989 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -160,7 +160,6 @@ private void setupConfig(JsonObject config) { config.put("identity_scope", getIdentityScope().toString()); config.put("advertising_token_v3", getTokenVersion() == TokenVersion.V3); config.put("advertising_token_v4_percentage", getTokenVersion() == TokenVersion.V4 ? 100 : 0); - config.put("site_ids_using_v4_tokens", ""); config.put("identity_v3", useIdentityV3()); config.put("client_side_token_generate", true); config.put("key_sharing_endpoint_provide_app_names", true); diff --git a/src/test/java/com/uid2/operator/service/TokenUtilsTest.java b/src/test/java/com/uid2/operator/service/TokenUtilsTest.java deleted file mode 100644 index 2fb7af1fd..000000000 --- a/src/test/java/com/uid2/operator/service/TokenUtilsTest.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.uid2.operator.service; - -import com.uid2.shared.cloud.CloudStorageException; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static com.uid2.operator.service.TokenUtils.getSiteIdsUsingV4Tokens; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -public class TokenUtilsTest { - Set siteIdsV4TokensSet = new HashSet<>(Arrays.asList(127, 128)); - @Test - void getSiteIdsUsingV4Tokens_multipleSiteIds() { - Set actualSiteIdsV4TokensSet = getSiteIdsUsingV4Tokens("127, 128"); - assertEquals(siteIdsV4TokensSet, actualSiteIdsV4TokensSet); - } - - @Test - void getSiteIdsUsingV4Tokens_oneSiteIds() { - Set actualSiteIdsV4TokensSet = getSiteIdsUsingV4Tokens("127"); - assertEquals(new HashSet<>(List.of(127)), actualSiteIdsV4TokensSet); - } - - @Test - void getSiteIdsUsingV4Tokens_emptyInput() { - Set actualSiteIdsV4TokensSet = getSiteIdsUsingV4Tokens(""); - assertEquals(new HashSet<>(), actualSiteIdsV4TokensSet); - } - - @Test - void getSiteIdsUsingV4Tokens_inputContainsSpaces() { - Set actualSiteIdsV4TokensSet = getSiteIdsUsingV4Tokens(" 127 ,128 "); - assertEquals(siteIdsV4TokensSet, actualSiteIdsV4TokensSet); - } - - @Test - void getSiteIdsUsingV4Tokens_inputContainsInvalidInteger() { - assertThrows(IllegalArgumentException.class, - () -> getSiteIdsUsingV4Tokens(" 1 27 ,128 ")); - } -} From 9f0f28b58612bc74206fb667e3df09ad1cc8e840 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Tue, 3 Dec 2024 16:25:13 +1100 Subject: [PATCH 29/36] UID2-3804 Remove `advertising_token_v4_percentage` env var (#1180) * Remove `advertising_token_v4_percentage` env var * Always return advertising token v4 in tests * Return token version v4 for EUID as well * Remove duplicated code * Remove PRNG * Remove advertising_token_v3 --- conf/docker-config.json | 1 - conf/local-config.json | 2 -- conf/local-e2e-docker-private-config.json | 1 - conf/local-e2e-docker-public-config.json | 1 - conf/local-e2e-private-config.json | 1 - conf/local-e2e-public-config.json | 1 - ...alidator-latest-e2e-docker-public-config.json | 1 - scripts/aws/conf/default-config.json | 1 - scripts/aws/conf/prod-euid-config.json | 1 - scripts/azure-cc/conf/default-config.json | 1 - scripts/gcp-oidc/conf/default-config.json | 1 - .../operator/service/UIDOperatorService.java | 16 +--------------- .../uid2/operator/EUIDOperatorVerticleTest.java | 3 --- .../operator/EUIDOperatorVerticleV4Test.java | 15 --------------- .../uid2/operator/UIDOperatorServiceTest.java | 12 +----------- .../uid2/operator/UIDOperatorVerticleTest.java | 4 +--- .../uid2/operator/UidOperatorVerticleV4Test.java | 14 -------------- 17 files changed, 3 insertions(+), 73 deletions(-) delete mode 100644 src/test/java/com/uid2/operator/EUIDOperatorVerticleV4Test.java delete mode 100644 src/test/java/com/uid2/operator/UidOperatorVerticleV4Test.java diff --git a/conf/docker-config.json b/conf/docker-config.json index 648b922a8..25f38e6ae 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -4,7 +4,6 @@ "storage_mock": true, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/local-config.json b/conf/local-config.json index 7a3200f12..6a357dba1 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -12,8 +12,6 @@ "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, - "advertising_token_v4_percentage": 0, "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index ef05b8772..8637e6da3 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -14,7 +14,6 @@ "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 60f0abd92..a145c4d17 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -16,7 +16,6 @@ "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index e9d3f8b53..4ab52330f 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -16,7 +16,6 @@ "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index cb635b103..bfdc8e394 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -16,7 +16,6 @@ "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index cabf23380..8f82b01a4 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -17,7 +17,6 @@ "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/scripts/aws/conf/default-config.json b/scripts/aws/conf/default-config.json index 1836f1aa7..35c3be58c 100644 --- a/scripts/aws/conf/default-config.json +++ b/scripts/aws/conf/default-config.json @@ -34,6 +34,5 @@ "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, - "advertising_token_v4_percentage": 100, "operator_type": "private" } diff --git a/scripts/aws/conf/prod-euid-config.json b/scripts/aws/conf/prod-euid-config.json index 581ad0ae9..0fbf5d69c 100644 --- a/scripts/aws/conf/prod-euid-config.json +++ b/scripts/aws/conf/prod-euid-config.json @@ -24,7 +24,6 @@ "refresh_identity_token_after_seconds": 3600, "allow_legacy_api": false, "identity_scope": "euid", - "advertising_token_v3": true, "refresh_token_v3": true, "enable_phone_support": true, "enable_v1_phone_support": false, diff --git a/scripts/azure-cc/conf/default-config.json b/scripts/azure-cc/conf/default-config.json index c47eef8f6..4870b2fda 100644 --- a/scripts/azure-cc/conf/default-config.json +++ b/scripts/azure-cc/conf/default-config.json @@ -38,6 +38,5 @@ "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, - "advertising_token_v4_percentage": 100, "operator_type": "private" } diff --git a/scripts/gcp-oidc/conf/default-config.json b/scripts/gcp-oidc/conf/default-config.json index aaea43364..6a65ee2d0 100644 --- a/scripts/gcp-oidc/conf/default-config.json +++ b/scripts/gcp-oidc/conf/default-config.json @@ -38,6 +38,5 @@ "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, - "advertising_token_v4_percentage": 100, "operator_type": "private" } diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 7e6450829..6d4ff86d0 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -46,8 +46,6 @@ public class UIDOperatorService implements IUIDOperatorService { private final Duration refreshIdentityAfter; private final OperatorIdentity operatorIdentity; - protected final TokenVersion tokenVersionToUseIfNotV4; - protected final int advertisingTokenV4Percentage; private final TokenVersion refreshTokenVersion; private final boolean identityV3Enabled; @@ -91,9 +89,6 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); } - this.advertisingTokenV4Percentage = config.getInteger("advertising_token_v4_percentage", 0); //0 indicates token v4 will not be used - this.tokenVersionToUseIfNotV4 = config.getBoolean("advertising_token_v3", false) ? TokenVersion.V3 : TokenVersion.V2; - this.refreshTokenVersion = TokenVersion.V3; this.identityV3Enabled = config.getBoolean("identity_v3", false); } @@ -267,16 +262,7 @@ private RefreshToken createRefreshToken(PublisherIdentity publisherIdentity, Use } private AdvertisingToken createAdvertisingToken(PublisherIdentity publisherIdentity, UserIdentity userIdentity, Instant now) { - TokenVersion tokenVersion; - int pseudoRandomNumber = 1; - final var rawUid = userIdentity.id; - if (rawUid.length > 2) - { - int hash = ((rawUid[0] & 0xFF) << 12) | ((rawUid[1] & 0xFF) << 4) | ((rawUid[2] & 0xFF) & 0xF); //using same logic as ModBasedSaltEntryIndexer.getIndex() in uid2-shared - pseudoRandomNumber = (hash % 100) + 1; //1 to 100 - } - tokenVersion = (pseudoRandomNumber <= this.advertisingTokenV4Percentage) ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; - return new AdvertisingToken(tokenVersion, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, publisherIdentity, userIdentity); + return new AdvertisingToken(TokenVersion.V4, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, publisherIdentity, userIdentity); } static protected class GlobalOptoutResult { diff --git a/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java index 8d168d42e..138e17777 100644 --- a/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java @@ -18,9 +18,6 @@ public class EUIDOperatorVerticleTest extends UIDOperatorVerticleTest { public EUIDOperatorVerticleTest() throws IOException { } - @Override - protected TokenVersion getTokenVersion() {return TokenVersion.V3;} - @Override protected IdentityScope getIdentityScope() { return IdentityScope.EUID; } @Override diff --git a/src/test/java/com/uid2/operator/EUIDOperatorVerticleV4Test.java b/src/test/java/com/uid2/operator/EUIDOperatorVerticleV4Test.java deleted file mode 100644 index fb5ff985b..000000000 --- a/src/test/java/com/uid2/operator/EUIDOperatorVerticleV4Test.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.uid2.operator; - -import com.uid2.shared.model.TokenVersion; - -import java.io.IOException; - -public class EUIDOperatorVerticleV4Test extends EUIDOperatorVerticleTest { - public EUIDOperatorVerticleV4Test() throws IOException { - } - - @Override - protected TokenVersion getTokenVersion() { - return TokenVersion.V4; - } -} diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 4a8aef9d1..37eeef36f 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -57,11 +57,6 @@ class ExtendedUIDOperatorService extends UIDOperatorService { public ExtendedUIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, IdentityScope identityScope, Handler saltRetrievalResponseHandler) { super(config, optOutStore, saltProvider, encoder, clock, identityScope, saltRetrievalResponseHandler); } - - public TokenVersion getAdvertisingTokenVersionForTests() { - assert this.advertisingTokenV4Percentage == 0 || this.advertisingTokenV4Percentage == 100; //we want tests to be deterministic - return this.advertisingTokenV4Percentage == 100 ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; - } } @BeforeEach @@ -93,8 +88,6 @@ void setup() throws Exception { uid2Config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - uid2Config.put("advertising_token_v4_percentage", 100); - uid2Config.put("advertising_token_v3", false); // prod is using v2 token version for now uid2Config.put("identity_v3", false); uid2Service = new ExtendedUIDOperatorService( @@ -111,8 +104,6 @@ void setup() throws Exception { euidConfig.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - euidConfig.put("advertising_token_v4_percentage", 0); - euidConfig.put("advertising_token_v3", true); euidConfig.put("identity_v3", true); euidService = new ExtendedUIDOperatorService( @@ -148,8 +139,7 @@ private UserIdentity createUserIdentity(String rawIdentityHash, IdentityScope sc } private AdvertisingToken validateAndGetToken(EncryptedTokenEncoder tokenEncoder, String advertisingTokenString, IdentityScope scope, IdentityType type, int siteId) { - TokenVersion tokenVersion = (scope == IdentityScope.UID2) ? uid2Service.getAdvertisingTokenVersionForTests() : euidService.getAdvertisingTokenVersionForTests(); - UIDOperatorVerticleTest.validateAdvertisingToken(advertisingTokenString, tokenVersion, scope, type); + UIDOperatorVerticleTest.validateAdvertisingToken(advertisingTokenString, TokenVersion.V4, scope, type); return tokenEncoder.decodeAdvertisingToken(advertisingTokenString); } diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 6ef9ba989..d9a91ae01 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -158,8 +158,6 @@ private void setupConfig(JsonObject config) { config.put(Const.Config.SharingTokenExpiryProp, 60 * 60 * 24 * 30); config.put("identity_scope", getIdentityScope().toString()); - config.put("advertising_token_v3", getTokenVersion() == TokenVersion.V3); - config.put("advertising_token_v4_percentage", getTokenVersion() == TokenVersion.V4 ? 100 : 0); config.put("identity_v3", useIdentityV3()); config.put("client_side_token_generate", true); config.put("key_sharing_endpoint_provide_app_names", true); @@ -665,7 +663,7 @@ private JsonObject setupIdentityMapServiceLinkTest() { return req; } - protected TokenVersion getTokenVersion() {return TokenVersion.V2;} + protected TokenVersion getTokenVersion() {return TokenVersion.V4;} final boolean useIdentityV3() { return getTokenVersion() != TokenVersion.V2; } protected IdentityScope getIdentityScope() { return IdentityScope.UID2; } diff --git a/src/test/java/com/uid2/operator/UidOperatorVerticleV4Test.java b/src/test/java/com/uid2/operator/UidOperatorVerticleV4Test.java deleted file mode 100644 index 7a040427e..000000000 --- a/src/test/java/com/uid2/operator/UidOperatorVerticleV4Test.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.uid2.operator; - -import com.uid2.shared.model.TokenVersion; - -import java.io.IOException; - -public class UidOperatorVerticleV4Test extends UIDOperatorVerticleTest { - public UidOperatorVerticleV4Test() throws IOException { - } - - @Override - protected TokenVersion getTokenVersion() {return TokenVersion.V4;} - -} From 7d2ad652270ea14737ec98782b7947111788b8e2 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 3 Dec 2024 05:27:48 +0000 Subject: [PATCH 30/36] [CI Pipeline] Released Patch version: 5.42.17 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 13a9c3676..f5fb728c5 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.7-alpha-139-SNAPSHOT + 5.42.17 UTF-8 From 242b08d037d6e949529e640ddb39ef72a4067121 Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:42:38 -0800 Subject: [PATCH 31/36] xuy-UID2-4341-change-log-level Refine response logging (#1183) Refine response logging, client error, invalid http origin, and invalid app name responses are logged at info level --- .../uid2/operator/service/JsonParseUtils.java | 2 +- .../uid2/operator/service/ResponseUtil.java | 91 +++++++++---------- .../operator/vertx/UIDOperatorVerticle.java | 70 +++++++------- .../uid2/operator/vertx/V2PayloadHandler.java | 10 +- .../operator/service/ResponseUtilTest.java | 41 +++++---- 5 files changed, 107 insertions(+), 107 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/JsonParseUtils.java b/src/main/java/com/uid2/operator/service/JsonParseUtils.java index 8860c6fd9..4255b799f 100644 --- a/src/main/java/com/uid2/operator/service/JsonParseUtils.java +++ b/src/main/java/com/uid2/operator/service/JsonParseUtils.java @@ -10,7 +10,7 @@ public static JsonArray parseArray(JsonObject object, String key, RoutingContext try { outArray = object.getJsonArray(key); } catch (ClassCastException e) { - ResponseUtil.ClientError(rc, String.format("%s must be an array", key)); + ResponseUtil.LogInfoAndSend400Response(rc, String.format("%s must be an array", key)); return null; } return outArray; diff --git a/src/main/java/com/uid2/operator/service/ResponseUtil.java b/src/main/java/com/uid2/operator/service/ResponseUtil.java index 5f59eab96..a1842c275 100644 --- a/src/main/java/com/uid2/operator/service/ResponseUtil.java +++ b/src/main/java/com/uid2/operator/service/ResponseUtil.java @@ -1,7 +1,6 @@ package com.uid2.operator.service; import com.uid2.operator.monitoring.TokenResponseStatsCollector; -import com.uid2.operator.vertx.UIDOperatorVerticle; import com.uid2.shared.model.TokenVersion; import com.uid2.shared.store.ISiteStore; import io.vertx.core.http.HttpHeaders; @@ -64,19 +63,28 @@ public static void SuccessV2(RoutingContext rc, Object body) { rc.data().put("response", json); } - public static void ClientError(RoutingContext rc, String message) { - Warning(ResponseStatus.ClientError, 400, rc, message); + public static void LogInfoAndSend400Response(RoutingContext rc, String message) { + LogInfoAndSendResponse(ResponseStatus.ClientError, 400, rc, message); } public static void SendClientErrorResponseAndRecordStats(String errorStatus, int statusCode, RoutingContext rc, String message, Integer siteId, TokenResponseStatsCollector.Endpoint endpoint, TokenResponseStatsCollector.ResponseStatus responseStatus, ISiteStore siteProvider, TokenResponseStatsCollector.PlatformType platformType) { - Warning(errorStatus, statusCode, rc, message); + if (ResponseStatus.ClientError.equals(errorStatus) || + ResponseStatus.InvalidAppName.equals(errorStatus) || + ResponseStatus.InvalidHttpOrigin.equals(errorStatus)) + { + LogInfoAndSendResponse(errorStatus, statusCode, rc, message); + } + else { + LogWarningAndSendResponse(errorStatus, statusCode, rc, message); + } + recordTokenResponseStats(siteId, endpoint, responseStatus, siteProvider, null, platformType); } public static void SendServerErrorResponseAndRecordStats(RoutingContext rc, String message, Integer siteId, TokenResponseStatsCollector.Endpoint endpoint, TokenResponseStatsCollector.ResponseStatus responseStatus, ISiteStore siteProvider, Exception exception, TokenResponseStatsCollector.PlatformType platformType) { - Error(ResponseStatus.UnknownError, 500, rc, message, exception); + LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, message, exception); rc.fail(500); recordTokenResponseStats(siteId, endpoint, responseStatus, siteProvider, null, platformType); } @@ -97,62 +105,40 @@ public static JsonObject Response(String status, String message) { return json; } - public static void Error(String errorStatus, int statusCode, RoutingContext rc, String message) { - logError(errorStatus, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + public static void LogErrorAndSendResponse(String errorStatus, int statusCode, RoutingContext rc, String message) { + String msg = ComposeMessage(errorStatus, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + LOGGER.error(msg); final JsonObject json = Response(errorStatus, message); rc.response().setStatusCode(statusCode).putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(json.encode()); } - public static void Error(String errorStatus, int statusCode, RoutingContext rc, String message, Exception exception) { - logError(errorStatus, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress(), exception); + public static void LogErrorAndSendResponse(String errorStatus, int statusCode, RoutingContext rc, String message, Exception exception) { + String msg = ComposeMessage(errorStatus, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + LOGGER.error(msg, exception); final JsonObject json = Response(errorStatus, message); rc.response().setStatusCode(statusCode).putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(json.encode()); } - public static void Warning(String status, int statusCode, RoutingContext rc, String message) { - logWarning(status, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + public static void LogInfoAndSendResponse(String status, int statusCode, RoutingContext rc, String message) { + String msg = ComposeMessage(status, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + LOGGER.info(msg); final JsonObject json = Response(status, message); rc.response().setStatusCode(statusCode).putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(json.encode()); } - private static void logError(String errorStatus, int statusCode, String message, RoutingContextReader contextReader, String clientAddress) { - JsonObject errorJsonObj = JsonObject.of( - "errorStatus", errorStatus, - "contact", contextReader.getContact(), - "siteId", contextReader.getSiteId(), - "statusCode", statusCode, - "clientAddress", clientAddress, - "message", message - ); - final String linkName = contextReader.getLinkName(); - if (!linkName.isBlank()) { - errorJsonObj.put(SecureLinkValidatorService.SERVICE_LINK_NAME, linkName); - } - final String serviceName = contextReader.getServiceName(); - if (!serviceName.isBlank()) { - errorJsonObj.put(SecureLinkValidatorService.SERVICE_NAME, serviceName); - } - LOGGER.error("Error response to http request. " + errorJsonObj.encode()); - } - - private static void logError(String errorStatus, int statusCode, String message, RoutingContextReader contextReader, String clientAddress, Exception exception) { - String errorMessage = "Error response to http request. " + JsonObject.of( - "errorStatus", errorStatus, - "contact", contextReader.getContact(), - "siteId", contextReader.getSiteId(), - "path", contextReader.getPath(), - "statusCode", statusCode, - "clientAddress", clientAddress, - "message", message - ).encode(); - LOGGER.error(errorMessage, exception); + public static void LogWarningAndSendResponse(String status, int statusCode, RoutingContext rc, String message) { + String msg = ComposeMessage(status, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + LOGGER.warn(msg); + final JsonObject json = Response(status, message); + rc.response().setStatusCode(statusCode).putHeader(HttpHeaders.CONTENT_TYPE, "application/json") + .end(json.encode()); } - private static void logWarning(String status, int statusCode, String message, RoutingContextReader contextReader, String clientAddress) { - JsonObject warnMessageJsonObject = JsonObject.of( + private static String ComposeMessage(String status, int statusCode, String message, RoutingContextReader contextReader, String clientAddress) { + JsonObject msgJsonObject = JsonObject.of( "errorStatus", status, "contact", contextReader.getContact(), "siteId", contextReader.getSiteId(), @@ -165,14 +151,22 @@ private static void logWarning(String status, int statusCode, String message, Ro final String origin = contextReader.getOrigin(); if (statusCode >= 400 && statusCode < 500) { if (referer != null) { - warnMessageJsonObject.put("referer", referer); + msgJsonObject.put("referer", referer); } if (origin != null) { - warnMessageJsonObject.put("origin", origin); + msgJsonObject.put("origin", origin); } } - String warnMessage = "Warning response to http request. " + warnMessageJsonObject.encode(); - LOGGER.warn(warnMessage); + + final String linkName = contextReader.getLinkName(); + if (!linkName.isBlank()) { + msgJsonObject.put(SecureLinkValidatorService.SERVICE_LINK_NAME, linkName); + } + final String serviceName = contextReader.getServiceName(); + if (!serviceName.isBlank()) { + msgJsonObject.put(SecureLinkValidatorService.SERVICE_NAME, serviceName); + } + return "Response to http request. " + msgJsonObject.encode(); } public static class ResponseStatus { @@ -183,6 +177,7 @@ public static class ResponseStatus { public static final String InvalidToken = "invalid_token"; public static final String ExpiredToken = "expired_token"; public static final String GenericError = "error"; + public static final String InvalidClient = "invalid_client"; public static final String UnknownError = "unknown"; public static final String InsufficientUserConsent = "insufficient_user_consent"; public static final String InvalidHttpOrigin = "invalid_http_origin"; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index c3784ae38..19b14e927 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -569,7 +569,7 @@ private void handleKeysRequestCommon(RoutingContext rc, Handler onSuc final ClientKey clientKey = AuthMiddleware.getAuthClient(ClientKey.class, rc); final int clientSiteId = clientKey.getSiteId(); if (!clientKey.hasValidSiteId()) { - ResponseUtil.Warning("invalid_client", 401, rc, "Unexpected client site id " + Integer.toString(clientSiteId)); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidClient, 401, rc, "Unexpected client site id " + Integer.toString(clientSiteId)); return; } @@ -820,13 +820,13 @@ private void handleTokenRefreshV1(RoutingContext rc) { ResponseUtil.SuccessNoBody(ResponseStatus.OptOut, rc); } else if (!AuthMiddleware.isAuthenticated(rc)) { // unauthenticated clients get a generic error - ResponseUtil.Warning(ResponseStatus.GenericError, 400, rc, "Error refreshing token"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.GenericError, 400, rc, "Error refreshing token"); } else if (r.isInvalidToken()) { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + tokenList.get(0)); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + tokenList.get(0)); } else if (r.isExpired()) { - ResponseUtil.Warning(ResponseStatus.ExpiredToken, 400, rc, "Expired Token presented"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.ExpiredToken, 400, rc, "Expired Token presented"); } else { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown State"); } } else { ResponseUtil.Success(rc, toJsonV1(r.getTokens())); @@ -852,15 +852,15 @@ private void handleTokenRefreshV2(RoutingContext rc) { ResponseUtil.SuccessNoBodyV2(ResponseStatus.OptOut, rc); } else if (!AuthMiddleware.isAuthenticated(rc)) { // unauthenticated clients get a generic error - ResponseUtil.Warning(ResponseStatus.GenericError, 400, rc, "Error refreshing token"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.GenericError, 400, rc, "Error refreshing token"); } else if (r.isInvalidToken()) { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented"); } else if (r.isExpired()) { - ResponseUtil.Warning(ResponseStatus.ExpiredToken, 400, rc, "Expired Token presented"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.ExpiredToken, 400, rc, "Expired Token presented"); } else if (r.noActiveKey()) { SendServerErrorResponseAndRecordStats(rc, "No active encryption key available", siteId, TokenResponseStatsCollector.Endpoint.RefreshV2, TokenResponseStatsCollector.ResponseStatus.NoActiveKey, siteProvider, new KeyManager.NoActiveKeyException("No active encryption key available"), platformType); } else { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown State"); } } else { ResponseUtil.SuccessV2(rc, toJsonV1(r.getTokens())); @@ -894,7 +894,7 @@ private void handleTokenValidateV1(RoutingContext rc) { ResponseUtil.Success(rc, Boolean.FALSE); } } catch (ClientInputValidationException cie) { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented"); } catch (Exception e) { LOGGER.error("Unknown error while validating token", e); rc.fail(500); @@ -1120,7 +1120,7 @@ private void handleLogoutAsync(RoutingContext rc) { } }); } else { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); } } @@ -1143,7 +1143,7 @@ private Future handleLogoutAsyncV2(RoutingContext rc) { }); return promise.future(); } else { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); return Future.failedFuture(""); } } @@ -1165,7 +1165,7 @@ private void handleOptOutGet(RoutingContext rc) { rc.fail(500); } } else { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); } } @@ -1178,7 +1178,7 @@ private void handleBucketsV1(RoutingContext rc) { sinceTimestamp = ld.toInstant(ZoneOffset.UTC); LOGGER.info(String.format("identity bucket endpoint is called with since_timestamp %s and site id %s", ld, AuthMiddleware.getAuthClient(rc).getSiteId())); } catch (Exception e) { - ResponseUtil.ClientError(rc, "invalid date, must conform to ISO 8601"); + ResponseUtil.LogInfoAndSend400Response(rc, "invalid date, must conform to ISO 8601"); return; } final List modified = this.idService.getModifiedBuckets(sinceTimestamp); @@ -1195,7 +1195,7 @@ private void handleBucketsV1(RoutingContext rc) { ResponseUtil.Success(rc, resp); } } else { - ResponseUtil.ClientError(rc, "missing parameter since_timestamp"); + ResponseUtil.LogInfoAndSend400Response(rc, "missing parameter since_timestamp"); } } @@ -1210,7 +1210,7 @@ private void handleBucketsV2(RoutingContext rc) { sinceTimestamp = ld.toInstant(ZoneOffset.UTC); LOGGER.info(String.format("identity bucket endpoint is called with since_timestamp %s and site id %s", ld, AuthMiddleware.getAuthClient(rc).getSiteId())); } catch (Exception e) { - ResponseUtil.ClientError(rc, "invalid date, must conform to ISO 8601"); + ResponseUtil.LogInfoAndSend400Response(rc, "invalid date, must conform to ISO 8601"); return; } final List modified = this.idService.getModifiedBuckets(sinceTimestamp); @@ -1227,7 +1227,7 @@ private void handleBucketsV2(RoutingContext rc) { ResponseUtil.SuccessV2(rc, resp); } } else { - ResponseUtil.ClientError(rc, "missing parameter since_timestamp"); + ResponseUtil.LogInfoAndSend400Response(rc, "missing parameter since_timestamp"); } } @@ -1245,7 +1245,7 @@ private void handleIdentityMapV1(RoutingContext rc) { jsonObject.put("bucket_id", mappedIdentity.bucketId); ResponseUtil.Success(rc, jsonObject); } catch (Exception e) { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State", e); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown State", e); } } @@ -1359,10 +1359,10 @@ private InputUtil.InputVal getTokenInputV1(RoutingContext rc) { private boolean isTokenInputValid(InputUtil.InputVal input, RoutingContext rc) { if (input == null) { String message = this.phoneSupport ? ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT : ERROR_INVALID_INPUT_EMAIL_MISSING; - ResponseUtil.ClientError(rc, message); + ResponseUtil.LogInfoAndSend400Response(rc, message); return false; } else if (!input.isValid()) { - ResponseUtil.ClientError(rc, "Invalid Identifier"); + ResponseUtil.LogInfoAndSend400Response(rc, "Invalid Identifier"); return false; } return true; @@ -1374,11 +1374,11 @@ private InputUtil.InputVal[] getIdentityBulkInput(RoutingContext rc) { final JsonArray emailHashes = obj.getJsonArray("email_hash"); // FIXME TODO. Avoid Double Iteration. Turn to a decorator pattern if (emails == null && emailHashes == null) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); return null; } else if (emails != null && !emails.isEmpty()) { if (emailHashes != null && !emailHashes.isEmpty()) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_TWICE); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_EMAIL_TWICE); return null; } return createInputList(emails, false); @@ -1391,7 +1391,7 @@ private InputUtil.InputVal[] getIdentityBulkInput(RoutingContext rc) { private InputUtil.InputVal[] getIdentityBulkInputV1(RoutingContext rc) { final JsonObject obj = rc.body().asJsonObject(); if(obj.isEmpty()) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); return null; } final JsonArray emails = JsonParseUtils.parseArray(obj, "email", rc); @@ -1423,7 +1423,7 @@ private InputUtil.InputVal[] getIdentityBulkInputV1(RoutingContext rc) { } if (validInputs == 0 || nonEmptyInputs > 1) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); return null; } @@ -1495,7 +1495,7 @@ private void handleIdentityMapBatchV1(RoutingContext rc) { final JsonObject resp = handleIdentityMapCommon(rc, inputList); ResponseUtil.Success(rc, resp); } catch (Exception e) { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown error while mapping batched identity", e); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown error while mapping batched identity", e); } } @@ -1504,22 +1504,22 @@ private void handleIdentityMapV2(RoutingContext rc) { final InputUtil.InputVal[] inputList = getIdentityMapV2Input(rc); if (inputList == null) { if (this.phoneSupport) - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); else - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); return; } JsonObject requestJsonObject = (JsonObject) rc.data().get(REQUEST); if (!this.secureLinkValidatorService.validateRequest(rc, requestJsonObject, Role.MAPPER)) { - ResponseUtil.Error(ResponseStatus.Unauthorized, HttpStatus.SC_UNAUTHORIZED, rc, "Invalid link_id"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.Unauthorized, HttpStatus.SC_UNAUTHORIZED, rc, "Invalid link_id"); return; } final JsonObject resp = handleIdentityMapCommon(rc, inputList); ResponseUtil.SuccessV2(rc, resp); } catch (Exception e) { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown error while mapping identity v2", e); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown error while mapping identity v2", e); } } @@ -1572,11 +1572,11 @@ private void handleIdentityMapBatch(RoutingContext rc) { final JsonArray emails = obj.getJsonArray("email"); final JsonArray emailHashes = obj.getJsonArray("email_hash"); if (emails == null && emailHashes == null) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); return; } else if (emails != null && !emails.isEmpty()) { if (emailHashes != null && !emailHashes.isEmpty()) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_TWICE); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_EMAIL_TWICE); return; } inputList = createInputList(emails, false); @@ -1678,16 +1678,16 @@ private void recordIdentityMapStatsForServiceLinks(RoutingContext rc, String api private List parseOptoutStatusRequestPayload(RoutingContext rc) { final JsonObject requestObj = (JsonObject) rc.data().get("request"); if (requestObj == null) { - ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Invalid request body"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Invalid request body"); return null; } final JsonArray rawUidsJsonArray = requestObj.getJsonArray("advertising_ids"); if (rawUidsJsonArray == null) { - ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Required Parameter Missing: advertising_ids"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Required Parameter Missing: advertising_ids"); return null; } if (rawUidsJsonArray.size() > optOutStatusMaxRequestSize) { - ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Request payload is too large"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Request payload is too large"); return null; } List rawUID2sInputList = new ArrayList<>(rawUidsJsonArray.size()); @@ -1721,7 +1721,7 @@ private void handleOptoutStatus(RoutingContext rc) { ResponseUtil.SuccessV2(rc, bodyJsonObj); recordOptOutStatusEndpointStats(rc, rawUID2sInput.size(), optedOutJsonArray.size()); } catch (Exception e) { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown error while getting optout status", e); } } diff --git a/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java b/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java index 07ab3ff58..10627ab10 100644 --- a/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java +++ b/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java @@ -51,7 +51,7 @@ public void handle(RoutingContext rc, Handler apiHandler) { V2RequestUtil.V2Request request = V2RequestUtil.parseRequest(rc.body().asString(), AuthMiddleware.getAuthClient(ClientKey.class, rc), new InstantClock()); if (!request.isValid()) { - ResponseUtil.ClientError(rc, request.errorMessage); + ResponseUtil.LogInfoAndSend400Response(rc, request.errorMessage); return; } rc.data().put("request", request.payload); @@ -69,7 +69,7 @@ public void handleAsync(RoutingContext rc, Function apiH V2RequestUtil.V2Request request = V2RequestUtil.parseRequest(rc.body().asString(), AuthMiddleware.getAuthClient(ClientKey.class, rc), new InstantClock()); if (!request.isValid()) { - ResponseUtil.ClientError(rc, request.errorMessage); + ResponseUtil.LogInfoAndSend400Response(rc, request.errorMessage); return; } rc.data().put("request", request.payload); @@ -110,7 +110,7 @@ public void handleTokenGenerate(RoutingContext rc, Handler apiHa } catch (Exception ex){ LOGGER.error("Failed to generate token", ex); - ResponseUtil.Error(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); + ResponseUtil.LogErrorAndSendResponse(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); } } @@ -163,7 +163,7 @@ public void handleTokenRefresh(RoutingContext rc, Handler apiHan } catch (Exception ex){ LOGGER.error("Failed to refresh token", ex); - ResponseUtil.Error(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); + ResponseUtil.LogErrorAndSendResponse(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); } } @@ -199,7 +199,7 @@ private void handleResponse(RoutingContext rc, V2RequestUtil.V2Request request) writeResponse(rc, request.nonce, respJson, request.encryptionKey); } catch (Exception ex) { LOGGER.error("Failed to generate response", ex); - ResponseUtil.Error(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); + ResponseUtil.LogErrorAndSendResponse(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); } } } diff --git a/src/test/java/com/uid2/operator/service/ResponseUtilTest.java b/src/test/java/com/uid2/operator/service/ResponseUtilTest.java index 103dd73a6..77f848cc3 100644 --- a/src/test/java/com/uid2/operator/service/ResponseUtilTest.java +++ b/src/test/java/com/uid2/operator/service/ResponseUtilTest.java @@ -42,12 +42,13 @@ void tearDown() { @Test void logsErrorWithNoExtraDetails() { - ResponseUtil.Error("Some error status", 500, rc, "Some error message"); + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc, "Some error message"); - String expected = "Error response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":null," + "\"message\":\"Some error message\"" + @@ -65,12 +66,13 @@ void logsErrorWithExtraDetailsFromAuthorizable() { when(mockAuthorizable.getSiteId()).thenReturn(10); when(rc.data().get("api-client")).thenReturn(mockAuthorizable); - ResponseUtil.Error("Some error status", 500, rc, "Some error message"); + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc, "Some error message"); - String expected = "Error response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":\"Test Contract\"," + "\"siteId\":10," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":null," + "\"message\":\"Some error message\"" + @@ -83,12 +85,13 @@ void logsErrorWithExtraDetailsFromAuthorizable() { void logsErrorWithSiteIdFromContext() { when(rc.get(Const.RoutingContextData.SiteId)).thenReturn(20); - ResponseUtil.Error("Some error status", 500, rc, "Some error message"); + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc, "Some error message"); - String expected = "Error response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":20," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":null," + "\"message\":\"Some error message\"" + @@ -104,12 +107,13 @@ void logsErrorWithClientAddress() { when(rc.request().remoteAddress()).thenReturn(socket); - ResponseUtil.Error("Some error status", 500, rc, "Some error message"); + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc, "Some error message"); - String expected = "Error response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":\"192.168.10.10\"," + "\"message\":\"Some error message\"" + @@ -124,11 +128,12 @@ void logsErrorWithServiceAndServiceLinkNames() { when(rc1.get(SecureLinkValidatorService.SERVICE_LINK_NAME, "")).thenReturn("TestLink1"); when(rc1.get(SecureLinkValidatorService.SERVICE_NAME, "")).thenReturn("TestService1"); - ResponseUtil.Error("Some error status", 500, rc1, "Some error message"); - String expected = "Error response to http request. {" + + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc1, "Some error message"); + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":null," + "\"message\":\"Some error message\"," + @@ -144,9 +149,9 @@ void logsWarningWithOrigin() { when(request.getHeader("origin")).thenReturn("testOriginHeader"); when(rc.request()).thenReturn(request); - ResponseUtil.Warning("Some error status", 400, rc, "Some error message"); + ResponseUtil.LogInfoAndSendResponse("Some error status", 400, rc, "Some error message"); - String expected = "Warning response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + @@ -165,9 +170,9 @@ void logsWarningWithOriginNull() { when(request.getHeader("origin")).thenReturn(null); when(rc.request()).thenReturn(request); - ResponseUtil.Warning("Some error status", 400, rc, "Some error message"); + ResponseUtil.LogWarningAndSendResponse("Some error status", 400, rc, "Some error message"); - String expected = "Warning response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + @@ -185,9 +190,9 @@ void logsWarningWithReferer() { when(request.getHeader("referer")).thenReturn("testRefererHeader"); when(rc.request()).thenReturn(request); - ResponseUtil.Warning("Some error status", 400, rc, "Some error message"); + ResponseUtil.LogInfoAndSendResponse("Some error status", 400, rc, "Some error message"); - String expected = "Warning response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + @@ -206,9 +211,9 @@ void logsWarningWithRefererNull() { when(request.getHeader("referer")).thenReturn(null); when(rc.request()).thenReturn(request); - ResponseUtil.Warning("Some error status", 400, rc, "Some error message"); + ResponseUtil.LogWarningAndSendResponse("Some error status", 400, rc, "Some error message"); - String expected = "Warning response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + From 4a1af7de1ed658dc0ea6f76ba98db5cf400de8d7 Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Thu, 5 Dec 2024 10:02:34 -0800 Subject: [PATCH 32/36] Xuy UI d2 2578 update share lib to add site name to metrics (#1179) * Xuy UI d2 2578 Update share uid2 lib to 8.0.9, which added site name into metrics --- pom.xml | 2 +- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index f5fb728c5..2450f1529 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.0.6 + 8.0.9 ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 19b14e927..20472f9ec 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -221,7 +221,7 @@ private Router createRoutesSetup() throws IOException { final Router router = Router.router(vertx); router.allowForward(AllowForwardHeaders.X_FORWARD); - router.route().handler(new RequestCapturingHandler()); + router.route().handler(new RequestCapturingHandler(siteProvider)); router.route().handler(new ClientVersionCapturingHandler("static/js", "*.js", clientKeyProvider)); router.route().handler(CorsHandler.create() .addRelativeOrigin(".*.") From 06d49c8840e9e4dbed150a7cc1b2d0138f8dd06f Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 5 Dec 2024 21:13:19 +0000 Subject: [PATCH 33/36] [CI Pipeline] Released Minor version: 5.43.0 --- pom.xml | 2 +- version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2450f1529..d817ec3dd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.17 + 5.43.0 UTF-8 diff --git a/version.json b/version.json index ad32c1ceb..61e9e46d1 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.42", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.43", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } From e5dea0e019116f772235449e5fd286c3d48321c2 Mon Sep 17 00:00:00 2001 From: Vishal Egbert Date: Tue, 10 Dec 2024 15:15:06 +1100 Subject: [PATCH 34/36] Increase max form buffer --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 20472f9ec..617074b9b 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -40,6 +40,7 @@ import io.vertx.core.Promise; import io.vertx.core.buffer.Buffer; import io.vertx.core.http.HttpHeaders; +import io.vertx.core.http.HttpServerOptions; import io.vertx.core.http.HttpServerResponse; import io.vertx.core.json.DecodeException; import io.vertx.core.json.JsonArray; @@ -201,7 +202,7 @@ public void start(Promise startPromise) throws Exception { final Router router = createRoutesSetup(); final int port = Const.Port.ServicePortForOperator + Utils.getPortOffset(); - vertx.createHttpServer() + vertx.createHttpServer(new HttpServerOptions().setMaxFormBufferedBytes((int) MAX_REQUEST_BODY_SIZE)) .requestHandler(router) .listen(port, result -> { if (result.succeeded()) { From 1b72e558f3c3bb9debaa2392ae98d6a99301d8ff Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 10 Dec 2024 04:36:08 +0000 Subject: [PATCH 35/36] [CI Pipeline] Released Patch version: 5.43.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d817ec3dd..69634018d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.43.0 + 5.43.4 UTF-8 From b7922328eb244933037e0d3aa97fe5592232dbdb Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Tue, 10 Dec 2024 17:16:26 +1100 Subject: [PATCH 36/36] Use correct raw UID version for UID2 UIDOperatorVerticleTest and rename identityV3Enabled to rawUidV3Enabled (#1190) * Use correct raw UID version for UID2's UIDOperatorVerticleTest * renamed the variable/method useIdentityV3/identityV3Enabled to useRawUidV3/rawUidV3Enabled to be up to date with latest terminlogies * Improved TokenEncodingTest#testAdvertisingTokenEncodings to tests all combo's of raw UID and ad token versions --- .../operator/service/UIDOperatorService.java | 7 +++--- .../operator/EUIDOperatorVerticleTest.java | 2 ++ .../com/uid2/operator/TokenEncodingTest.java | 23 ++++++++++++------ .../operator/UIDOperatorVerticleTest.java | 24 +++++++++---------- 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 6d4ff86d0..5e66dd70c 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -47,7 +47,8 @@ public class UIDOperatorService implements IUIDOperatorService { private final OperatorIdentity operatorIdentity; private final TokenVersion refreshTokenVersion; - private final boolean identityV3Enabled; + // if we use Raw UID v3 format for the raw UID2/EUIDs generated in this operator + private final boolean rawUidV3Enabled; private final Handler saltRetrievalResponseHandler; @@ -90,7 +91,7 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv } this.refreshTokenVersion = TokenVersion.V3; - this.identityV3Enabled = config.getBoolean("identity_v3", false); + this.rawUidV3Enabled = config.getBoolean("identity_v3", false); } @Override @@ -230,7 +231,7 @@ private MappedIdentity getAdvertisingId(UserIdentity firstLevelHashIdentity, Ins final SaltEntry rotatingSalt = getSaltProviderSnapshot(asOf).getRotatingSalt(firstLevelHashIdentity.id); return new MappedIdentity( - this.identityV3Enabled + this.rawUidV3Enabled ? TokenUtils.getAdvertisingIdV3(firstLevelHashIdentity.identityScope, firstLevelHashIdentity.identityType, firstLevelHashIdentity.id, rotatingSalt.getSalt()) : TokenUtils.getAdvertisingIdV2(firstLevelHashIdentity.id, rotatingSalt.getSalt()), rotatingSalt.getHashedId()); diff --git a/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java index 138e17777..7c894fba6 100644 --- a/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java @@ -21,6 +21,8 @@ public EUIDOperatorVerticleTest() throws IOException { @Override protected IdentityScope getIdentityScope() { return IdentityScope.EUID; } @Override + protected boolean useRawUidV3() { return true; } + @Override protected void addAdditionalTokenGenerateParams(JsonObject payload) { if (payload != null && !payload.containsKey("tcf_consent_string")) { payload.put("tcf_consent_string", "CPehNtWPehNtWABAMBFRACBoALAAAEJAAIYgAKwAQAKgArABAAqAAA"); diff --git a/src/test/java/com/uid2/operator/TokenEncodingTest.java b/src/test/java/com/uid2/operator/TokenEncodingTest.java index c77c81b78..73e11309c 100644 --- a/src/test/java/com/uid2/operator/TokenEncodingTest.java +++ b/src/test/java/com/uid2/operator/TokenEncodingTest.java @@ -16,6 +16,7 @@ import io.vertx.core.json.JsonObject; import org.junit.Assert; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.EnumSource; import java.time.Instant; @@ -86,15 +87,23 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { } @ParameterizedTest - @EnumSource(TokenVersion.class) - public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { + @CsvSource({"false, V4", //same as current UID2 prod (as at 2024-12-10) + "true, V4", //same as current EUID prod (as at 2024-12-10) + //the following combinations aren't used in any UID2/EUID environments but just testing them regardless + "false, V3", + "true, V3", + "false, V2", + "true, V2", + } + ) + public void testAdvertisingTokenEncodings(boolean useRawUIDv3, TokenVersion adTokenVersion) { final EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(this.keyManager); final Instant now = EncodingUtils.NowUTCMillis(); - final byte[] rawUid = UIDOperatorVerticleTest.getRawUid(IdentityType.Email, "test@example.com", IdentityScope.UID2, tokenVersion != TokenVersion.V2); + final byte[] rawUid = UIDOperatorVerticleTest.getRawUid(IdentityType.Email, "test@example.com", IdentityScope.UID2, useRawUIDv3); final AdvertisingToken token = new AdvertisingToken( - tokenVersion, + adTokenVersion, now, now.plusSeconds(60), new OperatorIdentity(101, OperatorType.Service, 102, 103), @@ -103,9 +112,9 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { ); final byte[] encodedBytes = encoder.encode(token, now); - final AdvertisingToken decoded = encoder.decodeAdvertisingToken(EncryptedTokenEncoder.bytesToBase64Token(encodedBytes, tokenVersion)); + final AdvertisingToken decoded = encoder.decodeAdvertisingToken(EncryptedTokenEncoder.bytesToBase64Token(encodedBytes, adTokenVersion)); - assertEquals(tokenVersion, decoded.version); + assertEquals(adTokenVersion, decoded.version); assertEquals(token.createdAt, decoded.createdAt); assertEquals(token.expiresAt, decoded.expiresAt); assertTrue(token.userIdentity.matches(decoded.userIdentity)); @@ -114,7 +123,7 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { assertEquals(token.publisherIdentity.siteId, decoded.publisherIdentity.siteId); Buffer b = Buffer.buffer(encodedBytes); - int keyId = b.getInt(tokenVersion == TokenVersion.V2 ? 1 : 2); //TODO - extract master key from token should be a helper function + int keyId = b.getInt(adTokenVersion == TokenVersion.V2 ? 1 : 2); //TODO - extract master key from token should be a helper function assertEquals(Data.MasterKeySiteId, keyManager.getSiteIdFromKeyId(keyId)); } } diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index d9a91ae01..82ab057d0 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -13,7 +13,6 @@ import com.uid2.operator.util.Tuple; import com.uid2.operator.vertx.OperatorShutdownHandler; import com.uid2.operator.vertx.UIDOperatorVerticle; -import com.uid2.operator.vertx.ClientInputValidationException; import com.uid2.shared.Utils; import com.uid2.shared.auth.ClientKey; import com.uid2.shared.auth.Keyset; @@ -27,9 +26,7 @@ import com.uid2.shared.secret.KeyHasher; import com.uid2.shared.store.*; import com.uid2.shared.store.reader.RotatingKeysetProvider; -import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.search.MeterNotFoundException; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.vertx.core.AsyncResult; import io.vertx.core.Future; @@ -158,7 +155,7 @@ private void setupConfig(JsonObject config) { config.put(Const.Config.SharingTokenExpiryProp, 60 * 60 * 24 * 30); config.put("identity_scope", getIdentityScope().toString()); - config.put("identity_v3", useIdentityV3()); + config.put("identity_v3", useRawUidV3()); config.put("client_side_token_generate", true); config.put("key_sharing_endpoint_provide_app_names", true); config.put("client_side_token_generate_log_invalid_http_origins", true); @@ -622,23 +619,23 @@ private void assertTokenStatusMetrics(Integer siteId, TokenResponseStatsCollecto } private byte[] getAdvertisingIdFromIdentity(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUid(identityType, identityString, firstLevelSalt, rotatingSalt, getIdentityScope(), useIdentityV3()); + return getRawUid(identityType, identityString, firstLevelSalt, rotatingSalt, getIdentityScope(), useRawUidV3()); } - private static byte[] getRawUid(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt, IdentityScope identityScope, boolean useIdentityV3) { - return !useIdentityV3 + private static byte[] getRawUid(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt, IdentityScope identityScope, boolean useRawUidV3) { + return !useRawUidV3 ? TokenUtils.getAdvertisingIdV2FromIdentity(identityString, firstLevelSalt, rotatingSalt) : TokenUtils.getAdvertisingIdV3FromIdentity(identityScope, identityType, identityString, firstLevelSalt, rotatingSalt); } - public static byte[] getRawUid(IdentityType identityType, String identityString, IdentityScope identityScope, boolean useIdentityV3) { - return !useIdentityV3 + public static byte[] getRawUid(IdentityType identityType, String identityString, IdentityScope identityScope, boolean useRawUidV3) { + return !useRawUidV3 ? TokenUtils.getAdvertisingIdV2FromIdentity(identityString, firstLevelSalt, rotatingSalt123.getSalt()) : TokenUtils.getAdvertisingIdV3FromIdentity(identityScope, identityType, identityString, firstLevelSalt, rotatingSalt123.getSalt()); } private byte[] getAdvertisingIdFromIdentityHash(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt) { - return !useIdentityV3() + return !useRawUidV3() ? TokenUtils.getAdvertisingIdV2FromIdentityHash(identityString, firstLevelSalt, rotatingSalt) : TokenUtils.getAdvertisingIdV3FromIdentityHash(getIdentityScope(), identityType, identityString, firstLevelSalt, rotatingSalt); } @@ -665,7 +662,7 @@ private JsonObject setupIdentityMapServiceLinkTest() { protected TokenVersion getTokenVersion() {return TokenVersion.V4;} - final boolean useIdentityV3() { return getTokenVersion() != TokenVersion.V2; } + protected boolean useRawUidV3() { return false; } protected IdentityScope getIdentityScope() { return IdentityScope.UID2; } protected void addAdditionalTokenGenerateParams(JsonObject payload) {} @@ -816,7 +813,10 @@ private AdvertisingToken validateAndGetToken(EncryptedTokenEncoder encoder, Json final String advertisingTokenString = body.getString("advertising_token"); validateAdvertisingToken(advertisingTokenString, getTokenVersion(), getIdentityScope(), identityType); AdvertisingToken advertisingToken = encoder.decodeAdvertisingToken(advertisingTokenString); - if (getTokenVersion() == TokenVersion.V4) { + + // without useIdentityV3() the assert will be trigger as there's no IdentityType in v4 token generated with + // a raw UID v2 as old raw UID format doesn't store the identity type (and scope) + if (useRawUidV3() && getTokenVersion() == TokenVersion.V4) { assertEquals(identityType, advertisingToken.userIdentity.identityType); } return advertisingToken;