Merge remote-tracking branch 'origin/main' into syw-UID2-4159-token-g…

…en-code-refactoring-UserIdentity
IABTechLab · Dec 9, 2024 · 78a70d4 · 78a70d4
2 parents 2ae26b1 + e148266
commit 78a70d4
Show file tree

Hide file tree

Showing 40 changed files with 247 additions and 290 deletions.
diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml
@@ -3,7 +3,7 @@ on: [pull_request, push, workflow_dispatch]
 
 jobs:
   build:
-    uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-build-and-test.yaml@v2
+    uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-build-and-test.yaml@v3
     with:
       java_version: 21
     secrets: inherit
diff --git a/.github/workflows/publish-all-operators.yaml b/.github/workflows/publish-all-operators.yaml
@@ -55,7 +55,7 @@ jobs:
           fetch-depth: 0
 
       - name: Scan vulnerabilities
-        uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan_filesystem@v2
+        uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan_filesystem@v3
         with:
           scan_severity: HIGH,CRITICAL
           failure_severity: CRITICAL

diff --git a/.github/workflows/validate-image.yaml b/.github/workflows/validate-image.yaml
@@ -19,15 +19,15 @@ on:
 
 jobs:
   build-publish-docker-default:
-    uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2
+    uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3
     with: 
       failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }}
       fail_on_error: ${{ inputs.fail_on_error || true }}
       cloud_provider: 'default'
       java_version: 21
     secrets: inherit
   build-publish-docker-aws:
-    uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2
+    uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3
     with: 
       failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }}
       fail_on_error: ${{ inputs.fail_on_error || true }}
@@ -36,7 +36,7 @@ jobs:
     secrets: inherit
     needs: [build-publish-docker-default]
   build-publish-docker-gcp:
-    uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2
+    uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3
     with: 
       failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }}
       fail_on_error: ${{ inputs.fail_on_error || true }}
@@ -45,7 +45,7 @@ jobs:
     secrets: inherit
     needs: [build-publish-docker-aws]
   build-publish-docker-azure:
-    uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2
+    uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3
     with: 
       failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }}
       fail_on_error: ${{ inputs.fail_on_error || true }}

diff --git a/.trivyignore b/.trivyignore
@@ -2,4 +2,6 @@
 # See https://aquasecurity.github.io/trivy/v0.35/docs/vulnerability/examples/filter/ 
 # for more details
 # e.g. 
-# CVE-2022-3996
+
+# https://thetradedesk.atlassian.net/browse/UID2-4460
+CVE-2024-47535
diff --git a/Makefile.eif b/Makefile.eif
@@ -14,11 +14,11 @@ all: build_eif
 build_eif: uid2operator.eif euidoperator.eif
 
 uid2operator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile build/load_config.py build/make_config.py
-	cd build; docker build -t uid2operator . --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./uid2operator.tar uid2operator; docker cp ./uid2operator.tar amazonlinux:/uid2operator.tar
+	cd build; docker build -t uid2operator . --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./uid2operator.tar uid2operator; docker cp ./uid2operator.tar amazonlinux:/uid2operator.tar; rm -f ./uid2operator.tar
 	docker exec amazonlinux bash aws_nitro_eif.sh uid2operator
 
 euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile build/load_config.py build/make_config.py
-	cd build; docker build -t euidoperator . --build-arg IDENTITY_SCOPE='EUID' --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./euidoperator.tar euidoperator; docker cp ./euidoperator.tar amazonlinux:/euidoperator.tar
+	cd build; docker build -t euidoperator . --build-arg IDENTITY_SCOPE='EUID' --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./euidoperator.tar euidoperator; docker cp ./euidoperator.tar amazonlinux:/euidoperator.tar; rm -f ./euidoperator.tar
 	docker exec amazonlinux bash aws_nitro_eif.sh euidoperator
 
 ##################################################################################################################################################################

diff --git a/conf/docker-config.json b/conf/docker-config.json
@@ -4,7 +4,6 @@
   "storage_mock": true,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
-  "advertising_token_v3": false,
   "refresh_token_v3": false,
   "identity_v3": false,
   "identity_scope": "uid2",

diff --git a/conf/local-config.json b/conf/local-config.json
@@ -12,9 +12,6 @@
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
-  "advertising_token_v3": false,
-  "advertising_token_v4_percentage": 0,
-  "site_ids_using_v4_tokens": "",
   "refresh_token_v3": false,
   "identity_v3": false,
   "identity_scope": "uid2",

diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json
@@ -14,7 +14,6 @@
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
-  "advertising_token_v3": false,
   "refresh_token_v3": true,
   "identity_v3": false,
   "identity_scope": "uid2",

diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json
@@ -16,7 +16,6 @@
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
-  "advertising_token_v3": false,
   "refresh_token_v3": true,
   "identity_v3": false,
   "identity_scope": "uid2",

diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json
@@ -16,7 +16,6 @@
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
-  "advertising_token_v3": false,
   "refresh_token_v3": true,
   "identity_v3": false,
   "identity_scope": "uid2",

diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json
@@ -16,7 +16,6 @@
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
-  "advertising_token_v3": false,
   "refresh_token_v3": true,
   "identity_v3": false,
   "identity_scope": "uid2",

diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json
@@ -17,7 +17,6 @@
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
-  "advertising_token_v3": false,
   "refresh_token_v3": true,
   "identity_v3": false,
   "identity_scope": "uid2",

diff --git a/pom.xml b/pom.xml
@@ -6,11 +6,11 @@
 
     <groupId>com.uid2</groupId>
     <artifactId>uid2-operator</artifactId>
-    <version>5.41.0</version>
+    <version>5.43.0</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <vertx.version>4.5.3</vertx.version>
+        <vertx.version>4.5.11</vertx.version>
         <vertx-maven-plugin.version>1.0.22</vertx-maven-plugin.version>
         <junit-jupiter.version>5.11.2</junit-jupiter.version>
         <junit-vintage.version>5.11.2</junit-vintage.version>
@@ -22,7 +22,7 @@
         <enclave-aws.version>2.1.0</enclave-aws.version>
         <enclave-azure.version>2.1.0</enclave-azure.version>
         <enclave-gcp.version>2.1.0</enclave-gcp.version>
-        <uid2-shared.version>7.20.0</uid2-shared.version>
+        <uid2-shared.version>8.0.9</uid2-shared.version>
         <image.version>${project.version}</image.version>
         <maven.compiler.source>21</maven.compiler.source>
         <maven.compiler.target>21</maven.compiler.target>

diff --git a/scripts/aws/conf/default-config.json b/scripts/aws/conf/default-config.json
@@ -34,7 +34,5 @@
   "failure_shutdown_wait_hours": 120,
   "sharing_token_expiry_seconds": 2592000,
   "validate_service_links": false,
-  "advertising_token_v4_percentage": 100,
-  "site_ids_using_v4_tokens": "",
   "operator_type": "private"
 }
diff --git a/scripts/aws/conf/prod-euid-config.json b/scripts/aws/conf/prod-euid-config.json
@@ -24,9 +24,8 @@
   "refresh_identity_token_after_seconds": 3600,
   "allow_legacy_api": false,
   "identity_scope": "euid",
-  "advertising_token_v3": true,
   "refresh_token_v3": true,
-  "enable_phone_support": false,
+  "enable_phone_support": true,
   "enable_v1_phone_support": false,
   "enable_v2_encryption": true
 }
diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh
@@ -27,7 +27,7 @@ function setup_vsockproxy() {
     echo "setup_vsockproxy"
     VSOCK_PROXY=${VSOCK_PROXY:-/home/vsockpx}
     VSOCK_CONFIG=${VSOCK_CONFIG:-/home/proxies.host.yaml}
-    VSOCK_THREADS=${VSOCK_THREADS:-$(( $(nproc) * 2 )) }
+    VSOCK_THREADS=${VSOCK_THREADS:-$(( ( $(nproc) + 1 ) / 2 )) }
     VSOCK_LOG_LEVEL=${VSOCK_LOG_LEVEL:-3}
     echo "starting vsock proxy at $VSOCK_PROXY with $VSOCK_THREADS worker threads..."
     $VSOCK_PROXY -c $VSOCK_CONFIG --workers $VSOCK_THREADS --log-level $VSOCK_LOG_LEVEL --daemon

diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh
@@ -16,7 +16,7 @@ ifconfig lo 127.0.0.1
 
 # -- start vsock proxy
 echo "Starting vsock proxy..."
-/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 3
+/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3
 
 # -- load config from identity service
 echo "Loading config from identity service via proxy..."

diff --git a/scripts/aws/make_config.py b/scripts/aws/make_config.py
@@ -26,7 +26,7 @@ def apply_override(config, overrides, key, type):
 config['optout_api_token'] = overrides['api_token']
 
 # number of threads
-config['service_instances'] = thread_count
+config['service_instances'] = int((thread_count + 1) * 2 / 3)
 
 # environment
 if overrides.get('environment') == 'integ':

diff --git a/scripts/aws/pipeline/amazonlinux2023.Dockerfile b/scripts/aws/pipeline/amazonlinux2023.Dockerfile
@@ -4,8 +4,9 @@ FROM amazonlinux:2023
 RUN dnf update -y
     # systemd is not a hard requirement for Amazon ECS Anywhere, but the installation script currently only supports systemd to run.
     # Amazon ECS Anywhere can be used without systemd, if you set up your nodes and register them into your ECS cluster **without** the installation script.
-RUN dnf -y groupinstall "Development Tools"
-RUN dnf -y install systemd vim-common wget git tar libstdc++-static.x86_64 cmake cmake3 aws-nitro-enclaves-cli aws-nitro-enclaves-cli-devel
+RUN dnf -y groupinstall "Development Tools" \
+    && dnf -y install systemd vim-common wget git tar libstdc++-static.x86_64 cmake cmake3 aws-nitro-enclaves-cli aws-nitro-enclaves-cli-devel \
+    && dnf clean all
 
 RUN systemctl enable docker
 
@@ -14,12 +15,14 @@ RUN wget https://www.inet.no/dante/files/dante-1.4.3.tar.gz \
     && sha256sum --check dante_checksum \
     && tar -xf dante-1.4.3.tar.gz \
     && cd dante-1.4.3; ./configure; make; cd .. \
-    && cp dante-1.4.3/sockd/sockd ./
+    && cp dante-1.4.3/sockd/sockd ./ \
+    && rm -rf dante-1.4.3 dante-1.4.3.tar.gz
 
 RUN git clone https://github.com/IABTechLab/uid2-aws-enclave-vsockproxy.git \
     && mkdir uid2-aws-enclave-vsockproxy/build \
     && cd uid2-aws-enclave-vsockproxy/build; cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo; make; cd ../.. \
-    && cp uid2-aws-enclave-vsockproxy/build/vsock-bridge/src/vsock-bridge ./vsockpx
+    && cp uid2-aws-enclave-vsockproxy/build/vsock-bridge/src/vsock-bridge ./vsockpx \
+    && rm -rf uid2-aws-enclave-vsockproxy
 
 COPY ./scripts/aws/pipeline/aws_nitro_eif.sh /aws_nitro_eif.sh
 

diff --git a/scripts/aws/pipeline/aws_nitro_eif.sh b/scripts/aws/pipeline/aws_nitro_eif.sh
@@ -10,5 +10,6 @@ while (! docker stats --no-stream >/dev/null 2>&1); do
     sleep 1
 done
 docker load -i $1.tar
+rm -f $1.tar
 nitro-cli build-enclave --docker-uri $1 --output-file $1.eif
 nitro-cli describe-eif --eif-path $1.eif | jq -r '.Measurements.PCR0' | xxd -r -p | base64 > pcr0.txt
diff --git a/scripts/aws/start.sh b/scripts/aws/start.sh
@@ -81,7 +81,7 @@ function update_allocation() {
 function setup_vsockproxy() {
     VSOCK_PROXY=${VSOCK_PROXY:-/usr/bin/vsockpx}
     VSOCK_CONFIG=${VSOCK_CONFIG:-/etc/uid2operator/proxy.yaml}
-    VSOCK_THREADS=${VSOCK_THREADS:-$(( $(nproc) * 2 )) }
+    VSOCK_THREADS=${VSOCK_THREADS:-$(( ( $(nproc) + 1 ) / 2 )) }
     VSOCK_LOG_LEVEL=${VSOCK_LOG_LEVEL:-3}
     echo "starting vsock proxy at $VSOCK_PROXY with $VSOCK_THREADS worker threads..."
     $VSOCK_PROXY -c $VSOCK_CONFIG --workers $VSOCK_THREADS --log-level $VSOCK_LOG_LEVEL --daemon

diff --git a/scripts/azure-cc/conf/default-config.json b/scripts/azure-cc/conf/default-config.json
@@ -38,7 +38,5 @@
   "failure_shutdown_wait_hours": 120,
   "sharing_token_expiry_seconds": 2592000,
   "validate_service_links": false,
-  "advertising_token_v4_percentage": 100,
-  "site_ids_using_v4_tokens": "",
   "operator_type": "private"
 }
diff --git a/scripts/gcp-oidc/conf/default-config.json b/scripts/gcp-oidc/conf/default-config.json
@@ -38,7 +38,5 @@
   "failure_shutdown_wait_hours": 120,
   "sharing_token_expiry_seconds": 2592000,
   "validate_service_links": false,
-  "advertising_token_v4_percentage": 100,
-  "site_ids_using_v4_tokens": "",
   "operator_type": "private"
 }
diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java
@@ -414,7 +414,7 @@ private static Vertx createVertx() {
     }
 
     private static void setupMetrics(MicrometerMetricsOptions metricOptions) {
-        BackendRegistries.setupBackend(metricOptions);
+        BackendRegistries.setupBackend(metricOptions, null);
 
         MeterRegistry backendRegistry = BackendRegistries.getDefaultNow();
         if (backendRegistry instanceof PrometheusMeterRegistry) {

diff --git a/src/main/java/com/uid2/operator/service/JsonParseUtils.java b/src/main/java/com/uid2/operator/service/JsonParseUtils.java
@@ -10,7 +10,7 @@ public static JsonArray parseArray(JsonObject object, String key, RoutingContext
         try {
             outArray = object.getJsonArray(key);
         } catch (ClassCastException e) {
-            ResponseUtil.ClientError(rc, String.format("%s must be an array", key));
+            ResponseUtil.LogInfoAndSend400Response(rc, String.format("%s must be an array", key));
             return null;
         }
         return outArray;