From 15f377195ab1632a47fa7d601fd5e438fc4addc4 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Thu, 8 Aug 2024 13:30:48 +1000 Subject: [PATCH] UID2-3506 Add functionality on oncall page to list related keysets and rotate them (#329) * Add backend change for listing related keysets * Improve logic on backend filtering * Add frontend to highlight related keysets * Add functionality to rotate keysets * Get ClientTypes from backend * Remove condition for force check * Check if a site has any client key that has ID_READER role * Add helper text to explain min age and force option * Check ID_READER role with correct type * Add tests for related keyset api * Update comments * Update highlights for client types * Revert changes for keyset.json * Revert unused functions * Revert unused clock * Add changes to show rotation result * Update `min_age_seconds` to 1 * Use `Collections.disjoint` instead of customised `ContainAny` * Modify logic for verifying site id * Add comments for moving checking keyset logic to shared * Fix logic of disjoint * Add prompt to confirm rotation * Update wordings for the note --- src/main/java/com/uid2/admin/Main.java | 2 +- .../admin/store/parser/AdminKeysetParser.java | 1 - .../admin/vertx/service/SharingService.java | 65 ++++++- .../admin/vertx/ClientKeyServiceTest.java | 2 +- .../uid2/admin/vertx/SharingServiceTest.java | 159 +++++++++++++++++- webroot/adm/oncall/participant-summary.html | 26 +++ webroot/js/participantSummary.js | 65 ++++++- 7 files changed, 307 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/uid2/admin/Main.java b/src/main/java/com/uid2/admin/Main.java index 5568852e..1f5472f0 100644 --- a/src/main/java/com/uid2/admin/Main.java +++ b/src/main/java/com/uid2/admin/Main.java @@ -251,7 +251,7 @@ public void run() { new EnclaveIdService(auth, writeLock, enclaveStoreWriter, enclaveIdProvider, clock), encryptionKeyService, new KeyAclService(auth, writeLock, keyAclStoreWriter, keyAclProvider, siteProvider, encryptionKeyService), - new SharingService(auth, writeLock, adminKeysetProvider, keysetManager, siteProvider, enableKeysets), + new SharingService(auth, writeLock, adminKeysetProvider, keysetManager, siteProvider, enableKeysets, clientKeyProvider), clientSideKeypairService, new ServiceService(auth, writeLock, serviceStoreWriter, serviceProvider, siteProvider, serviceLinkProvider), new ServiceLinkService(auth, writeLock, serviceLinkStoreWriter, serviceLinkProvider, serviceProvider, siteProvider), diff --git a/src/main/java/com/uid2/admin/store/parser/AdminKeysetParser.java b/src/main/java/com/uid2/admin/store/parser/AdminKeysetParser.java index f57cdad9..fe896ff9 100644 --- a/src/main/java/com/uid2/admin/store/parser/AdminKeysetParser.java +++ b/src/main/java/com/uid2/admin/store/parser/AdminKeysetParser.java @@ -6,7 +6,6 @@ import com.uid2.shared.store.parser.Parser; import com.uid2.shared.store.parser.ParsingResult; import com.uid2.shared.Utils; -import com.uid2.shared.auth.KeysetSnapshot; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; diff --git a/src/main/java/com/uid2/admin/vertx/service/SharingService.java b/src/main/java/com/uid2/admin/vertx/service/SharingService.java index cef0692d..6fd6466d 100644 --- a/src/main/java/com/uid2/admin/vertx/service/SharingService.java +++ b/src/main/java/com/uid2/admin/vertx/service/SharingService.java @@ -2,11 +2,15 @@ import com.uid2.admin.auth.AdminAuthMiddleware; import com.uid2.admin.auth.AdminKeyset; +import com.uid2.admin.legacy.LegacyClientKey; +import com.uid2.admin.legacy.RotatingLegacyClientKeyProvider; import com.uid2.admin.store.reader.RotatingAdminKeysetStore; +import com.uid2.admin.vertx.RequestUtil; import com.uid2.admin.vertx.WriteLock; import com.uid2.admin.managers.KeysetManager; import com.uid2.admin.vertx.ResponseUtil; import com.uid2.shared.Const; +import com.uid2.shared.auth.KeysetSnapshot; import com.uid2.shared.auth.Role; import com.uid2.shared.model.ClientType; import com.uid2.shared.model.SiteUtil; @@ -30,6 +34,7 @@ public class SharingService implements IService { private final RotatingAdminKeysetStore keysetProvider; private final RotatingSiteStore siteProvider; private final KeysetManager keysetManager; + private final RotatingLegacyClientKeyProvider clientKeyProvider; private static final Logger LOGGER = LoggerFactory.getLogger(SharingService.class); private final boolean enableKeysets; @@ -39,13 +44,15 @@ public SharingService(AdminAuthMiddleware auth, RotatingAdminKeysetStore keysetProvider, KeysetManager keysetManager, RotatingSiteStore siteProvider, - boolean enableKeyset) { + boolean enableKeyset, + RotatingLegacyClientKeyProvider clientKeyProvider) { this.auth = auth; this.writeLock = writeLock; this.keysetProvider = keysetProvider; this.keysetManager = keysetManager; this.siteProvider = siteProvider; this.enableKeysets = enableKeyset; + this.clientKeyProvider = clientKeyProvider; } @Override @@ -70,6 +77,9 @@ public void setupRoutes(Router router) { router.get("/api/sharing/keyset/:keyset_id").handler( auth.handle(this::handleListKeyset, Role.MAINTAINER) ); + router.get("/api/sharing/keysets/related").handler( + auth.handle(this::handleListAllKeysetsRelated, Role.MAINTAINER) + ); } private void handleSetKeyset(RoutingContext rc) { @@ -150,6 +160,59 @@ private void handleSetKeyset(RoutingContext rc) { } } + private void handleListAllKeysetsRelated(RoutingContext rc) { + try { + // Get value for site id + final Optional siteIdOpt = RequestUtil.getSiteId(rc, "site_id"); + if (!siteIdOpt.isPresent()) { + ResponseUtil.error(rc, 400, "must specify a site id"); + return; + } + final int siteId = siteIdOpt.get(); + + if (!SiteUtil.isValidSiteId(siteId)) { + ResponseUtil.error(rc, 400, "must specify a valid site id"); + return; + } + + // Get value for client type from the backend + Set clientTypes = this.siteProvider.getSite(siteId).getClientTypes(); + + // Check if this site has any client key that has an ID_READER role + boolean isIdReaderRole = false; + for (LegacyClientKey c : this.clientKeyProvider.getAll()) { + if (c.getRoles().contains(Role.ID_READER)) { + isIdReaderRole = true; + } + } + + // Get the keyset ids that need to be rotated + final JsonArray ja = new JsonArray(); + Map collection = this.keysetProvider.getSnapshot().getAllKeysets(); + for (Map.Entry keyset : collection.entrySet()) { + // The keysets meet any of the below conditions ALL need to be rotated: + // a. Keysets where allowed_types include any of the clientTypes of the site + // b. If this participant has a client key with ID_READER role, we want to rotate all the keysets where allowed_sites is set to null + // c. Keysets where allowed_sites include the leaked site + // d. Keysets belonging to the leaked site itself + if (!Collections.disjoint(keyset.getValue().getAllowedTypes(), clientTypes) || + isIdReaderRole && keyset.getValue().getAllowedSites() == null || + keyset.getValue().getAllowedSites() != null && keyset.getValue().getAllowedSites().contains(siteId) || + keyset.getValue().getSiteId() == siteId) { + // TODO: We have functions below which check if a keysetkey is accessible by a client. We should move the logic of checking keyset to shared as well. + // https://github.com/IABTechLab/uid2-shared/blob/19edb010c6a4d753d03c89268c238be10a8f6722/src/main/java/com/uid2/shared/auth/KeysetSnapshot.java#L13 + ja.add(jsonFullKeyset(keyset.getValue())); + } + } + + rc.response() + .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") + .end(ja.encode()); + } catch (Exception e) { + rc.fail(500, e); + } + } + // Returns if a keyset is one of the reserved ones private static boolean isSpecialKeyset(int keysetId) { return keysetId == Const.Data.MasterKeysetId || keysetId == Const.Data.RefreshKeysetId diff --git a/src/test/java/com/uid2/admin/vertx/ClientKeyServiceTest.java b/src/test/java/com/uid2/admin/vertx/ClientKeyServiceTest.java index a6ada25e..6f321641 100644 --- a/src/test/java/com/uid2/admin/vertx/ClientKeyServiceTest.java +++ b/src/test/java/com/uid2/admin/vertx/ClientKeyServiceTest.java @@ -460,7 +460,7 @@ private static void assertAddedClientKeyEquals(ClientKey expected, ClientKey act .isEqualTo(expected); } - private static class LegacyClientBuilder { + public static class LegacyClientBuilder { private String name = "test_client"; private String contact = "test_contact"; private int siteId = 999; diff --git a/src/test/java/com/uid2/admin/vertx/SharingServiceTest.java b/src/test/java/com/uid2/admin/vertx/SharingServiceTest.java index a41ff1c5..3c3002de 100644 --- a/src/test/java/com/uid2/admin/vertx/SharingServiceTest.java +++ b/src/test/java/com/uid2/admin/vertx/SharingServiceTest.java @@ -21,23 +21,19 @@ import org.junit.jupiter.params.provider.ValueSource; import java.time.Instant; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.*; public class SharingServiceTest extends ServiceTestBase { @Override protected IService createService() { KeysetManager keysetManager = new KeysetManager(adminKeysetProvider, adminKeysetWriter, keysetKeyManager, true); - return new SharingService(auth, writeLock, adminKeysetProvider, keysetManager, siteProvider, true); + return new SharingService(auth, writeLock, adminKeysetProvider, keysetManager, siteProvider, true, clientKeyProvider); } private void compareKeysetListToResult(AdminKeyset keyset, JsonArray actualList) { @@ -1259,4 +1255,153 @@ void KeysetSetNewWithType(Vertx vertx, VertxTestContext testContext) { testContext.completeNow(); }); } + + @Test + void RelatedKeysetSetsWithClientTypes(Vertx vertx, VertxTestContext testContext) { + fakeAuth(Role.MAINTAINER); + + AdminKeyset adminKeyset1 = new AdminKeyset(3, 5, "test", Set.of(4,6,7), Instant.now().getEpochSecond(),true, true, new HashSet<>(Arrays.asList(ClientType.DSP, ClientType.PUBLISHER))); + AdminKeyset adminKeyset2 = new AdminKeyset(4, 7, "test", Set.of(12), Instant.now().getEpochSecond(),true, true, new HashSet<>(Arrays.asList(ClientType.DSP))); + AdminKeyset adminKeyset3 = new AdminKeyset(5, 4, "test", Set.of(5), Instant.now().getEpochSecond(),true, true, new HashSet<>()); + + Map keysets = new HashMap() {{ + put(3, adminKeyset1); + put(4, adminKeyset2); + put(5, adminKeyset3); + }}; + + setAdminKeysets(keysets); + mockSiteExistence(5, 7, 4, 8, 22, 25, 6); + doReturn(new Site(8, "test-name", true, new HashSet<>(Arrays.asList(ClientType.DSP)), null)).when(siteProvider).getSite(8); + + + get(vertx, testContext, "/api/sharing/keysets/related?site_id=8", response -> { + assertEquals(200, response.statusCode()); + + Set expectedKeysetIds = new HashSet<>(Arrays.asList(adminKeyset1.getKeysetId(), adminKeyset2.getKeysetId())); + + Set actualKeysetIds = new HashSet<>(); + JsonArray responseArray = response.bodyAsJsonArray(); + for (int i = 0; i < responseArray.size(); i++) { + JsonObject item = responseArray.getJsonObject(i); + int keysetId = item.getInteger("keyset_id"); + actualKeysetIds.add(keysetId); + } + assertEquals(true, actualKeysetIds.containsAll(expectedKeysetIds)); + testContext.completeNow(); + }); + } + + @Test + void RelatedKeysetSetsWithAllowedSites(Vertx vertx, VertxTestContext testContext) { + fakeAuth(Role.MAINTAINER); + + AdminKeyset adminKeyset1 = new AdminKeyset(3, 1, "test", Set.of(4,8), Instant.now().getEpochSecond(),true, true, new HashSet<>()); + AdminKeyset adminKeyset2 = new AdminKeyset(4, 2, "test", Set.of(5,8), Instant.now().getEpochSecond(),true, true, new HashSet<>()); + AdminKeyset adminKeyset3 = new AdminKeyset(5, 3, "test", Set.of(6), Instant.now().getEpochSecond(),true, true, new HashSet<>()); + + Map keysets = new HashMap() {{ + put(3, adminKeyset1); + put(4, adminKeyset2); + put(5, adminKeyset3); + }}; + + setAdminKeysets(keysets); + mockSiteExistence(1,2,3,4,5,6,8); + doReturn(new Site(8, "test-name", true,null)).when(siteProvider).getSite(8); + + + get(vertx, testContext, "/api/sharing/keysets/related?site_id=8", response -> { + assertEquals(200, response.statusCode()); + + Set expectedKeysetIds = new HashSet<>(Arrays.asList(adminKeyset1.getKeysetId(), adminKeyset2.getKeysetId())); + + Set actualKeysetIds = new HashSet<>(); + JsonArray responseArray = response.bodyAsJsonArray(); + for (int i = 0; i < responseArray.size(); i++) { + JsonObject item = responseArray.getJsonObject(i); + int keysetId = item.getInteger("keyset_id"); + actualKeysetIds.add(keysetId); + } + assertEquals(true, actualKeysetIds.containsAll(expectedKeysetIds)); + testContext.completeNow(); + }); + } + + @Test + void RelatedKeysetSetsWithSameSiteId(Vertx vertx, VertxTestContext testContext) { + fakeAuth(Role.MAINTAINER); + + AdminKeyset adminKeyset1 = new AdminKeyset(3, 1, "test", Set.of(4), Instant.now().getEpochSecond(),true, true, new HashSet<>()); + AdminKeyset adminKeyset2 = new AdminKeyset(4, 2, "test", Set.of(5), Instant.now().getEpochSecond(),true, true, new HashSet<>()); + AdminKeyset adminKeyset3 = new AdminKeyset(5, 8, "test", Set.of(6), Instant.now().getEpochSecond(),true, true, new HashSet<>()); + + Map keysets = new HashMap() {{ + put(3, adminKeyset1); + put(4, adminKeyset2); + put(5, adminKeyset3); + }}; + + setAdminKeysets(keysets); + mockSiteExistence(1,2,4,5,6,8); + doReturn(new Site(8, "test-name", true,null)).when(siteProvider).getSite(8); + + + get(vertx, testContext, "/api/sharing/keysets/related?site_id=8", response -> { + assertEquals(200, response.statusCode()); + + Set expectedKeysetIds = new HashSet<>(Arrays.asList(adminKeyset3.getKeysetId())); + + Set actualKeysetIds = new HashSet<>(); + JsonArray responseArray = response.bodyAsJsonArray(); + for (int i = 0; i < responseArray.size(); i++) { + JsonObject item = responseArray.getJsonObject(i); + int keysetId = item.getInteger("keyset_id"); + actualKeysetIds.add(keysetId); + } + assertEquals(true, actualKeysetIds.containsAll(expectedKeysetIds)); + testContext.completeNow(); + }); + } + + @Test + void RelatedKeysetSetsWithAllowSiteNull(Vertx vertx, VertxTestContext testContext) { + fakeAuth(Role.MAINTAINER); + + AdminKeyset adminKeyset1 = new AdminKeyset(3, 1, "test", Set.of(4), Instant.now().getEpochSecond(),true, true, new HashSet<>()); + AdminKeyset adminKeyset2 = new AdminKeyset(4, 2, "test", Set.of(5), Instant.now().getEpochSecond(),true, true, new HashSet<>()); + AdminKeyset adminKeyset3 = new AdminKeyset(5, 3, "test", null, Instant.now().getEpochSecond(),true, true, new HashSet<>()); + + Map keysets = new HashMap() {{ + put(3, adminKeyset1); + put(4, adminKeyset2); + put(5, adminKeyset3); + }}; + + setAdminKeysets(keysets); + mockSiteExistence(1,2,3,4,5,8); + doReturn(new Site(8, "test-name", true,null)).when(siteProvider).getSite(8); + setClientKeys( + new ClientKeyServiceTest.LegacyClientBuilder() + .withRoles(new HashSet<>(Arrays.asList(Role.ID_READER))) + .withSiteId(8) + .build()); + + + get(vertx, testContext, "/api/sharing/keysets/related?site_id=8", response -> { + assertEquals(200, response.statusCode()); + + Set expectedKeysetIds = new HashSet<>(Arrays.asList(adminKeyset3.getKeysetId())); + + Set actualKeysetIds = new HashSet<>(); + JsonArray responseArray = response.bodyAsJsonArray(); + for (int i = 0; i < responseArray.size(); i++) { + JsonObject item = responseArray.getJsonObject(i); + int keysetId = item.getInteger("keyset_id"); + actualKeysetIds.add(keysetId); + } + assertEquals(true, actualKeysetIds.containsAll(expectedKeysetIds)); + testContext.completeNow(); + }); + } } diff --git a/webroot/adm/oncall/participant-summary.html b/webroot/adm/oncall/participant-summary.html index 16cfe1fe..e3067322 100644 --- a/webroot/adm/oncall/participant-summary.html +++ b/webroot/adm/oncall/participant-summary.html @@ -82,6 +82,32 @@
Participant Opt-out Webhook

                     
                 
+                

+                    

+                        
Participant Related Keysets

+                        
A keyset is related to the participant if it matches below criteria:

+                            a. Keysets where allowed_types include any of the clientTypes of the site

+                            b. If this participant has a client key with ID_READER role, we want to rotate all the keysets where allowed_sites is set to null

+                            c. Keysets where allowed_sites include the leaked site

+                            d. Keysets belonging to the leaked site itself

+                        

+                        

+
+                        

+                            Rotate Keysets
+                            

+                                Normally, keys don't become active for 24 hours when rotated, which gives participants 24 hours before they need to call sdk.refresh().
+                                However in this case, rotation will make the new keys active immediately.
+                                This means participants will not be able to decrypt newly created UID tokens until they have called sdk.refresh().
+                                Note that we recommend calling sdk.refresh() once per hour (see documentation)
+                            

+                            

+                                

+                                

+                            

+                        

+                    

+                

             
         
 
diff --git a/webroot/js/participantSummary.js b/webroot/js/participantSummary.js
index ca40dba3..adc7467f 100644
--- a/webroot/js/participantSummary.js
+++ b/webroot/js/participantSummary.js
@@ -7,9 +7,19 @@ function participantSummaryErrorHandler(err, divContainer) {
 };
 
 function loadAllSitesCallback(result) {
-    siteList = JSON.parse(result).map((item) => { return { name: item.name, id: item.id } });
+    siteList = JSON.parse(result).map((item) => { return { name: item.name, id: item.id, clientTypes: item.clientTypes } });
 };
 
+function rotateKeysetsCallback(result, keyset_id) {
+    const resultJson = JSON.parse(result);
+    const formatted = prettifyJson(JSON.stringify(resultJson));
+    $('#rotateKeysetsStandardOutput').append(`keyset_id: ${keyset_id} rotated: 
${formatted}
`);
+}
+
+function rotateKeysetsErrorHandler(err, keyset_id) {
+    $('#rotateKeysetsErrorOutput').append(`keyset_id: ${keyset_id} rotation failed: 
${err}
`);
+}
+
 function loadSiteCallback(result) {
     const resultJson = JSON.parse(result);
 
@@ -88,6 +98,37 @@ function loadOptoutWebhooksCallback(result, siteName) {
     $('#webhooksStandardOutput').html(formatted);
 };
 
+function loadRelatedKeysetsCallback(result, siteId, clientTypes) {
+    const resultJson = JSON.parse(result);
+    resultJson.forEach(obj => {
+        // Keysets where allowed_types include any of the clientTypes that belows to the site
+        obj.allowed_types = obj.allowed_types.map(item => {
+            return clientTypes.includes(item) ? `` : item;
+        });
+
+        // Keysets where allowed_sites include the leaked site. As it's an integer object, change it to a placeholder and replace later.
+        if (obj.allowed_sites) {
+            obj.allowed_sites = obj.allowed_sites.map(item => {
+                return item === siteId ? "" : item;
+            });
+        }
+    });
+    const formatted = prettifyJson(JSON.stringify(resultJson));
+    let highlightedText = formatted;
+    // Highlight ketsets where allowed_sites is set to null
+    highlightedText = highlightedText.replaceAll(`"allowed_sites": null`, '' + `"allowed_sites": null` + '');
+    // Highlight keysets where allowed_types include the site client_types
+    highlightedText = highlightedText.replaceAll(`""`, `"DSP"`);
+    highlightedText = highlightedText.replaceAll(`""`, `"ADVERTISER"`);
+    highlightedText = highlightedText.replaceAll(`""`, `"DATA_PROVIDER"`);
+    highlightedText = highlightedText.replaceAll(`""`, `"PUBLISHER"`);
+    // Highlight keysets where allowed_sites include the leaked site
+    highlightedText = highlightedText.replaceAll(`""`, `${siteId}`);
+    // Highlight keysets belonging to the leaked site itself
+    highlightedText = highlightedText.replaceAll(`"site_id": ${siteId}`, '' + `"site_id": ${siteId}` + '');
+    $('#relatedKeysetsStandardOutput').html(highlightedText);
+};
+
 $(document).ready(() => {
     const sitesUrl = '/api/site/list';
     doApiCallWithCallback('GET', sitesUrl, loadAllSitesCallback, null);
@@ -123,7 +164,27 @@ $(document).ready(() => {
 
         url = '/api/partner_config/get';
         doApiCallWithCallback('GET', url, (r) => { loadOptoutWebhooksCallback(r, site.name) }, (err) => { participantSummaryErrorHandler(err, '#webhooksErrorOutput') });
-    
+
+        url = `/api/sharing/keysets/related?site_id=${site.id}`;
+        doApiCallWithCallback('GET', url, (r) => { loadRelatedKeysetsCallback(r, site.id, site.clientTypes) }, (err) => { participantSummaryErrorHandler(err, '#relatedKeysetsErrorOutput') });
         $('.section').show();
     });
+
+    $('#doRotateKeysets').on('click', () => {
+        if (!confirm("Are you sure?")) {
+            return;
+        }
+
+        var keysets = $('#relatedKeysetsStandardOutput').text();
+        const ja = JSON.parse(keysets);
+        var rotateKeysetsMessage = '';
+        ja.forEach((keyset) => {
+            var url = `/api/key/rotate_keyset_key?min_age_seconds=1&keyset_id=${keyset.keyset_id}&force=true`;
+            doApiCallWithCallback(
+                'POST',
+                url,
+                (r) => { rotateKeysetsCallback(r, keyset.keyset_id) },
+                (err) => { rotateKeysetsErrorHandler(err, '#rotateKeysetsErrorOutput') });
+        });
+    });
 });
\ No newline at end of file