From d972f7e66b1808874f9558e8c7c84528afbfc9bc Mon Sep 17 00:00:00 2001 From: Christer Edvartsen Date: Tue, 26 Nov 2024 12:43:54 +0100 Subject: [PATCH] Fix outdated GraphQL query, and attempt to introduce pagination NAIS API just released v1, and as a result of that the queries in "Teamkatalogen" must be updated. v1 of NAIS API implements the Cursor Connections Specification instead of the limit / offset based pagination we previously had. Basically we fetch a page of teams, and the pageInfo object holds information about how to continue to fetch more. The query for single teams has not changed in v1, so no changes needed to be made to that. Also, the GraphQL endpoint has been changed from "/query" to "/graphql". More info: - https://github.com/nais/api/blob/main/docs/graphql_practices.md#pagination - https://relay.dev/graphql/connections.htm --- .../data/team/naisteam/NaisConsoleClient.java | 38 +++++++++++++------ .../no/nav/data/team/naisteam/NaisTeam.java | 5 ++- .../src/main/resources/application.yaml | 2 +- .../no/nav/data/team/WiremockExtension.java | 2 +- .../resources/application-test.properties | 2 +- 5 files changed, 33 insertions(+), 16 deletions(-) diff --git a/apps/backend/src/main/java/no/nav/data/team/naisteam/NaisConsoleClient.java b/apps/backend/src/main/java/no/nav/data/team/naisteam/NaisConsoleClient.java index 242ad2a4a..290e9a06d 100644 --- a/apps/backend/src/main/java/no/nav/data/team/naisteam/NaisConsoleClient.java +++ b/apps/backend/src/main/java/no/nav/data/team/naisteam/NaisConsoleClient.java @@ -15,6 +15,7 @@ import org.springframework.web.reactive.function.client.WebClient; import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -86,21 +87,36 @@ public boolean naisTeamExists(String teamId) { } private List fetchAllNaisTeams() { - var paginationLimit = 1000; - - var out = client.document(NaisTeam.TEAMS_QUERY) - .variable("limit", paginationLimit) - .variable("offset", 0) + List allTeams = new ArrayList<>(); + String cursor = ""; + boolean hasNextPage = true; + + while (hasNextPage) { + var response = client.document(NaisTeam.TEAMS_QUERY) + .variable("first", 100) + .variable("after", cursor) .execute() - .map(response -> response.field("teams.nodes").toEntity(new ParameterizedTypeReference>() { - })) .block(); - if(out != null && out.size() > (2*paginationLimit/3)){ - log.error("fetchAllNaisTeams: The amount of nais-teams fetched is approaching the pagination limit, {} / {}. Consider implementing proper support for pagination.", out.size(), paginationLimit); + if (response != null && response.isValid()) { + var teams = response + .field("teams.nodes") + .toEntity(new ParameterizedTypeReference>() {}); + + if (teams != null) { + allTeams.addAll(teams); + } + + cursor = response.field("teams.pageInfo.endCursor").toEntity(String.class); + hasNextPage = Boolean.TRUE.equals(response.field("teams.pageInfo.hasNextPage").toEntity(Boolean.class)); + } else { + log.error("fetchAllNaisTeams: Received a null response from the GraphQL query."); + break; + } } - return out; + log.info("fetchAllNaisTeams: Fetched {} nais-teams in total.", allTeams.size()); + return allTeams; } private NaisTeam fetchNaisTeam(String slug) { @@ -109,6 +125,6 @@ private NaisTeam fetchNaisTeam(String slug) { .execute() .block(); - return response.isValid() ? response.field("team").toEntity(NaisTeam.class) : null; + return response != null && response.isValid() ? response.field("team").toEntity(NaisTeam.class) : null; } } diff --git a/apps/backend/src/main/java/no/nav/data/team/naisteam/NaisTeam.java b/apps/backend/src/main/java/no/nav/data/team/naisteam/NaisTeam.java index 208e5b9a7..793070ef5 100644 --- a/apps/backend/src/main/java/no/nav/data/team/naisteam/NaisTeam.java +++ b/apps/backend/src/main/java/no/nav/data/team/naisteam/NaisTeam.java @@ -9,8 +9,8 @@ public record NaisTeam( @SuppressWarnings("GraphQLUnresolvedReference") public final static String TEAMS_QUERY = //language=graphql """ - query ($limit: Int, $offset: Int) { - teams(limit: $limit, offset: $offset) { + query ($first: Int, $after: Cursor!) { + teams(first: $first, after: $after) { nodes { slackChannel purpose @@ -18,6 +18,7 @@ public record NaisTeam( } pageInfo { hasNextPage + endCursor } } } diff --git a/apps/backend/src/main/resources/application.yaml b/apps/backend/src/main/resources/application.yaml index 7bdec0ad1..426f86523 100644 --- a/apps/backend/src/main/resources/application.yaml +++ b/apps/backend/src/main/resources/application.yaml @@ -88,7 +88,7 @@ client: console: auth: token: ${NAIS_CONSOLE_TOKEN} - base-url: https://console.nav.cloud.nais.io/query + base-url: https://console.nav.cloud.nais.io/graphql nom: graphql: url: https://nom/graphql diff --git a/apps/backend/src/test/java/no/nav/data/team/WiremockExtension.java b/apps/backend/src/test/java/no/nav/data/team/WiremockExtension.java index 24c674193..c8556d763 100644 --- a/apps/backend/src/test/java/no/nav/data/team/WiremockExtension.java +++ b/apps/backend/src/test/java/no/nav/data/team/WiremockExtension.java @@ -50,7 +50,7 @@ public void afterEach(ExtensionContext context) { } private void stubCommon() { - getWiremock().stubFor(post("/console/query").willReturn(okJson(toJson(consoleMockResponse())))); // Stub Nais Console Team Query + getWiremock().stubFor(post("/console/graphql").willReturn(okJson(toJson(consoleMockResponse())))); // Stub Nais Console Team Query getWiremock().stubFor(get(urlMatching("/datacatgraph/node/out/.*")).willReturn(notFound())); getWiremock().stubFor(get(urlMatching("/datacatgraph/node/in/.*")).willReturn(notFound())); diff --git a/apps/backend/src/test/resources/application-test.properties b/apps/backend/src/test/resources/application-test.properties index 421c742d1..f5ff15866 100644 --- a/apps/backend/src/test/resources/application-test.properties +++ b/apps/backend/src/test/resources/application-test.properties @@ -3,7 +3,7 @@ spring.kafka.security.protocol=PLAINTEXT spring.main.allow-bean-definition-overriding=true spring.main.lazy-initialization=true -client.nais.console.base-url=http://localhost:${wiremock.server.port:8080}/console/query +client.nais.console.base-url=http://localhost:${wiremock.server.port:8080}/console/graphql client.process-cat.base-url=http://localhost:${wiremock.server.port:8080}/processcat client.nom.graphql.url=http://localhost:${wiremock.server.port:8080}/nomgraphql team-catalog.envlevel=primary