diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java index 88963e60d415d0..a9cf82866d127f 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java @@ -13,10 +13,16 @@ import io.swagger.v3.oas.annotations.OpenAPIDefinition; import io.swagger.v3.oas.annotations.info.Info; import io.swagger.v3.oas.annotations.servers.Server; +import io.swagger.v3.oas.models.Components; import io.swagger.v3.oas.models.OpenAPI; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.springdoc.core.models.GroupedOpenApi; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -38,8 +44,6 @@ public class SpringWebConfig implements WebMvcConfigurer { private static final Set V1_PACKAGES = Set.of("io.datahubproject.openapi.v1"); private static final Set V2_PACKAGES = Set.of("io.datahubproject.openapi.v2"); private static final Set V3_PACKAGES = Set.of("io.datahubproject.openapi.v3"); - private static final Set SCHEMA_REGISTRY_PACKAGES = - Set.of("io.datahubproject.openapi.schema.registry"); private static final Set OPENLINEAGE_PACKAGES = Set.of("io.datahubproject.openapi.openlineage"); @@ -74,14 +78,28 @@ public void addFormatters(FormatterRegistry registry) { public GroupedOpenApi v3OpenApiGroup(final EntityRegistry entityRegistry) { return GroupedOpenApi.builder() .group("10-openapi-v3") - .displayName("DataHub Entities v3 (OpenAPI)") + .displayName("DataHub v3 (OpenAPI)") .addOpenApiCustomizer( openApi -> { OpenAPI v3OpenApi = OpenAPIV3Generator.generateOpenApiSpec(entityRegistry); openApi.setInfo(v3OpenApi.getInfo()); openApi.setTags(Collections.emptyList()); - openApi.setPaths(v3OpenApi.getPaths()); - openApi.setComponents(v3OpenApi.getComponents()); + openApi.getPaths().putAll(v3OpenApi.getPaths()); + // Merge components. Swagger does not provide append method to add components. + final Components components = new Components(); + final Components oComponents = openApi.getComponents(); + final Components v3Components = v3OpenApi.getComponents(); + components.callbacks(concat(oComponents::getCallbacks, v3Components::getCallbacks)) + .examples(concat(oComponents::getExamples, v3Components::getExamples)) + .extensions(concat(oComponents::getExtensions, v3Components::getExtensions)) + .headers(concat(oComponents::getHeaders, v3Components::getHeaders)) + .links(concat(oComponents::getLinks, v3Components::getLinks)) + .parameters(concat(oComponents::getParameters, v3Components::getParameters)) + .requestBodies(concat(oComponents::getRequestBodies, v3Components::getRequestBodies)) + .responses(concat(oComponents::getResponses, v3Components::getResponses)) + .schemas(concat(oComponents::getSchemas, v3Components::getSchemas)) + .securitySchemes(concat(oComponents::getSecuritySchemes, v3Components::getSecuritySchemes)); + openApi.setComponents(components); }) .packagesToScan(V3_PACKAGES.toArray(String[]::new)) .build(); @@ -122,4 +140,13 @@ public GroupedOpenApi openlineageOpenApiGroup() { .packagesToScan(OPENLINEAGE_PACKAGES.toArray(String[]::new)) .build(); } + + /** + * Concatenates two maps. + */ + private Map concat(Supplier> a, Supplier> b) { + return a.get() == null ? b.get() : b.get() == null ? a.get() + : Stream.concat(a.get().entrySet().stream(), b.get().entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } } diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericRelationshipController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericRelationshipController.java new file mode 100644 index 00000000000000..7110cc2280ed0d --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericRelationshipController.java @@ -0,0 +1,225 @@ +package io.datahubproject.openapi.controller; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.authorization.AuthUtil; +import com.datahub.authorization.AuthorizerChain; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.models.graph.Edge; +import com.linkedin.metadata.aspect.models.graph.RelatedEntities; +import com.linkedin.metadata.aspect.models.graph.RelatedEntitiesScrollResult; +import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.metadata.query.filter.RelationshipFilter; +import com.linkedin.metadata.search.utils.QueryUtils; +import io.datahubproject.openapi.exception.UnauthorizedException; +import io.datahubproject.openapi.models.GenericScrollResult; +import io.datahubproject.openapi.v2.models.GenericRelationship; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.linkedin.metadata.authorization.ApiGroup.RELATIONSHIP; +import static com.linkedin.metadata.authorization.ApiOperation.READ; + +public abstract class GenericRelationshipController { + + @Autowired private EntityRegistry entityRegistry; + @Autowired private ElasticSearchGraphService graphService; + @Autowired private AuthorizerChain authorizationChain; + + /** + * Returns relationship edges by type + * + * @param relationshipType the relationship type + * @param count number of results + * @param scrollId scrolling id + * @return list of relation edges + */ + @GetMapping(value = "/{relationshipType}", produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Scroll relationships of the given type.") + public ResponseEntity> getRelationshipsByType( + @PathVariable("relationshipType") String relationshipType, + @RequestParam(value = "count", defaultValue = "10") Integer count, + @RequestParam(value = "scrollId", required = false) String scrollId) { + + Authentication authentication = AuthenticationContext.getAuthentication(); + if (!AuthUtil.isAPIAuthorized(authentication, authorizationChain, RELATIONSHIP, READ)) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + + " is unauthorized to " + + READ + + " " + + RELATIONSHIP); + } + + RelatedEntitiesScrollResult result = + graphService.scrollRelatedEntities( + null, + null, + null, + null, + List.of(relationshipType), + new RelationshipFilter().setDirection(RelationshipDirection.UNDIRECTED), + Edge.EDGE_SORT_CRITERION, + scrollId, + count, + null, + null); + + if (!AuthUtil.isAPIAuthorizedUrns( + authentication, + authorizationChain, + RELATIONSHIP, + READ, + result.getEntities().stream() + .flatMap( + edge -> + Stream.of( + UrnUtils.getUrn(edge.getSourceUrn()), + UrnUtils.getUrn(edge.getDestinationUrn()))) + .collect(Collectors.toSet()))) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + + " is unauthorized to " + + READ + + " " + + RELATIONSHIP); + } + + return ResponseEntity.ok( + GenericScrollResult.builder() + .results(toGenericRelationships(result.getEntities())) + .scrollId(result.getScrollId()) + .build()); + } + + /** + * Returns edges for a given urn + * + * @param relationshipTypes types of edges + * @param direction direction of the edges + * @param count number of results + * @param scrollId scroll id + * @return urn edges + */ + @GetMapping(value = "/{entityName}/{entityUrn}", produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Scroll relationships from a given entity.") + public ResponseEntity> getRelationshipsByEntity( + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @RequestParam(value = "relationshipType[]", required = false, defaultValue = "*") + String[] relationshipTypes, + @RequestParam(value = "direction", defaultValue = "OUTGOING") String direction, + @RequestParam(value = "count", defaultValue = "10") Integer count, + @RequestParam(value = "scrollId", required = false) String scrollId) { + + final RelatedEntitiesScrollResult result; + + Authentication authentication = AuthenticationContext.getAuthentication(); + if (!AuthUtil.isAPIAuthorizedUrns( + authentication, + authorizationChain, + RELATIONSHIP, + READ, + List.of(UrnUtils.getUrn(entityUrn)))) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + + " is unauthorized to " + + READ + + " " + + RELATIONSHIP); + } + + switch (RelationshipDirection.valueOf(direction.toUpperCase())) { + case INCOMING -> result = + graphService.scrollRelatedEntities( + null, + null, + null, + null, + relationshipTypes.length > 0 && !relationshipTypes[0].equals("*") + ? Arrays.stream(relationshipTypes).toList() + : List.of(), + new RelationshipFilter() + .setDirection(RelationshipDirection.UNDIRECTED) + .setOr(QueryUtils.newFilter("destination.urn", entityUrn).getOr()), + Edge.EDGE_SORT_CRITERION, + scrollId, + count, + null, + null); + case OUTGOING -> result = + graphService.scrollRelatedEntities( + null, + null, + null, + null, + relationshipTypes.length > 0 && !relationshipTypes[0].equals("*") + ? Arrays.stream(relationshipTypes).toList() + : List.of(), + new RelationshipFilter() + .setDirection(RelationshipDirection.UNDIRECTED) + .setOr(QueryUtils.newFilter("source.urn", entityUrn).getOr()), + Edge.EDGE_SORT_CRITERION, + scrollId, + count, + null, + null); + default -> throw new IllegalArgumentException("Direction must be INCOMING or OUTGOING"); + } + + if (!AuthUtil.isAPIAuthorizedUrns( + authentication, + authorizationChain, + RELATIONSHIP, + READ, + result.getEntities().stream() + .flatMap( + edge -> + Stream.of( + UrnUtils.getUrn(edge.getSourceUrn()), + UrnUtils.getUrn(edge.getDestinationUrn()))) + .collect(Collectors.toSet()))) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + + " is unauthorized to " + + READ + + " " + + RELATIONSHIP); + } + + return ResponseEntity.ok( + GenericScrollResult.builder() + .results(toGenericRelationships(result.getEntities())) + .scrollId(result.getScrollId()) + .build()); + } + + private List toGenericRelationships(List relatedEntities) { + return relatedEntities.stream() + .map( + result -> { + Urn source = UrnUtils.getUrn(result.getSourceUrn()); + Urn dest = UrnUtils.getUrn(result.getDestinationUrn()); + return GenericRelationship.builder() + .relationshipType(result.getRelationshipType()) + .source(GenericRelationship.GenericNode.fromUrn(source)) + .destination(GenericRelationship.GenericNode.fromUrn(dest)) + .build(); + }) + .collect(Collectors.toList()); + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/RelationshipController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/RelationshipController.java index 3e46e10857fbd8..b08177a9da9d5e 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/RelationshipController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/RelationshipController.java @@ -1,40 +1,10 @@ package io.datahubproject.openapi.v2.controller; -import static com.linkedin.metadata.authorization.ApiGroup.RELATIONSHIP; -import static com.linkedin.metadata.authorization.ApiOperation.READ; - -import com.datahub.authentication.Authentication; -import com.datahub.authentication.AuthenticationContext; -import com.datahub.authorization.AuthUtil; -import com.datahub.authorization.AuthorizerChain; -import com.linkedin.common.urn.Urn; -import com.linkedin.common.urn.UrnUtils; -import com.linkedin.metadata.aspect.models.graph.Edge; -import com.linkedin.metadata.aspect.models.graph.RelatedEntities; -import com.linkedin.metadata.aspect.models.graph.RelatedEntitiesScrollResult; -import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; -import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.query.filter.RelationshipDirection; -import com.linkedin.metadata.query.filter.RelationshipFilter; -import com.linkedin.metadata.search.utils.QueryUtils; -import io.datahubproject.openapi.exception.UnauthorizedException; -import io.datahubproject.openapi.models.GenericScrollResult; -import io.datahubproject.openapi.v2.models.GenericRelationship; -import io.swagger.v3.oas.annotations.Operation; +import io.datahubproject.openapi.controller.GenericRelationshipController; import io.swagger.v3.oas.annotations.tags.Tag; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @RestController @@ -42,194 +12,8 @@ @RequestMapping("/v2/relationship") @Slf4j @Tag( - name = "Generic Relationships", + name = "V2 Generic Relationships", description = "APIs for ingesting and accessing entity relationships.") -public class RelationshipController { - - @Autowired private EntityRegistry entityRegistry; - @Autowired private ElasticSearchGraphService graphService; - @Autowired private AuthorizerChain authorizationChain; - - /** - * Returns relationship edges by type - * - * @param relationshipType the relationship type - * @param count number of results - * @param scrollId scrolling id - * @return list of relation edges - */ - @GetMapping(value = "/{relationshipType}", produces = MediaType.APPLICATION_JSON_VALUE) - @Operation(summary = "Scroll relationships of the given type.") - public ResponseEntity> getRelationshipsByType( - @PathVariable("relationshipType") String relationshipType, - @RequestParam(value = "count", defaultValue = "10") Integer count, - @RequestParam(value = "scrollId", required = false) String scrollId) { - - Authentication authentication = AuthenticationContext.getAuthentication(); - if (!AuthUtil.isAPIAuthorized(authentication, authorizationChain, RELATIONSHIP, READ)) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() - + " is unauthorized to " - + READ - + " " - + RELATIONSHIP); - } - - RelatedEntitiesScrollResult result = - graphService.scrollRelatedEntities( - null, - null, - null, - null, - List.of(relationshipType), - new RelationshipFilter().setDirection(RelationshipDirection.UNDIRECTED), - Edge.EDGE_SORT_CRITERION, - scrollId, - count, - null, - null); - - if (!AuthUtil.isAPIAuthorizedUrns( - authentication, - authorizationChain, - RELATIONSHIP, - READ, - result.getEntities().stream() - .flatMap( - edge -> - Stream.of( - UrnUtils.getUrn(edge.getSourceUrn()), - UrnUtils.getUrn(edge.getDestinationUrn()))) - .collect(Collectors.toSet()))) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() - + " is unauthorized to " - + READ - + " " - + RELATIONSHIP); - } - - return ResponseEntity.ok( - GenericScrollResult.builder() - .results(toGenericRelationships(result.getEntities())) - .scrollId(result.getScrollId()) - .build()); - } - - /** - * Returns edges for a given urn - * - * @param relationshipTypes types of edges - * @param direction direction of the edges - * @param count number of results - * @param scrollId scroll id - * @return urn edges - */ - @GetMapping(value = "/{entityName}/{entityUrn}", produces = MediaType.APPLICATION_JSON_VALUE) - @Operation(summary = "Scroll relationships from a given entity.") - public ResponseEntity> getRelationshipsByEntity( - @PathVariable("entityName") String entityName, - @PathVariable("entityUrn") String entityUrn, - @RequestParam(value = "relationshipType[]", required = false, defaultValue = "*") - String[] relationshipTypes, - @RequestParam(value = "direction", defaultValue = "OUTGOING") String direction, - @RequestParam(value = "count", defaultValue = "10") Integer count, - @RequestParam(value = "scrollId", required = false) String scrollId) { - - final RelatedEntitiesScrollResult result; - - Authentication authentication = AuthenticationContext.getAuthentication(); - if (!AuthUtil.isAPIAuthorizedUrns( - authentication, - authorizationChain, - RELATIONSHIP, - READ, - List.of(UrnUtils.getUrn(entityUrn)))) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() - + " is unauthorized to " - + READ - + " " - + RELATIONSHIP); - } - - switch (RelationshipDirection.valueOf(direction.toUpperCase())) { - case INCOMING -> result = - graphService.scrollRelatedEntities( - null, - null, - null, - null, - relationshipTypes.length > 0 && !relationshipTypes[0].equals("*") - ? Arrays.stream(relationshipTypes).toList() - : List.of(), - new RelationshipFilter() - .setDirection(RelationshipDirection.UNDIRECTED) - .setOr(QueryUtils.newFilter("destination.urn", entityUrn).getOr()), - Edge.EDGE_SORT_CRITERION, - scrollId, - count, - null, - null); - case OUTGOING -> result = - graphService.scrollRelatedEntities( - null, - null, - null, - null, - relationshipTypes.length > 0 && !relationshipTypes[0].equals("*") - ? Arrays.stream(relationshipTypes).toList() - : List.of(), - new RelationshipFilter() - .setDirection(RelationshipDirection.UNDIRECTED) - .setOr(QueryUtils.newFilter("source.urn", entityUrn).getOr()), - Edge.EDGE_SORT_CRITERION, - scrollId, - count, - null, - null); - default -> throw new IllegalArgumentException("Direction must be INCOMING or OUTGOING"); - } - - if (!AuthUtil.isAPIAuthorizedUrns( - authentication, - authorizationChain, - RELATIONSHIP, - READ, - result.getEntities().stream() - .flatMap( - edge -> - Stream.of( - UrnUtils.getUrn(edge.getSourceUrn()), - UrnUtils.getUrn(edge.getDestinationUrn()))) - .collect(Collectors.toSet()))) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() - + " is unauthorized to " - + READ - + " " - + RELATIONSHIP); - } - - return ResponseEntity.ok( - GenericScrollResult.builder() - .results(toGenericRelationships(result.getEntities())) - .scrollId(result.getScrollId()) - .build()); - } - - private List toGenericRelationships(List relatedEntities) { - return relatedEntities.stream() - .map( - result -> { - Urn source = UrnUtils.getUrn(result.getSourceUrn()); - Urn dest = UrnUtils.getUrn(result.getDestinationUrn()); - return GenericRelationship.builder() - .relationshipType(result.getRelationshipType()) - .source(GenericRelationship.GenericNode.fromUrn(source)) - .destination(GenericRelationship.GenericNode.fromUrn(dest)) - .build(); - }) - .collect(Collectors.toList()); - } +public class RelationshipController extends GenericRelationshipController { + // Supports same methods as GenericRelationshipController. } diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/RelationshipController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/RelationshipController.java new file mode 100644 index 00000000000000..8f317e86227239 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/RelationshipController.java @@ -0,0 +1,19 @@ +package io.datahubproject.openapi.v3.controller; + +import io.datahubproject.openapi.controller.GenericRelationshipController; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController("RelationshipControllerV3") +@RequiredArgsConstructor +@RequestMapping("/v3/relationship") +@Slf4j +@Tag( + name = "Generic Relationships", + description = "APIs for ingesting and accessing entity relationships.") +public class RelationshipController extends GenericRelationshipController { + // Supports same methods as GenericRelationshipController. +}