Skip to content

Commit

Permalink
Improve JDBI integration with Alpine
Browse files Browse the repository at this point in the history
Adds an `AlpineRequest`-aware JDBI `StatementCustomizer`, that transparently enriches statement contexts with query template variables and parameter bindings for:

* Filtering
* Ordering
* Pagination
* Portfolio Access Control

This removes the need for explicit `@DefineOrdering` and `@DefinePagination` parameters in JDBI queries.

Functionality-wise, this matches what Alpine is providing via [`AbstractAlpineQueryManager#decorate`](https://github.com/stevespringett/Alpine/blob/c50e5253c6d93387b0c1fd4c4058d25f7dc56abe/alpine-infra/src/main/java/alpine/persistence/AbstractAlpineQueryManager.java#L200-L246) for ordering and pagination, and Dependency-Track via [`QueryManager#preprocessACLs`](https://github.com/DependencyTrack/dependency-track/blob/09fdec3b39d825844d6a08f25cedb149f39385d9/src/main/java/org/dependencytrack/persistence/ProjectQueryManager.java#L897-L933) for portfolio ACLs.

This change will reduce code redundancies once we start serving more REST API requests from JDBI queries (i.e. DependencyTrack/hyades#1292).

Signed-off-by: nscuro <[email protected]>
  • Loading branch information
nscuro committed Jun 7, 2024
1 parent 5580c72 commit c37c46c
Show file tree
Hide file tree
Showing 31 changed files with 1,490 additions and 702 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
import org.dependencytrack.model.WorkflowStatus;
import org.dependencytrack.model.WorkflowStep;
import org.dependencytrack.notification.NotificationConstants;
import org.dependencytrack.persistence.QueryManager;
import org.dependencytrack.persistence.jdbi.NotificationSubjectDao;
import org.dependencytrack.persistence.jdbi.VulnerabilityScanDao;
import org.dependencytrack.persistence.jdbi.WorkflowDao;
Expand All @@ -62,7 +61,7 @@

import static java.lang.Math.toIntExact;
import static org.dependencytrack.common.ConfigKey.TMP_DELAY_BOM_PROCESSED_NOTIFICATION;
import static org.dependencytrack.persistence.jdbi.JdbiFactory.jdbi;
import static org.dependencytrack.persistence.jdbi.JdbiFactory.useJdbiTransaction;
import static org.dependencytrack.proto.notification.v1.Group.GROUP_BOM_PROCESSED;
import static org.dependencytrack.proto.notification.v1.Group.GROUP_PROJECT_VULN_ANALYSIS_COMPLETE;
import static org.dependencytrack.proto.notification.v1.Level.LEVEL_INFORMATIONAL;
Expand Down Expand Up @@ -96,16 +95,14 @@ public void process(final List<ConsumerRecord<String, ScanResult>> records) thro

final var completedVulnScans = new ArrayList<VulnerabilityScan>();
final var notifications = new ArrayList<KafkaEvent<?, ?>>();
try (final var qm = new QueryManager()) {
jdbi(qm).useTransaction(jdbiHandle -> {
completedVulnScans.addAll(processScanResults(jdbiHandle, records));
notifications.addAll(createVulnAnalysisCompleteNotifications(jdbiHandle, completedVulnScans));
useJdbiTransaction(handle -> {
completedVulnScans.addAll(processScanResults(handle, records));
notifications.addAll(createVulnAnalysisCompleteNotifications(handle, completedVulnScans));

if (shouldDispatchBomProcessedNotification) {
notifications.addAll(createBomProcessedNotifications(jdbiHandle, completedVulnScans));
}
});
}
if (shouldDispatchBomProcessedNotification) {
notifications.addAll(createBomProcessedNotifications(handle, completedVulnScans));
}
});

eventDispatcher.dispatchAll(notifications);
LOGGER.debug("Dispatched %d notifications".formatted(notifications.size()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,9 @@
import static org.dependencytrack.common.MdcKeys.MDC_COMPONENT_UUID;
import static org.dependencytrack.common.MdcKeys.MDC_SCAN_TOKEN;
import static org.dependencytrack.parser.dependencytrack.ModelConverterCdxToVuln.convert;
import static org.dependencytrack.persistence.jdbi.JdbiFactory.jdbi;
import static org.dependencytrack.persistence.jdbi.JdbiFactory.inJdbiTransaction;
import static org.dependencytrack.persistence.jdbi.JdbiFactory.useJdbiHandle;
import static org.dependencytrack.persistence.jdbi.JdbiFactory.withJdbiHandle;
import static org.dependencytrack.proto.notification.v1.Group.GROUP_NEW_VULNERABILITY;
import static org.dependencytrack.proto.notification.v1.Group.GROUP_NEW_VULNERABLE_DEPENDENCY;
import static org.dependencytrack.proto.notification.v1.Group.GROUP_PROJECT_AUDIT_CHANGE;
Expand Down Expand Up @@ -165,7 +167,7 @@ private void processInternal(final ScanKey scanKey,
qm.getPersistenceManager().setProperty(PROPERTY_RETAIN_VALUES, "true");
qm.getPersistenceManager().setProperty(PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT, "false");

final Component component = jdbi(qm).withExtension(Dao.class, dao -> dao.getComponentByUuid(UUID.fromString(scanKey.getComponentUuid())));
final Component component = withJdbiHandle(handle -> handle.attach(Dao.class).getComponentByUuid(UUID.fromString(scanKey.getComponentUuid())));
if (component == null) {
LOGGER.warn("Received result for component, but it does not exist");
return;
Expand Down Expand Up @@ -405,8 +407,8 @@ private Map<UUID, VulnerabilityPolicy> maybeEvaluateVulnPolicies(final Component
private List<Vulnerability> synchronizeFindingsAndAnalyses(final QueryManager qm, final Component component,
final Collection<Vulnerability> vulns, final Scanner scanner,
final Map<UUID, VulnerabilityPolicy> policiesByVulnUuid) {
return jdbi(qm).inTransaction(jdbiHandle -> {
final var dao = jdbiHandle.attach(Dao.class);
return inJdbiTransaction(handle -> {
final var dao = handle.attach(Dao.class);

// Bulk-create new findings and corresponding scanner attributions.
final List<Long> newFindingVulnIds = dao.createFindings(component, vulns);
Expand Down Expand Up @@ -652,9 +654,7 @@ private void maybeQueueProjectAuditChangeNotification(final QueryManager qm, fin
return;
}

jdbi(qm)
.withExtension(NotificationSubjectDao.class,
dao -> dao.getForProjectAuditChange(component.uuid(), vuln.getUuid(), policyAnalysis.state, policyAnalysis.suppressed))
withJdbiHandle(handle -> handle.attach(NotificationSubjectDao.class).getForProjectAuditChange(component.uuid(), vuln.getUuid(), policyAnalysis.state, policyAnalysis.suppressed))
.map(subject -> org.dependencytrack.proto.notification.v1.Notification.newBuilder()
.setScope(SCOPE_PORTFOLIO)
.setGroup(GROUP_PROJECT_AUDIT_CHANGE)
Expand Down Expand Up @@ -710,7 +710,9 @@ private void maybeQueueNotifications(final QueryManager qm, final Component comp
}

final Timestamp notificationTimestamp = Timestamps.now();
jdbi(qm).useExtension(NotificationSubjectDao.class, dao -> {
useJdbiHandle(handle -> {
final var dao = handle.attach(NotificationSubjectDao.class);

if (isNewComponent) {
dao.getForNewVulnerableDependency(component.uuid())
.map(subject -> org.dependencytrack.proto.notification.v1.Notification.newBuilder()
Expand Down
24 changes: 0 additions & 24 deletions src/main/java/org/dependencytrack/persistence/QueryManager.java
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,6 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -420,29 +419,6 @@ private IntegrityAnalysisQueryManager getIntegrityAnalysisQueryManager() {
return integrityAnalysisQueryManager;
}

/**
* Get the IDs of the {@link Team}s a given {@link Principal} is a member of.
*
* @return A {@link Set} of {@link Team} IDs
*/
protected Set<Long> getTeamIds(final Principal principal) {
final Set<Long> principalTeamIds = new HashSet<>();

if (principal instanceof final UserPrincipal userPrincipal
&& userPrincipal.getTeams() != null) {
for (final Team userInTeam : userPrincipal.getTeams()) {
principalTeamIds.add(userInTeam.getId());
}
} else if (principal instanceof final ApiKey apiKey
&& apiKey.getTeams() != null) {
for (final Team userInTeam : apiKey.getTeams()) {
principalTeamIds.add(userInTeam.getId());
}
}

return principalTeamIds;
}

private void disableL2Cache() {
pm.setProperty(PropertyNames.PROPERTY_CACHE_L2_TYPE, "none");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@
import java.util.function.Function;
import java.util.stream.Collectors;

import static org.dependencytrack.util.PrincipalUtil.getPrincipalTeamIds;

final class VulnerabilityQueryManager extends QueryManager implements IQueryManager {

/**
Expand Down Expand Up @@ -535,7 +537,7 @@ public List<Project> getProjects(final Vulnerability vulnerability, final Set<St
&& :teamIds.contains(team.id)
VARIABLES alpine.model.Team team
""";
params.put("teamIds", getTeamIds(super.principal));
params.put("teamIds", getPrincipalTeamIds(super.principal));
}

// TODO: This query should support pagination
Expand Down Expand Up @@ -621,7 +623,7 @@ SELECT COUNT(DISTINCT this.project.id)
&& :teamIds.contains(team.id)
VARIABLES alpine.model.Team team
""";
params.put("teamIds", getTeamIds(super.principal));
params.put("teamIds", getPrincipalTeamIds(super.principal));
}

final Query<?> query = pm.newQuery(Query.JDOQL, queryStr);
Expand Down
100 changes: 100 additions & 0 deletions src/main/java/org/dependencytrack/persistence/jdbi/AllowOrdering.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
* This file is part of Dependency-Track.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* Copyright (c) OWASP Foundation. All Rights Reserved.
*/
package org.dependencytrack.persistence.jdbi;

import org.jdbi.v3.core.config.ConfigRegistry;
import org.jdbi.v3.core.extension.SimpleExtensionConfigurer;
import org.jdbi.v3.core.extension.annotation.UseExtensionConfigurer;

import java.lang.annotation.Annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.Arrays;
import java.util.stream.Collectors;

import static org.apache.commons.lang3.StringUtils.trimToNull;

/**
* @since 5.5.0
*/
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
@UseExtensionConfigurer(AllowOrdering.ExtensionConfigurer.class)
public @interface AllowOrdering {

/**
* Columns to allow ordering by.
*/
Column[] by();

/**
* Name of the column that should always be included in the {@code ORDER BY}
* clause. A corresponding {@link Column} must be provided to {@link #by()}.
* <p>
* Can optionally include the ordering direction as {@code asc} or {@code desc}.
*/
String alwaysBy() default "";

@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
@interface Column {

/**
* Name of the column to allow ordering by.
* <p>
* It will be quoted automatically with double quotes before insertion to the query.
*/
String name();

/**
* An optional, <strong>raw</strong> name of the column, as used in the query.
* <p>
* When provided, this name will be used instead of {@link #name()},
* when ordering by a column matching {@link #name()} is requested.
* <p>
* This name will not be quoted automatically.
*/
String queryName() default "";

}

final class ExtensionConfigurer extends SimpleExtensionConfigurer {

@Override
public void configure(final ConfigRegistry configRegistry, final Annotation annotation, final Class<?> extensionType) {
final var allowOrderingAnnotation = (AllowOrdering) annotation;

final var config = configRegistry.get(AlpineRequestConfig.class);
config.setOrderingAlwaysBy(allowOrderingAnnotation.alwaysBy());
config.setOrderingAllowedColumns(Arrays.stream(allowOrderingAnnotation.by())
.map(column -> new AlpineRequestConfig.OrderingColumn(
column.name(),
trimToNull(column.queryName())
))
.collect(Collectors.toSet()));
}

}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
* This file is part of Dependency-Track.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* Copyright (c) OWASP Foundation. All Rights Reserved.
*/
package org.dependencytrack.persistence.jdbi;

import org.jdbi.v3.core.config.JdbiConfig;

import java.util.Collections;
import java.util.Optional;
import java.util.Set;

/**
* @since 5.5.0
*/
public class AlpineRequestConfig implements JdbiConfig<AlpineRequestConfig> {

private Set<OrderingColumn> orderingAllowedColumns = Collections.emptySet();
private String orderingAlwaysBy = "";

// TODO: Make this configurable via annotation when needed (similar to @AllowOrdering).
// In some queries the PROJECT table may be aliased (e.g. as P).
private String projectAclProjectTableName = "PROJECT";

@SuppressWarnings("unused")
public AlpineRequestConfig() {
// Used by JDBI to instantiate the class via reflection.
}

private AlpineRequestConfig(final AlpineRequestConfig that) {
this.orderingAllowedColumns = Set.copyOf(that.orderingAllowedColumns);
this.orderingAlwaysBy = that.orderingAlwaysBy;
this.projectAclProjectTableName = that.projectAclProjectTableName;
}

@Override
public AlpineRequestConfig createCopy() {
return new AlpineRequestConfig(this);
}

Optional<OrderingColumn> orderingAllowedColumn(final String name) {
return orderingAllowedColumns.stream()
.filter(column -> column.name().equals(name))
.findAny();
}

Set<OrderingColumn> orderingAllowedColumns() {
return orderingAllowedColumns;
}

public void setOrderingAllowedColumns(final Set<OrderingColumn> orderingAllowedColumns) {
this.orderingAllowedColumns = orderingAllowedColumns;
}

String orderingAlwaysBy() {
return orderingAlwaysBy;
}

public void setOrderingAlwaysBy(final String orderingAlwaysBy) {
this.orderingAlwaysBy = orderingAlwaysBy;
}

String projectAclProjectTableName() {
return projectAclProjectTableName;
}

public record OrderingColumn(String name, String queryName) {

public OrderingColumn(final String name) {
this(name, null);
}

}

}
Loading

0 comments on commit c37c46c

Please sign in to comment.