Skip to content

Commit

Permalink
Merge branch 'master' into feature/tableau-ingestion-access-permissions
Browse files Browse the repository at this point in the history
  • Loading branch information
haeniya authored Sep 30, 2024
2 parents 5048136 + 0187fc6 commit cdac760
Show file tree
Hide file tree
Showing 871 changed files with 31,431 additions and 20,997 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/airflow-plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,16 +43,16 @@ jobs:
extra_pip_requirements: "apache-airflow~=2.4.3"
extra_pip_extras: plugin-v2,test-airflow24
- python-version: "3.10"
extra_pip_requirements: 'apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
extra_pip_extras: plugin-v2
- python-version: "3.10"
extra_pip_requirements: 'apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
extra_pip_extras: plugin-v2
- python-version: "3.10"
extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
extra_pip_extras: plugin-v2
- python-version: "3.11"
extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt'
extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
extra_pip_extras: plugin-v2
fail-fast: false
steps:
Expand All @@ -73,7 +73,7 @@ jobs:
run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
- name: pip freeze show list installed
if: always()
run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze
run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze
- uses: actions/upload-artifact@v3
if: ${{ always() && matrix.python-version == '3.10' && matrix.extra_pip_requirements == 'apache-airflow>=2.7.0' }}
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/dagster-plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ jobs:
run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:dagster-plugin:lint :metadata-ingestion-modules:dagster-plugin:testQuick
- name: pip freeze show list installed
if: always()
run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && pip freeze
run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze
- uses: actions/upload-artifact@v3
if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }}
with:
Expand Down
6 changes: 4 additions & 2 deletions .github/workflows/docker-unified.yml
Original file line number Diff line number Diff line change
Expand Up @@ -990,7 +990,7 @@ jobs:
DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }}
ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }}
ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5"
ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor] acryl-datahub-actions"
ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"
run: |
./smoke-test/run-quickstart.sh
Expand Down Expand Up @@ -1030,6 +1030,7 @@ jobs:
TEST_STRATEGY: ${{ matrix.test_strategy }}
run: |
echo "$DATAHUB_VERSION"
./gradlew --stop
./smoke-test/smoke.sh
- name: Disk Check
run: df -h . && docker images
Expand All @@ -1043,8 +1044,9 @@ jobs:
uses: actions/upload-artifact@v3
if: failure()
with:
name: docker logs
name: docker-logs-${{ matrix.test_strategy }}
path: "docker_logs/*.log"
retention-days: 5
- name: Upload screenshots
uses: actions/upload-artifact@v3
if: failure()
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/gx-plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ jobs:
run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:gx-plugin:lint :metadata-ingestion-modules:gx-plugin:testQuick
- name: pip freeze show list installed
if: always()
run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && pip freeze
run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && uv pip freeze
- uses: actions/upload-artifact@v3
if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'great-expectations~=0.17.0' }}
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/metadata-ingestion.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ jobs:
- name: Debug info
if: always()
run: |
source metadata-ingestion/venv/bin/activate && pip freeze
source metadata-ingestion/venv/bin/activate && uv pip freeze
set -x
df -hl
docker image ls
Expand Down
8 changes: 2 additions & 6 deletions .github/workflows/pr-labeler.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,18 +61,14 @@ jobs:
contains(
fromJson('[
"siladitya2",
"sgomezvillamor",
"ngamanda",
"HarveyLeo",
"frsann",
"bossenti",
"nikolakasev",
"PatrickfBraz",
"cuong-pham",
"sudhakarast",
"tkdrahn",
"rtekal",
"sgm44"
"mikeburke24",
"DSchmidtDev"
]'),
github.actor
)
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/prefect-plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ jobs:
with:
distribution: "zulu"
java-version: 17
- uses: gradle/gradle-build-action@v2
- uses: gradle/actions/setup-gradle@v3
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
Expand All @@ -55,7 +55,7 @@ jobs:
run: ./gradlew :metadata-ingestion-modules:prefect-plugin:lint :metadata-ingestion-modules:prefect-plugin:testQuick
- name: pip freeze show list installed
if: always()
run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && pip freeze
run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze
- uses: actions/upload-artifact@v3
if: ${{ always() && matrix.python-version == '3.10'}}
with:
Expand Down
23 changes: 14 additions & 9 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ buildscript {
// Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
ext.pegasusVersion = '29.57.0'
ext.mavenVersion = '3.6.3'
ext.springVersion = '6.1.6'
ext.springBootVersion = '3.2.6'
ext.springVersion = '6.1.13'
ext.springBootVersion = '3.2.9'
ext.springKafkaVersion = '3.1.6'
ext.openTelemetryVersion = '1.18.0'
ext.neo4jVersion = '5.14.0'
Expand All @@ -52,7 +52,7 @@ buildscript {
ext.hadoop3Version = '3.3.6'
ext.kafkaVersion = '5.5.15'
ext.hazelcastVersion = '5.3.6'
ext.ebeanVersion = '12.16.1'
ext.ebeanVersion = '15.5.2'
ext.googleJavaFormatVersion = '1.18.1'
ext.openLineageVersion = '1.19.0'
ext.logbackClassicJava8 = '1.2.12'
Expand All @@ -63,7 +63,7 @@ buildscript {
buildscript.repositories.addAll(project.repositories)
dependencies {
classpath 'com.linkedin.pegasus:gradle-plugins:' + pegasusVersion
classpath 'com.github.node-gradle:gradle-node-plugin:7.0.1'
classpath 'com.github.node-gradle:gradle-node-plugin:7.0.2'
classpath 'io.acryl.gradle.plugin:gradle-avro-plugin:0.2.0'
classpath 'org.springframework.boot:spring-boot-gradle-plugin:' + springBootVersion
classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0"
Expand Down Expand Up @@ -104,8 +104,8 @@ project.ext.spec = [

project.ext.externalDependency = [
'akkaHttp': 'com.typesafe.akka:akka-http-core_2.12:10.2.10',
'antlr4Runtime': 'org.antlr:antlr4-runtime:4.7.2',
'antlr4': 'org.antlr:antlr4:4.7.2',
'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
'antlr4': 'org.antlr:antlr4:4.9.3',
'assertJ': 'org.assertj:assertj-core:3.11.1',
'avro': 'org.apache.avro:avro:1.11.3',
'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3',
Expand All @@ -129,8 +129,10 @@ project.ext.externalDependency = [
'dropwizardMetricsCore': 'io.dropwizard.metrics:metrics-core:4.2.3',
'dropwizardMetricsJmx': 'io.dropwizard.metrics:metrics-jmx:4.2.3',
'ebean': 'io.ebean:ebean:' + ebeanVersion,
'ebeanTest': 'io.ebean:ebean-test:' + ebeanVersion,
'ebeanAgent': 'io.ebean:ebean-agent:' + ebeanVersion,
'ebeanDdl': 'io.ebean:ebean-ddl-generator:' + ebeanVersion,
'ebeanQueryBean': 'io.ebean:querybean-generator:' + ebeanVersion,
'elasticSearchRest': 'org.opensearch.client:opensearch-rest-high-level-client:' + elasticsearchVersion,
'elasticSearchJava': 'org.opensearch.client:opensearch-java:2.6.0',
'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1',
Expand Down Expand Up @@ -220,10 +222,10 @@ project.ext.externalDependency = [
'playServer': "com.typesafe.play:play-server_2.12:$playVersion",
'playTest': "com.typesafe.play:play-test_2.12:$playVersion",
'playFilters': "com.typesafe.play:filters-helpers_2.12:$playVersion",
'pac4j': 'org.pac4j:pac4j-oidc:4.5.7',
'pac4j': 'org.pac4j:pac4j-oidc:4.5.8',
'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2',
'postgresql': 'org.postgresql:postgresql:42.3.9',
'protobuf': 'com.google.protobuf:protobuf-java:3.19.6',
'protobuf': 'com.google.protobuf:protobuf-java:3.25.5',
'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0',
'reflections': 'org.reflections:reflections:0.9.9',
Expand Down Expand Up @@ -265,7 +267,7 @@ project.ext.externalDependency = [
'testContainersOpenSearch': 'org.opensearch:opensearch-testcontainers:2.0.0',
'typesafeConfig':'com.typesafe:config:1.4.1',
'wiremock':'com.github.tomakehurst:wiremock:2.10.0',
'zookeeper': 'org.apache.zookeeper:zookeeper:3.7.2',
'zookeeper': 'org.apache.zookeeper:zookeeper:3.8.4',
'wire': 'com.squareup.wire:wire-compiler:3.7.1',
'charle': 'com.charleskorn.kaml:kaml:0.53.0',
'common': 'commons-io:commons-io:2.7',
Expand Down Expand Up @@ -359,6 +361,9 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
exclude group: "org.slf4j", module: "slf4j-log4j12"
exclude group: "org.slf4j", module: "slf4j-nop"
exclude group: "org.slf4j", module: "slf4j-ext"

resolutionStrategy.force externalDependency.antlr4Runtime
resolutionStrategy.force externalDependency.antlr4
}
}

Expand Down
6 changes: 3 additions & 3 deletions datahub-frontend/app/auth/AuthModule.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import java.util.Collections;

import io.datahubproject.metadata.context.ActorContext;
import io.datahubproject.metadata.context.AuthorizerContext;
import io.datahubproject.metadata.context.AuthorizationContext;
import io.datahubproject.metadata.context.EntityRegistryContext;
import io.datahubproject.metadata.context.OperationContext;
import io.datahubproject.metadata.context.OperationContextConfig;
Expand Down Expand Up @@ -183,10 +183,10 @@ protected OperationContext provideOperationContext(
return OperationContext.builder()
.operationContextConfig(systemConfig)
.systemActorContext(systemActorContext)
// Authorizer.EMPTY is fine since it doesn't actually apply to system auth
.authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
.searchContext(SearchContext.EMPTY)
.entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
// Authorizer.EMPTY doesn't actually apply to system auth
.authorizerContext(AuthorizerContext.builder().authorizer(Authorizer.EMPTY).build())
.build(systemAuthentication);
}

Expand Down
3 changes: 3 additions & 0 deletions datahub-frontend/app/auth/sso/oidc/OidcConfigs.java
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,9 @@ public Builder from(final com.typesafe.config.Config configs, final String ssoSe
Optional.ofNullable(getOptional(configs, OIDC_PREFERRED_JWS_ALGORITHM, null));
}

grantType = Optional.ofNullable(getOptional(configs, OIDC_GRANT_TYPE, null));
acrValues = Optional.ofNullable(getOptional(configs, OIDC_ACR_VALUES, null));

return this;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ public CustomOidcClient(final OidcConfiguration configuration) {
protected void clientInit() {
CommonHelper.assertNotNull("configuration", getConfiguration());
getConfiguration().init();
defaultRedirectionActionBuilder(new OidcRedirectionActionBuilder(getConfiguration(), this));
defaultRedirectionActionBuilder(new CustomOidcRedirectionActionBuilder(getConfiguration(), this));
defaultCredentialsExtractor(new OidcExtractor(getConfiguration(), this));
defaultAuthenticator(new CustomOidcAuthenticator(this));
defaultProfileCreator(new OidcProfileCreator<>(getConfiguration(), this));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package auth.sso.oidc.custom;

import java.util.Map;
import java.util.Optional;
import org.pac4j.core.context.WebContext;
import org.pac4j.core.exception.http.RedirectionAction;
import org.pac4j.core.exception.http.RedirectionActionHelper;
import org.pac4j.oidc.client.OidcClient;
import org.pac4j.oidc.config.OidcConfiguration;
import org.pac4j.oidc.redirect.OidcRedirectionActionBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


public class CustomOidcRedirectionActionBuilder extends OidcRedirectionActionBuilder {

private static final Logger logger = LoggerFactory.getLogger(OidcRedirectionActionBuilder.class);
public CustomOidcRedirectionActionBuilder(OidcConfiguration configuration, OidcClient client) {
super(configuration, client);
}

@Override
public Optional<RedirectionAction> getRedirectionAction(WebContext context) {
Map<String, String> params = this.buildParams();
String computedCallbackUrl = this.client.computeFinalCallbackUrl(context);
params.put("redirect_uri", computedCallbackUrl);
this.addStateAndNonceParameters(context, params);
if (this.configuration.getMaxAge() != null) {
params.put("max_age", this.configuration.getMaxAge().toString());
}

String location = this.buildAuthenticationRequestUrl(params);

logger.debug("Custom parameters: {}", this.configuration.getCustomParams());

String acrValues = this.configuration.getCustomParam("acr_values");

if (acrValues != null && !location.contains("acr_values=")) {
location += (location.contains("?") ? "&" : "?") + "acr_values=" + acrValues;
}

logger.debug("Authentication request url: {}", location);
return Optional.of(RedirectionActionHelper.buildRedirectUrlAction(context, location));
}

}
46 changes: 46 additions & 0 deletions datahub-frontend/app/controllers/Application.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,15 @@
import akka.util.ByteString;
import auth.Authenticator;
import com.datahub.authentication.AuthenticationConstants;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.linkedin.util.Pair;
import com.typesafe.config.Config;
import java.io.InputStream;
import java.net.URI;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand All @@ -33,6 +36,7 @@
import play.libs.ws.StandaloneWSClient;
import play.libs.ws.ahc.StandaloneAhcWSClient;
import play.mvc.Controller;
import play.mvc.Http.Cookie;
import play.mvc.Http;
import play.mvc.ResponseHeader;
import play.mvc.Result;
Expand Down Expand Up @@ -132,6 +136,9 @@ public CompletableFuture<Result> proxy(String path, Http.Request request)
headers.put(Http.HeaderNames.X_FORWARDED_PROTO, List.of(schema));
}

// Get the current time to measure the duration of the request
Instant start = Instant.now();

return _ws.url(
String.format(
"%s://%s:%s%s", protocol, metadataServiceHost, metadataServicePort, resolvedUri))
Expand Down Expand Up @@ -160,6 +167,15 @@ AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER, getDataHubActorHeader(req
.execute()
.thenApply(
apiResponse -> {
// Log the query if it takes longer than the configured threshold and verbose logging is enabled
boolean verboseGraphQLLogging = _config.getBoolean("graphql.verbose.logging");
int verboseGraphQLLongQueryMillis = _config.getInt("graphql.verbose.slowQueryMillis");
Instant finish = Instant.now();
long timeElapsed = Duration.between(start, finish).toMillis();
if (verboseGraphQLLogging && timeElapsed >= verboseGraphQLLongQueryMillis) {
logSlowQuery(request, resolvedUri, timeElapsed);
}

final ResponseHeader header =
new ResponseHeader(
apiResponse.getStatus(),
Expand Down Expand Up @@ -359,4 +375,34 @@ private String mapPath(@Nonnull final String path) {
// Otherwise, return original path
return path;
}


/**
* Called if verbose logging is enabled and request takes longer that the slow query milliseconds defined in the config
* @param request GraphQL request that was made
* @param resolvedUri URI that was requested
* @param duration How long the query took to complete
*/
private void logSlowQuery(Http.Request request, String resolvedUri, float duration) {
StringBuilder jsonBody = new StringBuilder();
Optional<Cookie> actorCookie = request.getCookie("actor");
String actorValue = actorCookie.isPresent() ? actorCookie.get().value() : "N/A";

try {
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = request.body().asJson();
((ObjectNode) jsonNode).remove("query");
jsonBody.append(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonNode));
}
catch (Exception e) {
_logger.info("GraphQL Request Received: {}, Unable to parse JSON body", resolvedUri);
}
String jsonBodyStr = jsonBody.toString();
_logger.info("Slow GraphQL Request Received: {}, Request query string: {}, Request actor: {}, Request JSON: {}, Request completed in {} ms",
resolvedUri,
request.queryString(),
actorValue,
jsonBodyStr,
duration);
}
}
8 changes: 7 additions & 1 deletion datahub-frontend/conf/application.conf
Original file line number Diff line number Diff line change
Expand Up @@ -298,4 +298,10 @@ entityClient.numRetries = ${?ENTITY_CLIENT_NUM_RETRIES}
entityClient.restli.get.batchSize = 50
entityClient.restli.get.batchSize = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE}
entityClient.restli.get.batchConcurrency = 2
entityClient.restli.get.batchConcurrency = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY}
entityClient.restli.get.batchConcurrency = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY}

# Enable verbose authentication logging
graphql.verbose.logging = false
graphql.verbose.logging = ${?GRAPHQL_VERBOSE_LOGGING}
graphql.verbose.slowQueryMillis = 2500
graphql.verbose.slowQueryMillis = ${?GRAPHQL_VERBOSE_LONG_QUERY_MILLIS}
Loading

0 comments on commit cdac760

Please sign in to comment.