Merge branch 'master' into feature/add-exclusive-project-path-pattern

haeniya · Sep 16, 2024 · 59300c2 · 59300c2
2 parents 135428e + dc35465
commit 59300c2
Show file tree

Hide file tree

Showing 314 changed files with 7,575 additions and 2,616 deletions.
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -43,16 +43,16 @@ jobs:
             extra_pip_requirements: "apache-airflow~=2.4.3"
             extra_pip_extras: plugin-v2,test-airflow24
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
+            extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
+            extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
+            extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
             extra_pip_extras: plugin-v2
           - python-version: "3.11"
-            extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt'
+            extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
             extra_pip_extras: plugin-v2
       fail-fast: false
     steps:
@@ -73,7 +73,7 @@ jobs:
         run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
       - name: pip freeze show list installed
         if: always()
-        run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze
+        run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v3
         if: ${{ always() && matrix.python-version == '3.10' && matrix.extra_pip_requirements == 'apache-airflow>=2.7.0' }}
         with:

diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml
@@ -55,7 +55,7 @@ jobs:
         run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:dagster-plugin:lint :metadata-ingestion-modules:dagster-plugin:testQuick
       - name: pip freeze show list installed
         if: always()
-        run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && pip freeze
+        run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v3
         if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }}
         with:

diff --git a/.github/workflows/gx-plugin.yml b/.github/workflows/gx-plugin.yml
@@ -57,7 +57,7 @@ jobs:
         run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:gx-plugin:lint :metadata-ingestion-modules:gx-plugin:testQuick
       - name: pip freeze show list installed
         if: always()
-        run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && pip freeze
+        run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v3
         if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'great-expectations~=0.17.0' }}
         with:

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -83,7 +83,7 @@ jobs:
       - name: Debug info
         if: always()
         run: |
-          source metadata-ingestion/venv/bin/activate && pip freeze
+          source metadata-ingestion/venv/bin/activate && uv pip freeze
           set -x
           df -hl
           docker image ls

diff --git a/.github/workflows/prefect-plugin.yml b/.github/workflows/prefect-plugin.yml
@@ -43,7 +43,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
@@ -55,7 +55,7 @@ jobs:
         run: ./gradlew :metadata-ingestion-modules:prefect-plugin:lint :metadata-ingestion-modules:prefect-plugin:testQuick
       - name: pip freeze show list installed
         if: always()
-        run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && pip freeze
+        run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v3
         if: ${{ always() && matrix.python-version == '3.10'}}
         with:

diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java
@@ -25,7 +25,7 @@
 import java.util.Collections;
 
 import io.datahubproject.metadata.context.ActorContext;
-import io.datahubproject.metadata.context.AuthorizerContext;
+import io.datahubproject.metadata.context.AuthorizationContext;
 import io.datahubproject.metadata.context.EntityRegistryContext;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.metadata.context.OperationContextConfig;
@@ -183,10 +183,10 @@ protected OperationContext provideOperationContext(
     return OperationContext.builder()
             .operationContextConfig(systemConfig)
             .systemActorContext(systemActorContext)
+            // Authorizer.EMPTY is fine since it doesn't actually apply to system auth
+            .authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
             .searchContext(SearchContext.EMPTY)
             .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
-            // Authorizer.EMPTY doesn't actually apply to system auth
-            .authorizerContext(AuthorizerContext.builder().authorizer(Authorizer.EMPTY).build())
             .build(systemAuthentication);
   }
 

diff --git a/datahub-frontend/app/controllers/Application.java b/datahub-frontend/app/controllers/Application.java
@@ -9,12 +9,15 @@
 import akka.util.ByteString;
 import auth.Authenticator;
 import com.datahub.authentication.AuthenticationConstants;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.linkedin.util.Pair;
 import com.typesafe.config.Config;
 import java.io.InputStream;
 import java.net.URI;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -33,6 +36,7 @@
 import play.libs.ws.StandaloneWSClient;
 import play.libs.ws.ahc.StandaloneAhcWSClient;
 import play.mvc.Controller;
+import play.mvc.Http.Cookie;
 import play.mvc.Http;
 import play.mvc.ResponseHeader;
 import play.mvc.Result;
@@ -132,6 +136,9 @@ public CompletableFuture<Result> proxy(String path, Http.Request request)
       headers.put(Http.HeaderNames.X_FORWARDED_PROTO, List.of(schema));
     }
 
+    // Get the current time to measure the duration of the request
+    Instant start = Instant.now();
+
     return _ws.url(
             String.format(
                 "%s://%s:%s%s", protocol, metadataServiceHost, metadataServicePort, resolvedUri))
@@ -160,6 +167,15 @@ AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER, getDataHubActorHeader(req
         .execute()
         .thenApply(
             apiResponse -> {
+              // Log the query if it takes longer than the configured threshold and verbose logging is enabled
+              boolean verboseGraphQLLogging = _config.getBoolean("graphql.verbose.logging");
+              int verboseGraphQLLongQueryMillis = _config.getInt("graphql.verbose.slowQueryMillis");
+              Instant finish = Instant.now();
+              long timeElapsed = Duration.between(start, finish).toMillis();
+              if (verboseGraphQLLogging && timeElapsed >= verboseGraphQLLongQueryMillis) {
+                logSlowQuery(request, resolvedUri, timeElapsed);
+              }
+
               final ResponseHeader header =
                   new ResponseHeader(
                       apiResponse.getStatus(),
@@ -359,4 +375,34 @@ private String mapPath(@Nonnull final String path) {
     // Otherwise, return original path
     return path;
   }
+
+
+  /**
+   * Called if verbose logging is enabled and request takes longer that the slow query milliseconds defined in the config
+   * @param request GraphQL request that was made
+   * @param resolvedUri URI that was requested
+   * @param duration How long the query took to complete
+   */
+  private void logSlowQuery(Http.Request request, String resolvedUri, float duration) {
+    StringBuilder jsonBody = new StringBuilder();
+    Optional<Cookie> actorCookie = request.getCookie("actor");
+    String actorValue = actorCookie.isPresent() ? actorCookie.get().value() : "N/A";
+
+    try {
+      ObjectMapper mapper = new ObjectMapper();
+      JsonNode jsonNode = request.body().asJson();
+      ((ObjectNode) jsonNode).remove("query");
+      jsonBody.append(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonNode));
+    }
+    catch (Exception e) {
+      _logger.info("GraphQL Request Received: {}, Unable to parse JSON body", resolvedUri);
+    }
+    String jsonBodyStr = jsonBody.toString();
+    _logger.info("Slow GraphQL Request Received: {}, Request query string: {}, Request actor: {}, Request JSON: {}, Request completed in {} ms",
+            resolvedUri,
+            request.queryString(),
+            actorValue,
+            jsonBodyStr,
+            duration);
+  }
 }
diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf
@@ -298,4 +298,10 @@ entityClient.numRetries = ${?ENTITY_CLIENT_NUM_RETRIES}
 entityClient.restli.get.batchSize = 50
 entityClient.restli.get.batchSize = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE}
 entityClient.restli.get.batchConcurrency = 2
-entityClient.restli.get.batchConcurrency = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY}
+entityClient.restli.get.batchConcurrency = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY}
+
+# Enable verbose authentication logging
+graphql.verbose.logging = false
+graphql.verbose.logging = ${?GRAPHQL_VERBOSE_LOGGING}
+graphql.verbose.slowQueryMillis = 2500
+graphql.verbose.slowQueryMillis = ${?GRAPHQL_VERBOSE_LONG_QUERY_MILLIS}