Merge pull request #32 from dqops/1.5.0

1.5.0

.run/dqo run.run.xml

@@ -5,7 +5,7 @@
       <option name="region" />
       <option name="useCurrentConnection" value="false" />
     </extension>
-    <option name="JAR_PATH" value="$PROJECT_DIR$/dqops/target/dqo-dqops-1.4.1.jar" />
+    <option name="JAR_PATH" value="$PROJECT_DIR$/dqops/target/dqo-dqops-1.5.0.jar" />
     <option name="VM_PARAMETERS" value="-XX:MaxRAMPercentage=60.0 --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED" />
     <option name="PROGRAM_PARAMETERS" value="--server.port=8888" />
     <option name="WORKING_DIRECTORY" value="C:\dev\dqoado" />

CHANGELOG.md

@@ -1,6 +1,7 @@
-# 1.4.1
-* Small fixes to the code that calculates the current data quality status of tables and columns
-* Fixed permalinks to screens
-* Small changes to handling empty tables in column level data quality checks
-* Incident generation fixed for monitoring checks (full table scan checks)
-
+# 1.5.0
+* Error sampling
+* DuckDB 1.0.0 upgrade
+* Global incident management screen
+* Support for analyzing flat files on GCP
+* Freshness anomaly check
+* Filtering tables by name on the table import screen

VERSION

@@ -1 +1 @@
-1.4.1
+1.5.0

distribution/pom.xml

@@ -11,7 +11,7 @@
 
 	<groupId>com.dqops</groupId>
 	<artifactId>dqo-distribution</artifactId>
-    <version>1.4.1</version>  <!-- DQOps Version, do not touch (changed automatically) -->
+    <version>1.5.0</version>  <!-- DQOps Version, do not touch (changed automatically) -->
 	<name>dqo-distribution</name>
 	<description>DQOps Data Quality Operations Center final assembly</description>
 	<packaging>pom</packaging>

distribution/python/dqops/airflow/collect_statistics/dqops_collect_statistics_operator.py

@@ -39,7 +39,7 @@ class DqopsCollectStatisticsOperator(BaseOperator):
     def __init__(
         self,
         *,
-        connection: (Union[Unset, str]) = UNSET,
+        connection: Union[Unset, str] = UNSET,
         full_table_name: Union[Unset, str] = UNSET,
         enabled: Union[Unset, bool] = UNSET,
         labels: Union[Unset, List[str]] = UNSET,

distribution/python/dqops/airflow/wait_for_job/dqops_wait_for_job_operator.py

@@ -122,9 +122,9 @@ def execute(self, context):
             self.retries = 0
             raise DqopsJobFailedException(context["ti"], job_result.to_dict())
 
-        run_check_result: Union[
-            RunChecksResult, None
-        ] = job_result.parameters.run_checks_parameters.run_checks_result
+        run_check_result: Union[RunChecksResult, None] = (
+            job_result.parameters.run_checks_parameters.run_checks_result
+        )
 
         if (
             job_result.job_type == DqoJobType.RUN_CHECKS

distribution/python/dqops/client/__init__.py

@@ -1,4 +1,5 @@
 """ A client library for accessing DQOps """
+
 from .client import AuthenticatedClient, Client
 
 __all__ = (

distribution/python/dqops/client/api/check_results/get_column_monitoring_checks_results.py

@@ -26,6 +26,7 @@ def _get_kwargs(
     table_comparison: Union[Unset, None, str] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/check_results/get_column_partitioned_checks_results.py

@@ -26,6 +26,7 @@ def _get_kwargs(
     table_comparison: Union[Unset, None, str] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/check_results/get_column_profiling_checks_results.py

@@ -24,6 +24,7 @@ def _get_kwargs(
     table_comparison: Union[Unset, None, str] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/check_results/get_table_data_quality_status.py

@@ -30,6 +30,7 @@ def _get_kwargs(
     table_comparison: Union[Unset, None, str] = UNSET,
     quality_dimension: Union[Unset, None, str] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/check_results/get_table_monitoring_checks_results.py

@@ -25,6 +25,7 @@ def _get_kwargs(
     table_comparison: Union[Unset, None, str] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/check_results/get_table_partitioned_checks_results.py

@@ -25,6 +25,7 @@ def _get_kwargs(
     table_comparison: Union[Unset, None, str] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/check_results/get_table_profiling_checks_results.py

@@ -23,6 +23,7 @@ def _get_kwargs(
     table_comparison: Union[Unset, None, str] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/check_results_overview/get_column_monitoring_checks_overview.py

@@ -20,6 +20,7 @@ def _get_kwargs(
     category: Union[Unset, None, str] = UNSET,
     check_name: Union[Unset, None, str] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/check_results_overview/get_column_partitioned_checks_overview.py

@@ -20,6 +20,7 @@ def _get_kwargs(
     category: Union[Unset, None, str] = UNSET,
     check_name: Union[Unset, None, str] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/check_results_overview/get_column_profiling_checks_overview.py

@@ -18,6 +18,7 @@ def _get_kwargs(
     category: Union[Unset, None, str] = UNSET,
     check_name: Union[Unset, None, str] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/check_results_overview/get_table_monitoring_checks_overview.py

@@ -19,6 +19,7 @@ def _get_kwargs(
     category: Union[Unset, None, str] = UNSET,
     check_name: Union[Unset, None, str] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/check_results_overview/get_table_partitioned_checks_overview.py

@@ -19,6 +19,7 @@ def _get_kwargs(
     category: Union[Unset, None, str] = UNSET,
     check_name: Union[Unset, None, str] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/check_results_overview/get_table_profiling_checks_overview.py

@@ -17,6 +17,7 @@ def _get_kwargs(
     category: Union[Unset, None, str] = UNSET,
     check_name: Union[Unset, None, str] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}

distribution/python/dqops/client/api/checks/create_check.py

@@ -14,6 +14,7 @@ def _get_kwargs(
     *,
     json_body: CheckDefinitionModel,
 ) -> Dict[str, Any]:
+
     pass
 
     json_json_body = json_body.to_dict()

distribution/python/dqops/client/api/checks/delete_check.py

@@ -11,6 +11,7 @@
 def _get_kwargs(
     full_check_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/checks/get_all_checks.py

@@ -10,6 +10,7 @@
 
 
 def _get_kwargs() -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/checks/get_check.py

@@ -12,6 +12,7 @@
 def _get_kwargs(
     full_check_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/checks/get_check_folder_tree.py

@@ -10,6 +10,7 @@
 
 
 def _get_kwargs() -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/checks/update_check.py

@@ -14,6 +14,7 @@ def _get_kwargs(
     *,
     json_body: CheckDefinitionModel,
 ) -> Dict[str, Any]:
+
     pass
 
     json_json_body = json_body.to_dict()

distribution/python/dqops/client/api/columns/create_column.py

@@ -17,6 +17,7 @@ def _get_kwargs(
     *,
     json_body: ColumnSpec,
 ) -> Dict[str, Any]:
+
     pass
 
     json_json_body = json_body.to_dict()

distribution/python/dqops/client/api/columns/delete_column.py

@@ -15,6 +15,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column.py

@@ -15,6 +15,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_basic.py

@@ -15,6 +15,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_comments.py

@@ -15,6 +15,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_labels.py

@@ -14,6 +14,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_monitoring_checks_basic_model.py

@@ -17,6 +17,7 @@ def _get_kwargs(
     column_name: str,
     time_scale: CheckTimeScale,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_monitoring_checks_daily.py

@@ -17,6 +17,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_monitoring_checks_model.py

@@ -17,6 +17,7 @@ def _get_kwargs(
     column_name: str,
     time_scale: CheckTimeScale,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_monitoring_checks_model_filter.py

@@ -19,6 +19,7 @@ def _get_kwargs(
     check_category: str,
     check_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_monitoring_checks_monthly.py

@@ -17,6 +17,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_partitioned_checks_basic_model.py

@@ -17,6 +17,7 @@ def _get_kwargs(
     column_name: str,
     time_scale: CheckTimeScale,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_partitioned_checks_daily.py

@@ -17,6 +17,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_partitioned_checks_model.py

@@ -17,6 +17,7 @@ def _get_kwargs(
     column_name: str,
     time_scale: CheckTimeScale,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_partitioned_checks_model_filter.py

@@ -19,6 +19,7 @@ def _get_kwargs(
     check_category: str,
     check_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_partitioned_checks_monthly.py

@@ -17,6 +17,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_profiling_checks.py

@@ -17,6 +17,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_profiling_checks_basic_model.py

@@ -15,6 +15,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_profiling_checks_model.py

@@ -15,6 +15,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_profiling_checks_model_filter.py

@@ -17,6 +17,7 @@ def _get_kwargs(
     check_category: str,
     check_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_column_statistics.py

@@ -15,6 +15,7 @@ def _get_kwargs(
     table_name: str,
     column_name: str,
 ) -> Dict[str, Any]:
+
     pass
 
     return {

distribution/python/dqops/client/api/columns/get_columns.py

@@ -18,6 +18,7 @@ def _get_kwargs(
     data_quality_status: Union[Unset, None, bool] = UNSET,
     check_type: Union[Unset, None, CheckType] = UNSET,
 ) -> Dict[str, Any]:
+
     pass
 
     params: Dict[str, Any] = {}