Merge pull request #43 from dqops/1.9.0

1.9.0

.github/workflows/main-publish-release.yml

@@ -42,7 +42,7 @@ jobs:
           python-version: 3.8
 
       - name: Build with Maven
-        run: mvn package -DskipTests -DcompressZip=true -Pbuild-with-jdk-11 -Prun-npm
+        run: mvn package -DskipTests -DcompressZip=true -Pdisable-duckdb-extensions-download -Pbuild-with-jdk-11 -Prun-npm
 
       - uses: juliangruber/read-file-action@v1
         name: Read VERSION
@@ -177,7 +177,9 @@ jobs:
           headers: |-
             content-type: application/zip
 
-      - uses: docker-practice/actions-setup-docker@master
+      - uses: docker/setup-qemu-action@v3
+
+      - uses: docker/setup-buildx-action@v3
 
       - uses: mr-smithers-excellent/docker-build-push@v6
         name: Build and Push Docker Image

.github/workflows/mvn-build-develop.yml

@@ -57,7 +57,7 @@ jobs:
           key: 'npm'
 
       - name: Build with Maven
-        run: mvn package -DskipTests -Pbuild-with-jdk-11 -Pdevelopment-build -Prun-npm
+        run: mvn package -DskipTests -Pbuild-with-jdk-11 -Pdisable-duckdb-extensions-download -Pdevelopment-build -Prun-npm
 
       - name: Save cached npm packages
         id: save-cached-npm-packages
@@ -68,7 +68,9 @@ jobs:
             dqops/src/main/frontend/node_modules
           key: 'npm'
 
-      - uses: docker-practice/actions-setup-docker@master
+      - uses: docker/setup-qemu-action@v3
+
+      - uses: docker/setup-buildx-action@v3
 
       - uses: mr-smithers-excellent/docker-build-push@v6
         name: Build and Push Docker Image

.run/dqo run.run.xml

@@ -5,10 +5,10 @@
       <option name="region" />
       <option name="useCurrentConnection" value="false" />
     </extension>
-    <option name="JAR_PATH" value="$PROJECT_DIR$/dqops/target/dqo-dqops-1.8.0.jar" />
+    <option name="JAR_PATH" value="$PROJECT_DIR$/dqops/target/dqo-dqops-1.9.0.jar" />
     <option name="VM_PARAMETERS" value="-XX:MaxRAMPercentage=60.0 --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED" />
     <option name="PROGRAM_PARAMETERS" value="--server.port=8888" />
-    <option name="WORKING_DIRECTORY" value="C:\dev\dqoado" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
     <option name="ALTERNATIVE_JRE_PATH" value="temurin-17" />
     <envs>
       <env name="DQO_HOME" value="$PROJECT_DIR$/home" />

.run/run app with paid extensions.run.xml

@@ -0,0 +1,28 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="run app with paid extensions" type="Application" factoryName="Application">
+    <option name="ALTERNATIVE_JRE_PATH" value="temurin-17" />
+    <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="true" />
+    <classpathModifications>
+      <entry path="$PROJECT_DIR$/userhome/jars/paid/dqops-paid-bin.jar" />
+    </classpathModifications>
+    <envs>
+      <env name="DQO_HOME" value="$PROJECT_DIR$/home" />
+      <env name="DQO_USER_HOME" value="$PROJECT_DIR$/userhome" />
+      <env name="HADOOP_HOME" value="$PROJECT_DIR$/home" />
+      <env name="AZURE_ENABLE_HTTP_CLIENT_SHARING" value="true" />
+    </envs>
+    <option name="MAIN_CLASS_NAME" value="com.dqops.cli.CliApplication" />
+    <module name="dqo-dqops" />
+    <option name="PROGRAM_PARAMETERS" value="--server.port=8888 --dqo.webserver.authentication-method=oauth2" />
+    <shortenClasspath name="ARGS_FILE" />
+    <option name="VM_PARAMETERS" value="-XX:MaxRAMPercentage=60.0 --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED" />
+    <extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
+      <option name="credential" />
+      <option name="region" />
+      <option name="useCurrentConnection" value="false" />
+    </extension>
+    <method v="2">
+      <option name="Make" enabled="false" />
+    </method>
+  </configuration>
+</component>

CHANGELOG.md

@@ -1,9 +1,13 @@
-# 1.8.0
+# 1.9.0
 
-* Small layout fixes in the UI, mostly related to the colors of issue severity colors
-* Documentation for the data quality process and rule mining
-* Duplicate record detection check
-* Some column-level checks fixed to skip NULL values, instead of treating them as invalid (not passing checks)
-* Store data lineage definition (links to source tables)
-* Default observability check patterns renamed to data quality policies
-* Default quality policy named "default" deleted, and replaced with multiple smaller policies to allow more control 
+* SAP HANA support
+* IBM DB2 support
+* Improvements to the anomaly rules
+* Missing UI for the data domains added
+* Incident histogram shows time periods, not the execution day
+* Incident notifications fixed to send notifications used a dedicated queue and work even when fallback addresses are not configured
+* Data lineage flow diagrams
+* Table similarity search to build data lineage
+* Table and column screens have a new primary tab to show the data observability status, which is a shortened status
+* Basic statistics are scheduled by CRON and reuse the profiling schedule
+* Connections support configuring a CRON expression to import new tables

Dockerfile

@@ -76,4 +76,6 @@ COPY --from=dqo-libs /workspace/app/lib/target/output/dqo-lib/jars /dqo/app/lib
 COPY --from=dqo-libs ${DEPENDENCY}/BOOT-INF/lib /dqo/app/lib
 COPY --from=dqo-libs ${DEPENDENCY}/META-INF /dqo/app/META-INF
 COPY --from=dqo-libs ${DEPENDENCY}/BOOT-INF/classes /dqo/app
+
+WORKDIR /dqo/userhome
 ENTRYPOINT ["/dqo/home/bin/dqo_docker_entrypoint.sh"]

Dockerfile-fast

@@ -56,4 +56,6 @@ COPY --from=dqo-libs ${DEPENDENCY}/jars /dqo/app/lib
 COPY --from=dqo-libs ${DEPENDENCY}/expanded/BOOT-INF/lib /dqo/app/lib
 COPY --from=dqo-libs ${DEPENDENCY}/expanded/META-INF /dqo/app/META-INF
 COPY --from=dqo-libs ${DEPENDENCY}/expanded/BOOT-INF/classes /dqo/app
+
+WORKDIR /dqo/userhome
 ENTRYPOINT ["/dqo/home/bin/dqo_docker_entrypoint.sh"]

VERSION

@@ -1 +1 @@
-1.8.0
+1.9.0

distribution/dqo_docker_entrypoint.sh

@@ -27,7 +27,7 @@ _term() {
 trap _term SIGTERM
 trap _term SIGINT
 
-exec 3<&0 java $DQO_JAVA_OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED -cp /dqo/app:/dqo/app/lib/* com.dqops.cli.CliApplication $* <&3 &
+exec 3<&0 java $DQO_JAVA_OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED -cp /dqo/app:/dqo/app/lib/*:/dqo/home/jars/*:/dqo/userhome/jars/*:/dqo/userhome/jars/:/dqo/userhome/jars/paid/* com.dqops.cli.CliApplication $* <&3 &
 
 child=$!
 wait $child

distribution/pom.xml

@@ -11,7 +11,7 @@
 
 	<groupId>com.dqops</groupId>
 	<artifactId>dqo-distribution</artifactId>
-    <version>1.8.0</version>  <!-- DQOps Version, do not touch (changed automatically) -->
+    <version>1.9.0</version>  <!-- DQOps Version, do not touch (changed automatically) -->
 	<name>dqo-distribution</name>
 	<description>DQOps Data Quality Operations Center final assembly</description>
 	<packaging>pom</packaging>

distribution/python/dqops/client/api/check_results/get_column_monitoring_checks_results.py

@@ -6,6 +6,7 @@
 
 from ... import errors
 from ...client import AuthenticatedClient, Client
+from ...models.check_results_detailed_load_mode import CheckResultsDetailedLoadMode
 from ...models.check_results_list_model import CheckResultsListModel
 from ...models.check_time_scale import CheckTimeScale
 from ...types import UNSET, Response, Unset
@@ -24,6 +25,7 @@ def _get_kwargs(
     check_name: Union[Unset, None, str] = UNSET,
     category: Union[Unset, None, str] = UNSET,
     table_comparison: Union[Unset, None, str] = UNSET,
+    load_mode: Union[Unset, None, CheckResultsDetailedLoadMode] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Dict[str, Any]:
 
@@ -50,6 +52,12 @@ def _get_kwargs(
 
     params["tableComparison"] = table_comparison
 
+    json_load_mode: Union[Unset, None, str] = UNSET
+    if not isinstance(load_mode, Unset):
+        json_load_mode = load_mode.value if load_mode else None
+
+    params["loadMode"] = json_load_mode
+
     params["maxResultsPerCheck"] = max_results_per_check
 
     params = {k: v for k, v in params.items() if v is not UNSET and v is not None}
@@ -110,6 +118,7 @@ def sync_detailed(
     check_name: Union[Unset, None, str] = UNSET,
     category: Union[Unset, None, str] = UNSET,
     table_comparison: Union[Unset, None, str] = UNSET,
+    load_mode: Union[Unset, None, CheckResultsDetailedLoadMode] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Response[List["CheckResultsListModel"]]:
     """getColumnMonitoringChecksResults
@@ -129,6 +138,7 @@ def sync_detailed(
         check_name (Union[Unset, None, str]):
         category (Union[Unset, None, str]):
         table_comparison (Union[Unset, None, str]):
+        load_mode (Union[Unset, None, CheckResultsDetailedLoadMode]):
         max_results_per_check (Union[Unset, None, int]):
 
     Raises:
@@ -151,6 +161,7 @@ def sync_detailed(
         check_name=check_name,
         category=category,
         table_comparison=table_comparison,
+        load_mode=load_mode,
         max_results_per_check=max_results_per_check,
     )
 
@@ -175,6 +186,7 @@ def sync(
     check_name: Union[Unset, None, str] = UNSET,
     category: Union[Unset, None, str] = UNSET,
     table_comparison: Union[Unset, None, str] = UNSET,
+    load_mode: Union[Unset, None, CheckResultsDetailedLoadMode] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Optional[List["CheckResultsListModel"]]:
     """getColumnMonitoringChecksResults
@@ -194,6 +206,7 @@ def sync(
         check_name (Union[Unset, None, str]):
         category (Union[Unset, None, str]):
         table_comparison (Union[Unset, None, str]):
+        load_mode (Union[Unset, None, CheckResultsDetailedLoadMode]):
         max_results_per_check (Union[Unset, None, int]):
 
     Raises:
@@ -217,6 +230,7 @@ def sync(
         check_name=check_name,
         category=category,
         table_comparison=table_comparison,
+        load_mode=load_mode,
         max_results_per_check=max_results_per_check,
     ).parsed
 
@@ -235,6 +249,7 @@ async def asyncio_detailed(
     check_name: Union[Unset, None, str] = UNSET,
     category: Union[Unset, None, str] = UNSET,
     table_comparison: Union[Unset, None, str] = UNSET,
+    load_mode: Union[Unset, None, CheckResultsDetailedLoadMode] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Response[List["CheckResultsListModel"]]:
     """getColumnMonitoringChecksResults
@@ -254,6 +269,7 @@ async def asyncio_detailed(
         check_name (Union[Unset, None, str]):
         category (Union[Unset, None, str]):
         table_comparison (Union[Unset, None, str]):
+        load_mode (Union[Unset, None, CheckResultsDetailedLoadMode]):
         max_results_per_check (Union[Unset, None, int]):
 
     Raises:
@@ -276,6 +292,7 @@ async def asyncio_detailed(
         check_name=check_name,
         category=category,
         table_comparison=table_comparison,
+        load_mode=load_mode,
         max_results_per_check=max_results_per_check,
     )
 
@@ -298,6 +315,7 @@ async def asyncio(
     check_name: Union[Unset, None, str] = UNSET,
     category: Union[Unset, None, str] = UNSET,
     table_comparison: Union[Unset, None, str] = UNSET,
+    load_mode: Union[Unset, None, CheckResultsDetailedLoadMode] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Optional[List["CheckResultsListModel"]]:
     """getColumnMonitoringChecksResults
@@ -317,6 +335,7 @@ async def asyncio(
         check_name (Union[Unset, None, str]):
         category (Union[Unset, None, str]):
         table_comparison (Union[Unset, None, str]):
+        load_mode (Union[Unset, None, CheckResultsDetailedLoadMode]):
         max_results_per_check (Union[Unset, None, int]):
 
     Raises:
@@ -341,6 +360,7 @@ async def asyncio(
             check_name=check_name,
             category=category,
             table_comparison=table_comparison,
+            load_mode=load_mode,
             max_results_per_check=max_results_per_check,
         )
     ).parsed

distribution/python/dqops/client/api/check_results/get_column_partitioned_checks_results.py

@@ -6,6 +6,7 @@
 
 from ... import errors
 from ...client import AuthenticatedClient, Client
+from ...models.check_results_detailed_load_mode import CheckResultsDetailedLoadMode
 from ...models.check_results_list_model import CheckResultsListModel
 from ...models.check_time_scale import CheckTimeScale
 from ...types import UNSET, Response, Unset
@@ -24,6 +25,7 @@ def _get_kwargs(
     check_name: Union[Unset, None, str] = UNSET,
     category: Union[Unset, None, str] = UNSET,
     table_comparison: Union[Unset, None, str] = UNSET,
+    load_mode: Union[Unset, None, CheckResultsDetailedLoadMode] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Dict[str, Any]:
 
@@ -50,6 +52,12 @@ def _get_kwargs(
 
     params["tableComparison"] = table_comparison
 
+    json_load_mode: Union[Unset, None, str] = UNSET
+    if not isinstance(load_mode, Unset):
+        json_load_mode = load_mode.value if load_mode else None
+
+    params["loadMode"] = json_load_mode
+
     params["maxResultsPerCheck"] = max_results_per_check
 
     params = {k: v for k, v in params.items() if v is not UNSET and v is not None}
@@ -110,6 +118,7 @@ def sync_detailed(
     check_name: Union[Unset, None, str] = UNSET,
     category: Union[Unset, None, str] = UNSET,
     table_comparison: Union[Unset, None, str] = UNSET,
+    load_mode: Union[Unset, None, CheckResultsDetailedLoadMode] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Response[List["CheckResultsListModel"]]:
     """getColumnPartitionedChecksResults
@@ -129,6 +138,7 @@ def sync_detailed(
         check_name (Union[Unset, None, str]):
         category (Union[Unset, None, str]):
         table_comparison (Union[Unset, None, str]):
+        load_mode (Union[Unset, None, CheckResultsDetailedLoadMode]):
         max_results_per_check (Union[Unset, None, int]):
 
     Raises:
@@ -151,6 +161,7 @@ def sync_detailed(
         check_name=check_name,
         category=category,
         table_comparison=table_comparison,
+        load_mode=load_mode,
         max_results_per_check=max_results_per_check,
     )
 
@@ -175,6 +186,7 @@ def sync(
     check_name: Union[Unset, None, str] = UNSET,
     category: Union[Unset, None, str] = UNSET,
     table_comparison: Union[Unset, None, str] = UNSET,
+    load_mode: Union[Unset, None, CheckResultsDetailedLoadMode] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Optional[List["CheckResultsListModel"]]:
     """getColumnPartitionedChecksResults
@@ -194,6 +206,7 @@ def sync(
         check_name (Union[Unset, None, str]):
         category (Union[Unset, None, str]):
         table_comparison (Union[Unset, None, str]):
+        load_mode (Union[Unset, None, CheckResultsDetailedLoadMode]):
         max_results_per_check (Union[Unset, None, int]):
 
     Raises:
@@ -217,6 +230,7 @@ def sync(
         check_name=check_name,
         category=category,
         table_comparison=table_comparison,
+        load_mode=load_mode,
         max_results_per_check=max_results_per_check,
     ).parsed
 
@@ -235,6 +249,7 @@ async def asyncio_detailed(
     check_name: Union[Unset, None, str] = UNSET,
     category: Union[Unset, None, str] = UNSET,
     table_comparison: Union[Unset, None, str] = UNSET,
+    load_mode: Union[Unset, None, CheckResultsDetailedLoadMode] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Response[List["CheckResultsListModel"]]:
     """getColumnPartitionedChecksResults
@@ -254,6 +269,7 @@ async def asyncio_detailed(
         check_name (Union[Unset, None, str]):
         category (Union[Unset, None, str]):
         table_comparison (Union[Unset, None, str]):
+        load_mode (Union[Unset, None, CheckResultsDetailedLoadMode]):
         max_results_per_check (Union[Unset, None, int]):
 
     Raises:
@@ -276,6 +292,7 @@ async def asyncio_detailed(
         check_name=check_name,
         category=category,
         table_comparison=table_comparison,
+        load_mode=load_mode,
         max_results_per_check=max_results_per_check,
     )
 
@@ -298,6 +315,7 @@ async def asyncio(
     check_name: Union[Unset, None, str] = UNSET,
     category: Union[Unset, None, str] = UNSET,
     table_comparison: Union[Unset, None, str] = UNSET,
+    load_mode: Union[Unset, None, CheckResultsDetailedLoadMode] = UNSET,
     max_results_per_check: Union[Unset, None, int] = UNSET,
 ) -> Optional[List["CheckResultsListModel"]]:
     """getColumnPartitionedChecksResults
@@ -317,6 +335,7 @@ async def asyncio(
         check_name (Union[Unset, None, str]):
         category (Union[Unset, None, str]):
         table_comparison (Union[Unset, None, str]):
+        load_mode (Union[Unset, None, CheckResultsDetailedLoadMode]):
         max_results_per_check (Union[Unset, None, int]):
 
     Raises:
@@ -341,6 +360,7 @@ async def asyncio(
             check_name=check_name,
             category=category,
             table_comparison=table_comparison,
+            load_mode=load_mode,
             max_results_per_check=max_results_per_check,
         )
     ).parsed