Skip to content

Commit

Permalink
feat: add a new tab called 'CSV Via HTTPs Data Quality' under the 'Da…
Browse files Browse the repository at this point in the history
…ta Quality' section tech-by-design#975
  • Loading branch information
megin1989 committed Jan 1, 2025
1 parent ad2ec4e commit 745c96e
Show file tree
Hide file tree
Showing 5 changed files with 181 additions and 4 deletions.
Binary file modified hub-prime/lib/techbd-udi-jooq-ingress.auto.jar
Binary file not shown.
2 changes: 1 addition & 1 deletion hub-prime/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
</parent>
<groupId>org.techbd</groupId>
<artifactId>hub-prime</artifactId>
<version>0.414.0</version>
<version>0.415.0</version>
<packaging>war</packaging>
<name>Tech by Design Hub (Prime)</name>
<description>Tech by Design Hub (Primary)</description>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,20 @@ public String diagnosticsFhirValidationIssues(final Model model, final HttpServl
return presentation.populateModel("page/diagnostics/fhir-validation-issues", model, request);
}

@GetMapping("/data-quality/https-via-csv-validations")
@RouteMapping(label = "CSV Via HTTPs Data Quality", title = "CSV Via HTTPs Data Quality", siblingOrder = 40)
public String diagnosticsHttpsViaCsvValidationIssues(final Model model, final HttpServletRequest request) {
return presentation.populateModel("page/diagnostics/https-via-csv-validations", model, request);
}

@GetMapping("/data-quality/ig-publication-issues")
@RouteMapping(label = "IG Publication Issues", title = "IG Publication Issues", siblingOrder = 40)
@RouteMapping(label = "IG Publication Issues", title = "IG Publication Issues", siblingOrder = 50)
public String igPublicationIssues(final Model model, final HttpServletRequest request) {
return presentation.populateModel("page/diagnostics/ig-publication-issues", model, request);
}

@GetMapping("/data-quality/fhir-rules")
@RouteMapping(label = "FHIR Rules", title = "FHIR Rules", siblingOrder = 50)
@RouteMapping(label = "FHIR Rules", title = "FHIR Rules", siblingOrder = 60)
public String fhirRules(final Model model, final HttpServletRequest request) {
return presentation.populateModel("page/diagnostics/fhir-rules", model, request);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
<!DOCTYPE html>
<html lang="en" xmlns:th="http://www.thymeleaf.org" xmlns:layout="http://www.ultraq.net.nz/thymeleaf/layout"
layout:decorate="~{layout/prime}">

<head>
<link rel="stylesheet" href="https://unpkg.com/ag-grid-community/styles/ag-grid.css">
<link rel="stylesheet" href="https://unpkg.com/ag-grid-community/styles/ag-theme-alpine.css">

<!-- if JSON Viewer is not already in the layout, add the following -->
<!-- <script src="https://unpkg.com/@alenaksu/[email protected]/dist/json-viewer.bundle.js"></script> -->

<script src="https://unpkg.com/ag-grid-enterprise/dist/ag-grid-enterprise.js"></script>
<script type="module">
import { AGGridAide, AGGridAideBuilder } from '@presentation/shell/aggrid-aide.js';
import ModalAide from '@presentation/shell/modal-aide.js';

const schemaName = 'techbd_udi_ingress';
const viewName = 'https_csv_validation_errors';
document.addEventListener('DOMContentLoaded', function () {
const modalAide = new ModalAide();
const agGridInstance = new AGGridAideBuilder()
.withColumnDefs([
{
headerName: "Created Time",
field: "created_at",
sortable: true,
sort: "desc",
filter: "agDateColumnFilter",
headerTooltip: "The timestamp when the CSV validation error was recorded."
},
{
headerName: "TechBD Tenant ID",
field: "tenant_id",
filter: "agTextColumnFilter",
headerTooltip: "The unique identifier for the TechBD tenant associated with the CSV file."
},
{
headerName: "URI",
field: "uri",
filter: "agTextColumnFilter",
headerTooltip: "The URI where the CSV file was submitted or validated."
},
{
headerName: "File Name",
field: "file_name",
filter: "agTextColumnFilter",
headerTooltip: "The name of the CSV file where the error occurred."
},
{
headerName: "Field Name",
field: "fieldname",
filter: "agTextColumnFilter",
headerTooltip: "The name of the field in the CSV file where the error occurred."
},
{
headerName: "Cell Value",
field: "value",
filter: "agTextColumnFilter",
headerTooltip: "The value of the cell in the CSV file that triggered the error."
},
{
headerName: "Error Type",
field: "error_type",
filter: "agTextColumnFilter",
headerTooltip: "The category or type of validation error encountered."
},
{
headerName: "Error Message",
field: "error",
filter: "agTextColumnFilter",
headerTooltip: "The detailed error message describing the validation issue."
},
{
headerName: "Description",
field: "description",
filter: "agTextColumnFilter",
headerTooltip: "A brief description of the validation error."
},
{
headerName: "Row Number",
field: "rownumber",
filter: "agNumberColumnFilter",
headerTooltip: "The row number in the CSV file where the error occurred."
},
{
headerName: "Field Number",
field: "fieldnumber",
filter: "agNumberColumnFilter",
headerTooltip: "The field number in the CSV file where the error occurred."
}
])
.withServerSideDatasource(
window.shell.serverSideUrl(`/api/ux/tabular/jooq/${schemaName}/${viewName}.json`),
(data, valueCols) => {
return valueCols.map(col => ({
headerName: col.displayName,
field: col.field
}));
},
)
.withModalAide(modalAide)
.withGridDivStyles({ height: "750px", width: "100%" })
.build();

agGridInstance.init('serverDataGrid');
});
</script>
</head>

<body>
<div layout:fragment="content">
<div class="grid-description">
This data grid provides an in-depth view of validation errors identified during the HTTPS-based CSV validation process. Each row in the grid represents a specific issue detected within the submitted CSV files, including details such as the problematic field, the corresponding error message, and the exact row and field number where the error occurred.
Additionally, the grid highlights the section of the CSV being validated, the associated file name, and the tenant information for accurate context. This information is critical for identifying and addressing errors in the ingestion process, ensuring data quality and compliance with validation standards. Users can hover over column headers for additional details about each field. </div>
<div id="serverDataGrid" class="ag-theme-alpine"></div>
</div>
</body>

</html>
Original file line number Diff line number Diff line change
Expand Up @@ -3923,4 +3923,56 @@ AS SELECT sat_interaction_fhir_request_id,
(SELECT string_agg(concat_ws(': ', key, COALESCE(value, 'NULL')), ', ') FROM jsonb_each_text((payload->>'responseBody')::jsonb) AS json_data(key, value)) AS response_body_details,
source_hub_interaction_id
FROM techbd_udi_ingress.sat_interaction_fhir_request intr_hreq
WHERE to_state = 'FAIL'::text AND source_type = 'CSV';
WHERE to_state = 'FAIL'::text AND source_type = 'CSV';


/*****************************************
This view extracts error details from CSV validation payloads in `sat_interaction_flat_file_csv_request`.
It includes fields like `fieldName`, `cell`, `title`, `message`, `description`, `rowNumber`, and `fieldNumber`.
The `file_name` is determined based on the section type. Only validation results with errors are included,
sorted by `created_at` in descending order.
*****************************************/


DROP VIEW IF EXISTS techbd_udi_ingress.https_csv_validation_errors CASCADE;
CREATE OR REPLACE VIEW techbd_udi_ingress.https_csv_validation_errors
AS
SELECT
sifcr.created_at,
sifcr.tenant_id,
sifcr.uri,
sifcr.hub_interaction_id,
tasks.value ->> 'name' AS section,
error_payload.value ->> 'fieldName'::text AS fieldname,
error_payload.value ->> 'cell'::text AS value,
error_payload.value ->> 'title'::text AS error_type,
error_payload.value ->> 'message'::text AS error,
error_payload.value ->> 'description'::text AS description,
(error_payload.value ->> 'rowNumber'::text)::integer AS rownumber,
(error_payload.value ->> 'fieldNumber'::text)::integer AS fieldnumber,
CASE
WHEN tasks.value ->> 'name' = 'qe_admin_data' THEN original_csv.qe_admin_data_file_name
WHEN tasks.value ->> 'name' = 'demographic_data' THEN original_csv.demographic_data_file_name
WHEN tasks.value ->> 'name' = 'screening_observation_data' THEN original_csv.screening_observation_data_file_name
WHEN tasks.value ->> 'name' = 'screening_profile_data' THEN original_csv.screening_profile_data_file_name
ELSE NULL
END AS file_name
FROM
techbd_udi_ingress.sat_interaction_flat_file_csv_request sifcr
LEFT JOIN techbd_udi_ingress.sat_interaction_flat_file_csv_request original_csv
ON sifcr.hub_interaction_id = original_csv.hub_interaction_id
AND original_csv.nature = 'Original Flat File CSV',
LATERAL jsonb_array_elements(
((sifcr.validation_result_payload -> 'validationResults'::text) -> 'report'::text) -> 'tasks'::text
) tasks(value),
LATERAL jsonb_array_elements(tasks.value -> 'errors'::text) error_payload(value)
WHERE
sifcr.nature = 'CSV Validation Result'::text
AND sifcr.uri = ANY (ARRAY[
'/flatfile/csv/Bundle'::text,
'/flatfile/csv/Bundle/'::text,
'/flatfile/csv/Bundle/$validate'::text,
'/flatfile/csv/Bundle/$validate/'::text
])
AND (((((sifcr.validation_result_payload -> 'validationResults'::text) -> 'report'::text) -> 'stats'::text) ->> 'errors'::text)::integer) > 0
order by sifcr.created_at DESC ;

0 comments on commit 745c96e

Please sign in to comment.