Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add a new tab called 'CSV Via HTTPs Data Quality' under the 'Data Quality' section #975 #976

Merged
merged 1 commit into from
Jan 1, 2025
Merged

feat: add a new tab called 'CSV Via HTTPs Data Quality' under the 'Data Quality' section #975 #976

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified hub-prime/lib/techbd-udi-jooq-ingress.auto.jar
View file
Open in desktop
Binary file not shown.
2 changes: 1 addition & 1 deletion hub-prime/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
</parent>
<groupId>org.techbd</groupId>
<artifactId>hub-prime</artifactId>
<version>0.414.0</version>
<version>0.415.0</version>
<packaging>war</packaging>
<name>Tech by Design Hub (Prime)</name>
<description>Tech by Design Hub (Primary)</description>
Expand Down
10 changes: 8 additions & 2 deletions hub-prime/src/main/java/org/techbd/service/http/hub/prime/ux/DataQualityController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,20 @@ public String diagnosticsFhirValidationIssues(final Model model, final HttpServl
return presentation.populateModel("page/diagnostics/fhir-validation-issues", model, request);
}

@GetMapping("/data-quality/https-via-csv-validations")
@RouteMapping(label = "CSV Via HTTPs Data Quality", title = "CSV Via HTTPs Data Quality", siblingOrder = 40)
public String diagnosticsHttpsViaCsvValidationIssues(final Model model, final HttpServletRequest request) {
return presentation.populateModel("page/diagnostics/https-via-csv-validations", model, request);
}

@GetMapping("/data-quality/ig-publication-issues")
@RouteMapping(label = "IG Publication Issues", title = "IG Publication Issues", siblingOrder = 40)
@RouteMapping(label = "IG Publication Issues", title = "IG Publication Issues", siblingOrder = 50)
public String igPublicationIssues(final Model model, final HttpServletRequest request) {
return presentation.populateModel("page/diagnostics/ig-publication-issues", model, request);
}

@GetMapping("/data-quality/fhir-rules")
@RouteMapping(label = "FHIR Rules", title = "FHIR Rules", siblingOrder = 50)
@RouteMapping(label = "FHIR Rules", title = "FHIR Rules", siblingOrder = 60)
public String fhirRules(final Model model, final HttpServletRequest request) {
return presentation.populateModel("page/diagnostics/fhir-rules", model, request);
}
Expand Down
119 changes: 119 additions & 0 deletions hub-prime/src/main/resources/templates/page/diagnostics/https-via-csv-validations.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
<!DOCTYPE html>
<html lang="en" xmlns:th="http://www.thymeleaf.org" xmlns:layout="http://www.ultraq.net.nz/thymeleaf/layout"
layout:decorate="~{layout/prime}">

<head>
<link rel="stylesheet" href="https://unpkg.com/ag-grid-community/styles/ag-grid.css">
<link rel="stylesheet" href="https://unpkg.com/ag-grid-community/styles/ag-theme-alpine.css">

<!-- if JSON Viewer is not already in the layout, add the following -->
<!-- <script src="https://unpkg.com/@alenaksu/json-viewer@2.0.0/dist/json-viewer.bundle.js"></script> -->

<script src="https://unpkg.com/ag-grid-enterprise/dist/ag-grid-enterprise.js"></script>
<script type="module">
import { AGGridAide, AGGridAideBuilder } from '@presentation/shell/aggrid-aide.js';
import ModalAide from '@presentation/shell/modal-aide.js';

const schemaName = 'techbd_udi_ingress';
const viewName = 'https_csv_validation_errors';
document.addEventListener('DOMContentLoaded', function () {
const modalAide = new ModalAide();
const agGridInstance = new AGGridAideBuilder()
.withColumnDefs([
{
headerName: "Created Time",
field: "created_at",
sortable: true,
sort: "desc",
filter: "agDateColumnFilter",
headerTooltip: "The timestamp when the CSV validation error was recorded."
},
{
headerName: "TechBD Tenant ID",
field: "tenant_id",
filter: "agTextColumnFilter",
headerTooltip: "The unique identifier for the TechBD tenant associated with the CSV file."
},
{
headerName: "URI",
field: "uri",
filter: "agTextColumnFilter",
headerTooltip: "The URI where the CSV file was submitted or validated."
},
{
headerName: "File Name",
field: "file_name",
filter: "agTextColumnFilter",
headerTooltip: "The name of the CSV file where the error occurred."
},
{
headerName: "Field Name",
field: "fieldname",
filter: "agTextColumnFilter",
headerTooltip: "The name of the field in the CSV file where the error occurred."
},
{
headerName: "Cell Value",
field: "value",
filter: "agTextColumnFilter",
headerTooltip: "The value of the cell in the CSV file that triggered the error."
},
{
headerName: "Error Type",
field: "error_type",
filter: "agTextColumnFilter",
headerTooltip: "The category or type of validation error encountered."
},
{
headerName: "Error Message",
field: "error",
filter: "agTextColumnFilter",
headerTooltip: "The detailed error message describing the validation issue."
},
{
headerName: "Description",
field: "description",
filter: "agTextColumnFilter",
headerTooltip: "A brief description of the validation error."
},
{
headerName: "Row Number",
field: "rownumber",
filter: "agNumberColumnFilter",
headerTooltip: "The row number in the CSV file where the error occurred."
},
{
headerName: "Field Number",
field: "fieldnumber",
filter: "agNumberColumnFilter",
headerTooltip: "The field number in the CSV file where the error occurred."
}
])
.withServerSideDatasource(
window.shell.serverSideUrl(`/api/ux/tabular/jooq/${schemaName}/${viewName}.json`),
(data, valueCols) => {
return valueCols.map(col => ({
headerName: col.displayName,
field: col.field
}));
},
)
.withModalAide(modalAide)
.withGridDivStyles({ height: "750px", width: "100%" })
.build();

agGridInstance.init('serverDataGrid');
});
</script>
</head>

<body>
<div layout:fragment="content">
<div class="grid-description">
This data grid provides an in-depth view of validation errors identified during the HTTPS-based CSV validation process. Each row in the grid represents a specific issue detected within the submitted CSV files, including details such as the problematic field, the corresponding error message, and the exact row and field number where the error occurred.
Additionally, the grid highlights the section of the CSV being validated, the associated file name, and the tenant information for accurate context. This information is critical for identifying and addressing errors in the ingestion process, ensuring data quality and compliance with validation standards. Users can hover over column headers for additional details about each field. </div>
<div id="serverDataGrid" class="ag-theme-alpine"></div>
</div>
</body>

</html>
54 changes: 53 additions & 1 deletion udi-prime/src/main/postgres/ingestion-center/001_idempotent_interaction.psql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3923,4 +3923,56 @@ AS SELECT sat_interaction_fhir_request_id,
(SELECT string_agg(concat_ws(': ', key, COALESCE(value, 'NULL')), ', ') FROM jsonb_each_text((payload->>'responseBody')::jsonb) AS json_data(key, value)) AS response_body_details,
source_hub_interaction_id
FROM techbd_udi_ingress.sat_interaction_fhir_request intr_hreq
WHERE to_state = 'FAIL'::text AND source_type = 'CSV';
WHERE to_state = 'FAIL'::text AND source_type = 'CSV';


/*****************************************
This view extracts error details from CSV validation payloads in `sat_interaction_flat_file_csv_request`.
It includes fields like `fieldName`, `cell`, `title`, `message`, `description`, `rowNumber`, and `fieldNumber`.
The `file_name` is determined based on the section type. Only validation results with errors are included,
sorted by `created_at` in descending order.
*****************************************/


DROP VIEW IF EXISTS techbd_udi_ingress.https_csv_validation_errors CASCADE;
CREATE OR REPLACE VIEW techbd_udi_ingress.https_csv_validation_errors
AS
SELECT
sifcr.created_at,
sifcr.tenant_id,
sifcr.uri,
sifcr.hub_interaction_id,
tasks.value ->> 'name' AS section,
error_payload.value ->> 'fieldName'::text AS fieldname,
error_payload.value ->> 'cell'::text AS value,
error_payload.value ->> 'title'::text AS error_type,
error_payload.value ->> 'message'::text AS error,
error_payload.value ->> 'description'::text AS description,
(error_payload.value ->> 'rowNumber'::text)::integer AS rownumber,
(error_payload.value ->> 'fieldNumber'::text)::integer AS fieldnumber,
CASE
WHEN tasks.value ->> 'name' = 'qe_admin_data' THEN original_csv.qe_admin_data_file_name
WHEN tasks.value ->> 'name' = 'demographic_data' THEN original_csv.demographic_data_file_name
WHEN tasks.value ->> 'name' = 'screening_observation_data' THEN original_csv.screening_observation_data_file_name
WHEN tasks.value ->> 'name' = 'screening_profile_data' THEN original_csv.screening_profile_data_file_name
ELSE NULL
END AS file_name
FROM
techbd_udi_ingress.sat_interaction_flat_file_csv_request sifcr
LEFT JOIN techbd_udi_ingress.sat_interaction_flat_file_csv_request original_csv
ON sifcr.hub_interaction_id = original_csv.hub_interaction_id
AND original_csv.nature = 'Original Flat File CSV',
LATERAL jsonb_array_elements(
((sifcr.validation_result_payload -> 'validationResults'::text) -> 'report'::text) -> 'tasks'::text
) tasks(value),
LATERAL jsonb_array_elements(tasks.value -> 'errors'::text) error_payload(value)
WHERE
sifcr.nature = 'CSV Validation Result'::text
AND sifcr.uri = ANY (ARRAY[
'/flatfile/csv/Bundle'::text,
'/flatfile/csv/Bundle/'::text,
'/flatfile/csv/Bundle/$validate'::text,
'/flatfile/csv/Bundle/$validate/'::text
])
AND (((((sifcr.validation_result_payload -> 'validationResults'::text) -> 'report'::text) -> 'stats'::text) ->> 'errors'::text)::integer) > 0
order by sifcr.created_at DESC ;
Loading