Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add a summary view for identifying data errors from the fhir submissions #1057 #1078

Merged
merged 1 commit into from
Jan 21, 2025
Merged

feat: add a summary view for identifying data errors from the fhir submissions #1057 #1078

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified hub-prime/lib/techbd-udi-jooq-ingress.auto.jar
View file
Open in desktop
Binary file not shown.
147 changes: 138 additions & 9 deletions hub-prime/src/main/resources/templates/page/diagnostics/fhir-validation-issues.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,12 @@
<head>
<link rel="stylesheet" href="https://unpkg.com/ag-grid-community/styles/ag-grid.css">
<link rel="stylesheet" href="https://unpkg.com/ag-grid-community/styles/ag-theme-alpine.css">

<style>
.grid-description{
font-size: 14px;
margin: 5px 0px 5px 15px;
}
</style>
<!-- if JSON Viewer is not already in the layout, add the following -->
<!-- <script src="https://unpkg.com/@alenaksu/json-viewer@2.0.0/dist/json-viewer.bundle.js"></script> -->

Expand All @@ -16,18 +21,127 @@

const schemaName = 'techbd_udi_ingress';
const viewName = 'fhir_session_diagnostics';
const fhirValidationIssue = 'fhir_session_diagnostics_details';
const fhirValidationIssueColumnDefs = [
{ headerName: "Session ID", field: "session_id", filter: "agTextColumnFilter", headerTooltip: "Identifies the ingestion session during which the issue occurred" },
{ headerName: "Bundle ID", field: "bundle_id", filter: "agTextColumnFilter", headerTooltip: "Identifies the bundle during which the issue occurred" },
{ headerName: "URI", field: "uri", filter: "agTextColumnFilter", headerTooltip: "The URI associated with the interaction" },
{ headerName: "Issue Line", field: "line", filter: "agTextColumnFilter", headerTooltip: "The line number where the issue occurred" },
{ headerName: "Issue Column", field: "column", filter: "agTextColumnFilter", headerTooltip: "The column number where the issue occurred" },
{ headerName: "Diagnostics", field: "diagnostics", filter: "agTextColumnFilter", headerTooltip: "The diagnostics associated with the issue" },
];

function transformFhirValidationIssues(response) {
const transformedData = response.data.map(issue => {
return {
validation_engine: issue.validation_engine , // Keep the validation engine
message: issue.message , // Default message if none exists
severity: issue.severity , // Default severity if none exists
encountered_date: issue.encountered_date , // Set to encountered date or default
ig_version: issue.ig_version , // Set to ig_version, default to "Unknown" if not available
encountered_date: issue.encountered_date , // Assuming encountered_date maps to issue_date
tenant_id: issue.tenant_id , // Mapping tenant_id
session_id: issue.session_id , // Mapping session_id
line: issue.line , // Mapping line
column: issue.column , // Mapping column
uri: issue.uri , // Mapping uri
diagnostics: issue.diagnostics , // Mapping diagnostics
bundle_id: issue.bundle_id , // Mapping bundle_id
elaboration: issue.elaboration // Mapping elaboration
};
});

return transformedData;
}

function getFhirValidationIssueGridData(params) {
const ig_version = params.data.ig_version ;
const message = params.data.message ;
const encountered_date = params.data.encountered_date ;
const severity = params.data.severity ;
const tenant_id = params.data.tenant_id ;
const validation_engine = params.data.validation_engine ;

const requestData = {
"startRow": 0,
"endRow": 500,
"rowGroupCols": [],
"valueCols": [],
"pivotCols": [],
"pivotMode": false,
"groupKeys": [],
"filterModel": {
"tenant_id": {
"filterType": "text",
"type": "equals",
"filter": tenant_id
},
"severity": {
"filterType": "text",
"type": "equals",
"filter": severity
},
"message": {
"filterType": "text",
"type": "equals",
"filter": message
},
"ig_version": {
"filterType": "text",
"type": "equals",
"filter": ig_version
},
"validation_engine": {
"filterType": "text",
"type": "equals",
"filter": validation_engine
}

},
"sortModel": [
{
"sort": "desc",
"colId": "encountered_date"
}
]
};

// Fetch with POST method
fetch(window.shell.serverSideUrl(`/api/ux/tabular/jooq/${schemaName}/${fhirValidationIssue}.json`), {
method: 'POST', // Use POST method
headers: {
'Content-Type': 'application/json' // Ensure it's sent as JSON
},
body: JSON.stringify(requestData) // Convert the request data to a JSON string
})
.then(response => {
if (response.url.includes('/?timeout=true')) {
window.location.href = '/?timeout=true'; // Redirect to login page
return null; // Prevent further processing of the response
}
return response.json(); // Parse the response as JSON
})
.then(response => {
// Transform the response data
const transformedResponse = transformFhirValidationIssues(response);
params.successCallback(transformedResponse); // Pass the response to the ag-Grid success callback
})
.catch(error => {
console.error('Error fetching details data: ' + error);
});

}
document.addEventListener('DOMContentLoaded', function () {
const modalAide = new ModalAide();
const agGridInstance = new AGGridAideBuilder()
.withColumnDefs([
{ headerName: "Encountered Time", field: "encounteredat", sortable: true, sort: "desc", filter: "agDateColumnFilter" },
{ headerName: "Session ID", field: "session_id", filter: "agTextColumnFilter" },
{ headerName: "TechBD Tenant ID", field: "tenant_id", filter: "agTextColumnFilter" },
{ headerName: "Severity", field: "severity", filter: "agTextColumnFilter" },
{ headerName: "Issue Message", field: "message", filter: "agTextColumnFilter" },
{ headerName: "Issue Line", field: "line", filter: "agNumberColumnFilter" },
{ headerName: "Issue Column", field: "column", filter: "agNumberColumnFilter" },
{ headerName: "Diagnostics", field: "diagnostics", filter: "agTextColumnFilter" }
{ headerName: "Encountered Time", field: "encountered_date", sortable: true, sort: "desc", filter: "agDateColumnFilter", cellRenderer: 'agGroupCellRenderer', headerTooltip: "Timestamp indicating when the issue was encountered." },
{ headerName: "TechBD Tenant ID", field: "tenant_id", filter: "agTextColumnFilter", headerTooltip: "The unique identifier for the TechBD tenant." },
{ headerName: "Severity", field: "severity", filter: "agTextColumnFilter", headerTooltip: "The severity of the issue." },
{ headerName: "Issue Message", field: "message", filter: "agTextColumnFilter", headerTooltip: "A description of the issue." },
{ headerName: "IG Version", field: "ig_version", filter: "agTextColumnFilter", headerTooltip: "The version of the Implementation Guide associated with the FHIR data and its validation." },
{ headerName: "HAPI Version", field: "validation_engine", filter: "agTextColumnFilter", headerTooltip: "The validation engine used to validate the FHIR data against the Implementation Guide (IG)." },
{ headerName: "Issue Count", field: "issue_count", filter: "agTextColumnFilter", headerTooltip: "The number of times this specific issue has been encountered across different interactions within a particular IG version." }
])
.withServerSideDatasource(
window.shell.serverSideUrl(`/api/ux/tabular/jooq/${schemaName}/${viewName}.json`),
Expand All @@ -38,6 +152,18 @@
}));
},
)
.withMasterDetail(true)
.withDetailCellRendererParams({
detailGridOptions: {
columnDefs: fhirValidationIssueColumnDefs,
defaultColDef: {
flex: 1
}
},
getDetailRowData: params => {
getFhirValidationIssueGridData(params);
}
})
.withModalAide(modalAide)
.withGridDivStyles({ height: "750px", width: "100%" })
.build();
Expand All @@ -49,6 +175,9 @@

<body>
<div layout:fragment="content">
<div class="grid-description">
This data grid provides a high-level summary view on the Data Quality page, highlighting errors in submissions to facilitate swift identification and resolution. Key fields such as Date, Tenant ID, Issue Message, Severity, IG Version, HAPI Version, and Issue Count are displayed, offering a comprehensive snapshot of data quality issues.
A drill-down feature enables users to explore detailed insights for each entry, including additional context like the associated Bundle ID and other relevant information. This functionality ensures efficient error tracing and resolution, streamlining the data validation process and enhancing overall submission quality.
<div id="serverDataGrid" class="ag-theme-alpine"></div>
</div>
</body>
Expand Down
16 changes: 1 addition & 15 deletions hub-prime/src/main/resources/templates/page/diagnostics/ig-publication-issues.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,21 +59,7 @@
{ headerName: "Issue Identified On", field: "issue_date", filter: "agDateColumnFilter", headerTooltip: "Indicates the most recent date when the issue was detected, helping users track the timeline of validation problems." },
{ headerName: "Validation Engine", field: "validation_engine", filter: "agTextColumnFilter", headerTooltip: "The validation engine used to validate the FHIR data against the Implementation Guide (IG)." },
{ headerName: "Count Of URLs With Issues", field: "distinct_issue_count", filter: "agNumberColumnFilter", headerTooltip: "Represents the number of URLs with issues found on the selected date, allowing users to understand the frequency and impact of each issue." }
])
.withMasterDetail(true)
.withDetailCellRendererParams({


detailGridOptions: {
columnDefs: igPublicationColumnDefs,
defaultColDef: {
flex: 1
}
},
getDetailRowData: params => {
getIgPublicationGridData(params);
}
})
])
.withServerSideDatasource(
window.shell.serverSideUrl(`/api/ux/tabular/jooq/${schemaName}/${viewName}.json`),
(data, valueCols) => {
Expand Down
74 changes: 74 additions & 0 deletions udi-prime/src/main/postgres/ingestion-center/001_idempotent_interaction.psql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4226,3 +4226,77 @@ SELECT
FROM
ranked_records
WHERE rn = 1;


/*****************************************
This view consolidates diagnostic data from the table 'sat_interaction_fhir_session_diagnostic'.
It transforms and organizes key fields, including 'tenant_id', 'severity', 'message', 'ig_version',
and 'validation_engine', along with converting the 'encountered_at' timestamp into a date format.

The view aggregates data to calculate the count of issues ('issue_count') for each combination of
'tenant_id', 'severity', 'message', 'ig_version', and 'validation_engine'. It provides an ordered
output by the most recent encountered dates ('encountered_date') and tenant IDs for streamlined
analysis or reporting.

If an existing view 'fhir_session_diagnostics' already exists, it is dropped before creating
this new view definition to ensure consistency.
******************************************************************************************/

drop view if exists techbd_udi_ingress.fhir_session_diagnostics cascade;
create or replace view techbd_udi_ingress.fhir_session_diagnostics
as
select
to_timestamp(encountered_at::double precision)::date as encountered_date,
tenant_id,
severity,
message as message,
ig_version,
validation_engine,
COUNT(*) as issue_count
from
techbd_udi_ingress.sat_interaction_fhir_session_diagnostic
where message != ''
group by
encountered_date,
tenant_id,
severity,
message,
ig_version,
validation_engine
order by
encountered_date desc;


/*****************************************
This view, 'fhir_session_diagnostics_details', extracts detailed diagnostic information
from the 'sat_interaction_fhir_session_diagnostic' table. It includes fields such as
'tenant_id', 'uri', 'bundle_id', 'session_id', 'severity', 'message', 'line', 'column',
'diagnostics', and 'elaboration'.

The 'encountered_at' timestamp is converted into a date format ('encountered_date')
to simplify analysis. This view is intended to provide a comprehensive, detailed view
of diagnostics for further investigation or reporting.

If an existing view 'fhir_session_diagnostics_details' already exists, it is dropped
before creating the new view to ensure the latest definition is applied.
******************************************************************************************/

drop view if exists techbd_udi_ingress.fhir_session_diagnostics_details cascade;
create or replace view techbd_udi_ingress.fhir_session_diagnostics_details
as
select
tenant_id,
uri,
bundle_id,
session_id,
severity,
message,
line,
"column",
ig_version,
validation_engine,
diagnostics,
to_char(to_timestamp(intr_diagno.encountered_at::double precision), 'MM-DD-YYYY'::text) AS encountered_date,
elaboration
from
techbd_udi_ingress.sat_interaction_fhir_session_diagnostic intr_diagno;
Loading