diff --git a/.gitignore b/.gitignore index 6fcbdf64..a0300533 100644 --- a/.gitignore +++ b/.gitignore @@ -32,9 +32,17 @@ codex-processes-ap1-docker-test-setup/**/bpe/last_event/time.file codex-processes-ap1-docker-test-setup/**/bpe/plugin/*.jar codex-processes-ap1-docker-test-setup/**/bpe/process/*.jar +codex-processes-ap1-docker-test-setup/dic/bpe/cache/**/*.json.gz + codex-processes-ap1-docker-test-setup/**/fhir/conf/bundle.xml codex-processes-ap1-docker-test-setup/**/fhir/log/*.log codex-processes-ap1-docker-test-setup/**/fhir/log/*.log.gz codex-processes-ap1-docker-test-setup/secrets/*.pem -codex-processes-ap1-docker-test-setup/.env \ No newline at end of file +codex-processes-ap1-docker-test-setup/.env +codex-processes-ap1-docker-test-setup/docker-compose.override.yml + +### +# codex-process-data-transfer ignores +### +codex-process-data-transfer/application.properties \ No newline at end of file diff --git a/codex-process-data-transfer/pom.xml b/codex-process-data-transfer/pom.xml index 99634cd3..fc82d444 100644 --- a/codex-process-data-transfer/pom.xml +++ b/codex-process-data-transfer/pom.xml @@ -1,6 +1,4 @@ - + 4.0.0 codex-process-data-transfer @@ -16,32 +14,23 @@ + org.highmed.dsf dsf-bpe-process-base provided - ca.uhn.hapi.fhir - hapi-fhir-client - provided + org.highmed.dsf + dsf-tools-documentation-generator + + - org.springframework - spring-web - provided + ca.uhn.hapi.fhir + hapi-fhir-client - - de.hs-heilbronn.mi - log4j2-utils - test - - - org.highmed.dsf - dsf-fhir-validation - test - org.mockito mockito-core @@ -51,6 +40,15 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + + **/ValidateDataLearningTest.java + + + org.apache.maven.plugins maven-dependency-plugin @@ -157,6 +155,26 @@ ../codex-processes-ap1-docker-test-setup/gth/bpe/plugin + + + copy-standalone-dependencies + install + + copy-dependencies + + + ${project.build.directory}/lib + hapi-fhir-base,hapi-fhir-client,hapi-fhir-converter,hapi-fhir-structures-r4,hapi-fhir-structures-r5,hapi-fhir-validation,hapi-fhir-validation-resources-r4, + hapi-fhir-validation-resources-r5,org.hl7.fhir.convertors,org.hl7.fhir.dstu2,org.hl7.fhir.dstu2016may,org.hl7.fhir.dstu3,org.hl7.fhir.r4,org.hl7.fhir.r5,org.hl7.fhir.utilities, + org.hl7.fhir.validation,jackson-annotations,jackson-core,jackson-databind,jackson-module-jaxb-annotations,caffeine,guava, + commons-codec,commons-io,crypto-utils,log4j2-utils,jakarta.activation,jakarta.annotation-api,jakarta.ws.rs-api,jakarta.xml.bind-api,commons-compress,commons-lang3,commons-text, + httpclient,httpcore,log4j-api,log4j-core,log4j-slf4j-impl,bcpkix-jdk15on,bcprov-jdk15on,bcutil-jdk15on,ucum,hk2-api,hk2-locator,hk2-utils,osgi-resource-locator, + aopalliance-repackaged,jakarta.inject,jersey-apache-connector,jersey-client,jersey-common,jersey-entity-filtering,jersey-hk2,jersey-media-jaxb,jersey-media-json-jackson, + dsf-bpe-process-base,dsf-fhir-rest-adapter,dsf-fhir-validation,dsf-openehr-model,jcl-over-slf4j, + slf4j-api,spring-aop,spring-beans,spring-context,spring-core,spring-expression,spring-jcl,thymeleaf,unbescape,xpp3,xpp3_xpath + + + @@ -209,6 +227,53 @@ + + maven-assembly-plugin + + false + + src/assembly/zip.xml + + + + + zip-assembly + install + + single + + + + + + org.codehaus.mojo + exec-maven-plugin + + + + exec + + prepare-package + + + + java + + -classpath + + + org.highmed.dsf.tools.generator.DocumentationGenerator + + + de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config + + + true + true + compile + ${project.basedir} + + \ No newline at end of file diff --git a/codex-process-data-transfer/src/assembly/zip.xml b/codex-process-data-transfer/src/assembly/zip.xml new file mode 100644 index 00000000..f0dadb48 --- /dev/null +++ b/codex-process-data-transfer/src/assembly/zip.xml @@ -0,0 +1,26 @@ + + + zip + + zip + + + + + ${project.build.directory} + + + *.jar + + + + ${project.build.directory}/lib + lib + + *.jar + + + + \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java index 08f72994..7511f1ac 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java @@ -64,4 +64,20 @@ public interface ConstantsDataTransfer * dic-source/dic-pseudonym-original */ String PSEUDONYM_PATTERN_STRING = "(?[^/]+)/(?[^/]+)"; + + String HAPI_USER_DATA_SOURCE_ID_ELEMENT = "source-id"; + + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR = "http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error"; + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED = "validation-failed"; + + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE = "http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-source"; + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_MEDIC = "MeDIC"; + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_GTH = "GTH"; + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_FTTP = "fTTP"; + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_CRR = "CRR"; + + String EXTENSION_ERROR_METADATA = "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/error-metadata"; + String EXTENSION_ERROR_METADATA_TYPE = "type"; + String EXTENSION_ERROR_METADATA_SOURCE = "source"; + String EXTENSION_ERROR_METADATA_REFERENCE = "reference"; } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java index cf2b32ad..5959137c 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java @@ -14,6 +14,8 @@ import org.highmed.dsf.fhir.resources.ResourceProvider; import org.highmed.dsf.fhir.resources.StructureDefinitionResource; import org.highmed.dsf.fhir.resources.ValueSetResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; import org.springframework.core.env.PropertyResolver; @@ -22,9 +24,13 @@ import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.client.GeccoClientFactory; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config.TransferDataConfig; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config.TransferDataSerializerConfig; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config.ValidationConfig; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactory; public class DataTransferProcessPluginDefinition implements ProcessPluginDefinition { + private static final Logger logger = LoggerFactory.getLogger(DataTransferProcessPluginDefinition.class); + public static final String VERSION = "0.5.0"; public static final LocalDate DATE = LocalDate.of(2021, 9, 6); @@ -55,7 +61,7 @@ public Stream getBpmnFiles() @Override public Stream> getSpringConfigClasses() { - return Stream.of(TransferDataConfig.class, TransferDataSerializerConfig.class); + return Stream.of(TransferDataConfig.class, TransferDataSerializerConfig.class, ValidationConfig.class); } @Override @@ -68,28 +74,34 @@ public ResourceProvider getResourceProvider(FhirContext fhirContext, ClassLoader var aRec = ActivityDefinitionResource.file("fhir/ActivityDefinition/num-codex-data-receive.xml"); var cD = CodeSystemResource.file("fhir/CodeSystem/num-codex-data-transfer.xml"); + var cDeS = CodeSystemResource.file("fhir/CodeSystem/num-codex-data-transfer-error-source.xml"); + var cDe = CodeSystemResource.file("fhir/CodeSystem/num-codex-data-transfer-error.xml"); var nD = NamingSystemResource.file("fhir/NamingSystem/num-codex-dic-pseudonym-identifier.xml"); var nC = NamingSystemResource.file("fhir/NamingSystem/num-codex-crr-pseudonym-identifier.xml"); var nB = NamingSystemResource.file("fhir/NamingSystem/num-codex-bloom-filter-identifier.xml"); + var sTexErMe = StructureDefinitionResource + .file("fhir/StructureDefinition/num-codex-extension-error-metadata.xml"); + var sTstaDrec = StructureDefinitionResource + .file("fhir/StructureDefinition/num-codex-task-start-data-receive.xml"); + var sTstaDsen = StructureDefinitionResource.file("fhir/StructureDefinition/num-codex-task-start-data-send.xml"); + var sTstaDtra = StructureDefinitionResource + .file("fhir/StructureDefinition/num-codex-task-start-data-translate.xml"); var sTstaDtri = StructureDefinitionResource .file("fhir/StructureDefinition/num-codex-task-start-data-trigger.xml"); var sTstoDtri = StructureDefinitionResource .file("fhir/StructureDefinition/num-codex-task-stop-data-trigger.xml"); - var sTstaDsen = StructureDefinitionResource.file("fhir/StructureDefinition/num-codex-task-start-data-send.xml"); - var sTstaDtra = StructureDefinitionResource - .file("fhir/StructureDefinition/num-codex-task-start-data-translate.xml"); - var sTstaDrec = StructureDefinitionResource - .file("fhir/StructureDefinition/num-codex-task-start-data-receive.xml"); var vD = ValueSetResource.file("fhir/ValueSet/num-codex-data-transfer.xml"); + var vDeS = ValueSetResource.file("fhir/ValueSet/num-codex-data-transfer-error-source.xml"); + var vDe = ValueSetResource.file("fhir/ValueSet/num-codex-data-transfer-error.xml"); Map> resourcesByProcessKeyAndVersion = Map.of( // "wwwnetzwerk-universitaetsmedizinde_dataTrigger/" + VERSION, Arrays.asList(aTri, cD, nD, sTstaDtri, sTstoDtri, vD), // "wwwnetzwerk-universitaetsmedizinde_dataSend/" + VERSION, - Arrays.asList(aSen, cD, nD, nB, sTstaDsen, vD), // + Arrays.asList(aSen, cD, cDeS, cDe, nD, nB, sTexErMe, sTstaDsen, vD, vDeS, vDe), // "wwwnetzwerk-universitaetsmedizinde_dataTranslate/" + VERSION, Arrays.asList(aTra, cD, nD, nC, sTstaDtra, vD), // "wwwnetzwerk-universitaetsmedizinde_dataReceive/" + VERSION, @@ -114,5 +126,18 @@ public void onProcessesDeployed(ApplicationContext pluginApplicationContext, Lis { pluginApplicationContext.getBean(FttpClientFactory.class).testConnection(); } + + if (activeProcesses.contains("wwwnetzwerk-universitaetsmedizinde_dataSend")) + { + boolean testOk = pluginApplicationContext.getBean(ValidationConfig.class) + .testConnectionToTerminologyServer(); + + if (testOk) + pluginApplicationContext.getBean(BundleValidatorFactory.class).init(); + else + logger.warn( + "Due to an error while testing the connection to the terminology server {} was not initialized, validation of bundles will be skipped.", + BundleValidatorFactory.class.getSimpleName()); + } } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClient.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClient.java index 8964267c..a5778b15 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClient.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClient.java @@ -8,6 +8,8 @@ public interface GeccoClient { + String getServerBase(); + FhirContext getFhirContext(); void testConnection(); diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientFactory.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientFactory.java index 3f97407d..1bb0c0fa 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientFactory.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientFactory.java @@ -39,21 +39,27 @@ private static final class GeccoClientStub implements GeccoClient } @Override - public void testConnection() + public String getServerBase() { - logger.warn("Stub implementation, no connection test performed"); + return null; } @Override - public GeccoFhirClient getFhirClient() + public FhirContext getFhirContext() { - return new GeccoFhirClientStub(this); + return fhirContext; } @Override - public FhirContext getFhirContext() + public void testConnection() { - return fhirContext; + logger.warn("Stub implementation, no connection test performed"); + } + + @Override + public GeccoFhirClient getFhirClient() + { + return new GeccoFhirClientStub(this); } @Override @@ -141,6 +147,11 @@ public GeccoClientFactory(Path trustStorePath, Path certificatePath, Path privat this.useChainedParameterNotLogicalReference = useChainedParameterNotLogicalReference; } + public String getServerBase() + { + return geccoServerBase; + } + public void testConnection() { try diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientImpl.java index 40962fba..b8eccfb7 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientImpl.java @@ -127,6 +127,18 @@ private void configureLoggingInterceptor(IGenericClient client) } } + @Override + public String getServerBase() + { + return geccoServerBase; + } + + @Override + public FhirContext getFhirContext() + { + return fhirContext; + } + @Override public void testConnection() { @@ -154,12 +166,6 @@ public GeccoFhirClient getFhirClient() } } - @Override - public FhirContext getFhirContext() - { - return fhirContext; - } - @Override public Path getSearchBundleOverride() { diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/fhir/AbstractFhirClient.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/fhir/AbstractFhirClient.java index 7f11e8cc..5bfe464d 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/fhir/AbstractFhirClient.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/fhir/AbstractFhirClient.java @@ -233,8 +233,9 @@ private Optional getIdentifierPatientReference(Patient patient private PatientReference getAbsoluteUrlPatientReference(Patient patient) { IdType idElement = patient.getIdElement(); - return PatientReference.from(new IdType(geccoClient.getGenericFhirClient().getServerBase(), - idElement.getResourceType(), idElement.getIdPart(), null).getValue()); + return PatientReference + .from(new IdType(geccoClient.getServerBase(), idElement.getResourceType(), idElement.getIdPart(), null) + .getValue()); } private Stream getPatients(Bundle bundle) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/error/ErrorOutputParameterGenerator.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/error/ErrorOutputParameterGenerator.java new file mode 100644 index 00000000..ac481a8a --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/error/ErrorOutputParameterGenerator.java @@ -0,0 +1,59 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.error; + +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_MEDIC; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.EXTENSION_ERROR_METADATA; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.EXTENSION_ERROR_METADATA_REFERENCE; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.EXTENSION_ERROR_METADATA_SOURCE; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.EXTENSION_ERROR_METADATA_TYPE; +import static org.highmed.dsf.bpe.ConstantsBase.CODESYSTEM_HIGHMED_BPMN; +import static org.highmed.dsf.bpe.ConstantsBase.CODESYSTEM_HIGHMED_BPMN_VALUE_ERROR; + +import java.util.stream.Stream; + +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Extension; +import org.hl7.fhir.r4.model.IdType; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity; +import org.hl7.fhir.r4.model.OperationOutcome.OperationOutcomeIssueComponent; +import org.hl7.fhir.r4.model.Reference; +import org.hl7.fhir.r4.model.StringType; +import org.hl7.fhir.r4.model.Task.TaskOutputComponent; + +public class ErrorOutputParameterGenerator +{ + public Stream createMeDicValidationError(IdType sourceId, OperationOutcome outcome) + { + return outcome.getIssue().stream() + .filter(i -> IssueSeverity.FATAL.equals(i.getSeverity()) || IssueSeverity.ERROR.equals(i.getSeverity())) + .map(i -> createMeDicValidationError(sourceId, i)); + } + + private TaskOutputComponent createMeDicValidationError(IdType sourceId, OperationOutcomeIssueComponent i) + { + TaskOutputComponent output = new TaskOutputComponent(); + output.getType().getCodingFirstRep().setSystem(CODESYSTEM_HIGHMED_BPMN) + .setCode(CODESYSTEM_HIGHMED_BPMN_VALUE_ERROR); + + Extension metaData = output.addExtension(); + metaData.setUrl(EXTENSION_ERROR_METADATA); + metaData.addExtension().setUrl(EXTENSION_ERROR_METADATA_TYPE) + .setValue(new Coding().setSystem(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR) + .setCode(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED)); + metaData.addExtension().setUrl(EXTENSION_ERROR_METADATA_SOURCE) + .setValue(new Coding().setSystem(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE) + .setCode(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_MEDIC)); + + if (sourceId != null) + metaData.addExtension().setUrl(EXTENSION_ERROR_METADATA_REFERENCE) + .setValue(new Reference().setReferenceElement(sourceId)); + + output.setValue(new StringType( + "Validation faild at " + i.getLocation().stream().map(StringType::getValue).findFirst().orElse("?"))); + + return output; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/logging/ErrorLogger.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/logging/ErrorLogger.java new file mode 100644 index 00000000..8dd25985 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/logging/ErrorLogger.java @@ -0,0 +1,17 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.logging; + +import org.hl7.fhir.r4.model.IdType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ErrorLogger +{ + private static final Logger validationLogger = LoggerFactory.getLogger("validation-error-logger"); + + public void logValidationFailed(IdType taskId) + { + validationLogger.debug( + "Validation of FHIR resources faild during execution of data-send process started by Task {}", + taskId.getValue()); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/FindNewData.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/FindNewData.java index bea445af..414b00c5 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/FindNewData.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/FindNewData.java @@ -133,6 +133,11 @@ protected PatientReferenceList searchForPatientReferencesWithNewData(DateWithPre GeccoFhirClient fhirClient = geccoClientFactory.getGeccoClient().getFhirClient(); - return fhirClient.getPatientReferencesWithNewData(exportFrom, exportTo); + PatientReferenceList references = fhirClient.getPatientReferencesWithNewData(exportFrom, exportTo); + + logger.info("Found {} patient{} with changes to transport", references.getReferences().size(), + references.getReferences().size() != 1 ? "s" : ""); + + return references; } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ReadData.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ReadData.java index 78b9c703..b416c717 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ReadData.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ReadData.java @@ -5,6 +5,7 @@ import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_VALUE_EXPORT_FROM; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_VALUE_EXPORT_TO; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.HAPI_USER_DATA_SOURCE_ID_ELEMENT; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.IDENTIFIER_NUM_CODEX_DIC_PSEUDONYM_TYPE_CODE; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.IDENTIFIER_NUM_CODEX_DIC_PSEUDONYM_TYPE_SYSTEM; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.NAMING_SYSTEM_NUM_CODEX_DIC_PSEUDONYM; @@ -36,6 +37,7 @@ import org.hl7.fhir.r4.model.DateTimeType; import org.hl7.fhir.r4.model.DiagnosticReport; import org.hl7.fhir.r4.model.DomainResource; +import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.Identifier; import org.hl7.fhir.r4.model.Immunization; import org.hl7.fhir.r4.model.InstantType; @@ -70,7 +72,8 @@ public class ReadData extends AbstractServiceDelegate private final GeccoClientFactory geccoClientFactory; public ReadData(FhirWebserviceClientProvider clientProvider, TaskHelper taskHelper, - ReadAccessHelper readAccessHelper, FhirContext fhirContext, GeccoClientFactory geccoClientFactory) + ReadAccessHelper readAccessHelper, FhirContext fhirContext, GeccoClientFactory geccoClientFactory, + String geccoServerBase) { super(clientProvider, taskHelper, readAccessHelper); @@ -156,9 +159,14 @@ protected Bundle toBundle(String pseudonym, Stream resources) List entries = resources.map(r -> { BundleEntryComponent entry = b.addEntry(); + + // storing original resource reference for validation error tracking + entry.setUserData(HAPI_USER_DATA_SOURCE_ID_ELEMENT, getAbsoluteId(r)); + entry.setFullUrl("urn:uuid:" + UUID.randomUUID()); entry.getRequest().setMethod(HTTPVerb.PUT).setUrl(getConditionalUpdateUrl(pseudonym, r)); entry.setResource(setSubjectOrIdentifier(clean(r), pseudonym)); + return entry; }).collect(Collectors.toList()); @@ -166,6 +174,15 @@ protected Bundle toBundle(String pseudonym, Stream resources) return b; } + private IdType getAbsoluteId(DomainResource r) + { + if (r == null) + return null; + + return r.getIdElement().isAbsolute() ? r.getIdElement() + : r.getIdElement().withServerBase(geccoClientFactory.getServerBase(), r.getResourceType().name()); + } + private DomainResource clean(DomainResource r) { r.setIdElement(null); diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java index 1332fa7c..9f9af31f 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java @@ -1,32 +1,213 @@ package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.BPMN_EXECUTION_VARIABLE_BUNDLE; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.HAPI_USER_DATA_SOURCE_ID_ELEMENT; + +import java.util.Objects; + import org.camunda.bpm.engine.delegate.BpmnError; import org.camunda.bpm.engine.delegate.DelegateExecution; import org.highmed.dsf.bpe.delegate.AbstractServiceDelegate; import org.highmed.dsf.fhir.authorization.read.ReadAccessHelper; import org.highmed.dsf.fhir.client.FhirWebserviceClientProvider; import org.highmed.dsf.fhir.task.TaskHelper; +import org.highmed.dsf.fhir.variables.FhirResourceValues; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; +import org.hl7.fhir.r4.model.Bundle.BundleEntryResponseComponent; +import org.hl7.fhir.r4.model.IdType; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity; +import org.hl7.fhir.r4.model.OperationOutcome.OperationOutcomeIssueComponent; +import org.hl7.fhir.r4.model.StringType; +import org.hl7.fhir.r4.model.Task; +import org.hl7.fhir.r4.model.Task.TaskStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.error.ErrorOutputParameterGenerator; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.logging.ErrorLogger; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactory; + public class ValidateData extends AbstractServiceDelegate { private static final Logger logger = LoggerFactory.getLogger(ValidateData.class); + private final BundleValidatorFactory bundleValidatorSupplier; + private final ErrorOutputParameterGenerator errorOutputParameterGenerator; + private final ErrorLogger errorLogger; + public ValidateData(FhirWebserviceClientProvider clientProvider, TaskHelper taskHelper, - ReadAccessHelper readAccessHelper) + ReadAccessHelper readAccessHelper, BundleValidatorFactory bundleValidatorSupplier, + ErrorOutputParameterGenerator errorOutputParameterGenerator, ErrorLogger errorLogger) { super(clientProvider, taskHelper, readAccessHelper); + + this.bundleValidatorSupplier = bundleValidatorSupplier; + this.errorOutputParameterGenerator = errorOutputParameterGenerator; + this.errorLogger = errorLogger; + } + + @Override + public void afterPropertiesSet() throws Exception + { + super.afterPropertiesSet(); + + Objects.requireNonNull(bundleValidatorSupplier, "bundleValidatorSupplier"); + Objects.requireNonNull(errorOutputParameterGenerator, "errorOutputParameterGenerator"); + Objects.requireNonNull(errorLogger, "errorLogger"); } @Override protected void doExecute(DelegateExecution execution) throws BpmnError, Exception { - // Bundle bundle = (Bundle) execution.getVariable(BPMN_EXECUTION_VARIABLE_BUNDLE); + if (!bundleValidatorSupplier.isEnabled()) + { + logger.warn("Validation disabled, skipping validation. Modify configuration to enable validation"); + return; + } + + bundleValidatorSupplier.create().ifPresentOrElse(validator -> + { + Bundle bundle = (Bundle) execution.getVariable(BPMN_EXECUTION_VARIABLE_BUNDLE); + + logger.info("Validating bundle with {} entr{}", bundle.getEntry().size(), + bundle.getEntry().size() == 1 ? "y" : "ies"); + + bundle = validator.validate(bundle); + + if (bundle.hasEntry()) + { + if (bundle.getEntry().stream().anyMatch(e -> !e.hasResponse() || !e.getResponse().hasOutcome() + || !(e.getResponse().getOutcome() instanceof OperationOutcome))) + { + logger.warn( + "Validation result bundle has entries wihout response.outcome instance of OperationOutcome"); + } + else + { + logValidationDetails(bundle); + + long resourcesWithErrorCount = bundle.getEntry().stream().filter(BundleEntryComponent::hasResponse) + .map(BundleEntryComponent::getResponse).filter(BundleEntryResponseComponent::hasOutcome) + .map(BundleEntryResponseComponent::getOutcome).filter(r -> r instanceof OperationOutcome) + .map(o -> (OperationOutcome) o).map( + o -> o.getIssue().stream() + .anyMatch(i -> IssueSeverity.FATAL.equals(i.getSeverity()) + || IssueSeverity.ERROR.equals(i.getSeverity()))) + .filter(b -> b).count(); + + if (resourcesWithErrorCount > 0) + { + logger.error("Validation of transfer bundle failed, {} resource{} with error", + resourcesWithErrorCount, resourcesWithErrorCount != 1 ? "s" : ""); - // TODO validate against fhir profiles - // TODO maybe check only one pseudonym used + addErrorsToTaskAndSetFailed(bundle); + errorLogger.logValidationFailed(getLeadingTaskFromExecutionVariables().getIdElement() + .withServerBase(getFhirWebserviceClientProvider().getLocalBaseUrl(), + getLeadingTaskFromExecutionVariables().getIdElement().getResourceType())); - logger.warn("Bundle validation not implemented"); + throw new BpmnError(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED, + "Validation of transfer bundle failed, " + resourcesWithErrorCount + " resource" + + (resourcesWithErrorCount != 1 ? "s" : "") + " with error"); + } + else + { + removeValidationResultsAndUserData(bundle); + } + } + } + else + { + logger.warn("Validation result bundle has no entries"); + } + }, () -> + { + logger.warn( + "{} not initialized, skipping validation. This is likley due to an error during startup of the process plugin", + BundleValidatorFactory.class.getSimpleName()); + }); + } + + private void logValidationDetails(Bundle bundle) + { + bundle.getEntry().stream().filter(e -> e.hasResponse() && e.getResponse().hasOutcome() + && (e.getResponse().getOutcome() instanceof OperationOutcome)).forEach(entry -> + { + IdType sourceId = (IdType) entry.getUserData(HAPI_USER_DATA_SOURCE_ID_ELEMENT); + OperationOutcome outcome = (OperationOutcome) entry.getResponse().getOutcome(); + + outcome.getIssue().forEach(i -> logValidationDetails(sourceId, i)); + }); + } + + private void logValidationDetails(IdType sourceId, OperationOutcomeIssueComponent i) + { + if (i.getSeverity() != null) + { + switch (i.getSeverity()) + { + case FATAL: + case ERROR: + logger.error( + "Validation error for {}{}: {}", sourceId.getValue(), i.getLocation().stream() + .map(StringType::getValue).findFirst().map(l -> " location " + l).orElse(""), + i.getDiagnostics()); + break; + case WARNING: + logger.warn( + "Validation warning for {}{}: {}", sourceId.getValue(), i.getLocation().stream() + .map(StringType::getValue).findFirst().map(l -> " location " + l).orElse(""), + i.getDiagnostics()); + break; + case INFORMATION: + case NULL: + default: + logger.info( + "Validation info for {}{}: {}", sourceId.getValue(), i.getLocation().stream() + .map(StringType::getValue).findFirst().map(l -> " location " + l).orElse(""), + i.getDiagnostics()); + break; + } + } + else + { + logger.info( + "Validation info for {}{}: {}", sourceId.getValue(), i.getLocation().stream() + .map(StringType::getValue).findFirst().map(l -> " location " + l).orElse(""), + i.getDiagnostics()); + } + } + + private void addErrorsToTaskAndSetFailed(Bundle bundle) + { + Task task = getLeadingTaskFromExecutionVariables(); + + task.setStatus(TaskStatus.FAILED); + bundle.getEntry().stream() + .filter(e -> e.hasResponse() && e.getResponse().hasOutcome() + && (e.getResponse().getOutcome() instanceof OperationOutcome) + && ((OperationOutcome) e.getResponse().getOutcome()).getIssue().stream() + .anyMatch(i -> IssueSeverity.FATAL.equals(i.getSeverity()) + || IssueSeverity.ERROR.equals(i.getSeverity()))) + .forEach(entry -> + { + IdType sourceId = (IdType) entry.getUserData(HAPI_USER_DATA_SOURCE_ID_ELEMENT); + OperationOutcome outcome = (OperationOutcome) entry.getResponse().getOutcome(); + + errorOutputParameterGenerator.createMeDicValidationError(sourceId, outcome) + .forEach(task::addOutput); + }); + } + + private void removeValidationResultsAndUserData(Bundle bundle) + { + bundle.getEntry().stream().forEach(e -> + { + e.clearUserData(HAPI_USER_DATA_SOURCE_ID_ELEMENT); + e.setResponse(null); + }); + execution.setVariable(BPMN_EXECUTION_VARIABLE_BUNDLE, FhirResourceValues.create(bundle)); } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java index f3132489..ecd2b221 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java @@ -10,6 +10,7 @@ import org.highmed.dsf.fhir.organization.EndpointProvider; import org.highmed.dsf.fhir.organization.OrganizationProvider; import org.highmed.dsf.fhir.task.TaskHelper; +import org.highmed.dsf.tools.generator.ProcessDocumentation; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -22,6 +23,8 @@ import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.client.fhir.GeccoFhirClient; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.crypto.CrrKeyProvider; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.crypto.CrrKeyProviderImpl; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.error.ErrorOutputParameterGenerator; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.logging.ErrorLogger; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.message.StartReceiveProcess; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.message.StartSendProcess; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.message.StartTranslateProcess; @@ -44,6 +47,7 @@ import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service.StoreDataForCrr; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service.StoreDataForTransferHub; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service.ValidateData; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactory; @Configuration public class TransferDataConfig @@ -66,126 +70,224 @@ public class TransferDataConfig @Autowired private FhirContext fhirContext; + @Autowired + private BundleValidatorFactory bundleValidatorFactory; + + @ProcessDocumentation(description = "PEM encoded file with trusted certificates to validate the server-certificate of the GECCO FHIR server", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, recommendation = "Use docker secret file to configure", example = "/run/secrets/gecco_fhir_server_ca.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.trust.certificates:#{null}}") private String fhirStoreTrustStore; + @ProcessDocumentation(description = "PEM encoded file with client-certificate, if GECCO FHIR server requires mutual TLS authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, recommendation = "Use docker secret file to configure", example = "/run/secrets/gecco_fhir_server_client_certificate.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.certificate:#{null}}") private String fhirStoreCertificate; + @ProcessDocumentation(description = "PEM encoded file with private-key for the client-certificate defined via `de.netzwerk.universitaetsmedizin.codex.gecco.server.certificate`", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, recommendation = "Use docker secret file to configure", example = "/run/secrets/gecco_fhir_server_client_certificate_private_key.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.private.key:#{null}}") private String fhirStorePrivateKey; + @ProcessDocumentation(description = "Password to decrypt the private-key defined via `de.netzwerk.universitaetsmedizin.codex.gecco.server.private.key`", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/gecco_fhir_server_client_certificate_private_key.pem.password") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.private.key.password:#{null}}") private char[] fhirStorePrivateKeyPassword; + @ProcessDocumentation(description = "Base URL of the GECCO FHIR server", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, example = "http://foo.bar/fhir") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.base.url:#{null}}") private String fhirStoreBaseUrl; + @ProcessDocumentation(description = "Basic authentication username to authenticate against the GECCO FHIR server, set if the server requests authentication using basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.basicauth.username:#{null}}") private String fhirStoreUsername; + @ProcessDocumentation(description = "Basic authentication password to authenticate against the GECCO FHIR server, set if the server requests authentication using basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/gecco_fhir_server_basicauth.password") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.basicauth.password:#{null}}") private String fhirStorePassword; + @ProcessDocumentation(description = "Bearer token to authenticate against the GECCO FHIR server, set if the server requests authentication using bearer token, cannot be set using docker secrets", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.bearer.token:#{null}}") private String fhirStoreBearerToken; + @ProcessDocumentation(description = "Connection timeout in milliseconds used when accessing the GECCO FHIR server, time until a connection needs to be established before aborting", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.timeout.connect:10000}") private int fhirStoreConnectTimeout; - @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.timeout.socket:10000}") - private int fhirStoreSocketTimeout; - + @ProcessDocumentation(description = "Connection request timeout in milliseconds used when requesting a connection from the connection manager while accessing the GECCO FHIR server, time until a connection needs to be established before aborting", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.timeout.connection.request:10000}") private int fhirStoreConnectionRequestTimeout; + @ProcessDocumentation(description = "Maximum period of inactivity in milliseconds between two consecutive data packets from the GECCO FHIR server, time until the server needs to send a data packet before aborting", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.timeout.socket:10000}") + private int fhirStoreSocketTimeout; + + @ProcessDocumentation(description = "GECCO FHIR Server client implementation class", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.client:de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.client.fhir.FhirBridgeClient}") private String fhirStoreClientClass; + @ProcessDocumentation(description = "To enable verbose logging of requests and replies to the GECCO FHIR server set to `true`", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.client.hapi.verbose:false}") private boolean fhirStoreHapiClientVerbose; + @ProcessDocumentation(description = "Forwarding proxy server url, set if the GECCO FHIR server can only be reached via a proxy server", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, example = "http://proxy.foo:8080") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.proxy.url:#{null}}") private String fhirStoreProxyUrl; + @ProcessDocumentation(description = "Forwarding proxy server basic authentication username, set if the GECCO FHIR server can only be reached via a proxy server that requires basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.proxy.username:#{null}}") private String fhirStoreProxyUsername; + @ProcessDocumentation(description = "Forwarding proxy server basic authentication password, set if the GECCO FHIR server can only be reached via a proxy server that requires basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/gecco_fhir_server_proxy_basicauth.password") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.proxy.password:#{null}}") private String fhirStoreProxyPassword; + @ProcessDocumentation(description = "To enable the use of logical references instead of chained parameters (`patient:identifier` instead of `patient.identifier`) while searching for Patients in the GECCO FHIR server set to `true`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataReceive") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.use.chained.parameter.not.logical.reference:true}") private boolean fhirStoreUseChainedParameterNotLogicalReference; + @ProcessDocumentation(description = "Location of a FHIR batch bundle used to override the internally provided bundle used to search for GECCO FHIR ressources", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.search.bundle.override:#{null}}") private String fhirStoreSearchBundleOverride; + @ProcessDocumentation(description = "Location of the CRR public-key e.g. [crr_public-key-pre-prod.pem](https://keys.num-codex.de/crr_public-key-pre-prod.pem) used to RSA encrypt FHIR bundles for the central repository", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Ask central repository management for the correct public key regarding the test and production environments", example = "/run/secrets/crr_public-key-pre-prod.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.crr.public.key:#{null}}") private String crrPublicKeyFile; + @ProcessDocumentation(description = "Location of the CRR private-key used to RSA decrypt FHIR bundles for the central repository", processNames = "wwwnetzwerk-universitaetsmedizinde_dataReceive", example = "/run/secrets/crr_private-key-pre-prod.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.crr.private.key:#{null}}") private String crrPrivateKeyFile; + @ProcessDocumentation(description = "DSF organization identifier of the GECCO Transfer Hub", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gth.identifier.value:hs-heilbronn.de}") private String geccoTransferHubIdentifierValue; + @ProcessDocumentation(description = "DSF organization identifier of the central research repository", processNames = "wwwnetzwerk-universitaetsmedizinde_dataTranslate") @Value("${de.netzwerk.universitaetsmedizin.codex.crr.identifier.value:num-codex.de}") private String crrIdentifierValue; - @Value("#{'${de.netzwerk.universitaetsmedizin.codex.consent.granted.oids.mdat.transfer:2.16.840.1.113883.3.1937.777.24.5.3.8,2.16.840.1.113883.3.1937.777.24.5.3.9,2.16.840.1.113883.3.1937.777.24.5.3.33,2.16.840.1.113883.3.1937.777.24.5.3.34}'.split(',')}") + @ProcessDocumentation(description = "List of expected consent provision permit codes for exporting GECCO resources", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.consent.granted.oids.mdat.transfer:" + + "2.16.840.1.113883.3.1937.777.24.5.3.8," + "2.16.840.1.113883.3.1937.777.24.5.3.9," + + "2.16.840.1.113883.3.1937.777.24.5.3.33," + "2.16.840.1.113883.3.1937.777.24.5.3.34" + + "}'.trim().split('(,[ ]?)|(\\n)')}") private List mdatTransferGrantedOids; - @Value("#{'${de.netzwerk.universitaetsmedizin.codex.consent.granted.oids.idat.merge:2.16.840.1.113883.3.1937.777.24.5.3.4}'.split(',')}") + @ProcessDocumentation(description = "List of expected consent provision permit codes for requesting a pseudonym based on a bloom filter from the fTTP", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.consent.granted.oids.idat.merge:" + + "2.16.840.1.113883.3.1937.777.24.5.3.4}'.trim().split('(,[ ]?)|(\\n)')}") private List idatMergeGrantedOids; + @ProcessDocumentation(description = "PEM encoded file with trusted certificates to validate the server-certificate of the fTTP server", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Use docker secret file to configure", example = "/run/secrets/fttp_server_ca.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.trust.certificates:#{null}}") private String fttpTrustStore; + @ProcessDocumentation(description = "PEM encoded file with client-certificate used to authenticated against fTTP server", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Use docker secret file to configure", example = "/run/secrets/fttp_server_client_certificate.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.certificate:#{null}}") private String fttpCertificate; + @ProcessDocumentation(description = "PEM encoded file with private-key for the client-certificate defined via `de.netzwerk.universitaetsmedizin.codex.fttp.certificate`", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Use docker secret file to configure", example = "/run/secrets/fttp_server_client_certificate_private_key.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.private.key:#{null}}") private String fttpPrivateKey; + @ProcessDocumentation(description = "Password to decrypt the private-key defined via `de.netzwerk.universitaetsmedizin.codex.fttp.private.key`", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/fttp_server_client_certificate_private_key.pem.password") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.private.key.password:#{null}}") private char[] fttpPrivateKeyPassword; + @ProcessDocumentation(description = "Connection timeout in milliseconds used when accessing the fTTP server, time until a connection needs to be established before aborting", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.timeout.connect:10000}") private int fttpConnectTimeout; - @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.timeout.socket:10000}") - private int fttpSocketTimeout; - + @ProcessDocumentation(description = "Connection request timeout in milliseconds used when requesting a connection from the connection manager while accessing the fTTP server, time until a connection needs to be established before aborting", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.timeout.connection.request:10000}") private int fttpConnectionRequestTimeout; + @ProcessDocumentation(description = "Maximum period of inactivity in milliseconds between two consecutive data packets from the fTTP server, time until the server needs to send a data packet before aborting", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) + @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.timeout.socket:10000}") + private int fttpSocketTimeout; + + @ProcessDocumentation(description = "Basic authentication username to authenticate against the fTTP server, set if the server requests authentication using basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.basicauth.username:#{null}}") private String fttpBasicAuthUsername; + @ProcessDocumentation(description = "Basic authentication password to authenticate against the fTTP server, set if the server requests authentication using basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/fttp_server_basicauth.password") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.basicauth.password:#{null}}") private String fttpBasicAuthPassword; + @ProcessDocumentation(description = "TODO", processNames = { "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Specify if you are using the send process to request pseudonyms from the fTTP") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.server.base.url:#{null}}") private String fttpServerBase; + @ProcessDocumentation(description = "Your organizations API key provided by the fTTP, the fTTP API key can not be defined via docker secret file and needs to be defined directly via the environment variable", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Specify if you are using the send process to request pseudonyms from the fTTP") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.api.key:#{null}}") private String fttpApiKey; + @ProcessDocumentation(description = "Study identifier specified by the fTTP", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.study:num}") private String fttpStudy; + @ProcessDocumentation(description = "Pseudonymization domain target identifier specified by the fTTP", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, example = "dic_heidelberg", recommendation = "Specify if you are using the send process to request pseudonyms from the fTTP") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.target:codex}") private String fttpTarget; + @ProcessDocumentation(description = "To enable verbose logging of requests and replies to the fTTP server set to `true`", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.client.hapi.verbose:false}") private boolean fttpHapiClientVerbose; + @ProcessDocumentation(description = "Forwarding proxy server url, set if the fTTP server can only be reached via a proxy server", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, example = "http://proxy.foo:8080") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.proxy.url:#{null}}") private String fttpProxyUrl; + @ProcessDocumentation(description = "Forwarding proxy server basic authentication username, set if the fTTPserver can only be reached via a proxy server that requires basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.proxy.username:#{null}}") private String fttpProxyUsername; + @ProcessDocumentation(description = "Forwarding proxy server basic authentication password, set if the fTTP server can only be reached via a proxy server that requires basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/fttp_server_proxy_basicauth.password") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.proxy.password:#{null}}") private String fttpProxyPassword; @@ -328,13 +430,27 @@ public HandleNoConsentIdatMerge handleNoConsentIdatMerge() @Bean public ReadData readData() { - return new ReadData(fhirClientProvider, taskHelper, readAccessHelper, fhirContext, geccoClientFactory()); + return new ReadData(fhirClientProvider, taskHelper, readAccessHelper, fhirContext, geccoClientFactory(), + fhirStoreBaseUrl); } @Bean public ValidateData validateData() { - return new ValidateData(fhirClientProvider, taskHelper, readAccessHelper); + return new ValidateData(fhirClientProvider, taskHelper, readAccessHelper, bundleValidatorFactory, + errorOutputParameterGenerator(), errorLogger()); + } + + @Bean + public ErrorOutputParameterGenerator errorOutputParameterGenerator() + { + return new ErrorOutputParameterGenerator(); + } + + @Bean + public ErrorLogger errorLogger() + { + return new ErrorLogger(); } @Bean diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java new file mode 100644 index 00000000..2a8dff7a --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java @@ -0,0 +1,494 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.EnumSet; +import java.util.List; +import java.util.Objects; +import java.util.UUID; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import javax.ws.rs.WebApplicationException; + +import org.bouncycastle.pkcs.PKCSException; +import org.highmed.dsf.fhir.json.ObjectMapperFactory; +import org.highmed.dsf.fhir.validation.SnapshotGenerator; +import org.highmed.dsf.fhir.validation.ValueSetExpander; +import org.highmed.dsf.fhir.validation.ValueSetExpanderImpl; +import org.highmed.dsf.tools.generator.ProcessDocumentation; +import org.hl7.fhir.r4.model.CapabilityStatement; +import org.hl7.fhir.r4.model.Enumerations.BindingStrength; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.support.IValidationSupport; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactory; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactoryImpl; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorImpl; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithFileSystemCache; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithModifiers; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClient; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClientJersey; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClientWithFileSystemCache; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageIdentifier; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageManager; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageManagerImpl; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpanderWithFileSystemCache; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClient; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClientJersey; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClientWithFileSystemCache; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.StructureDefinitionModifier; +import de.rwh.utils.crypto.CertificateHelper; +import de.rwh.utils.crypto.io.CertificateReader; +import de.rwh.utils.crypto.io.PemIo; + +@Configuration +public class ValidationConfig +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationConfig.class); + + @ProcessDocumentation(description = "Enables/disables local FHIR validation, set to `false` to skip validation", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation:true}") + private boolean validationEnabled; + + @ProcessDocumentation(description = "FHIR implementation guide package used to validated resources, specify as `name|version`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package:de.gecco|1.0.5}") + private String validationPackage; + + @ProcessDocumentation(description = "FHIR implementation guide packages that do not need to be downloaded, list with `name|version` values", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.noDownload:hl7.fhir.r4.core|4.0.1}'.trim().split('(,[ ]?)|(\\n)')}") + private List noDownloadPackages; + + @ProcessDocumentation(description = "Folder for storing downloaded FHIR implementation guide packages", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.cacheFolder:${java.io.tmpdir}/codex_gecco_validation_cache/Package}") + private String packageCacheFolder; + + @ProcessDocumentation(description = "Base The base address of the FHIR repository containing GECCO data.URL of the FHIR implementation guide package server to download validation packages from", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.server.baseUrl:https://packages.simplifier.net}") + private String packageServerBaseUrl; + + @ProcessDocumentation(description = "PEM encoded file with trusted certificates to validate the server-certificate of the FHIR implementation guide package server, uses the JVMs trust store if not specified", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure", example = "/run/secrets/validation_package_server_ca.pem") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.trust.certificates:#{null}}") + private String packageClientTrustCertificates; + + @ProcessDocumentation(description = "PEM encoded file with client-certificate, if the FHIR implementation guide package server requires mutual TLS authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure", example = "/run/secrets/validation_package_server_client_certificate.pem") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate:#{null}}") + private String packageClientCertificate; + + @ProcessDocumentation(description = "PEM encoded file with private-key for the client-certificate defined via `de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure", example = "/run/secrets/validation_package_server_client_certificate_private_key.pem") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate.private.key:#{null}}") + private String packageClientCertificatePrivateKey; + + @ProcessDocumentation(description = "Password to decrypt the private-key defined via `de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate.private.key`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/validation_package_server_client_certificate_private_key.pem.password") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate.private.key.password:#{null}}") + private char[] packageClientCertificatePrivateKeyPassword; + + @ProcessDocumentation(description = "Basic authentication username to authenticate against the FHIR implementation guide package server, set if the server requests authentication using basic authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.basic.username:#{null}}") + private String packageClientBasicAuthUsername; + + @ProcessDocumentation(description = "Basic authentication password to authenticate against the FHIR implementation guide package server, set if the server requests authentication using basic authentication ", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/validation_package_server_basicauth.password") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.basic.password:#{null}}") + private char[] packageClientBasicAuthPassword; + + @ProcessDocumentation(description = "Forwarding proxy server url, set if the FHIR implementation guide package server can only be reached via a proxy server", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", example = "http://proxy.foo:8080") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.proxy.schemeHostPort:#{null}}") + private String packageClientProxySchemeHostPort; + + @ProcessDocumentation(description = "Forwarding proxy server basic authentication username, set if the FHIR implementation guide package server can only be reached via a proxy server that requires basic authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.proxy.username:#{null}}") + private String packageClientProxyUsername; + + @ProcessDocumentation(description = "Forwarding proxy server basic authentication password, set if the FHIR implementation guide package server can only be reached via a proxy server that requires basic authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/validation_package_server_proxy_basicauth.password") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.proxy.password:#{null}}") + private char[] packageClientProxyPassword; + + @ProcessDocumentation(description = "Connection timeout in milliseconds used when accessing the FHIR implementation guide package server, time until a connection needs to be established before aborting", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.timeout.connect:10000}") + private int packageClientConnectTimeout; + + @ProcessDocumentation(description = "Read timeout in milliseconds used when accessing the FHIR implementation guide package server, time until the server needs to send a reply before aborting", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.timeout.read:300000}") + private int packageClientReadTimeout; + + @ProcessDocumentation(description = "To enable verbose logging of requests and replies to the FHIR implementation guide package server set to `true`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.verbose:false}") + private boolean packageClientVerbose; + + @ProcessDocumentation(description = "ValueSets found in the StructureDefinitions with the specified binding strength will be expanded", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.bindingStrength:required,extensible,preferred,example}'.trim().split('(,[ ]?)|(\\n)')}") + private List valueSetExpansionBindingStrengths; + + @ProcessDocumentation(description = "Folder for storing expanded ValueSets", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.cacheFolder:${java.io.tmpdir}/codex_gecco_validation_cache/ValueSet}") + private String valueSetCacheFolder; + + @ProcessDocumentation(description = "Base URL of the terminology server used to expand FHIR ValueSets", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Specify a local terminology server to improve ValueSet expansion speed") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.server.baseUrl:https://terminology-highmed.medic.medfak.uni-koeln.de/fhir}") + private String valueSetExpansionServerBaseUrl; + + @ProcessDocumentation(description = "PEM encoded file with trusted certificates to validate the server-certificate of the terminology server, uses the JVMs trust store if not specified", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure", example = "/run/secrets/terminology_server_ca.pem") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.trust.certificates:#{null}}") + private String valueSetExpansionClientTrustCertificates; + + @ProcessDocumentation(description = "PEM encoded file with client-certificate, if the terminology server requires mutual TLS authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure", example = "/run/secrets/terminology_server_client_certificate.pem") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate:#{null}}") + private String valueSetExpansionClientCertificate; + + @ProcessDocumentation(description = "PEM encoded file with private-key for the client-certificate defined via `de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure", example = "/run/secrets/terminology_server_client_certificate_private_key.pem") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate.private.key:#{null}}") + private String valueSetExpansionClientCertificatePrivateKey; + + @ProcessDocumentation(description = "Password to decrypt the private-key defined via `de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate.private.key`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/terminology_server_client_certificate_private_key.pem.password") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate.private.key.password:#{null}}") + private char[] valueSetExpansionClientCertificatePrivateKeyPassword; + + @ProcessDocumentation(description = "Basic authentication username to authenticate against the terminology server, set if the server requests authentication using basic authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.basic.username:#{null}}") + private String valueSetExpansionClientBasicAuthUsername; + + @ProcessDocumentation(description = "Basic authentication password to authenticate against the terminology server, set if the server requests authentication using basic authentication ", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/terminology_server_basicauth.password") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.basic.password:#{null}}") + private char[] valueSetExpansionClientBasicAuthPassword; + + @ProcessDocumentation(description = "Forwarding proxy server url, set if the terminology server can only be reached via a proxy server", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", example = "http://proxy.foo:8080") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.proxy.schemeHostPort:#{null}}") + private String valueSetExpansionClientProxySchemeHostPort; + + @ProcessDocumentation(description = "Forwarding proxy server basic authentication username, set if the terminology server can only be reached via a proxy server that requires basic authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.proxy.username:#{null}}") + private String valueSetExpansionClientProxyUsername; + + @ProcessDocumentation(description = "Forwarding proxy server basic authentication password, set if the terminology server can only be reached via a proxy server that requires basic authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/terminology_server_proxy_basicauth.password") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.proxy.password:#{null}}") + private char[] valueSetExpansionClientProxyPassword; + + @ProcessDocumentation(description = "Connection timeout in milliseconds used when accessing the terminology server, time until a connection needs to be established before aborting", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.timeout.connect:10000}") + private int valueSetExpansionClientConnectTimeout; + + @ProcessDocumentation(description = "Read timeout in milliseconds used when accessing the terminology server, time until the server needs to send a reply before aborting", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.timeout.read:300000}") + private int valueSetExpansionClientReadTimeout; + + @ProcessDocumentation(description = "To enable verbose logging of requests and replies to the terminology server set to `true`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.verbose:false}") + private boolean valueSetExpansionClientVerbose; + + @ProcessDocumentation(description = "List of StructureDefinition modifier classes, modifiers are executed before atempting to generate a StructureDefinition snapshot", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.structuredefinition.modifierClasses:" + + "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover," + + "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.MiiModuleLabObservationLab10IdentifierRemover," + + "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.GeccoRadiologyProceduresCodingSliceMinFixer" + + "}'.trim().split('(,[ ]?)|(\\n)')}") + private List structureDefinitionModifierClasses; + + @ProcessDocumentation(description = "Folder for storing generated StructureDefinition snapshots", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.structuredefinition.cacheFolder:${java.io.tmpdir}/codex_gecco_validation_cache/StructureDefinition}") + private String structureDefinitionCacheFolder; + + @Value("${java.io.tmpdir}") + private String systemTempFolder; + + @Autowired + private FhirContext fhirContext; + + @Bean + public ValidationPackageIdentifier validationPackageIdentifier() + { + if (validationPackage == null || validationPackage.isBlank()) + throw new IllegalArgumentException("Validation package not specified"); + + return ValidationPackageIdentifier.fromString(validationPackage); + } + + @Bean + public ValidationPackageManager validationPackageManager() + { + List noDownload = noDownloadPackages.stream() + .map(ValidationPackageIdentifier::fromString).collect(Collectors.toList()); + EnumSet bindingStrengths = EnumSet.copyOf( + valueSetExpansionBindingStrengths.stream().map(BindingStrength::fromCode).collect(Collectors.toList())); + + return new ValidationPackageManagerImpl(validationPackageClient(), valueSetExpansionClient(), + validationObjectMapper(), fhirContext, internalSnapshotGeneratorFactory(), + internalValueSetExpanderFactory(), noDownload, bindingStrengths); + } + + private StructureDefinitionModifier createStructureDefinitionModifier(String className) + { + try + { + Class modifierClass = Class.forName(className); + if (StructureDefinitionModifier.class.isAssignableFrom(modifierClass)) + return (StructureDefinitionModifier) modifierClass.getConstructor().newInstance(); + else + throw new IllegalArgumentException( + "Class " + className + " not compatible with " + StructureDefinitionModifier.class.getName()); + } + catch (ClassNotFoundException | InstantiationException | IllegalAccessException | InvocationTargetException + | NoSuchMethodException | SecurityException e) + { + throw new RuntimeException(e); + } + } + + @Bean + public BiFunction internalSnapshotGeneratorFactory() + { + List structureDefinitionModifiers = structureDefinitionModifierClasses.stream() + .map(this::createStructureDefinitionModifier).collect(Collectors.toList()); + + return (fc, vs) -> new PluginSnapshotGeneratorWithFileSystemCache(structureDefinitionCacheFolder(), fc, + new PluginSnapshotGeneratorWithModifiers(new PluginSnapshotGeneratorImpl(fc, vs), + structureDefinitionModifiers)); + } + + @Bean + public Path structureDefinitionCacheFolder() + { + return cacheFolder("StructureDefinition", structureDefinitionCacheFolder); + } + + @Bean + public BiFunction internalValueSetExpanderFactory() + { + return (fc, vs) -> new ValueSetExpanderWithFileSystemCache(valueSetCacheFolder(), fc, + new ValueSetExpanderImpl(fc, vs)); + } + + private Path cacheFolder(String cacheFolderType, String cacheFolder) + { + Objects.requireNonNull(cacheFolder, "cacheFolder"); + Path cacheFolderPath = Paths.get(cacheFolder); + + try + { + if (cacheFolderPath.startsWith(systemTempFolder)) + { + Files.createDirectories(cacheFolderPath); + logger.debug("Cache folder for type {} created at {}", cacheFolderType, + cacheFolderPath.toAbsolutePath().toString()); + } + + if (!Files.isWritable(cacheFolderPath)) + throw new IOException("Cache folder for type " + cacheFolderType + " + at " + + cacheFolderPath.toAbsolutePath().toString() + " not writable"); + else + return cacheFolderPath; + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + private Path checkReadable(String file) + { + if (file == null) + return null; + else + { + Path path = Paths.get(file); + + if (!Files.isReadable(path)) + throw new RuntimeException(path.toString() + " not readable"); + + return path; + } + } + + private KeyStore trustStore(String trustStoreType, String trustCertificatesFile) + { + if (trustCertificatesFile == null) + return null; + + Path trustCertificatesPath = checkReadable(trustCertificatesFile); + + try + { + logger.debug("Creating trust-store for {} from {}", trustStoreType, trustCertificatesPath.toString()); + return CertificateReader.allFromCer(trustCertificatesPath); + } + catch (NoSuchAlgorithmException | CertificateException | KeyStoreException | IOException e) + { + throw new RuntimeException(e); + } + } + + private KeyStore keyStore(String keyStoreType, String clientCertificateFile, String clientCertificatePrivateKeyFile, + char[] clientCertificatePrivateKeyPassword, char[] keyStorePassword) + { + if ((clientCertificateFile != null) != (clientCertificatePrivateKeyFile != null)) + throw new IllegalArgumentException(keyStoreType + " certificate or private-key not specified"); + else if (clientCertificateFile == null && clientCertificatePrivateKeyFile == null) + return null; + + Path clientCertificatePath = checkReadable(clientCertificateFile); + Path clientCertificatePrivateKeyPath = checkReadable(clientCertificatePrivateKeyFile); + + try + { + PrivateKey privateKey = PemIo.readPrivateKeyFromPem(clientCertificatePrivateKeyPath, + clientCertificatePrivateKeyPassword); + X509Certificate certificate = PemIo.readX509CertificateFromPem(clientCertificatePath); + + logger.debug("Creating key-store for {} from {} and {} with password {}", keyStoreType, + clientCertificatePath.toString(), clientCertificatePrivateKeyPath.toString(), + clientCertificatePrivateKeyPassword != null ? "***" : "null"); + return CertificateHelper.toJksKeyStore(privateKey, new Certificate[] { certificate }, + UUID.randomUUID().toString(), keyStorePassword); + } + catch (NoSuchAlgorithmException | CertificateException | KeyStoreException | IOException | PKCSException e) + { + throw new RuntimeException(e); + } + } + + @Bean + public ValidationPackageClient validationPackageClient() + { + return new ValidationPackageClientWithFileSystemCache(packageCacheFolder(), validationObjectMapper(), + validationPackageClientJersey()); + } + + @Bean + public Path packageCacheFolder() + { + return cacheFolder("Package", packageCacheFolder); + } + + private ValidationPackageClientJersey validationPackageClientJersey() + { + if ((packageClientBasicAuthUsername != null) != (packageClientBasicAuthPassword != null)) + { + throw new IllegalArgumentException( + "Package client basic authentication username or password not specified"); + } + + if ((packageClientProxyUsername != null) != (packageClientProxyPassword != null)) + { + throw new IllegalArgumentException("Package client proxy username or password not specified"); + } + + KeyStore packageClientTrustStore = trustStore("FHIR package client", packageClientTrustCertificates); + char[] packageClientKeyStorePassword = UUID.randomUUID().toString().toCharArray(); + KeyStore packageClientKeyStore = keyStore("FHIR package client", packageClientCertificate, + packageClientCertificatePrivateKey, packageClientCertificatePrivateKeyPassword, + packageClientKeyStorePassword); + + return new ValidationPackageClientJersey(packageServerBaseUrl, packageClientTrustStore, packageClientKeyStore, + packageClientKeyStore == null ? null : packageClientKeyStorePassword, packageClientBasicAuthUsername, + packageClientBasicAuthPassword, packageClientProxySchemeHostPort, packageClientProxyUsername, + packageClientProxyPassword, packageClientConnectTimeout, packageClientReadTimeout, + packageClientVerbose); + } + + @Bean + public ValueSetExpansionClient valueSetExpansionClient() + { + return new ValueSetExpansionClientWithFileSystemCache(valueSetCacheFolder(), fhirContext, + valueSetExpansionClientJersey()); + } + + @Bean + public Path valueSetCacheFolder() + { + return cacheFolder("ValueSet", valueSetCacheFolder); + } + + private ValueSetExpansionClient valueSetExpansionClientJersey() + { + if ((valueSetExpansionClientBasicAuthUsername != null) != (valueSetExpansionClientBasicAuthPassword != null)) + { + throw new IllegalArgumentException( + "ValueSet expansion client basic authentication username or password not specified"); + } + + if ((valueSetExpansionClientProxyUsername != null) != (valueSetExpansionClientProxyPassword != null)) + { + throw new IllegalArgumentException("ValueSet expansion client proxy username or password not specified"); + } + + KeyStore valueSetExpansionClientTrustStore = trustStore("ValueSet expansion client", + valueSetExpansionClientTrustCertificates); + char[] valueSetExpansionClientKeyStorePassword = UUID.randomUUID().toString().toCharArray(); + KeyStore valueSetExpansionClientKeyStore = keyStore("ValueSet expansion client", + valueSetExpansionClientCertificate, valueSetExpansionClientCertificatePrivateKey, + valueSetExpansionClientCertificatePrivateKeyPassword, valueSetExpansionClientKeyStorePassword); + + return new ValueSetExpansionClientJersey(valueSetExpansionServerBaseUrl, valueSetExpansionClientTrustStore, + valueSetExpansionClientKeyStore, + valueSetExpansionClientKeyStore == null ? null : valueSetExpansionClientKeyStorePassword, + valueSetExpansionClientBasicAuthUsername, valueSetExpansionClientBasicAuthPassword, + valueSetExpansionClientProxySchemeHostPort, valueSetExpansionClientProxyUsername, + valueSetExpansionClientProxyPassword, valueSetExpansionClientConnectTimeout, + valueSetExpansionClientReadTimeout, valueSetExpansionClientVerbose, validationObjectMapper(), + fhirContext); + } + + @Bean + public ObjectMapper validationObjectMapper() + { + return ObjectMapperFactory.createObjectMapper(fhirContext); + } + + public boolean testConnectionToTerminologyServer() + { + logger.info( + "Testing connection to terminology server with {trustStorePath: {}, certificatePath: {}, privateKeyPath: {}, privateKeyPassword: {}," + + " basicAuthUsername {}, basicAuthPassword {}, serverBase: {}, proxyUrl {}, proxyUsername, proxyPassword {}}", + valueSetExpansionClientTrustCertificates, valueSetExpansionClientCertificate, + valueSetExpansionClientCertificatePrivateKey, + valueSetExpansionClientCertificatePrivateKeyPassword != null ? "***" : "null", + valueSetExpansionClientBasicAuthUsername, + valueSetExpansionClientBasicAuthPassword != null ? "***" : "null", valueSetExpansionServerBaseUrl, + valueSetExpansionClientProxySchemeHostPort, valueSetExpansionClientProxyUsername, + valueSetExpansionClientProxyPassword != null ? "***" : "null"); + + try + { + CapabilityStatement metadata = valueSetExpansionClient().getMetadata(); + logger.info("Connection test OK: {} - {}", metadata.getSoftware().getName(), + metadata.getSoftware().getVersion()); + return true; + } + catch (Exception e) + { + if (e instanceof WebApplicationException) + { + String response = ((WebApplicationException) e).getResponse().readEntity(String.class); + logger.error("Connection test failed: {} - {}", e.getMessage(), response); + } + else + logger.error("Connection test failed: {}", e.getMessage()); + + return false; + } + } + + @Bean + public BundleValidatorFactory bundleValidatorFactory() + { + return new BundleValidatorFactoryImpl(validationEnabled, validationPackageManager(), + validationPackageIdentifier()); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFhirResourceFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFhirResourceFileSystemCache.java new file mode 100644 index 00000000..0ae7fcfc --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFhirResourceFileSystemCache.java @@ -0,0 +1,82 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Objects; +import java.util.function.Function; + +import org.hl7.fhir.r4.model.Resource; +import org.springframework.beans.factory.InitializingBean; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.model.api.annotation.ResourceDef; +import ca.uhn.fhir.parser.IParser; + +public abstract class AbstractFhirResourceFileSystemCache extends AbstractFileSystemCache + implements InitializingBean +{ + private final Class resourceType; + private final FhirContext fhirContext; + + /** + * For JSON content with gzip compression using the .json.gz file name suffix. + * + * @param cacheFolder + * not null + * @param resourceType + * not null + * @param fhirContext + * not null + * @see AbstractFileSystemCache#FILENAME_SUFFIX + * @see AbstractFileSystemCache#OUT_COMPRESSOR_FACTORY + * @see AbstractFileSystemCache#IN_COMPRESSOR_FACTORY + */ + public AbstractFhirResourceFileSystemCache(Path cacheFolder, Class resourceType, FhirContext fhirContext) + { + super(cacheFolder, AbstractFileSystemCache.FILENAME_SUFFIX, AbstractFileSystemCache.OUT_COMPRESSOR_FACTORY, + AbstractFileSystemCache.IN_COMPRESSOR_FACTORY); + + this.resourceType = resourceType; + this.fhirContext = fhirContext; + } + + public AbstractFhirResourceFileSystemCache(Path cacheFolder, String fileNameSuffix, + FunctionWithIoException outCompressorFactory, + FunctionWithIoException inCompressorFactory, Class resourceType, + FhirContext fhirContext) + { + super(cacheFolder, fileNameSuffix, outCompressorFactory, inCompressorFactory); + + this.resourceType = resourceType; + this.fhirContext = fhirContext; + } + + @Override + public void afterPropertiesSet() throws Exception + { + super.afterPropertiesSet(); + + Objects.requireNonNull(resourceType, "resourceType"); + Objects.requireNonNull(fhirContext, "fhirContext"); + } + + protected IParser getJsonParser() + { + return fhirContext.newJsonParser(); + } + + protected T readResourceFromCache(String url, String version, Function fromResource) throws IOException + { + return readFromCache(url + "|" + version, resourceType.getAnnotation(ResourceDef.class).name(), + reader -> getJsonParser().parseResource(resourceType, reader), fromResource); + } + + protected T writeRsourceToCache(T value, Function toResource, Function toUrl, + Function toVersion) throws IOException + { + return writeToCache(value, r -> toUrl.apply(r) + "|" + toVersion.apply(r), r -> r.getResourceType().name(), + (w, r) -> getJsonParser().encodeResourceToWriter(r, w), toResource); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFileSystemCache.java new file mode 100644 index 00000000..9a234665 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFileSystemCache.java @@ -0,0 +1,153 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Objects; +import java.util.function.Function; + +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +public abstract class AbstractFileSystemCache implements InitializingBean +{ + public static final String FILENAME_SUFFIX = ".json.gz"; + public static final FunctionWithIoException OUT_COMPRESSOR_FACTORY = GzipCompressorOutputStream::new; + public static final FunctionWithIoException IN_COMPRESSOR_FACTORY = GzipCompressorInputStream::new; + + @FunctionalInterface + public interface FunctionWithIoException + { + R apply(T t) throws IOException; + } + + @FunctionalInterface + public interface BiConsumerWithIoException + { + void accept(T t, U u) throws IOException; + } + + private static final Logger logger = LoggerFactory.getLogger(AbstractFileSystemCache.class); + + private final Path cacheFolder; + private final String filenameSuffix; + private final FunctionWithIoException outCompressorFactory; + private final FunctionWithIoException inCompressorFactory; + + /** + * For JSON content with gzip compression using the .json.gz file name suffix. + * + * @param cacheFolder + * not null + * @see #FILENAME_SUFFIX + * @see #OUT_COMPRESSOR_FACTORY + * @see #IN_COMPRESSOR_FACTORY + */ + public AbstractFileSystemCache(Path cacheFolder) + { + this(cacheFolder, FILENAME_SUFFIX, OUT_COMPRESSOR_FACTORY, IN_COMPRESSOR_FACTORY); + } + + public AbstractFileSystemCache(Path cacheFolder, String filenameSuffix, + FunctionWithIoException outCompressorFactory, + FunctionWithIoException inCompressorFactory) + { + this.cacheFolder = cacheFolder; + this.filenameSuffix = filenameSuffix; + this.outCompressorFactory = outCompressorFactory; + this.inCompressorFactory = inCompressorFactory; + } + + @Override + public void afterPropertiesSet() throws Exception + { + Objects.requireNonNull(cacheFolder, "cacheFolder"); + Objects.requireNonNull(filenameSuffix, "filenameSuffix"); + Objects.requireNonNull(outCompressorFactory, "outCompressorFactory"); + Objects.requireNonNull(inCompressorFactory, "inCompressorFactory"); + + if (!Files.isWritable(cacheFolder)) + throw new IOException("Folder " + cacheFolder.toAbsolutePath().toString() + "not writable"); + } + + private Path cacheFile(String cacheEntryId) + { + cacheEntryId = cacheEntryId.replace("://", "_").replaceAll("/", "_").replace(":", "_").replace("|", "_") + .replace("\\", "_"); + + return cacheFolder.resolve(cacheEntryId + filenameSuffix); + } + + protected final T readFromCache(String cacheEntryId, String cacheEntryType, + FunctionWithIoException decoder) throws IOException + { + return readFromCache(cacheEntryId, cacheEntryType, decoder, Function.identity()); + } + + protected final T readFromCache(String cacheEntryId, String cacheEntryType, + FunctionWithIoException decoder, Function fromResource) throws IOException + { + Path cacheFile = cacheFile(cacheEntryId); + + if (!Files.exists(cacheFile)) + { + logger.debug("Cache file for {} {} does not exist", cacheEntryType, cacheEntryId); + return null; + } + else if (Files.exists(cacheFile) && !Files.isReadable(cacheFile)) + { + logger.error("Cache file for {} {} exist in cache but is not readable", cacheEntryType, cacheEntryId); + return null; + } + + try (InputStream in = Files.newInputStream(cacheFile); + BufferedInputStream bIn = new BufferedInputStream(in); + InputStream cIn = inCompressorFactory.apply(bIn); + InputStreamReader reader = new InputStreamReader(cIn, StandardCharsets.UTF_8)) + { + logger.debug("Reading {} {} from cache at {}", cacheEntryType, cacheEntryId, cacheFile.toString()); + return fromResource.apply(decoder.apply(reader)); + } + } + + protected final T writeToCache(T value, Function toCacheId, Function toCacheEntryType, + BiConsumerWithIoException encoder) throws IOException + { + return writeToCache(value, toCacheId, toCacheEntryType, encoder, Function.identity()); + } + + protected final T writeToCache(T value, Function toCacheId, Function toCacheEntryType, + BiConsumerWithIoException encoder, Function toResource) throws IOException + { + R resource = toResource.apply(value); + String cacheId = toCacheId.apply(resource); + String cacheEntryType = toCacheEntryType.apply(resource); + + Path cacheFile = cacheFile(cacheId); + + try (OutputStream out = Files.newOutputStream(cacheFile, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + BufferedOutputStream bOut = new BufferedOutputStream(out); + OutputStream cOut = outCompressorFactory.apply(bOut); + OutputStreamWriter writer = new OutputStreamWriter(cOut, StandardCharsets.UTF_8)) + { + logger.debug("Writing {} {} to cache at {}", cacheEntryType, cacheId, cacheFile.toString()); + encoder.accept(writer, resource); + } + + return value; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidator.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidator.java new file mode 100644 index 00000000..b2d88d02 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidator.java @@ -0,0 +1,18 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import org.highmed.dsf.fhir.validation.ResourceValidator; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.OperationOutcome; + +public interface BundleValidator extends ResourceValidator +{ + /** + * Validated all bundle entries with a entry.resource. The validation result will be added as a + * {@link OperationOutcome} resource to the corresponding entry.response.outcome property. + * + * @param bundle + * not null + * @return given bundle with added entry.response.outcome properties + */ + Bundle validate(Bundle bundle); +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java new file mode 100644 index 00000000..275db6d6 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java @@ -0,0 +1,23 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.Optional; + +public interface BundleValidatorFactory +{ + /** + * @return true if validation is enabled + */ + boolean isEnabled(); + + /** + * Initializes the {@link BundleValidatorFactory} by downloading all necessary FHIR implementation guides, expanding + * ValueSets and generating StructureDefinition snapshots. + */ + void init(); + + /** + * @return {@link Optional#empty()} if this {@link BundleValidatorFactory} was not initialized + * @see BundleValidatorFactory#init() + */ + Optional create(); +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java new file mode 100644 index 00000000..2bbe86d4 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java @@ -0,0 +1,67 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.Objects; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +import ca.uhn.fhir.context.support.IValidationSupport; + +public class BundleValidatorFactoryImpl implements BundleValidatorFactory, InitializingBean +{ + private static final Logger logger = LoggerFactory.getLogger(BundleValidatorFactoryImpl.class); + + private final boolean validationEnabled; + private final ValidationPackageManager validationPackageManager; + private final ValidationPackageIdentifier validationPackageIdentifier; + + private IValidationSupport validationSupport; + private ValidationPackageWithDepedencies packageWithDependencies; + + public BundleValidatorFactoryImpl(boolean validationEnabled, ValidationPackageManager validationPackageManager, + ValidationPackageIdentifier validationPackageIdentifier) + { + this.validationEnabled = validationEnabled; + this.validationPackageManager = validationPackageManager; + this.validationPackageIdentifier = validationPackageIdentifier; + } + + @Override + public void afterPropertiesSet() throws Exception + { + Objects.requireNonNull(validationPackageManager, "validationPackageManager"); + Objects.requireNonNull(validationPackageIdentifier, "validationPackageIdentifier"); + } + + @Override + public boolean isEnabled() + { + return validationEnabled; + } + + @Override + public void init() + { + if (validationSupport != null) + return; + + logger.info("Downloading FHIR validation package {} and dependencies", validationPackageIdentifier.toString()); + packageWithDependencies = validationPackageManager.downloadPackageWithDependencies(validationPackageIdentifier); + + logger.info("Expanding ValueSets and generating StructureDefinition snapshots"); + validationSupport = validationPackageManager + .expandValueSetsAndGenerateStructureDefinitionSnapshots(packageWithDependencies); + } + + @Override + public Optional create() + { + if (validationPackageManager == null || validationSupport == null) + return Optional.empty(); + else + return Optional + .of(validationPackageManager.createBundleValidator(validationSupport, packageWithDependencies)); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java new file mode 100644 index 00000000..f4c05730 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java @@ -0,0 +1,108 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.highmed.dsf.fhir.validation.ResourceValidator; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; +import org.hl7.fhir.r4.model.CanonicalType; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.Resource; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.StructureDefinition.StructureDefinitionKind; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.validation.ResultSeverityEnum; +import ca.uhn.fhir.validation.SingleValidationMessage; +import ca.uhn.fhir.validation.ValidationResult; + +public class BundleValidatorImpl implements BundleValidator +{ + private static final Logger logger = LoggerFactory.getLogger(BundleValidatorImpl.class); + + private final FhirContext fhirContext; + private final ResourceValidator delegate; + private final Set expectedStructureDefinitionUrls; + private final Set expectedStructureDefinitionUrlsWithVersion; + + public BundleValidatorImpl(FhirContext fhirContext, ValidationPackageWithDepedencies packageWithDependencies, + ResourceValidator delegate) + { + this.fhirContext = Objects.requireNonNull(fhirContext, "fhirContext"); + + Set sds = Objects.requireNonNull(packageWithDependencies, "packageWithDependencies") + .getValidationSupportResources().getStructureDefinitions().stream() + .flatMap(sd -> Stream.concat(Stream.of(sd), + packageWithDependencies.getStructureDefinitionDependencies(sd).stream())) + .filter(sd -> StructureDefinitionKind.RESOURCE.equals(sd.getKind())) + .filter(sd -> !sd.hasAbstract() || !sd.getAbstract()).filter(StructureDefinition::hasUrl) + .collect(Collectors.toSet()); + + expectedStructureDefinitionUrls = sds.stream().map(StructureDefinition::getUrl).collect(Collectors.toSet()); + expectedStructureDefinitionUrlsWithVersion = sds.stream().filter(StructureDefinition::hasVersion) + .map(sd -> sd.getUrl() + "|" + sd.getVersion()).collect(Collectors.toSet()); + + this.delegate = Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public ValidationResult validate(Resource resource) + { + Objects.requireNonNull(resource, "resource"); + + Set profiles = resource.getMeta().getProfile().stream().map(CanonicalType::getValue) + .collect(Collectors.toSet()); + + if (!Collections.disjoint(profiles, expectedStructureDefinitionUrls) + || !Collections.disjoint(profiles, expectedStructureDefinitionUrlsWithVersion)) + { + // at least one supported profile claimed + return delegate.validate(resource); + } + else + { + SingleValidationMessage message = new SingleValidationMessage(); + message.setLocationString(resource.getResourceType().name() + ".meta.profile"); + + String messageText; + if (profiles.isEmpty()) + messageText = "No supported profile claimed"; + else + messageText = "No supported profile claimed, profile" + (profiles.size() == 1 ? "" : "s") + " " + + profiles.stream().sorted().collect(Collectors.joining(", ", "[", "]")) + " not supported"; + + message.setMessage(messageText); + message.setSeverity(ResultSeverityEnum.ERROR); + + logger.debug("Supported profiles {}", expectedStructureDefinitionUrlsWithVersion.stream().sorted() + .collect(Collectors.joining(", ", "[", "]"))); + + return new ValidationResult(fhirContext, List.of(message)); + } + } + + @Override + public Bundle validate(Bundle bundle) + { + Objects.requireNonNull(bundle, "bundle"); + + bundle.getEntry().stream().forEach(this::validateAndSetOutcome); + return bundle; + } + + private void validateAndSetOutcome(BundleEntryComponent entry) + { + if (entry.hasResource()) + { + ValidationResult validationResult = validate(entry.getResource()); + entry.getResponse().setOutcome((OperationOutcome) validationResult.toOperationOutcome()); + } + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/CodeValidatorForExpandedValueSets.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/CodeValidatorForExpandedValueSets.java new file mode 100644 index 00000000..80ce5bc2 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/CodeValidatorForExpandedValueSets.java @@ -0,0 +1,138 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import static org.apache.commons.lang3.StringUtils.defaultString; +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; + +import java.util.ArrayList; +import java.util.List; + +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.ValueSet; +import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionComponent; +import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent; +import org.hl7.fhir.utilities.validation.ValidationMessage; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.support.ConceptValidationOptions; +import ca.uhn.fhir.context.support.IValidationSupport; +import ca.uhn.fhir.context.support.ValidationSupportContext; +import ca.uhn.fhir.util.VersionIndependentConcept; + +public class CodeValidatorForExpandedValueSets implements IValidationSupport +{ + private final FhirContext fhirContext; + + public CodeValidatorForExpandedValueSets(FhirContext fhirContext) + { + this.fhirContext = fhirContext; + } + + @Override + public FhirContext getFhirContext() + { + return fhirContext; + } + + @Override + public boolean isValueSetSupported(ValidationSupportContext supportContext, String valueSetUrl) + { + return supportContext.getRootValidationSupport().fetchResource(ValueSet.class, valueSetUrl).hasExpansion(); + } + + @Override + public CodeValidationResult validateCodeInValueSet(ValidationSupportContext supportContext, + ConceptValidationOptions options, String codeSystem, String code, String display, IBaseResource valueSet) + { + if (valueSet == null || !(valueSet instanceof ValueSet) || !((ValueSet) valueSet).hasExpansion()) + return new CodeValidationResult().setSeverity(IssueSeverity.ERROR).setMessage("ValueSet not supported"); + + ValueSetExpansionComponent expansion = ((ValueSet) valueSet).getExpansion(); + + return doValidateCodeInValueSet(supportContext, options, codeSystem, code, display, expansion); + } + + public CodeValidationResult doValidateCodeInValueSet(ValidationSupportContext supportContext, + ConceptValidationOptions options, String targetCodeSystem, String targetCode, String targetDisplay, + ValueSetExpansionComponent expansion) + + { + boolean codeSystemCaseSensitive = true; + CodeSystem codeSystem = null; + + if (!options.isInferSystem() && isNotBlank(targetCodeSystem)) + codeSystem = (CodeSystem) supportContext.getRootValidationSupport().fetchCodeSystem(targetCodeSystem); + + List codes = new ArrayList<>(); + flatten(expansion.getContains(), codes); + + String codeSystemName = null; + String codeSystemVersion = null; + String codeSystemContentMode = null; + + if (codeSystem != null) + { + codeSystemCaseSensitive = codeSystem.getCaseSensitive(); + codeSystemName = codeSystem.getName(); + codeSystemVersion = codeSystem.getVersion(); + codeSystemContentMode = codeSystem.getContentElement().getValueAsString(); + } + + for (VersionIndependentConcept nextExpansionCode : codes) + { + boolean codeMatches; + if (codeSystemCaseSensitive) + codeMatches = defaultString(targetCode).equals(nextExpansionCode.getCode()); + else + codeMatches = defaultString(targetCode).equalsIgnoreCase(nextExpansionCode.getCode()); + + if (codeMatches) + { + if (options.isInferSystem() || nextExpansionCode.getSystem().equals(targetCodeSystem)) + { + if (!options.isValidateDisplay() || (isBlank(nextExpansionCode.getDisplay()) + || isBlank(targetDisplay) || nextExpansionCode.getDisplay().equals(targetDisplay))) + { + return new CodeValidationResult().setCode(targetCode).setDisplay(nextExpansionCode.getDisplay()) + .setCodeSystemName(codeSystemName).setCodeSystemVersion(codeSystemVersion); + } + else + { + return new CodeValidationResult().setSeverity(IssueSeverity.ERROR) + .setDisplay(nextExpansionCode.getDisplay()) + .setMessage("Concept Display \"" + targetDisplay + "\" does not match expected \"" + + nextExpansionCode.getDisplay() + "\"") + .setCodeSystemName(codeSystemName).setCodeSystemVersion(codeSystemVersion); + } + } + } + } + + ValidationMessage.IssueSeverity severity; + String message; + if ("fragment".equals(codeSystemContentMode)) + { + severity = ValidationMessage.IssueSeverity.WARNING; + message = "Unknown code in fragment CodeSystem '" + + (isNotBlank(targetCodeSystem) ? targetCodeSystem + "#" : "") + targetCode + "'"; + } + else + { + severity = ValidationMessage.IssueSeverity.ERROR; + message = "Unknown code '" + (isNotBlank(targetCodeSystem) ? targetCodeSystem + "#" : "") + targetCode + + "'"; + } + + return new CodeValidationResult().setSeverityCode(severity.toCode()).setMessage(message); + } + + private void flatten(List components, List concepts) + { + for (ValueSetExpansionContainsComponent next : components) + { + concepts.add(new VersionIndependentConcept(next.getSystem(), next.getCode(), next.getDisplay())); + flatten(next.getContains(), concepts); + } + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorImpl.java new file mode 100644 index 00000000..72ef5470 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorImpl.java @@ -0,0 +1,23 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import org.highmed.dsf.fhir.validation.SnapshotGeneratorImpl; +import org.hl7.fhir.r4.context.IWorkerContext; +import org.hl7.fhir.r4.hapi.ctx.HapiWorkerContext; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.support.IValidationSupport; + +public class PluginSnapshotGeneratorImpl extends SnapshotGeneratorImpl +{ + public PluginSnapshotGeneratorImpl(FhirContext fhirContext, IValidationSupport validationSupport) + { + super(fhirContext, validationSupport); + } + + protected IWorkerContext createWorker(FhirContext context, IValidationSupport validationSupport) + { + HapiWorkerContext workerContext = new HapiWorkerContext(context, validationSupport); + workerContext.setLocale(context.getLocalizer().getLocale()); + return new PluginWorkerContext(workerContext); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java new file mode 100644 index 00000000..bebd67a1 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java @@ -0,0 +1,130 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Objects; + +import org.highmed.dsf.fhir.validation.SnapshotGenerator; +import org.highmed.dsf.fhir.validation.SnapshotGenerator.SnapshotWithValidationMessages; +import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +import ca.uhn.fhir.context.FhirContext; + +public class PluginSnapshotGeneratorWithFileSystemCache + extends AbstractFhirResourceFileSystemCache + implements SnapshotGenerator, InitializingBean +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationPackageClientWithFileSystemCache.class); + + private final SnapshotGenerator delegate; + + /** + * For JSON content with gzip compression using the .json.xz file name suffix. + * + * @param cacheFolder + * not null + * @param fhirContext + * not null + * @param delegate + * not null + * @see AbstractFileSystemCache#FILENAME_SUFFIX + * @see AbstractFileSystemCache#OUT_COMPRESSOR_FACTORY + * @see AbstractFileSystemCache#IN_COMPRESSOR_FACTORY + */ + public PluginSnapshotGeneratorWithFileSystemCache(Path cacheFolder, FhirContext fhirContext, + SnapshotGenerator delegate) + { + super(cacheFolder, StructureDefinition.class, fhirContext); + + this.delegate = delegate; + } + + public PluginSnapshotGeneratorWithFileSystemCache(Path cacheFolder, String fileNameSuffix, + FunctionWithIoException outCompressorFactory, + FunctionWithIoException inCompressorFactory, FhirContext fhirContext, + SnapshotGenerator delegate) + { + super(cacheFolder, fileNameSuffix, outCompressorFactory, inCompressorFactory, StructureDefinition.class, + fhirContext); + + this.delegate = delegate; + } + + @Override + public void afterPropertiesSet() throws Exception + { + super.afterPropertiesSet(); + + Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public SnapshotWithValidationMessages generateSnapshot(StructureDefinition structureDefinition) + { + Objects.requireNonNull(structureDefinition, "differential"); + + if (structureDefinition.hasSnapshot()) + { + logger.debug("StructureDefinition {}|{} has snapshot", structureDefinition.getUrl(), + structureDefinition.getVersion()); + return new SnapshotWithValidationMessages(structureDefinition, Collections.emptyList()); + } + + Objects.requireNonNull(structureDefinition.getUrl(), "structureDefinition.url"); + Objects.requireNonNull(structureDefinition.getVersion(), "structureDefinition.version"); + + try + { + SnapshotWithValidationMessages read = readResourceFromCache(structureDefinition.getUrl(), + structureDefinition.getVersion(), + // needs to return original structureDefinition object with included snapshot + sd -> new SnapshotWithValidationMessages(structureDefinition.setSnapshot(sd.getSnapshot()), + Collections.emptyList())); + if (read != null) + return read; + else + return generateSnapshotAndWriteToCache(structureDefinition); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + private SnapshotWithValidationMessages generateSnapshotAndWriteToCache(StructureDefinition structureDefinition) + throws IOException + { + SnapshotWithValidationMessages snapshot = delegate.generateSnapshot(structureDefinition); + + if (PublicationStatus.DRAFT.equals(snapshot.getSnapshot().getStatus())) + { + logger.info("Not writing StructureDefinition {}|{} with snapshot and status {} to cache", + snapshot.getSnapshot().getUrl(), snapshot.getSnapshot().getVersion(), + snapshot.getSnapshot().getStatus()); + return snapshot; + } + else if (!snapshot.getSnapshot().hasSnapshot()) + { + logger.info("Not writing StructureDefinition {}|{} without snapshot to cache", + snapshot.getSnapshot().getUrl(), snapshot.getSnapshot().getVersion()); + return snapshot; + } + else + return writeRsourceToCache(snapshot, SnapshotWithValidationMessages::getSnapshot, + StructureDefinition::getUrl, StructureDefinition::getVersion); + } + + @Override + public SnapshotWithValidationMessages generateSnapshot(StructureDefinition differential, + String baseAbsoluteUrlPrefix) + { + throw new UnsupportedOperationException("not implemented"); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithModifiers.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithModifiers.java new file mode 100644 index 00000000..f1c48bc5 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithModifiers.java @@ -0,0 +1,71 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Objects; + +import org.highmed.dsf.fhir.validation.SnapshotGenerator; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.springframework.beans.factory.InitializingBean; + +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.GeccoRadiologyProceduresCodingSliceMinFixer; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.MiiModuleLabObservationLab10IdentifierRemover; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.StructureDefinitionModifier; + +public class PluginSnapshotGeneratorWithModifiers implements SnapshotGenerator, InitializingBean +{ + public static final StructureDefinitionModifier CLOSED_TYPE_SLICING_REMOVER = new ClosedTypeSlicingRemover(); + public static final StructureDefinitionModifier MII_MODULE_LAB_OBSERVATION_LAB_1_0_IDENTIFIER_REMOVER = new MiiModuleLabObservationLab10IdentifierRemover(); + public static final StructureDefinitionModifier GECCO_RADIOLOGY_PROCEDURES_CODING_SLICE_MIN_FIXER = new GeccoRadiologyProceduresCodingSliceMinFixer(); + + private final SnapshotGenerator delegate; + private final List structureDefinitionModifiers = new ArrayList<>(); + + public PluginSnapshotGeneratorWithModifiers(SnapshotGenerator delegate) + { + this(delegate, Arrays.asList(CLOSED_TYPE_SLICING_REMOVER, MII_MODULE_LAB_OBSERVATION_LAB_1_0_IDENTIFIER_REMOVER, + GECCO_RADIOLOGY_PROCEDURES_CODING_SLICE_MIN_FIXER)); + } + + public PluginSnapshotGeneratorWithModifiers(SnapshotGenerator delegate, + Collection structureDefinitionModifiers) + { + this.delegate = delegate; + + if (structureDefinitionModifiers != null) + this.structureDefinitionModifiers.addAll(structureDefinitionModifiers); + } + + @Override + public void afterPropertiesSet() throws Exception + { + Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public SnapshotWithValidationMessages generateSnapshot(StructureDefinition differential) + { + return delegate.generateSnapshot(modify(differential)); + } + + @Override + public SnapshotWithValidationMessages generateSnapshot(StructureDefinition differential, + String baseAbsoluteUrlPrefix) + { + return delegate.generateSnapshot(modify(differential), baseAbsoluteUrlPrefix); + } + + private StructureDefinition modify(StructureDefinition differential) + { + if (differential == null) + return null; + + for (StructureDefinitionModifier mod : structureDefinitionModifiers) + differential = mod.modify(differential); + + return differential; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginWorkerContext.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginWorkerContext.java new file mode 100644 index 00000000..826787b8 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginWorkerContext.java @@ -0,0 +1,366 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.List; +import java.util.Locale; +import java.util.Set; + +import org.fhir.ucum.UcumService; +import org.hl7.fhir.exceptions.DefinitionException; +import org.hl7.fhir.exceptions.FHIRException; +import org.hl7.fhir.exceptions.TerminologyServiceException; +import org.hl7.fhir.r4.context.IWorkerContext; +import org.hl7.fhir.r4.formats.IParser; +import org.hl7.fhir.r4.formats.ParserType; +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.ConceptMap; +import org.hl7.fhir.r4.model.ElementDefinition.ElementDefinitionBindingComponent; +import org.hl7.fhir.r4.model.MetadataResource; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Resource; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.StructureMap; +import org.hl7.fhir.r4.model.ValueSet; +import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent; +import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome; +import org.hl7.fhir.r4.utils.INarrativeGenerator; +import org.hl7.fhir.r4.utils.IResourceValidator; +import org.hl7.fhir.utilities.TranslationServices; +import org.hl7.fhir.utilities.validation.ValidationOptions; + +public class PluginWorkerContext implements IWorkerContext +{ + private final IWorkerContext delegate; + + public PluginWorkerContext(IWorkerContext delegate) + { + this.delegate = delegate; + } + + @Override + public String getVersion() + { + return delegate.getVersion(); + } + + @Override + public UcumService getUcumService() + { + return delegate.getUcumService(); + } + + @Override + public IParser getParser(ParserType type) + { + return delegate.getParser(type); + } + + @Override + public IParser getParser(String type) + { + return delegate.getParser(type); + } + + @Override + public IParser newJsonParser() + { + return delegate.newJsonParser(); + } + + @Override + public IParser newXmlParser() + { + return delegate.newXmlParser(); + } + + @Override + public INarrativeGenerator getNarrativeGenerator(String prefix, String basePath) + { + return delegate.getNarrativeGenerator(prefix, basePath); + } + + @Override + public IResourceValidator newValidator() throws FHIRException + { + return delegate.newValidator(); + } + + @Override + public T fetchResource(Class class_, String uri) + { + return delegate.fetchResource(class_, uri); + } + + @Override + public T fetchResourceWithException(Class class_, String uri) throws FHIRException + { + return delegate.fetchResourceWithException(class_, uri); + } + + @Override + public Resource fetchResourceById(String type, String uri) + { + return delegate.fetchResourceById(type, uri); + } + + @Override + public boolean hasResource(Class class_, String uri) + { + return delegate.hasResource(class_, uri); + } + + @Override + public void cacheResource(Resource res) throws FHIRException + { + delegate.cacheResource(res); + } + + @Override + public List getResourceNames() + { + return delegate.getResourceNames(); + } + + @Override + public Set getResourceNamesAsSet() + { + return delegate.getResourceNamesAsSet(); + } + + @Override + public List getTypeNames() + { + return delegate.getTypeNames(); + } + + @Override + public List allStructures() + { + return delegate.allStructures(); + } + + @Override + public List getStructures() + { + return delegate.getStructures(); + } + + @Override + public List allConformanceResources() + { + return delegate.allConformanceResources(); + } + + @Override + public void generateSnapshot(StructureDefinition p) throws DefinitionException, FHIRException + { + delegate.generateSnapshot(p); + } + + @Override + public Parameters getExpansionParameters() + { + return delegate.getExpansionParameters(); + } + + @Override + public void setExpansionProfile(Parameters expParameters) + { + delegate.setExpansionProfile(expParameters); + } + + @Override + public CodeSystem fetchCodeSystem(String system) + { + return delegate.fetchCodeSystem(system); + } + + @Override + public boolean supportsSystem(String system) throws TerminologyServiceException + { + return delegate.supportsSystem(system); + } + + @Override + public List findMapsForSource(String url) throws FHIRException + { + return delegate.findMapsForSource(url); + } + + @Override + public ValueSetExpansionOutcome expandVS(ValueSet source, boolean cacheOk, boolean heiarchical) + { + if (source.hasExpansion()) + return new ValueSetExpansionOutcome(source); + else + return new ValueSetExpansionOutcome(null); + } + + @Override + public ValueSetExpansionOutcome expandVS(ElementDefinitionBindingComponent binding, boolean cacheOk, + boolean heiarchical) throws FHIRException + { + return delegate.expandVS(binding, cacheOk, heiarchical); + } + + @Override + + public ValueSetExpansionOutcome expandVS(ConceptSetComponent inc, boolean heirarchical) + throws TerminologyServiceException + { + return delegate.expandVS(inc, heirarchical); + } + + @Override + public Locale getLocale() + { + return delegate.getLocale(); + } + + @Override + public void setLocale(Locale locale) + { + delegate.setLocale(locale); + } + + @Override + public String formatMessage(String theMessage, Object... theMessageArguments) + { + return delegate.formatMessage(theMessage, theMessageArguments); + } + + @Override + public void setValidationMessageLanguage(Locale locale) + { + delegate.setValidationMessageLanguage(locale); + } + + @Override + public ValidationResult validateCode(ValidationOptions options, String system, String code, String display) + { + return delegate.validateCode(options, system, code, display); + } + + @Override + public ValidationResult validateCode(ValidationOptions options, String system, String code, String display, + ValueSet vs) + { + return delegate.validateCode(options, system, code, display, vs); + } + + @Override + public ValidationResult validateCode(ValidationOptions options, String code, ValueSet vs) + { + return delegate.validateCode(options, code, vs); + } + + @Override + public ValidationResult validateCode(ValidationOptions options, Coding code, ValueSet vs) + { + return delegate.validateCode(options, code, vs); + } + + @Override + public ValidationResult validateCode(ValidationOptions options, CodeableConcept code, ValueSet vs) + { + return delegate.validateCode(options, code, vs); + } + + @Override + public ValidationResult validateCode(ValidationOptions options, String system, String code, String display, + ConceptSetComponent vsi) + { + return delegate.validateCode(options, system, code, display, vsi); + } + + @Override + public String getAbbreviation(String name) + { + return delegate.getAbbreviation(name); + } + + @Override + public Set typeTails() + { + return delegate.typeTails(); + } + + @Override + public String oid2Uri(String code) + { + return delegate.oid2Uri(code); + } + + @Override + public boolean hasCache() + { + return delegate.hasCache(); + } + + @Override + public void setLogger(ILoggingService logger) + { + delegate.setLogger(logger); + } + + @Override + public ILoggingService getLogger() + { + return delegate.getLogger(); + } + + @Override + public boolean isNoTerminologyServer() + { + return delegate.isNoTerminologyServer(); + } + + @Override + public TranslationServices translator() + { + return delegate.translator(); + } + + @Override + public List listTransforms() + { + return delegate.listTransforms(); + } + + @Override + public StructureMap getTransform(String url) + { + return delegate.getTransform(url); + } + + @Override + public String getOverrideVersionNs() + { + return delegate.getOverrideVersionNs(); + } + + @Override + public void setOverrideVersionNs(String value) + { + delegate.setOverrideVersionNs(value); + } + + @Override + public StructureDefinition fetchTypeDefinition(String typeName) + { + return delegate.fetchTypeDefinition(typeName); + } + + @Override + public void setUcumService(UcumService ucumService) + { + delegate.setUcumService(ucumService); + } + + @Override + public String getLinkForUrl(String corePath, String s) + { + return delegate.getLinkForUrl(corePath, s); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java new file mode 100644 index 00000000..ec110767 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java @@ -0,0 +1,258 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Locale; +import java.util.Objects; +import java.util.stream.Stream; + +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Resource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.EnumerablePropertySource; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.i18n.HapiLocalizer; +import ca.uhn.fhir.parser.IParser; +import ca.uhn.fhir.validation.ValidationResult; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config.ValidationConfig; + +public class ValidationMain implements InitializingBean +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationMain.class); + + private static final class FileNameAndResource + { + final String filename; + final Resource resource; + + FileNameAndResource(String filename, Resource resource) + { + this.filename = filename; + this.resource = resource; + } + + String getFilename() + { + return filename; + } + + Resource getResource() + { + return resource; + } + } + + @Configuration + @PropertySource(ignoreResourceNotFound = true, value = "file:application.properties") + public static class TestConfig + { + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.output:JSON}") + private Output output; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.output.pretty:true}") + private boolean outputPretty; + + @Autowired + private ValidationPackageManager packageManager; + + @Autowired + private ValidationPackageIdentifier validationPackage; + + @Autowired + private ValueSetExpansionClient valueSetExpansionClient; + + @Autowired + private ConfigurableEnvironment environment; + + @Bean + public FhirContext fhirContext() + { + FhirContext context = FhirContext.forR4(); + HapiLocalizer localizer = new HapiLocalizer() + { + @Override + public Locale getLocale() + { + return Locale.ROOT; + } + }; + context.setLocalizer(localizer); + return context; + } + + @Bean + public ValidationMain validatorMain() + { + return new ValidationMain(environment, fhirContext(), packageManager, validationPackage, output, + outputPretty, valueSetExpansionClient); + } + } + + public static enum Output + { + JSON, XML + } + + public static void main(String[] args) + { + if (args.length == 0) + { + logger.warn("No files to validated specified"); + System.exit(1); + } + + try (AnnotationConfigApplicationContext springContext = new AnnotationConfigApplicationContext(TestConfig.class, + ValidationConfig.class)) + { + ValidationConfig config = springContext.getBean(ValidationConfig.class); + boolean testOk = config.testConnectionToTerminologyServer(); + + if (testOk) + { + ValidationMain main = springContext.getBean(ValidationMain.class); + main.validate(args); + } + } + catch (Exception e) + { + logger.error("", e); + System.exit(1); + } + } + + private final ConfigurableEnvironment environment; + private final FhirContext fhirContext; + private final ValidationPackageManager packageManager; + private final ValidationPackageIdentifier validationPackage; + private final Output output; + private final boolean outputPretty; + private final ValueSetExpansionClient valueSetExpansionClient; + + public ValidationMain(ConfigurableEnvironment environment, FhirContext fhirContext, + ValidationPackageManager packageManager, ValidationPackageIdentifier validationPackage, Output output, + boolean outputPretty, ValueSetExpansionClient valueSetExpansionClient) + { + this.environment = environment; + this.fhirContext = fhirContext; + this.packageManager = packageManager; + this.validationPackage = validationPackage; + this.output = output; + this.outputPretty = outputPretty; + this.valueSetExpansionClient = valueSetExpansionClient; + } + + @Override + public void afterPropertiesSet() throws Exception + { + Objects.requireNonNull(environment, "environment"); + Objects.requireNonNull(fhirContext, "fhirContext"); + Objects.requireNonNull(packageManager, "packageManager"); + Objects.requireNonNull(validationPackage, "validationPackage"); + Objects.requireNonNull(output, "output"); + Objects.requireNonNull(valueSetExpansionClient, "valueSetExpansionClient"); + } + + public void validate(String[] files) + { + logger.info("Using validation package {}", validationPackage); + getAllNumProperties().forEach(c -> logger.debug("Config: {}", c)); + + BundleValidator validator = packageManager.createBundleValidator(validationPackage.getName(), + validationPackage.getVersion()); + + Arrays.stream(files).map(this::read).filter(r -> r != null).forEach(r -> + { + logger.info("Validating {} from {}", r.getResource().getResourceType().name(), r.getFilename()); + + if (r.getResource() instanceof Bundle) + { + Bundle validationResult = validator.validate((Bundle) r.getResource()); + System.out.println(getOutputParser().encodeResourceToString(validationResult)); + } + else + { + ValidationResult validationResult = validator.validate(r.getResource()); + System.out.println(getOutputParser().encodeResourceToString(validationResult.toOperationOutcome())); + } + }); + } + + private Stream getAllNumProperties() + { + return environment.getPropertySources().stream().filter(p -> p instanceof EnumerablePropertySource) + .map(p -> (EnumerablePropertySource) p) + .flatMap(p -> Arrays.stream(p.getPropertyNames()) + .filter(n -> n.startsWith("de.netzwerk.universitaetsmedizin")) + .map(k -> new String[] { k, Objects.toString(p.getProperty(k)) })) + .map(e -> e[0].contains("password") ? new String[] { e[0], "***" } : e).map(e -> e[0] + ": " + e[1]); + } + + private IParser getOutputParser() + { + switch (output) + { + case JSON: + return fhirContext.newJsonParser().setPrettyPrint(outputPretty); + case XML: + return fhirContext.newXmlParser().setPrettyPrint(outputPretty); + default: + throw new IllegalStateException("Output of type " + output + " not supported"); + } + } + + private FileNameAndResource read(String file) + { + if (file.endsWith(".json")) + return tryJson(file); + else if (file.endsWith(".xml")) + return tryXml(file); + else + { + logger.warn("File {} not supported, filename needs to end with .json or .xml", file); + return null; + } + } + + private FileNameAndResource tryJson(String file) + { + try (InputStream in = Files.newInputStream(Paths.get(file))) + { + IBaseResource resource = fhirContext.newJsonParser().parseResource(in); + logger.info("{} read from {}", resource.getClass().getSimpleName(), file); + return new FileNameAndResource(file, (Resource) resource); + } + catch (Exception e) + { + logger.warn("Unable to read " + file + " as JSON, {}: {}", e.getClass().getName(), e.getMessage()); + return null; + } + } + + private FileNameAndResource tryXml(String file) + { + try (InputStream in = Files.newInputStream(Paths.get(file))) + { + IBaseResource resource = fhirContext.newXmlParser().parseResource(in); + logger.info("{} read from {}", resource.getClass().getSimpleName(), file); + return new FileNameAndResource(file, (Resource) resource); + } + catch (Exception e) + { + logger.warn("Unable to read " + file + " as XML, {}: {}", e.getClass().getName(), e.getMessage()); + return null; + } + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackage.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackage.java new file mode 100644 index 00000000..3f14aba8 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackage.java @@ -0,0 +1,201 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.NamingSystem; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.StructureDefinition.StructureDefinitionKind; +import org.hl7.fhir.r4.model.ValueSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +import ca.uhn.fhir.context.FhirContext; + +public class ValidationPackage +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationPackage.class); + + private static final String PACKAGE_JSON_FILENAME = "package/package.json"; + + public static ValidationPackage from(String name, String version, InputStream in) throws IOException + { + try (BufferedInputStream bufferedIn = new BufferedInputStream(in); + GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(bufferedIn); + TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) + { + List entries = new ArrayList<>(); + + ArchiveEntry entry; + while ((entry = tarIn.getNextEntry()) != null) + { + ValidationPackageEntry pEntry = ValidationPackageEntry.from(entry, tarIn); + if (pEntry != null) + entries.add(pEntry); + } + + return new ValidationPackage(name, version, entries); + } + } + + private final String name; + private final String version; + private final List entries = new ArrayList<>(); + + private Map entriesByFileName; + + private ValidationSupportResources resources; + + /** + * @param name + * not null + * @param version + * not null + * @param entries + * may be null + */ + @JsonCreator + public ValidationPackage(@JsonProperty("name") String name, @JsonProperty("version") String version, + @JsonProperty("entries") Collection entries) + { + Objects.requireNonNull(name, "name"); + Objects.requireNonNull(version, "version"); + + this.name = name; + this.version = version; + + if (entries != null) + this.entries.addAll(entries); + } + + @JsonProperty("name") + public String getName() + { + return name; + } + + @JsonProperty("version") + public String getVersion() + { + return version; + } + + @JsonIgnore + public ValidationPackageIdentifier getIdentifier() + { + return new ValidationPackageIdentifier(name, version); + } + + @JsonProperty("entries") + public List getEntries() + { + return Collections.unmodifiableList(entries); + } + + @JsonIgnore + public Map getEntriesByFileName() + { + if (entriesByFileName == null) + entriesByFileName = getEntries().stream().collect(Collectors + .toUnmodifiableMap(ValidationPackageEntry::getFileName, Function.identity(), (e0, e1) -> e1)); + + return entriesByFileName; + } + + @JsonIgnore + public ValidationPackageDescriptor getDescriptor(ObjectMapper mapper) throws IOException + { + ValidationPackageEntry packageJson = getEntriesByFileName().get(PACKAGE_JSON_FILENAME); + return mapper.readValue(packageJson.getContent(), ValidationPackageDescriptor.class); + } + + public void parseResources(FhirContext context) + { + if (resources == null) + { + List codeSystems = new ArrayList<>(); + List namingSystems = new ArrayList<>(); + List structureDefinitions = new ArrayList<>(); + List valueSets = new ArrayList<>(); + + getEntries() + .forEach(doParseResources(context, codeSystems, namingSystems, structureDefinitions, valueSets)); + + resources = new ValidationSupportResources(codeSystems, namingSystems, structureDefinitions, valueSets); + } + } + + private Consumer doParseResources(FhirContext context, List codeSystems, + List namingSystems, List structureDefinitions, List valueSets) + { + return entry -> + { + if ("package/package.json".equals(entry.getFileName()) + || (entry.getFileName() != null && (entry.getFileName().startsWith("package/example") + || entry.getFileName().endsWith(".index.json") || !entry.getFileName().endsWith(".json")))) + { + logger.debug("Ignoring " + entry.getFileName()); + return; + } + + logger.debug("Reading " + entry.getFileName()); + + try + { + IBaseResource resource = context.newJsonParser() + .parseResource(new String(entry.getContent(), StandardCharsets.UTF_8)); + + if (resource instanceof CodeSystem) + codeSystems.add((CodeSystem) resource); + else if (resource instanceof NamingSystem) + namingSystems.add((NamingSystem) resource); + else if (resource instanceof StructureDefinition) + { + if (!StructureDefinitionKind.LOGICAL.equals(((StructureDefinition) resource).getKind())) + structureDefinitions.add((StructureDefinition) resource); + else + logger.debug("Ignoring StructureDefinition with kind = logical"); + } + else if (resource instanceof ValueSet) + valueSets.add((ValueSet) resource); + else + logger.debug("Ignoring resource of type {}", resource.getClass().getName()); + } + catch (Exception e) + { + logger.warn("Ignoring resource with error while parsing, {}: {}", e.getClass().getName(), + e.getMessage()); + } + }; + } + + @JsonIgnore + public ValidationSupportResources getValidationSupportResources() + { + if (resources == null) + throw new IllegalStateException("Resources not parsed"); + + return resources; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClient.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClient.java new file mode 100644 index 00000000..c3cbafa7 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClient.java @@ -0,0 +1,31 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; + +import javax.ws.rs.WebApplicationException; + +public interface ValidationPackageClient +{ + /** + * @param name + * not null + * @param version + * not null + * @return downloaded {@link ValidationPackage}, never null + * @throws IOException + * @throws WebApplicationException + */ + default ValidationPackage download(String name, String version) throws IOException, WebApplicationException + { + return download(new ValidationPackageIdentifier(name, version)); + } + + /** + * @param identifier + * not null + * @return downloaded {@link ValidationPackage}, never null + * @throws IOException + * @throws WebApplicationException + */ + ValidationPackage download(ValidationPackageIdentifier identifier) throws IOException, WebApplicationException; +} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java new file mode 100644 index 00000000..fc0f53c6 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java @@ -0,0 +1,101 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyStore; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; + +import javax.net.ssl.SSLContext; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; + +import org.glassfish.jersey.SslConfigurator; +import org.glassfish.jersey.apache.connector.ApacheConnectorProvider; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.ClientProperties; +import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; +import org.glassfish.jersey.logging.LoggingFeature; +import org.glassfish.jersey.logging.LoggingFeature.Verbosity; + +public class ValidationPackageClientJersey implements ValidationPackageClient +{ + private static final java.util.logging.Logger requestDebugLogger = java.util.logging.Logger + .getLogger(ValueSetExpansionClientJersey.class.getName()); + + private final Client client; + private final String baseUrl; + + public ValidationPackageClientJersey(String baseUrl) + { + this(baseUrl, null, null, null, null, null, null, null, null, 0, 0, false); + } + + public ValidationPackageClientJersey(String baseUrl, KeyStore trustStore, KeyStore keyStore, + char[] keyStorePassword, String basicAuthUsername, char[] basicAuthPassword, String proxySchemeHostPort, + String proxyUsername, char[] proxyPassword, int connectTimeout, int readTimeout, boolean logRequests) + { + SSLContext sslContext = null; + if (trustStore != null && keyStore == null && keyStorePassword == null) + sslContext = SslConfigurator.newInstance().trustStore(trustStore).createSSLContext(); + if (trustStore == null && keyStore != null && keyStorePassword != null) + sslContext = SslConfigurator.newInstance().keyStore(keyStore).keyStorePassword(keyStorePassword) + .createSSLContext(); + else if (trustStore != null && keyStore != null && keyStorePassword != null) + sslContext = SslConfigurator.newInstance().trustStore(trustStore).keyStore(keyStore) + .keyStorePassword(keyStorePassword).createSSLContext(); + + ClientBuilder builder = ClientBuilder.newBuilder(); + + if (sslContext != null) + builder = builder.sslContext(sslContext); + + if (basicAuthUsername != null && basicAuthPassword != null) + { + HttpAuthenticationFeature basicAuthFeature = HttpAuthenticationFeature.basic(basicAuthUsername, + String.valueOf(basicAuthPassword)); + builder = builder.register(basicAuthFeature); + } + + ClientConfig config = new ClientConfig(); + config.connectorProvider(new ApacheConnectorProvider()); + config.property(ClientProperties.PROXY_URI, proxySchemeHostPort); + config.property(ClientProperties.PROXY_USERNAME, proxyUsername); + config.property(ClientProperties.PROXY_PASSWORD, proxyPassword == null ? null : String.valueOf(proxyPassword)); + builder = builder.withConfig(config); + + builder = builder.readTimeout(readTimeout, TimeUnit.MILLISECONDS).connectTimeout(connectTimeout, + TimeUnit.MILLISECONDS); + + if (logRequests) + { + builder = builder.register(new LoggingFeature(requestDebugLogger, Level.FINE, Verbosity.PAYLOAD_ANY, + LoggingFeature.DEFAULT_MAX_ENTITY_SIZE)); + } + + client = builder.build(); + + this.baseUrl = baseUrl; + } + + private WebTarget getResource() + { + return client.target(baseUrl); + } + + @Override + public ValidationPackage download(ValidationPackageIdentifier identifier) + throws IOException, WebApplicationException + { + Objects.requireNonNull(identifier, "identifier"); + + try (InputStream in = getResource().path(identifier.getName()).path(identifier.getVersion()) + .request("application/tar+gzip").get(InputStream.class)) + { + return ValidationPackage.from(identifier.getName(), identifier.getVersion(), in); + } + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientWithFileSystemCache.java new file mode 100644 index 00000000..85f0615b --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientWithFileSystemCache.java @@ -0,0 +1,78 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Objects; + +import javax.ws.rs.WebApplicationException; + +import org.springframework.beans.factory.InitializingBean; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class ValidationPackageClientWithFileSystemCache extends AbstractFileSystemCache + implements ValidationPackageClient, InitializingBean +{ + private final ObjectMapper mapper; + private final ValidationPackageClient delegate; + + /** + * For JSON content with gzip compression using the .json.xz file name suffix. + * + * @param cacheFolder + * not null + * @param mapper + * not null + * @param delegate + * not null + * @see AbstractFileSystemCache#FILENAME_SUFFIX + * @see AbstractFileSystemCache#OUT_COMPRESSOR_FACTORY + * @see AbstractFileSystemCache#IN_COMPRESSOR_FACTORY + */ + public ValidationPackageClientWithFileSystemCache(Path cacheFolder, ObjectMapper mapper, + ValidationPackageClient delegate) + { + super(cacheFolder); + + this.mapper = mapper; + this.delegate = delegate; + } + + public ValidationPackageClientWithFileSystemCache(Path cacheFolder, String fileNameSuffix, + FunctionWithIoException outCompressorFactory, + FunctionWithIoException inCompressorFactory, ObjectMapper mapper, + ValidationPackageClient delegate) + { + super(cacheFolder, fileNameSuffix, outCompressorFactory, inCompressorFactory); + + this.mapper = mapper; + this.delegate = delegate; + } + + @Override + public void afterPropertiesSet() throws Exception + { + super.afterPropertiesSet(); + + Objects.requireNonNull(mapper, "mapper"); + Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public ValidationPackage download(ValidationPackageIdentifier identifier) + throws IOException, WebApplicationException + { + Objects.requireNonNull(identifier, "identifier"); + + ValidationPackage read = readFromCache(identifier.toString(), "validation package", + r -> mapper.readValue(r, ValidationPackage.class)); + + if (read != null) + return read; + else + return writeToCache(delegate.download(identifier), p -> p.getIdentifier().toString(), + p -> "validation package", mapper::writeValue); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptor.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptor.java new file mode 100644 index 00000000..78bbe34a --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptor.java @@ -0,0 +1,156 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ValidationPackageDescriptor +{ + private final String author; + private final String canonical; + private final Map dependencies = new HashMap<>(); + private final String description; + // fhir-version-list + private final List fhirVersions = new ArrayList<>(); + private final String jurisdiction; + private final List keywords = new ArrayList<>(); + private final String license; + private final List maintainers = new ArrayList<>(); + private final String name; + private final String title; + private final String url; + private final String version; + + @JsonCreator + public ValidationPackageDescriptor(@JsonProperty("author") String author, + @JsonProperty("canonical") String canonical, @JsonProperty("dependencies") Map dependencies, + @JsonProperty("description") String description, + @JsonProperty("fhirVersions") @JsonAlias("fhir-version-list") List fhirVersions, + @JsonProperty("jurisdiction") String jurisdiction, @JsonProperty("keywords") List keywords, + @JsonProperty("license") String license, + @JsonProperty("maintainers") List maintainers, + @JsonProperty("name") String name, @JsonProperty("title") String title, @JsonProperty("url") String url, + @JsonProperty("version") String version) + { + this.author = author; + this.canonical = canonical; + + if (dependencies != null) + this.dependencies.putAll(dependencies); + + this.description = description; + + if (fhirVersions != null) + this.fhirVersions.addAll(fhirVersions); + + this.jurisdiction = jurisdiction; + + if (keywords != null) + this.keywords.addAll(keywords); + + this.license = license; + + if (maintainers != null) + this.maintainers.addAll(maintainers); + + this.name = name; + this.title = title; + this.url = url; + this.version = version; + } + + @JsonProperty("author") + public String getAuthor() + { + return author; + } + + @JsonProperty("canonical") + public String getCanonical() + { + return canonical; + } + + @JsonProperty("dependencies") + public Map getDependencies() + { + return Collections.unmodifiableMap(dependencies); + } + + @JsonIgnore + public List getDependencyIdentifiers() + { + return dependencies.entrySet().stream().map(e -> new ValidationPackageIdentifier(e.getKey(), e.getValue())) + .collect(Collectors.toUnmodifiableList()); + } + + @JsonProperty("description") + public String getDescription() + { + return description; + } + + @JsonProperty("fhirVersions") + public List getFhirVersions() + { + return Collections.unmodifiableList(fhirVersions); + } + + @JsonProperty("jurisdiction") + public String getJurisdiction() + { + return jurisdiction; + } + + @JsonProperty("keywords") + public List getKeywords() + { + return Collections.unmodifiableList(keywords); + } + + @JsonProperty("license") + public String getLicense() + { + return license; + } + + @JsonProperty("maintainers") + public List getMaintainers() + { + return Collections.unmodifiableList(maintainers); + } + + @JsonProperty("name") + public String getName() + { + return name; + } + + @JsonProperty("title") + public String getTitle() + { + return title; + } + + @JsonProperty("url") + public String getUrl() + { + return url; + } + + @JsonProperty("version") + public String getVersion() + { + return version; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptorMaintainer.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptorMaintainer.java new file mode 100644 index 00000000..edd2d971 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptorMaintainer.java @@ -0,0 +1,31 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ValidationPackageDescriptorMaintainer +{ + private final String email; + private final String name; + + @JsonCreator + public ValidationPackageDescriptorMaintainer(@JsonProperty("email") String email, @JsonProperty("name") String name) + { + this.email = email; + this.name = name; + } + + @JsonProperty("email") + public String getEmail() + { + return email; + } + + @JsonProperty("name") + public String getName() + { + return name; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageEntry.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageEntry.java new file mode 100644 index 00000000..3ec33d73 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageEntry.java @@ -0,0 +1,72 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.io.IOUtils; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class ValidationPackageEntry +{ + /** + * Does not close the input stream. + * + * @param entry + * not null + * @param in + * not null + * @return {@link ValidationPackageEntry} for the given {@link ArchiveEntry} from the given + * {@link TarArchiveInputStream}, null if the given entry can't be read from the input stream. + * @throws IOException + * @see {@link TarArchiveInputStream#canReadEntryData(ArchiveEntry)} + */ + public static ValidationPackageEntry from(ArchiveEntry entry, TarArchiveInputStream in) throws IOException + { + Objects.requireNonNull(entry, "entry"); + Objects.requireNonNull(in, "in"); + + if (!in.canReadEntryData(entry)) + return null; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copy(in, out); + return new ValidationPackageEntry(entry.getName(), entry.getLastModifiedDate(), out.toByteArray()); + } + + private final String fileName; + private final Date lastModified; + private final byte[] content; + + @JsonCreator + public ValidationPackageEntry(@JsonProperty("fileName") String fileName, + @JsonProperty("lastModified") Date lastModified, @JsonProperty("content") byte[] content) + { + this.fileName = fileName; + this.lastModified = lastModified; + this.content = content; + } + + @JsonProperty("fileName") + public String getFileName() + { + return fileName; + } + + @JsonProperty("lastModified") + public Date getLastModified() + { + return lastModified; + } + + @JsonProperty("content") + public byte[] getContent() + { + return content; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageIdentifier.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageIdentifier.java new file mode 100644 index 00000000..eed830ac --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageIdentifier.java @@ -0,0 +1,78 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.Objects; + +public class ValidationPackageIdentifier +{ + public static ValidationPackageIdentifier fromString(String nameAndVersion) + { + String[] split = nameAndVersion.split("\\|"); + + if (split.length != 2) + throw new IllegalArgumentException("Validation package not specified as 'name|version'"); + + return new ValidationPackageIdentifier(split[0], split[1]); + } + + private final String name; + private final String version; + + public ValidationPackageIdentifier(String name, String version) + { + this.name = Objects.requireNonNull(name, "name"); + this.version = Objects.requireNonNull(version, "version"); + } + + public String getName() + { + return name; + } + + public String getVersion() + { + return version; + } + + @Override + public String toString() + { + return name + "|" + version; + } + + @Override + public int hashCode() + { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + ((version == null) ? 0 : version.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ValidationPackageIdentifier other = (ValidationPackageIdentifier) obj; + if (name == null) + { + if (other.name != null) + return false; + } + else if (!name.equals(other.name)) + return false; + if (version == null) + { + if (other.version != null) + return false; + } + else if (!version.equals(other.version)) + return false; + return true; + } +} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java new file mode 100644 index 00000000..d402e5fe --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java @@ -0,0 +1,90 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import org.hl7.fhir.r4.model.Enumerations.BindingStrength; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.ValueSet; + +import ca.uhn.fhir.context.support.IValidationSupport; + +public interface ValidationPackageManager +{ + /** + * Downloads the given FHIR package and all its dependencies. + * + * @param name + * not null + * @param version + * not null + * @return unmodifiable list of {@link ValidationPackage}s + */ + default ValidationPackageWithDepedencies downloadPackageWithDependencies(String name, String version) + { + return downloadPackageWithDependencies(new ValidationPackageIdentifier(name, version)); + } + + /** + * Downloads the given FHIR package and all its dependencies. + * + * @param name + * not null + * @param version + * not null + * @return unmodifiable list of {@link ValidationPackage}s + */ + ValidationPackageWithDepedencies downloadPackageWithDependencies(ValidationPackageIdentifier identifier); + + /** + * Will try to generate snapshots for all {@link StructureDefinition}s of the root package and its dependencies, + * will try to expand all {@link ValueSet}s with binding strength {@link BindingStrength#EXTENSIBLE}, + * {@link BindingStrength#PREFERRED} or {@link BindingStrength#REQUIRED} used by the {@link StructureDefinition} of + * the root package or their dependencies, before returning a {@link IValidationSupport}. + * + * @param packageWithDependencies + * not null + * @return validation support for the validator + */ + IValidationSupport expandValueSetsAndGenerateStructureDefinitionSnapshots( + ValidationPackageWithDepedencies packageWithDependencies); + + /** + * @param validationSupport + * not null + * @param packageWithDependencies + * not null + * @return {@link BundleValidator} for the given {@link IValidationSupport} and + * {@link ValidationPackageWithDepedencies} + */ + BundleValidator createBundleValidator(IValidationSupport validationSupport, + ValidationPackageWithDepedencies packageWithDependencies); + + /** + * Downloads the given FHIR package and all its dependencies. Will try to generate snapshots for all + * {@link StructureDefinition}s of the specified (root) package and its dependencies, will try to expand all + * {@link ValueSet}s with binding strength {@link BindingStrength#EXTENSIBLE}, {@link BindingStrength#PREFERRED} or + * {@link BindingStrength#REQUIRED} used by the {@link StructureDefinition} of the specified (root) package or their + * dependencies, before returning a {@link IValidationSupport}. + * + * @param name + * not null + * @param version + * not null + * @return {@link BundleValidator} for the specified FHIR package + */ + default BundleValidator createBundleValidator(String name, String version) + { + return createBundleValidator(new ValidationPackageIdentifier(name, version)); + } + + /** + * Downloads the given FHIR package and all its dependencies. Will try to generate snapshots for all + * {@link StructureDefinition}s of the specified (root) package and its dependencies, will try to expand all + * {@link ValueSet}s with binding strength {@link BindingStrength#EXTENSIBLE}, {@link BindingStrength#PREFERRED} or + * {@link BindingStrength#REQUIRED} used by the {@link StructureDefinition} of the specified (root) package or their + * dependencies, before returning a {@link IValidationSupport}. + * + * @param identifier + * not null + * @return {@link BundleValidator} for the specified FHIR package + */ + BundleValidator createBundleValidator(ValidationPackageIdentifier identifier); +} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java new file mode 100644 index 00000000..59b07950 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java @@ -0,0 +1,362 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import javax.ws.rs.WebApplicationException; + +import org.highmed.dsf.fhir.validation.ResourceValidatorImpl; +import org.highmed.dsf.fhir.validation.SnapshotGenerator; +import org.highmed.dsf.fhir.validation.SnapshotGenerator.SnapshotWithValidationMessages; +import org.highmed.dsf.fhir.validation.ValidationSupportWithCustomResources; +import org.highmed.dsf.fhir.validation.ValueSetExpander; +import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService; +import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport; +import org.hl7.fhir.common.hapi.validation.support.ValidationSupportChain; +import org.hl7.fhir.r4.model.Enumerations.BindingStrength; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.ValueSet; +import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent; +import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome; +import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.support.DefaultProfileValidationSupport; +import ca.uhn.fhir.context.support.IValidationSupport; + +public class ValidationPackageManagerImpl implements InitializingBean, ValidationPackageManager +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationPackageManagerImpl.class); + + public static final List DEFAULT_NO_PACKAGE_DOWNLOAD_LIST = List + .of(new ValidationPackageIdentifier("hl7.fhir.r4.core", "4.0.1")); + + public static final EnumSet DEFAULT_VALUE_SET_BINDING_STRENGTHS = EnumSet + .allOf(BindingStrength.class); + + private final ValidationPackageClient validationPackageClient; + private final ValueSetExpansionClient valueSetExpansionClient; + + private final ObjectMapper mapper; + private final FhirContext fhirContext; + + private final BiFunction internalSnapshotGeneratorFactory; + private final BiFunction internalValueSetExpanderFactory; + + private final List noDownloadPackages = new ArrayList<>(); + private final EnumSet valueSetBindingStrengths; + + public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, + ValueSetExpansionClient valueSetExpansionClient, ObjectMapper mapper, FhirContext fhirContext, + BiFunction internalSnapshotGeneratorFactory, + BiFunction internalValueSetExpanderFactory) + { + this(validationPackageClient, valueSetExpansionClient, mapper, fhirContext, internalSnapshotGeneratorFactory, + internalValueSetExpanderFactory, DEFAULT_NO_PACKAGE_DOWNLOAD_LIST, DEFAULT_VALUE_SET_BINDING_STRENGTHS); + } + + public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, + ValueSetExpansionClient valueSetExpansionClient, ObjectMapper mapper, FhirContext fhirContext, + BiFunction internalSnapshotGeneratorFactory, + BiFunction internalValueSetExpanderFactory, + Collection noDownloadPackages, + EnumSet valueSetBindingStrengths) + { + this.validationPackageClient = validationPackageClient; + this.valueSetExpansionClient = valueSetExpansionClient; + this.mapper = mapper; + this.fhirContext = fhirContext; + this.internalSnapshotGeneratorFactory = internalSnapshotGeneratorFactory; + this.internalValueSetExpanderFactory = internalValueSetExpanderFactory; + + if (noDownloadPackages != null) + this.noDownloadPackages.addAll(noDownloadPackages); + + this.valueSetBindingStrengths = valueSetBindingStrengths; + } + + @Override + public void afterPropertiesSet() throws Exception + { + Objects.requireNonNull(validationPackageClient, "validationPackageClient"); + Objects.requireNonNull(valueSetExpansionClient, "valueSetExpansionClient"); + + Objects.requireNonNull(mapper, "mapper"); + Objects.requireNonNull(fhirContext, "fhirContext"); + + Objects.requireNonNull(internalSnapshotGeneratorFactory, "internalSnapshotGeneratorFactory"); + Objects.requireNonNull(internalValueSetExpanderFactory, "internalValueSetExpanderFactory"); + } + + @Override + public ValidationPackageWithDepedencies downloadPackageWithDependencies(ValidationPackageIdentifier identifier) + { + Objects.requireNonNull(identifier, "identifier"); + + Map packagesByNameAndVersion = new HashMap<>(); + downloadPackageWithDependencies(identifier, packagesByNameAndVersion); + + return ValidationPackageWithDepedencies.from(packagesByNameAndVersion, identifier); + } + + @Override + public IValidationSupport expandValueSetsAndGenerateStructureDefinitionSnapshots( + ValidationPackageWithDepedencies packageWithDependencies) + { + Objects.requireNonNull(packageWithDependencies, "packageWithDependencies"); + + packageWithDependencies.parseResources(fhirContext); + + return withSnapshots(packageWithDependencies, withExpandedValueSets(packageWithDependencies)); + } + + @Override + public BundleValidator createBundleValidator(IValidationSupport validationSupport, + ValidationPackageWithDepedencies packageWithDependencies) + { + Objects.requireNonNull(validationSupport, "validationSupport"); + Objects.requireNonNull(packageWithDependencies, "packageWithDependencies"); + + BundleValidatorImpl validator = new BundleValidatorImpl(fhirContext, packageWithDependencies, + new ResourceValidatorImpl(fhirContext, validationSupport)); + + return validator; + } + + @Override + public BundleValidator createBundleValidator(ValidationPackageIdentifier identifier) + { + Objects.requireNonNull(identifier, "identifier"); + + ValidationPackageWithDepedencies packageWithDependencies = downloadPackageWithDependencies(identifier); + IValidationSupport validationSupport = expandValueSetsAndGenerateStructureDefinitionSnapshots( + packageWithDependencies); + return createBundleValidator(validationSupport, packageWithDependencies); + } + + private void downloadPackageWithDependencies(ValidationPackageIdentifier identifier, + Map packagesByNameAndVersion) + { + if (packagesByNameAndVersion.containsKey(identifier)) + { + // already downloaded + return; + } + else if (noDownloadPackages.contains(identifier)) + { + logger.debug("Not downloading package {}", identifier.toString()); + return; + } + + ValidationPackage vPackage = downloadAndHandleException(identifier); + packagesByNameAndVersion.put(identifier, vPackage); + + ValidationPackageDescriptor descriptor = getDescriptorAndHandleException(vPackage); + descriptor.getDependencyIdentifiers() + .forEach(i -> downloadPackageWithDependencies(i, packagesByNameAndVersion)); + } + + private ValidationPackage downloadAndHandleException(ValidationPackageIdentifier identifier) + { + try + { + logger.debug("Downloading validation package {}", identifier); + return validationPackageClient.download(identifier); + } + catch (WebApplicationException | IOException e) + { + throw new RuntimeException(e); + } + } + + private ValidationPackageDescriptor getDescriptorAndHandleException(ValidationPackage vPackage) + { + try + { + return vPackage.getDescriptor(mapper); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + private List withExpandedValueSets(ValidationPackageWithDepedencies packageWithDependencies) + { + List expandedValueSets = new ArrayList<>(); + ValueSetExpander expander = internalValueSetExpanderFactory.apply(fhirContext, + createSupportChain(fhirContext, packageWithDependencies, Collections.emptyList(), expandedValueSets)); + + packageWithDependencies.getValueSetsIncludingDependencies(valueSetBindingStrengths).forEach(v -> + { + logger.debug("Expanding ValueSet {}|{}", v.getUrl(), v.getVersion()); + + // ValueSet uses filter in compose + if (v.hasCompose() && (v.getCompose().hasInclude() || v.getCompose().hasExclude()) + && (v.getCompose().getInclude().stream().anyMatch(ConceptSetComponent::hasFilter) + || v.getCompose().getExclude().stream().anyMatch(ConceptSetComponent::hasFilter))) + { + expandExternal(expandedValueSets, v); + } + else + { + // will try external expansion if internal not successful + expandInternal(expandedValueSets, expander, v); + } + }); + + return expandedValueSets; + } + + private void expandExternal(List expandedValueSets, ValueSet v) + { + try + { + ValueSet expansion = valueSetExpansionClient.expand(v); + expandedValueSets.add(expansion); + } + catch (WebApplicationException e) + { + logger.error("Error while expanding ValueSet {}|{}: {} - {}", v.getUrl(), v.getVersion(), + e.getClass().getName(), e.getMessage()); + getOutcome(e).ifPresent(m -> logger.debug("Expansion error response: {}", m)); + logger.debug("ValueSet with error while expanding: {}", + fhirContext.newJsonParser().encodeResourceToString(v)); + } + catch (Exception e) + { + logger.error("Error while expanding ValueSet {}|{}: {} - {}", v.getUrl(), v.getVersion(), + e.getClass().getName(), e.getMessage()); + logger.debug("ValueSet with error while expanding: {}", + fhirContext.newJsonParser().encodeResourceToString(v)); + } + } + + private void expandInternal(List expandedValueSets, ValueSetExpander expander, ValueSet v) + { + try + { + ValueSetExpansionOutcome expansion = expander.expand(v); + + if (expansion.getError() != null) + logger.warn("Error while expanding ValueSet {}|{}: {}", v.getUrl(), v.getVersion(), + expansion.getError()); + else + expandedValueSets.add(expansion.getValueset()); + } + catch (Exception e) + { + logger.info( + "Error while expanding ValueSet {}|{}: {} - {}, trying to expand via external terminology server next", + v.getUrl(), v.getVersion(), e.getClass().getName(), e.getMessage()); + + expandExternal(expandedValueSets, v); + } + } + + private Optional getOutcome(WebApplicationException e) + { + if (e.getResponse().hasEntity()) + { + String response = e.getResponse().readEntity(String.class); + return Optional.of(response); + } + else + return Optional.empty(); + } + + private IValidationSupport withSnapshots(ValidationPackageWithDepedencies packageWithDependencies, + List expandedValueSets) + { + Map snapshots = new HashMap<>(); + ValidationSupportChain supportChain = createSupportChain(fhirContext, packageWithDependencies, + snapshots.values(), expandedValueSets); + + SnapshotGenerator generator = internalSnapshotGeneratorFactory.apply(fhirContext, supportChain); + + packageWithDependencies.getValidationSupportResources().getStructureDefinitions().stream() + .filter(s -> s.hasDifferential() && !s.hasSnapshot()) + .forEach(diff -> createSnapshot(packageWithDependencies, snapshots, generator, diff)); + + return supportChain; + } + + private void createSnapshot(ValidationPackageWithDepedencies packageWithDependencies, + Map snapshots, SnapshotGenerator generator, StructureDefinition diff) + { + if (snapshots.containsKey(diff.getUrl() + "|" + diff.getVersion())) + return; + + List definitions = new ArrayList<>(); + definitions.addAll(packageWithDependencies.getStructureDefinitionDependencies(diff)); + definitions.add(diff); + + logger.debug("Generating snapshot for {}|{}, base {}, dependencies {}", diff.getUrl(), diff.getVersion(), + diff.getBaseDefinition(), + definitions.stream() + .filter(sd -> !sd.equals(diff) && !sd.getUrl().equals(diff.getBaseDefinition()) + && !(sd.getUrl() + "|" + sd.getVersion()).equals(diff.getBaseDefinition())) + .map(sd -> sd.getUrl() + "|" + sd.getVersion()).sorted() + .collect(Collectors.joining(", ", "[", "]"))); + + definitions.stream().filter(sd -> sd.hasDifferential() && !sd.hasSnapshot() + && !snapshots.containsKey(sd.getUrl() + "|" + sd.getVersion())).forEach(sd -> + { + try + { + SnapshotWithValidationMessages snapshot = generator.generateSnapshot(sd); + + if (snapshot.getSnapshot().hasSnapshot()) + snapshots.put(snapshot.getSnapshot().getUrl() + "|" + snapshot.getSnapshot().getVersion(), + snapshot.getSnapshot()); + else + logger.error( + "Error while generating snapshot for {}|{}: Not snaphsot returned from generator", + diff.getUrl(), diff.getVersion()); + + snapshot.getMessages().forEach(m -> + { + if (EnumSet.of(IssueSeverity.FATAL, IssueSeverity.ERROR, IssueSeverity.WARNING) + .contains(m.getLevel())) + logger.warn("{}|{} {}: {}", diff.getUrl(), diff.getVersion(), m.getLevel(), + m.toString()); + else + logger.info("{}|{} {}: {}", diff.getUrl(), diff.getVersion(), m.getLevel(), + m.toString()); + }); + } + catch (Exception e) + { + logger.error("Error while generating snapshot for {}|{}: {} - {}", diff.getUrl(), + diff.getVersion(), e.getClass().getName(), e.getMessage()); + } + }); + } + + private ValidationSupportChain createSupportChain(FhirContext context, + ValidationPackageWithDepedencies packageWithDependencies, + Collection snapshots, Collection expandedValueSets) + { + return new ValidationSupportChain(new CodeValidatorForExpandedValueSets(context), + new InMemoryTerminologyServerValidationSupport(context), + new ValidationSupportWithCustomResources(context, snapshots, null, expandedValueSets), + new ValidationSupportWithCustomResources(context, packageWithDependencies.getAllStructureDefinitions(), + packageWithDependencies.getAllCodeSystems(), packageWithDependencies.getAllValueSets()), + new DefaultProfileValidationSupport(context), new CommonCodeSystemsTerminologyService(context)); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageWithDepedencies.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageWithDepedencies.java new file mode 100644 index 00000000..765b7dbc --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageWithDepedencies.java @@ -0,0 +1,232 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.ElementDefinition; +import org.hl7.fhir.r4.model.ElementDefinition.ElementDefinitionBindingComponent; +import org.hl7.fhir.r4.model.Enumerations.BindingStrength; +import org.hl7.fhir.r4.model.MetadataResource; +import org.hl7.fhir.r4.model.NamingSystem; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.StructureDefinition.StructureDefinitionDifferentialComponent; +import org.hl7.fhir.r4.model.UriType; +import org.hl7.fhir.r4.model.ValueSet; + +import ca.uhn.fhir.context.FhirContext; + +public class ValidationPackageWithDepedencies extends ValidationPackage +{ + public static ValidationPackageWithDepedencies from( + Map packagesByNameAndVersion, + ValidationPackageIdentifier rootPackageIdentifier) + { + Objects.requireNonNull(packagesByNameAndVersion, "packagesByNameAndVersion"); + Objects.requireNonNull(rootPackageIdentifier, "rootPackageIdentifier"); + + ValidationPackage rootPackage = packagesByNameAndVersion.get(rootPackageIdentifier); + if (rootPackage == null) + throw new IllegalArgumentException("root package not part of given map"); + + List packages = packagesByNameAndVersion.entrySet().stream() + .filter(e -> !rootPackageIdentifier.equals(e.getKey())).map(Entry::getValue) + .collect(Collectors.toList()); + + return new ValidationPackageWithDepedencies(rootPackage, packages); + } + + private final List dependencies = new ArrayList<>(); + + private Map> structureDefinitionsByUrl; + private Map structureDefinitionsByUrlAndVersion; + + public ValidationPackageWithDepedencies(ValidationPackage validationPackage, List dependencies) + { + super(validationPackage.getName(), validationPackage.getVersion(), validationPackage.getEntries()); + + if (dependencies != null) + this.dependencies.addAll(dependencies); + } + + public List getDependencies() + { + return Collections.unmodifiableList(dependencies); + } + + @Override + public void parseResources(FhirContext context) + { + super.parseResources(context); + + getDependencies().forEach(p -> p.parseResources(context)); + } + + private List getAll(Function> accessor) + { + return Stream.concat(Stream.of(this), getDependencies().stream()) + .map(ValidationPackage::getValidationSupportResources).map(accessor).flatMap(List::stream) + .collect(Collectors.toList()); + } + + public List getAllCodeSystems() + { + return getAll(ValidationSupportResources::getCodeSystems); + } + + public List getAllNamingSystems() + { + return getAll(ValidationSupportResources::getNamingSystems); + } + + public List getAllStructureDefinitions() + { + return getAll(ValidationSupportResources::getStructureDefinitions); + } + + public List getAllValueSets() + { + return getAll(ValidationSupportResources::getValueSets); + } + + public ValidationSupportResources getAllValidationSupportResources() + { + return new ValidationSupportResources(getAllCodeSystems(), getAllNamingSystems(), getAllStructureDefinitions(), + getAllValueSets()); + } + + private Map> getStructureDefinitionsByUrl() + { + if (structureDefinitionsByUrl == null) + structureDefinitionsByUrl = getAllStructureDefinitions().stream().filter(StructureDefinition::hasUrl) + .collect(Collectors.toMap(StructureDefinition::getUrl, Collections::singletonList, (sd1, sd2) -> + { + List sds = new ArrayList<>(); + sds.addAll(sd1); + sds.addAll(sd2); + return sds; + })); + + return structureDefinitionsByUrl; + } + + private Map getStructureDefinitionsByUrlAndVersion() + { + if (structureDefinitionsByUrlAndVersion == null) + structureDefinitionsByUrlAndVersion = getAllStructureDefinitions().stream() + .filter(StructureDefinition::hasUrl).filter(StructureDefinition::hasVersion) + .collect(Collectors.toMap(s -> s.getUrl() + "|" + s.getVersion(), Function.identity())); + + return structureDefinitionsByUrlAndVersion; + } + + public List getStructureDefinitionDependencies(StructureDefinition structureDefinition) + { + return doGetDependencies(structureDefinition, new HashSet<>()); + } + + private List doGetDependencies(StructureDefinition structureDefinition, Set visited) + { + if (visited.contains(structureDefinition.getUrl()) + || visited.contains(structureDefinition.getUrl() + "|" + structureDefinition.getVersion())) + return Collections.emptyList(); + else + { + visited.add(structureDefinition.getUrl()); + visited.add(structureDefinition.getUrl() + "|" + structureDefinition.getVersion()); + } + + List dependencies = new ArrayList<>(); + Set baseDefinitions = getStructureDefinitionsByUrl( + structureDefinition.getBaseDefinition()); + + baseDefinitions.forEach(sd -> dependencies.addAll(doGetDependencies(sd, visited))); + dependencies.addAll(baseDefinitions); + + structureDefinition.getDifferential().getElement().stream().forEach(e -> + { + if (e.hasPath() && "Extension.url".equals(e.getPath()) && e.hasFixed() && e.getFixed() instanceof UriType) + { + UriType t = (UriType) e.getFixed(); + Set extensions = getStructureDefinitionsByUrl(t.getValue()); + extensions.forEach(sd -> dependencies.addAll(doGetDependencies(sd, visited))); + dependencies.addAll(extensions); + } + + if (e.hasType()) + { + e.getType().forEach(t -> + { + if (t.hasProfile()) + { + t.getProfile().forEach(p -> + { + Set profiles = getStructureDefinitionsByUrl(p.getValue()); + profiles.forEach(sd -> dependencies.addAll(doGetDependencies(sd, visited))); + dependencies.addAll(profiles); + }); + } + + if (t.hasTargetProfile()) + { + t.getTargetProfile().forEach(p -> + { + Set targetProfiles = getStructureDefinitionsByUrl(p.getValue()); + targetProfiles.forEach(sd -> dependencies.addAll(doGetDependencies(sd, visited))); + dependencies.addAll(targetProfiles); + }); + } + }); + } + }); + + return dependencies; + } + + private Set getStructureDefinitionsByUrl(String sdUrl) + { + Set sds = new HashSet<>(); + + List byUrl = getStructureDefinitionsByUrl().get(sdUrl); + if (byUrl != null) + sds.addAll(byUrl); + + StructureDefinition byUrlAndVersion = getStructureDefinitionsByUrlAndVersion().get(sdUrl); + if (byUrlAndVersion != null) + sds.add(byUrlAndVersion); + + return sds; + } + + private Set findValueSetsWithBindingStrength(Stream sds, + EnumSet bindingStrengths) + { + return sds.filter(StructureDefinition::hasDifferential).map(StructureDefinition::getDifferential) + .filter(StructureDefinitionDifferentialComponent::hasElement) + .map(StructureDefinitionDifferentialComponent::getElement).flatMap(List::stream) + .filter(ElementDefinition::hasBinding).map(ElementDefinition::getBinding) + .filter(b -> bindingStrengths.contains(b.getStrength())) + .map(ElementDefinitionBindingComponent::getValueSet).collect(Collectors.toSet()); + } + + public List getValueSetsIncludingDependencies(EnumSet bindingStrengths) + { + Stream sds = getValidationSupportResources().getStructureDefinitions().stream() + .flatMap(sd -> Stream.concat(Stream.of(sd), getStructureDefinitionDependencies(sd).stream())) + .distinct(); + + Set neededValueSets = findValueSetsWithBindingStrength(sds, bindingStrengths); + return getAllValueSets().stream().filter(vs -> neededValueSets.contains(vs.getUrl()) + || neededValueSets.contains(vs.getUrl() + "|" + vs.getVersion())).collect(Collectors.toList()); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationSupportResources.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationSupportResources.java new file mode 100644 index 00000000..48406a53 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationSupportResources.java @@ -0,0 +1,51 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.NamingSystem; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.ValueSet; + +public class ValidationSupportResources +{ + private final List codeSystems = new ArrayList<>(); + private final List namingSystems = new ArrayList<>(); + private final List structureDefinitions = new ArrayList<>(); + private final List valueSets = new ArrayList<>(); + + public ValidationSupportResources(List codeSystems, List namingSystems, + List structureDefinitions, List valueSets) + { + if (codeSystems != null) + this.codeSystems.addAll(codeSystems); + if (namingSystems != null) + this.namingSystems.addAll(namingSystems); + if (structureDefinitions != null) + this.structureDefinitions.addAll(structureDefinitions); + if (valueSets != null) + this.valueSets.addAll(valueSets); + } + + public List getCodeSystems() + { + return Collections.unmodifiableList(codeSystems); + } + + public List getNamingSystems() + { + return Collections.unmodifiableList(namingSystems); + } + + public List getStructureDefinitions() + { + return Collections.unmodifiableList(structureDefinitions); + } + + public List getValueSets() + { + return Collections.unmodifiableList(valueSets); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpanderWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpanderWithFileSystemCache.java new file mode 100644 index 00000000..b33c1077 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpanderWithFileSystemCache.java @@ -0,0 +1,115 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Objects; + +import org.highmed.dsf.fhir.validation.ValueSetExpander; +import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; +import org.hl7.fhir.r4.model.ValueSet; +import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +import ca.uhn.fhir.context.FhirContext; + +public class ValueSetExpanderWithFileSystemCache + extends AbstractFhirResourceFileSystemCache + implements ValueSetExpander, InitializingBean +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationPackageClientWithFileSystemCache.class); + + private final ValueSetExpander delegate; + + /** + * For JSON content with gzip compression using the .json.xz file name suffix. + * + * @param cacheFolder + * not null + * @param resourceType + * not null + * @param fhirContext + * not null + * @param delegate + * not null + * @see AbstractFileSystemCache#FILENAME_SUFFIX + * @see AbstractFileSystemCache#OUT_COMPRESSOR_FACTORY + * @see AbstractFileSystemCache#IN_COMPRESSOR_FACTORY + */ + public ValueSetExpanderWithFileSystemCache(Path cacheFolder, FhirContext fhirContext, ValueSetExpander delegate) + { + super(cacheFolder, ValueSet.class, fhirContext); + + this.delegate = delegate; + } + + public ValueSetExpanderWithFileSystemCache(Path cacheFolder, String fileNameSuffix, + FunctionWithIoException outCompressorFactory, + FunctionWithIoException inCompressorFactory, FhirContext fhirContext, + ValueSetExpander delegate) + { + super(cacheFolder, fileNameSuffix, outCompressorFactory, inCompressorFactory, ValueSet.class, fhirContext); + + this.delegate = delegate; + } + + @Override + public void afterPropertiesSet() throws Exception + { + super.afterPropertiesSet(); + + Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public ValueSetExpansionOutcome expand(ValueSet valueSet) + { + Objects.requireNonNull(valueSet, "valueSet"); + + if (valueSet.hasExpansion()) + { + logger.debug("ValueSet {}|{} already expanded", valueSet.getUrl(), valueSet.getVersion()); + return new ValueSetExpansionOutcome(valueSet); + } + + Objects.requireNonNull(valueSet.getUrl(), "valueSet.url"); + Objects.requireNonNull(valueSet.getVersion(), "valueSet.version"); + + try + { + // ValueSetExpansionOutcome read = readFromCache(valueSet.getUrl(), valueSet.getVersion(), ValueSet.class, + // ValueSetExpansionOutcome::new); + ValueSetExpansionOutcome read = readResourceFromCache(valueSet.getUrl(), valueSet.getVersion(), + ValueSetExpansionOutcome::new); + + if (read != null) + return read; + else + return downloadAndWriteToCache(valueSet); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + private ValueSetExpansionOutcome downloadAndWriteToCache(ValueSet valueSet) throws IOException + { + ValueSetExpansionOutcome expanded = delegate.expand(valueSet); + + if (PublicationStatus.DRAFT.equals(expanded.getValueset().getStatus())) + { + logger.info("Not writing expanded ValueSet {}|{} with status {} to cache", expanded.getValueset().getUrl(), + expanded.getValueset().getVersion(), expanded.getValueset().getStatus()); + return expanded; + } + else + // return writeToCache(expanded, ValueSetExpansionOutcome::getValueset, ValueSet::getUrl, + // ValueSet::getVersion); + return writeRsourceToCache(expanded, ValueSetExpansionOutcome::getValueset, ValueSet::getUrl, + ValueSet::getVersion); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClient.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClient.java new file mode 100644 index 00000000..391cbfda --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClient.java @@ -0,0 +1,22 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; + +import javax.ws.rs.WebApplicationException; + +import org.hl7.fhir.r4.model.CapabilityStatement; +import org.hl7.fhir.r4.model.ValueSet; + +public interface ValueSetExpansionClient +{ + /** + * @param valueSet + * not null + * @return expanded {@link ValueSet}, never null + * @throws IOException + * @throws WebApplicationException + */ + ValueSet expand(ValueSet valueSet) throws IOException, WebApplicationException; + + CapabilityStatement getMetadata() throws WebApplicationException; +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java new file mode 100644 index 00000000..ca76021c --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java @@ -0,0 +1,148 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.security.KeyStore; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; + +import javax.net.ssl.SSLContext; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; + +import org.glassfish.jersey.SslConfigurator; +import org.glassfish.jersey.apache.connector.ApacheConnectorProvider; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.ClientProperties; +import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; +import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJaxbJsonProvider; +import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJsonProvider; +import org.glassfish.jersey.logging.LoggingFeature; +import org.glassfish.jersey.logging.LoggingFeature.Verbosity; +import org.highmed.dsf.fhir.adapter.CapabilityStatementJsonFhirAdapter; +import org.highmed.dsf.fhir.adapter.CapabilityStatementXmlFhirAdapter; +import org.highmed.dsf.fhir.adapter.OperationOutcomeJsonFhirAdapter; +import org.highmed.dsf.fhir.adapter.OperationOutcomeXmlFhirAdapter; +import org.highmed.dsf.fhir.adapter.ParametersJsonFhirAdapter; +import org.highmed.dsf.fhir.adapter.ParametersXmlFhirAdapter; +import org.highmed.dsf.fhir.adapter.ValueSetJsonFhirAdapter; +import org.highmed.dsf.fhir.adapter.ValueSetXmlFhirAdapter; +import org.hl7.fhir.r4.model.CapabilityStatement; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.ValueSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.rest.api.Constants; + +public class ValueSetExpansionClientJersey implements ValueSetExpansionClient +{ + private static final Logger logger = LoggerFactory.getLogger(ValueSetExpansionClientJersey.class); + private static final java.util.logging.Logger requestDebugLogger = java.util.logging.Logger + .getLogger(ValueSetExpansionClientJersey.class.getName()); + + private final Client client; + private final String baseUrl; + + public ValueSetExpansionClientJersey(String baseUrl, ObjectMapper objectMapper, FhirContext fhirContext) + { + this(baseUrl, null, null, null, null, null, null, null, null, 0, 0, false, objectMapper, fhirContext); + } + + public ValueSetExpansionClientJersey(String baseUrl, KeyStore trustStore, KeyStore keyStore, + char[] keyStorePassword, String basicAuthUsername, char[] basicAuthPassword, String proxySchemeHostPort, + String proxyUsername, char[] proxyPassword, int connectTimeout, int readTimeout, boolean logRequests, + ObjectMapper objectMapper, FhirContext fhirContext) + { + SSLContext sslContext = null; + if (trustStore != null && keyStore == null && keyStorePassword == null) + sslContext = SslConfigurator.newInstance().trustStore(trustStore).createSSLContext(); + else if (trustStore == null && keyStore != null && keyStorePassword != null) + sslContext = SslConfigurator.newInstance().keyStore(keyStore).keyStorePassword(keyStorePassword) + .createSSLContext(); + else if (trustStore != null && keyStore != null && keyStorePassword != null) + sslContext = SslConfigurator.newInstance().trustStore(trustStore).keyStore(keyStore) + .keyStorePassword(keyStorePassword).createSSLContext(); + + ClientBuilder builder = ClientBuilder.newBuilder(); + + if (sslContext != null) + builder = builder.sslContext(sslContext); + + if (basicAuthUsername != null && basicAuthPassword != null) + { + HttpAuthenticationFeature basicAuthFeature = HttpAuthenticationFeature.basic(basicAuthUsername, + String.valueOf(basicAuthPassword)); + builder = builder.register(basicAuthFeature); + } + + ClientConfig config = new ClientConfig(); + config.connectorProvider(new ApacheConnectorProvider()); + config.property(ClientProperties.PROXY_URI, proxySchemeHostPort); + config.property(ClientProperties.PROXY_USERNAME, proxyUsername); + config.property(ClientProperties.PROXY_PASSWORD, proxyPassword == null ? null : String.valueOf(proxyPassword)); + builder = builder.withConfig(config); + + builder = builder.readTimeout(readTimeout, TimeUnit.MILLISECONDS).connectTimeout(connectTimeout, + TimeUnit.MILLISECONDS); + + if (objectMapper != null) + { + JacksonJaxbJsonProvider p = new JacksonJaxbJsonProvider(JacksonJsonProvider.BASIC_ANNOTATIONS); + p.setMapper(objectMapper); + builder = builder.register(p); + } + + builder = builder.register(new CapabilityStatementJsonFhirAdapter(fhirContext)) + .register(new CapabilityStatementXmlFhirAdapter(fhirContext)) + .register(new OperationOutcomeJsonFhirAdapter(fhirContext)) + .register(new OperationOutcomeXmlFhirAdapter(fhirContext)) + .register(new ParametersJsonFhirAdapter(fhirContext)) + .register(new ParametersXmlFhirAdapter(fhirContext)).register(new ValueSetJsonFhirAdapter(fhirContext)) + .register(new ValueSetXmlFhirAdapter(fhirContext)); + + if (logRequests) + { + builder = builder.register(new LoggingFeature(requestDebugLogger, Level.FINE, Verbosity.PAYLOAD_ANY, + LoggingFeature.DEFAULT_MAX_ENTITY_SIZE)); + } + + client = builder.build(); + + this.baseUrl = baseUrl; + } + + private WebTarget getResource() + { + return client.target(baseUrl); + } + + @Override + public ValueSet expand(ValueSet valueSet) throws WebApplicationException + { + Objects.requireNonNull(valueSet, "valueSet"); + + if (valueSet.hasExpansion()) + { + logger.debug("ValueSet {}|{} already expanded", valueSet.getUrl(), valueSet.getVersion()); + return valueSet; + } + + Parameters parameters = new Parameters(); + parameters.addParameter().setName("valueSet").setResource(valueSet); + + return getResource().path("ValueSet").path("$expand").request(Constants.CT_FHIR_JSON_NEW) + .post(Entity.entity(parameters, Constants.CT_FHIR_JSON_NEW), ValueSet.class); + } + + @Override + public CapabilityStatement getMetadata() throws WebApplicationException + { + return getResource().path("metadata").request(Constants.CT_FHIR_JSON_NEW).get(CapabilityStatement.class); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java new file mode 100644 index 00000000..9f74d38b --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java @@ -0,0 +1,108 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Objects; +import java.util.function.Function; + +import javax.ws.rs.WebApplicationException; + +import org.hl7.fhir.r4.model.CapabilityStatement; +import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; +import org.hl7.fhir.r4.model.ValueSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +import ca.uhn.fhir.context.FhirContext; + +public class ValueSetExpansionClientWithFileSystemCache extends AbstractFhirResourceFileSystemCache + implements ValueSetExpansionClient, InitializingBean +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationPackageClientWithFileSystemCache.class); + + private final ValueSetExpansionClient delegate; + + /** + * For JSON content with gzip compression using the .json.xz file name suffix. + * + * @param cacheFolder + * not null + * @param fhirContext + * not null + * @param delegate + * not null + * @see AbstractFileSystemCache#FILENAME_SUFFIX + * @see AbstractFileSystemCache#OUT_COMPRESSOR_FACTORY + * @see AbstractFileSystemCache#IN_COMPRESSOR_FACTORY + */ + public ValueSetExpansionClientWithFileSystemCache(Path cacheFolder, FhirContext fhirContext, + ValueSetExpansionClient delegate) + { + super(cacheFolder, ValueSet.class, fhirContext); + + this.delegate = delegate; + } + + public ValueSetExpansionClientWithFileSystemCache(Path cacheFolder, String fileNameSuffix, + FunctionWithIoException outCompressorFactory, + FunctionWithIoException inCompressorFactory, FhirContext fhirContext, + ValueSetExpansionClient delegate) + { + super(cacheFolder, fileNameSuffix, outCompressorFactory, inCompressorFactory, ValueSet.class, fhirContext); + + this.delegate = delegate; + } + + @Override + public void afterPropertiesSet() throws Exception + { + super.afterPropertiesSet(); + + Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public ValueSet expand(ValueSet valueSet) throws IOException, WebApplicationException + { + Objects.requireNonNull(valueSet, "valueSet"); + + if (valueSet.hasExpansion()) + { + logger.debug("ValueSet {}|{} already expanded", valueSet.getUrl(), valueSet.getVersion()); + return valueSet; + } + + Objects.requireNonNull(valueSet.getUrl(), "valueSet.url"); + Objects.requireNonNull(valueSet.getVersion(), "valueSet.version"); + + ValueSet read = readResourceFromCache(valueSet.getUrl(), valueSet.getVersion(), Function.identity()); + + if (read != null) + return read; + else + return expandAndWriteToCache(valueSet); + } + + private ValueSet expandAndWriteToCache(ValueSet valueSet) throws IOException + { + ValueSet expanded = delegate.expand(valueSet); + + if (PublicationStatus.DRAFT.equals(expanded.getStatus())) + { + logger.info("Not writing expanded ValueSet {}|{} with status {} to cache", expanded.getUrl(), + expanded.getVersion(), expanded.getStatus()); + return expanded; + } + else + return writeRsourceToCache(expanded, Function.identity(), ValueSet::getUrl, ValueSet::getVersion); + } + + @Override + public CapabilityStatement getMetadata() throws WebApplicationException + { + return delegate.getMetadata(); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java new file mode 100644 index 00000000..7e7a957a --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java @@ -0,0 +1,41 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition; + +import org.hl7.fhir.r4.model.ElementDefinition; +import org.hl7.fhir.r4.model.ElementDefinition.DiscriminatorType; +import org.hl7.fhir.r4.model.ElementDefinition.ElementDefinitionSlicingComponent; +import org.hl7.fhir.r4.model.ElementDefinition.ElementDefinitionSlicingDiscriminatorComponent; +import org.hl7.fhir.r4.model.ElementDefinition.SlicingRules; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Closed type slicings result in error from the snapshot generator. + */ +public class ClosedTypeSlicingRemover implements StructureDefinitionModifier +{ + private static final Logger logger = LoggerFactory.getLogger(ClosedTypeSlicingRemover.class); + + @Override + public StructureDefinition modify(StructureDefinition sd) + { + sd.getDifferential().getElement().stream().filter(ElementDefinition::hasSlicing).forEach(e -> + { + ElementDefinitionSlicingComponent slicing = e.getSlicing(); + if (SlicingRules.OPEN.equals(slicing.getRules()) && slicing.getDiscriminator().size() == 1) + { + ElementDefinitionSlicingDiscriminatorComponent discriminator = slicing.getDiscriminator().get(0); + if (DiscriminatorType.TYPE.equals(discriminator.getType()) && "$this".equals(discriminator.getPath())) + { + logger.warn( + "Removing Type slicing with slicing.rules != closed from validation rule with id {} in StructureDefinition {}|{}", + e.getId(), sd.getUrl(), sd.getVersion()); + + e.setSlicing(null); + } + } + }); + + return sd; + } +} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/GeccoRadiologyProceduresCodingSliceMinFixer.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/GeccoRadiologyProceduresCodingSliceMinFixer.java new file mode 100644 index 00000000..165ff58f --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/GeccoRadiologyProceduresCodingSliceMinFixer.java @@ -0,0 +1,32 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition; + +import org.hl7.fhir.r4.model.StructureDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * HAPI snapshot generator adds min=1, if no min value specified in the parent StructureDefinition. + */ +public class GeccoRadiologyProceduresCodingSliceMinFixer implements StructureDefinitionModifier +{ + private static final Logger logger = LoggerFactory.getLogger(GeccoRadiologyProceduresCodingSliceMinFixer.class); + + @Override + public StructureDefinition modify(StructureDefinition sd) + { + if ("https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/radiology-procedures" + .equals(sd.getUrl()) && "1.0.5".equals(sd.getVersion())) + { + sd.getDifferential().getElement().stream().filter( + e -> "Procedure.code.coding".equals(e.getPath()) && e.hasMax() && e.hasSliceName() && !e.hasMin()) + .forEach(e -> + { + logger.warn("Adding min=0 to rule with id {} in StructureDefinition {}|{}", e.getId(), + sd.getUrl(), sd.getVersion()); + e.setMin(0); + }); + } + + return sd; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java new file mode 100644 index 00000000..d505280a --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java @@ -0,0 +1,41 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition; + +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.hl7.fhir.r4.model.ElementDefinition; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Mandatory identifier on ObservationLab not compatible with data protection rules with current pseudonymization. + */ +public class MiiModuleLabObservationLab10IdentifierRemover implements StructureDefinitionModifier +{ + private static final Logger logger = LoggerFactory.getLogger(MiiModuleLabObservationLab10IdentifierRemover.class); + + @Override + public StructureDefinition modify(StructureDefinition sd) + { + if ("https://www.medizininformatik-initiative.de/fhir/core/modul-labor/StructureDefinition/ObservationLab" + .equals(sd.getUrl()) && "1.0".equals(sd.getVersion())) + { + Predicate toRemove = e -> e.hasPath() + && e.getPath().startsWith("Observation.identifier"); + + List filteredRules = sd.getDifferential().getElement().stream().filter(toRemove.negate()) + .collect(Collectors.toList()); + + logger.warn("Removing validation rules with ids {} from StructureDefinition {}|{}", + sd.getDifferential().getElement().stream().filter(toRemove).map(ElementDefinition::getId) + .collect(Collectors.joining(", ", "[", "]")), + sd.getUrl(), sd.getVersion()); + + sd.getDifferential().setElement(filteredRules); + } + + return sd; + } +} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/StructureDefinitionModifier.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/StructureDefinitionModifier.java new file mode 100644 index 00000000..1312d867 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/StructureDefinitionModifier.java @@ -0,0 +1,9 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition; + +import org.hl7.fhir.r4.model.StructureDefinition; + +@FunctionalInterface +public interface StructureDefinitionModifier +{ + StructureDefinition modify(StructureDefinition sd); +} diff --git a/codex-process-data-transfer/src/main/resources/bpe/send.bpmn b/codex-process-data-transfer/src/main/resources/bpe/send.bpmn index 16d66505..46782498 100644 --- a/codex-process-data-transfer/src/main/resources/bpe/send.bpmn +++ b/codex-process-data-transfer/src/main/resources/bpe/send.bpmn @@ -1,5 +1,5 @@ - + Flow_1km61ly @@ -102,12 +102,21 @@ ${!idatMergeGranted} + + Flow_018z9ct + + + + Flow_018z9ct + + + @@ -184,6 +193,11 @@ + + + + + @@ -238,6 +252,12 @@ + + + + + + diff --git a/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-source.xml b/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-source.xml new file mode 100644 index 00000000..f63dbd87 --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-source.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + <!-- status managed by bpe --> + <status value="unknown" /> + <experimental value="false" /> + <!-- date managed by bpe --> + <date value="#{date}" /> + <publisher value="NUM-CODEX" /> + <description value="CodeSystem with error source values for the NUM-CODEX data-transfer processes" /> + <caseSensitive value="true" /> + <hierarchyMeaning value="grouped-by" /> + <versionNeeded value="false" /> + <content value="complete" /> + <concept> + <code value="MeDIC" /> + <display value="MeDIC" /> + <definition value="Error during process execution at the medical Data Integration Center" /> + </concept> + <concept> + <code value="GTH" /> + <display value="GTH" /> + <definition value="Error during process execution at the GECCO Transfer Hub" /> + </concept> + <concept> + <code value="fTTP" /> + <display value="fTTP" /> + <definition value="Error during process execution at the federated Trusted Third Party" /> + </concept> + <concept> + <code value="CRR" /> + <display value="CRR" /> + <definition value="Error during process execution in the Central Research Repository" /> + </concept> +</CodeSystem> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error.xml b/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error.xml new file mode 100644 index 00000000..0fce6339 --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error.xml @@ -0,0 +1,30 @@ +<CodeSystem xmlns="http://hl7.org/fhir"> + <meta> + <tag> + <system value="http://highmed.org/fhir/CodeSystem/read-access-tag" /> + <code value="ALL" /> + </tag> + </meta> + <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error" /> + <!-- version managed by bpe --> + <version value="#{version}" /> + <name value="NumCodexDataTransferError" /> + <title value="NUM-CODEX data-transfer error" /> + <!-- status managed by bpe --> + <status value="unknown" /> + <experimental value="false" /> + <!-- date managed by bpe --> + <date value="#{date}" /> + <publisher value="NUM-CODEX" /> + <description value="CodeSystem with error codes for the NUM-CODEX data-transfer processes" /> + <caseSensitive value="true" /> + <hierarchyMeaning value="grouped-by" /> + <versionNeeded value="false" /> + <content value="complete" /> + <concept> + <code value="validation-failed" /> + <display value="Validation Failed" /> + <definition value="Error or fatal error during validaton of FHIR resources" /> + </concept> + <!-- TODO add additional error codes --> +</CodeSystem> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-extension-error-metadata.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-extension-error-metadata.xml new file mode 100644 index 00000000..f4e1f9a4 --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-extension-error-metadata.xml @@ -0,0 +1,137 @@ +<StructureDefinition xmlns="http://hl7.org/fhir"> + <meta> + <tag> + <system value="http://highmed.org/fhir/CodeSystem/read-access-tag" /> + <code value="ALL" /> + </tag> + </meta> + <url value="https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/error-metadata" /> + <!-- version managed by bpe --> + <version value="#{version}" /> + <name value="ErrorMetadata" /> + <!-- status managed by bpe --> + <status value="unknown" /> + <experimental value="false" /> + <!-- date managed by bpe --> + <date value="#{date}" /> + <fhirVersion value="4.0.1" /> + <kind value="complex-type" /> + <abstract value="false" /> + <context> + <type value="element" /> + <expression value="Task.output" /> + </context> + <type value="Extension" /> + <baseDefinition value="http://hl7.org/fhir/StructureDefinition/Extension" /> + <derivation value="constraint" /> + <differential> + <element id="Extension"> + <path value="Extension" /> + <max value="1" /> + </element> + <element id="Extension.extension"> + <path value="Extension.extension" /> + <slicing> + <discriminator> + <type value="value" /> + <path value="url" /> + </discriminator> + <rules value="open" /> + </slicing> + <min value="2" /> + </element> + <element id="Extension.extension:type"> + <path value="Extension.extension" /> + <sliceName value="type" /> + <min value="1" /> + <max value="1" /> + <binding> + <strength value="required" /> + <valueSet value="http://www.netzwerk-universitaetsmedizin.de/fhir/ValueSet/data-transfer-error" /> + </binding> + </element> + <element id="Extension.extension:type.url"> + <path value="Extension.extension.url" /> + <fixedUri value="type" /> + </element> + <element id="Extension.extension:type.value[x]"> + <path value="Extension.extension.value[x]" /> + <min value="1" /> + <type> + <code value="Coding" /> + </type> + </element> + <element id="Extension.extension:type.value[x].system"> + <path value="Extension.extension.value[x].system" /> + <min value="1" /> + <fixedUri value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error" /> + </element> + <element id="Extension.extension:type.value[x].code"> + <path value="Extension.extension.value[x].code" /> + <min value="1" /> + </element> + <element id="Extension.extension:source"> + <path value="Extension.extension" /> + <sliceName value="source" /> + <min value="1" /> + <max value="1" /> + <binding> + <strength value="required" /> + <valueSet value="http://www.netzwerk-universitaetsmedizin.de/fhir/ValueSet/data-transfer-error-source" /> + </binding> + </element> + <element id="Extension.extension:source.url"> + <path value="Extension.extension.url" /> + <fixedUri value="source" /> + </element> + <element id="Extension.extension:source.value[x]"> + <path value="Extension.extension.value[x]" /> + <min value="1" /> + <type> + <code value="Coding" /> + </type> + </element> + <element id="Extension.extension:source.value[x].system"> + <path value="Extension.extension.value[x].system" /> + <min value="1" /> + <fixedUri value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-source" /> + </element> + <element id="Extension.extension:source.value[x].code"> + <path value="Extension.extension.value[x].code" /> + <min value="1" /> + </element> + <element id="Extension.extension:reference"> + <path value="Extension.extension" /> + <sliceName value="reference" /> + <min value="0" /> + <max value="1" /> + </element> + <element id="Extension.extension:reference.url"> + <path value="Extension.extension.url" /> + <fixedUri value="reference" /> + </element> + <element id="Extension.extension:reference.value[x]"> + <path value="Extension.extension.value[x]" /> + <min value="1" /> + <type> + <code value="Reference" /> + </type> + </element> + <element id="Extension.extension:reference.value[x].reference"> + <path value="Extension.extension.value[x].reference" /> + <min value="1" /> + </element> + <element id="Extension.extension:reference.value[x].identifier"> + <path value="Extension.extension.value[x].identifier" /> + <max value="0" /> + </element> + <element id="Extension.url"> + <path value="Extension.url" /> + <fixedUri value="https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/error-metadata" /> + </element> + <element id="Extension.value[x]"> + <path value="Extension.value[x]" /> + <max value="0" /> + </element> + </differential> +</StructureDefinition> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-receive.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-receive.xml index dd0227a0..bad90d7d 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-receive.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-receive.xml @@ -6,8 +6,10 @@ </tag> </meta> <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/task-start-data-receive" /> + <!-- version managed by bpe --> <version value="#{version}" /> <name value="TaskStartDataReceive" /> + <!-- status managed by bpe --> <status value="unknown" /> <experimental value="false" /> <!-- date managed by bpe --> diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-send.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-send.xml index 679aac08..03d12449 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-send.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-send.xml @@ -144,5 +144,30 @@ <code value="instant" /> </type> </element> + <element id="Task.output:error"> + <path value="Task.output" /> + <sliceName value="error" /> + </element> + <element id="Task.output:error.extension"> + <path value="Task.output.extension" /> + <slicing> + <discriminator> + <type value="value" /> + <path value="url" /> + </discriminator> + <rules value="open" /> + </slicing> + <min value="0" /> + </element> + <element id="Task.output:error.extension:error-metadata"> + <path value="Task.output.extension" /> + <sliceName value="error-metadata" /> + <min value="0" /> + <type> + <code value="Extension" /> + <profile value="https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/error-metadata" /> + </type> + <isModifier value="false" /> + </element> </differential> </StructureDefinition> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-translate.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-translate.xml index 9b05f064..34c6941a 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-translate.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-translate.xml @@ -6,8 +6,10 @@ </tag> </meta> <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/task-start-data-translate" /> + <!-- version managed by bpe --> <version value="#{version}" /> <name value="TaskStartDataTranslate" /> + <!-- status managed by bpe --> <status value="unknown" /> <experimental value="false" /> <!-- date managed by bpe --> diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-trigger.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-trigger.xml index 8912cfec..dae32234 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-trigger.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-trigger.xml @@ -6,8 +6,10 @@ </tag> </meta> <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/task-start-data-trigger" /> + <!-- version managed by bpe --> <version value="#{version}" /> <name value="TaskStartDataTrigger" /> + <!-- status managed by bpe --> <status value="unknown" /> <experimental value="false" /> <!-- date managed by bpe --> diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-stop-data-trigger.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-stop-data-trigger.xml index 28d49323..6c8a2b77 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-stop-data-trigger.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-stop-data-trigger.xml @@ -6,8 +6,10 @@ </tag> </meta> <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/task-stop-data-trigger" /> + <!-- version managed by bpe --> <version value="#{version}" /> <name value="TaskStopDataTrigger" /> + <!-- status managed by bpe --> <status value="unknown" /> <experimental value="false" /> <!-- date managed by bpe --> diff --git a/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-source.xml b/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-source.xml new file mode 100644 index 00000000..c7574a44 --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-source.xml @@ -0,0 +1,26 @@ +<ValueSet xmlns="http://hl7.org/fhir"> + <meta> + <tag> + <system value="http://highmed.org/fhir/CodeSystem/read-access-tag" /> + <code value="ALL" /> + </tag> + </meta> + <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/ValueSet/data-transfer-error-source"/> + <!-- version managed by bpe --> + <version value="#{version}" /> + <name value="NumCodexDataTransferErrorSource"/> + <title value="NUM-CODEX data-transfer error source"/> + <!-- status managed by bpe --> + <status value="unknown" /> + <experimental value="false"/> + <!-- date managed by bpe --> + <date value="#{date}"/> + <publisher value="NUM-CODEX"/> + <description value="CodeSystem with with error source values for the NUM-CODEX data-transfer processes"/> + <immutable value="true"/> + <compose> + <include> + <system value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-source"/> + </include> + </compose> +</ValueSet> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error.xml b/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error.xml new file mode 100644 index 00000000..c78602f2 --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error.xml @@ -0,0 +1,26 @@ +<ValueSet xmlns="http://hl7.org/fhir"> + <meta> + <tag> + <system value="http://highmed.org/fhir/CodeSystem/read-access-tag" /> + <code value="ALL" /> + </tag> + </meta> + <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/ValueSet/data-transfer-error"/> + <!-- version managed by bpe --> + <version value="#{version}" /> + <name value="NumCodexDataTransferError"/> + <title value="NUM-CODEX data-transfer error"/> + <!-- status managed by bpe --> + <status value="unknown" /> + <experimental value="false"/> + <!-- date managed by bpe --> + <date value="#{date}"/> + <publisher value="NUM-CODEX"/> + <description value="CodeSystem with with error codes for the NUM-CODEX data-transfer processes"/> + <immutable value="true"/> + <compose> + <include> + <system value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error"/> + </include> + </compose> +</ValueSet> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/log4j2.xml b/codex-process-data-transfer/src/main/resources/log4j2.xml new file mode 100644 index 00000000..ac70ba75 --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/log4j2.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Config for ValidationMain class --> +<Configuration status="INFO" monitorInterval="30" verbose="false"> + + <Appenders> + <Console name="CONSOLE" target="SYSTEM_ERR"> + <PatternLayout pattern="%p\t| %m%n" /> + </Console> + </Appenders> + + <Loggers> + <Logger name="de.netzwerk_universitaetsmedizin" level="DEBUG" /> + + <Root level="WARN"> + <AppenderRef ref="CONSOLE" /> + </Root> + </Loggers> +</Configuration> \ No newline at end of file diff --git a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidateDataLearningTest.java b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidateDataLearningTest.java new file mode 100644 index 00000000..f7e8ef74 --- /dev/null +++ b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidateDataLearningTest.java @@ -0,0 +1,301 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.highmed.dsf.fhir.json.ObjectMapperFactory; +import org.highmed.dsf.fhir.validation.SnapshotGenerator; +import org.highmed.dsf.fhir.validation.SnapshotGenerator.SnapshotWithValidationMessages; +import org.highmed.dsf.fhir.validation.ValidationSupportWithCustomResources; +import org.highmed.dsf.fhir.validation.ValueSetExpanderImpl; +import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService; +import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport; +import org.hl7.fhir.common.hapi.validation.support.ValidationSupportChain; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; +import org.hl7.fhir.r4.model.CanonicalType; +import org.hl7.fhir.r4.model.ElementDefinition; +import org.hl7.fhir.r4.model.ElementDefinition.TypeRefComponent; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.StringType; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.support.DefaultProfileValidationSupport; + +public class ValidateDataLearningTest +{ + private static final Logger logger = LoggerFactory.getLogger(ValidateDataLearningTest.class); + + private static final Path cacheFolder = Paths.get("target"); + private static final FhirContext fhirContext = FhirContext.forR4(); + private static final ObjectMapper mapper = ObjectMapperFactory.createObjectMapper(fhirContext); + + @Test + public void testDownloadTagGz() throws Exception + { + ValidationPackageClient client = new ValidationPackageClientJersey("https://packages.simplifier.net"); + + ValidationPackage validationPackage = client.download("de.gecco", "1.0.5"); + + validationPackage.getEntries().forEach(e -> + { + if ("package/package.json".equals(e.getFileName())) + logger.debug(new String(e.getContent(), StandardCharsets.UTF_8)); + }); + + ValidationPackageDescriptor descriptor = validationPackage.getDescriptor(mapper); + logger.debug(descriptor.getName() + "/" + descriptor.getVersion() + ":"); + descriptor.getDependencies().forEach((k, v) -> logger.debug("\t" + k + "/" + v)); + } + + @Test + public void testDownloadWithDependencies() throws Exception + { + ValidationPackageClient validationPackageClient = new ValidationPackageClientJersey( + "https://packages.simplifier.net"); + ValidationPackageClient validationPackageClientWithCache = new ValidationPackageClientWithFileSystemCache( + cacheFolder, mapper, validationPackageClient); + + ValueSetExpansionClient valueSetExpansionClient = new ValueSetExpansionClientJersey( + "https://r4.ontoserver.csiro.au/fhir", mapper, fhirContext); + ValueSetExpansionClient valueSetExpansionClientWithCache = new ValueSetExpansionClientWithFileSystemCache( + cacheFolder, fhirContext, valueSetExpansionClient); + + ValidationPackageManager manager = new ValidationPackageManagerImpl(validationPackageClientWithCache, + valueSetExpansionClientWithCache, mapper, fhirContext, PluginSnapshotGeneratorImpl::new, + ValueSetExpanderImpl::new); + + ValidationPackageWithDepedencies packageWithDependencies = manager.downloadPackageWithDependencies("de.gecco", + "1.0.5"); + packageWithDependencies.parseResources(fhirContext); + } + + @Test + public void testValidate() throws Exception + { + ValidationPackageClient validationPackageClient = new ValidationPackageClientJersey( + "https://packages.simplifier.net"); + ValidationPackageClient validationPackageClientWithCache = new ValidationPackageClientWithFileSystemCache( + cacheFolder, mapper, validationPackageClient); + ValueSetExpansionClient valueSetExpansionClient = new ValueSetExpansionClientJersey( + "https://r4.ontoserver.csiro.au/fhir", mapper, fhirContext); + ValueSetExpansionClient valueSetExpansionClientWithCache = new ValueSetExpansionClientWithFileSystemCache( + cacheFolder, fhirContext, valueSetExpansionClient); + ValidationPackageManager manager = new ValidationPackageManagerImpl(validationPackageClientWithCache, + valueSetExpansionClientWithCache, mapper, fhirContext, + (fc, vs) -> new PluginSnapshotGeneratorWithFileSystemCache(cacheFolder, fc, + new PluginSnapshotGeneratorImpl(fc, vs)), + (fc, vs) -> new ValueSetExpanderWithFileSystemCache(cacheFolder, fc, new ValueSetExpanderImpl(fc, vs))); + + BundleValidator validator = manager.createBundleValidator("de.gecco", "1.0.5"); + + logger.debug("---------- executing validation tests ----------"); + + Path bundleFolder = Paths.get("src/test/resources/fhir/Bundle"); + String[] bundles = { "dic_fhir_store_demo_bf_large.json", "dic_fhir_store_demo_bf.json", + "dic_fhir_store_demo_psn_large.json", "dic_fhir_store_demo_psn.json" }; + Arrays.stream(bundles).map(bundleFolder::resolve).forEach(validateWith(validator)); + } + + private Consumer<Path> validateWith(BundleValidator validator) + { + return file -> + { + logger.debug("----- {} -----", file.toString()); + + Bundle bundle; + try + { + byte[] bundleData = Files.readAllBytes(file); + bundle = fhirContext.newJsonParser().parseResource(Bundle.class, + new String(bundleData, StandardCharsets.UTF_8)); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + + Bundle validationResult = validator.validate(bundle); + + logger.info("Validation result bundle: {}", + fhirContext.newJsonParser().encodeResourceToString(validationResult)); + + for (int i = 0; i < validationResult.getEntry().size(); i++) + { + BundleEntryComponent entry = validationResult.getEntry().get(i); + OperationOutcome outcome = (OperationOutcome) entry.getResponse().getOutcome(); + + final int index = i; + outcome.getIssue().forEach(issue -> + { + if (OperationOutcome.IssueSeverity.FATAL.equals(issue.getSeverity())) + logger.error( + "Bundle index {} fatal validation error ({}): {}", index, issue.getLocation().stream() + .map(StringType::getValue).collect(Collectors.joining(", ")), + issue.getDiagnostics()); + else if (OperationOutcome.IssueSeverity.ERROR.equals(issue.getSeverity())) + logger.error( + "Bundle index {} validation error ({}): {}", index, issue.getLocation().stream() + .map(StringType::getValue).collect(Collectors.joining(", ")), + issue.getDiagnostics()); + else if (OperationOutcome.IssueSeverity.WARNING.equals(issue.getSeverity())) + logger.warn( + "Bundle index {} validation warning ({}): {}", index, issue.getLocation().stream() + .map(StringType::getValue).collect(Collectors.joining(", ")), + issue.getDiagnostics()); + else if (issue.hasLocation()) + logger.info( + "Bundle index {} validation info ({}): {}", index, issue.getLocation().stream() + .map(StringType::getValue).collect(Collectors.joining(", ")), + issue.getDiagnostics()); + else + logger.info("Bundle index {} validation info: {}", index, issue.getDiagnostics()); + }); + } + }; + } + + @Test + public void testGenerateSnapshots() throws Exception + { + ValidationPackageClient validationPackageClient = new ValidationPackageClientJersey( + "https://packages.simplifier.net"); + ValidationPackageClient validationPackageClientWithCache = new ValidationPackageClientWithFileSystemCache( + cacheFolder, mapper, validationPackageClient); + ValueSetExpansionClient valueSetExpansionClient = new ValueSetExpansionClientJersey( + "https://r4.ontoserver.csiro.au/fhir", mapper, fhirContext); + ValueSetExpansionClient valueSetExpansionClientWithCache = new ValueSetExpansionClientWithFileSystemCache( + cacheFolder, fhirContext, valueSetExpansionClient); + ValidationPackageManager manager = new ValidationPackageManagerImpl(validationPackageClientWithCache, + valueSetExpansionClientWithCache, mapper, fhirContext, PluginSnapshotGeneratorImpl::new, + ValueSetExpanderImpl::new); + + ValidationPackageWithDepedencies packageWithDependencies = manager.downloadPackageWithDependencies("de.gecco", + "1.0.5"); + packageWithDependencies.parseResources(fhirContext); + + packageWithDependencies.getAllStructureDefinitions().stream() + .sorted(Comparator.comparing(StructureDefinition::getUrl) + .thenComparing(Comparator.comparing(StructureDefinition::getVersion))) + .forEach(s -> logger.debug(s.getUrl() + " " + s.getVersion())); + + StructureDefinition miiRef = packageWithDependencies.getAllStructureDefinitions().stream() + .filter(s -> "https://www.medizininformatik-initiative.de/fhir/core/StructureDefinition/MII-Reference" + .equals(s.getUrl())) + .findFirst().get(); + + SnapshotGenerator sGen = new PluginSnapshotGeneratorImpl(fhirContext, + new ValidationSupportChain(new InMemoryTerminologyServerValidationSupport(fhirContext), + new ValidationSupportWithCustomResources(fhirContext, + packageWithDependencies.getAllStructureDefinitions(), + packageWithDependencies.getAllCodeSystems(), packageWithDependencies.getAllValueSets()), + new DefaultProfileValidationSupport(fhirContext), + new CommonCodeSystemsTerminologyService(fhirContext))); + + sGen = new PluginSnapshotGeneratorWithModifiers(sGen); + + SnapshotWithValidationMessages result = sGen.generateSnapshot(miiRef); + + result.getMessages().forEach(m -> + { + if (IssueSeverity.ERROR.equals(m.getLevel()) || IssueSeverity.FATAL.equals(m.getLevel())) + logger.error("Error while generating snapshot for {}|{}: {}", result.getSnapshot().getUrl(), + result.getSnapshot().getVersion(), m.toString()); + else if (IssueSeverity.WARNING.equals(m.getLevel())) + logger.warn("Warning while generating snapshot for {}|{}: {}", result.getSnapshot().getUrl(), + result.getSnapshot().getVersion(), m.toString()); + else + logger.info("Info while generating snapshot for {}|{}: {}", result.getSnapshot().getUrl(), + result.getSnapshot().getVersion(), m.toString()); + }); + + Map<String, StructureDefinition> sDefByUrl = packageWithDependencies.getAllStructureDefinitions().stream() + .collect(Collectors.toMap(StructureDefinition::getUrl, Function.identity())); + + packageWithDependencies.getAllStructureDefinitions().forEach(s -> + { + logger.info("StructureDefinition {}|{}:", s.getUrl(), s.getVersion()); + printTree(s, sDefByUrl); + logger.debug(""); + }); + } + + private void printTree(StructureDefinition def, Map<String, StructureDefinition> structureDefinitionsByUrl) + { + logger.debug(""); + + Set<String> profileDependencies = new HashSet<>(); + Set<String> targetProfileDependencies = new HashSet<>(); + printTree(def.getUrl(), def, structureDefinitionsByUrl, "", profileDependencies, targetProfileDependencies); + + if (!profileDependencies.isEmpty()) + { + logger.debug(""); + logger.debug(" Profile-Dependencies:"); + profileDependencies.stream().sorted().forEach(url -> logger.debug(" " + url)); + } + if (!targetProfileDependencies.isEmpty()) + { + logger.debug(""); + logger.debug(" TargetProfile-Dependencies:"); + targetProfileDependencies.stream().sorted().forEach(url -> logger.debug(" " + url)); + } + } + + private void printTree(String k, StructureDefinition def, + Map<String, StructureDefinition> structureDefinitionsByUrl, String indentation, + Set<String> profileDependencies, Set<String> targetProfileDependencies) + { + logger.debug(indentation + "Profile: " + k); + for (ElementDefinition element : def.getDifferential().getElement()) + { + if (element.getType().stream().filter(t -> !t.getProfile().isEmpty() || !t.getTargetProfile().isEmpty()) + .findAny().isPresent()) + { + logger.debug(indentation + " Element: " + element.getId() + " (Path: " + element.getPath() + ")"); + for (TypeRefComponent type : element.getType()) + { + if (!type.getProfile().isEmpty()) + { + for (CanonicalType profile : type.getProfile()) + { + profileDependencies.add(profile.getValue()); + + if (structureDefinitionsByUrl.containsKey(profile.getValue())) + printTree(profile.getValue(), structureDefinitionsByUrl.get(profile.getValue()), + structureDefinitionsByUrl, indentation + " ", profileDependencies, + targetProfileDependencies); + else + logger.debug(indentation + " Profile: " + profile.getValue() + " ?"); + } + } + if (!type.getTargetProfile().isEmpty()) + { + for (CanonicalType targetProfile : type.getTargetProfile()) + { + targetProfileDependencies.add(targetProfile.getValue()); + logger.debug(indentation + " TargetProfile: " + targetProfile.getValue()); + } + } + } + } + } + } +} diff --git a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/fhir/profile/TaskProfileTest.java b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/fhir/profile/TaskProfileTest.java index 0817ccd2..93e957ca 100644 --- a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/fhir/profile/TaskProfileTest.java +++ b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/fhir/profile/TaskProfileTest.java @@ -41,6 +41,8 @@ import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.Identifier; import org.hl7.fhir.r4.model.InstantType; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity; import org.hl7.fhir.r4.model.Reference; import org.hl7.fhir.r4.model.ResourceType; import org.hl7.fhir.r4.model.StringType; @@ -55,6 +57,7 @@ import ca.uhn.fhir.model.api.TemporalPrecisionEnum; import ca.uhn.fhir.validation.ResultSeverityEnum; import ca.uhn.fhir.validation.ValidationResult; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.error.ErrorOutputParameterGenerator; public class TaskProfileTest { @@ -62,13 +65,16 @@ public class TaskProfileTest @ClassRule public static final ValidationSupportRule validationRule = new ValidationSupportRule(VERSION, DATE, - Arrays.asList("highmed-task-base-0.5.0.xml", "num-codex-task-start-data-receive.xml", - "num-codex-task-start-data-send.xml", "num-codex-task-start-data-translate.xml", - "num-codex-task-start-data-trigger.xml", "num-codex-task-stop-data-trigger.xml"), + Arrays.asList("highmed-task-base-0.5.0.xml", "num-codex-extension-error-metadata.xml", + "num-codex-task-start-data-receive.xml", "num-codex-task-start-data-send.xml", + "num-codex-task-start-data-translate.xml", "num-codex-task-start-data-trigger.xml", + "num-codex-task-stop-data-trigger.xml"), Arrays.asList("highmed-read-access-tag-0.5.0.xml", "highmed-bpmn-message-0.5.0.xml", - "num-codex-data-transfer.xml"), + "num-codex-data-transfer.xml", "num-codex-data-transfer-error-source.xml", + "num-codex-data-transfer-error.xml"), Arrays.asList("highmed-read-access-tag-0.5.0.xml", "highmed-bpmn-message-0.5.0.xml", - "num-codex-data-transfer.xml")); + "num-codex-data-transfer.xml", "num-codex-data-transfer-error-source.xml", + "num-codex-data-transfer-error.xml")); private ResourceValidator resourceValidator = new ResourceValidatorImpl(validationRule.getFhirContext(), validationRule.getValidationSupport()); @@ -221,6 +227,27 @@ public void testTaskStartDataSendValidWithExportFrom() throws Exception || ResultSeverityEnum.FATAL.equals(m.getSeverity())).count()); } + @Test + public void testTaskStartDataSendValidWithValidationError() throws Exception + { + Task task = createValidTaskStartDataSendWithIdentifierReference(); + task.setStatus(TaskStatus.FAILED); + + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(IssueSeverity.ERROR).addLocation("Patient.identifier[0].system"); + new ErrorOutputParameterGenerator() + .createMeDicValidationError(new IdType("http://gecco.fhir.server/fhir", "Patient", "42", null), outcome) + .forEach(task::addOutput); + + logTask(task); + + ValidationResult result = resourceValidator.validate(task); + ValidationSupportRule.logValidationMessages(logger, result); + + assertEquals(0, result.getMessages().stream().filter(m -> ResultSeverityEnum.ERROR.equals(m.getSeverity()) + || ResultSeverityEnum.FATAL.equals(m.getSeverity())).count()); + } + private Task createValidTaskStartDataSendWithIdentifierReference() { Task task = new Task(); diff --git a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf.json b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf.json index 96e277b1..e9050085 100644 --- a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf.json +++ b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf.json @@ -16,8 +16,8 @@ "url": "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/ethnic-group", "valueCoding": { "system": "http://snomed.info/sct", - "code": "186019001", - "display": "Other ethnic, mixed origin" + "code": "26242008", + "display": "Mixed (qualifier value)" } }, { diff --git a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_create.json b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_create.json index b561f235..34a695d9 100644 --- a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_create.json +++ b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_create.json @@ -16,8 +16,8 @@ "url": "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/ethnic-group", "valueCoding": { "system": "http://snomed.info/sct", - "code": "186019001", - "display": "Other ethnic, mixed origin" + "code": "26242008", + "display": "Mixed (qualifier value)" } }, { diff --git a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_large.json b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_large.json index 0d46defe..a54fc226 100644 --- a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_large.json +++ b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_large.json @@ -21,8 +21,8 @@ "url": "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/ethnic-group", "valueCoding": { "system": "http://snomed.info/sct", - "code": "186019001", - "display": "Other ethnic, mixed origin" + "code": "26242008", + "display": "Mixed (qualifier value)" } }, { @@ -91,10 +91,6 @@ { "system":"http://loinc.org", "code":"8691-8" - }, - { - "system":"http://snomed.info/sct", - "code":"443846001" } ], "text":"History of Travel" @@ -907,18 +903,6 @@ "reference":"urn:uuid:e6230cf4-4022-486c-8a4a-000000000001" }, "effectiveDateTime":"2021-02-17T01:01:00.000+01:00", - "interpretation":[ - { - "coding":[ - { - "system":"http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", - "code":"L", - "display":"low" - } - ], - "text":"Below low normal" - } - ], "bodySite":{ "coding":[ { @@ -941,11 +925,6 @@ "system":"http://snomed.info/sct", "code":"271649006", "display":"Systolic blood pressure" - }, - { - "system":"http://acme.org/devices/clinical-codes", - "code":"bp-s", - "display":"Systolic Blood pressure" } ] }, @@ -954,19 +933,7 @@ "unit":"mmHg", "system":"http://unitsofmeasure.org", "code":"mm[Hg]" - }, - "interpretation":[ - { - "coding":[ - { - "system":"http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", - "code":"N", - "display":"normal" - } - ], - "text":"Normal" - } - ] + } }, { "code":{ @@ -975,27 +942,20 @@ "system":"http://loinc.org", "code":"8462-4", "display":"Diastolic blood pressure" + }, + { + "system": "http://snomed.info/sct", + "code": "271650006", + "display": "Diastolic blood pressure" } ] }, "valueQuantity":{ - "value":107, + "value":72, "unit":"mmHg", "system":"http://unitsofmeasure.org", "code":"mm[Hg]" - }, - "interpretation":[ - { - "coding":[ - { - "system":"http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", - "code":"L", - "display":"low" - } - ], - "text":"Below low normal" - } - ] + } } ] }, @@ -1090,9 +1050,8 @@ "code":{ "coding":[ { - "code":"01", - "display":"Is the patient in the intensive care unit?", - "system":"https://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/ecrf-parameter-codes" + "system": "http://loinc.org", + "code": "95420-6" } ] }, diff --git a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn.json b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn.json index 37921866..3a0165fc 100644 --- a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn.json +++ b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn.json @@ -16,8 +16,8 @@ "url": "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/ethnic-group", "valueCoding": { "system": "http://snomed.info/sct", - "code": "186019001", - "display": "Other ethnic, mixed origin" + "code": "26242008", + "display": "Mixed (qualifier value)" } }, { diff --git a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_create.json b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_create.json index 9cd04ad1..274a6ab7 100644 --- a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_create.json +++ b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_create.json @@ -16,8 +16,8 @@ "url": "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/ethnic-group", "valueCoding": { "system": "http://snomed.info/sct", - "code": "186019001", - "display": "Other ethnic, mixed origin" + "code": "26242008", + "display": "Mixed (qualifier value)" } }, { @@ -107,8 +107,8 @@ "coding": [ { "system": "http://snomed.info/sct", - "code": "413839001", - "display": "Chronic lung disease" + "code": "13645005", + "display": "Chronic obstructive lung disease (disorder)" } ] }, diff --git a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_large.json b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_large.json index 6a0d7be9..c967e484 100644 --- a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_large.json +++ b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_large.json @@ -21,8 +21,8 @@ "url": "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/ethnic-group", "valueCoding": { "system": "http://snomed.info/sct", - "code": "186019001", - "display": "Other ethnic, mixed origin" + "code": "26242008", + "display": "Mixed (qualifier value)" } }, { @@ -91,10 +91,6 @@ { "system":"http://loinc.org", "code":"8691-8" - }, - { - "system":"http://snomed.info/sct", - "code":"443846001" } ], "text":"History of Travel" @@ -907,18 +903,6 @@ "reference":"urn:uuid:e6230cf4-4022-486c-8a4a-000000000001" }, "effectiveDateTime":"2021-02-17T01:01:00.000+01:00", - "interpretation":[ - { - "coding":[ - { - "system":"http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", - "code":"L", - "display":"low" - } - ], - "text":"Below low normal" - } - ], "bodySite":{ "coding":[ { @@ -941,11 +925,6 @@ "system":"http://snomed.info/sct", "code":"271649006", "display":"Systolic blood pressure" - }, - { - "system":"http://acme.org/devices/clinical-codes", - "code":"bp-s", - "display":"Systolic Blood pressure" } ] }, @@ -954,19 +933,7 @@ "unit":"mmHg", "system":"http://unitsofmeasure.org", "code":"mm[Hg]" - }, - "interpretation":[ - { - "coding":[ - { - "system":"http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", - "code":"N", - "display":"normal" - } - ], - "text":"Normal" - } - ] + } }, { "code":{ @@ -975,27 +942,20 @@ "system":"http://loinc.org", "code":"8462-4", "display":"Diastolic blood pressure" + }, + { + "system": "http://snomed.info/sct", + "code": "271650006", + "display": "Diastolic blood pressure" } ] }, "valueQuantity":{ - "value":107, + "value":72, "unit":"mmHg", "system":"http://unitsofmeasure.org", "code":"mm[Hg]" - }, - "interpretation":[ - { - "coding":[ - { - "system":"http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", - "code":"L", - "display":"low" - } - ], - "text":"Below low normal" - } - ] + } } ] }, @@ -1090,9 +1050,8 @@ "code":{ "coding":[ { - "code":"01", - "display":"Is the patient in the intensive care unit?", - "system":"https://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/ecrf-parameter-codes" + "system": "http://loinc.org", + "code": "95420-6" } ] }, diff --git a/codex-process-data-transfer/src/test/resources/log4j2.xml b/codex-process-data-transfer/src/test/resources/log4j2.xml index 736faa29..73f8205a 100644 --- a/codex-process-data-transfer/src/test/resources/log4j2.xml +++ b/codex-process-data-transfer/src/test/resources/log4j2.xml @@ -51,7 +51,7 @@ <Logger name="com.sun.jersey" level="WARN"/> <Logger name="liquibase" level="WARN"/> <Logger name="ca.uhn.hl7v2" level="WARN"/> - <Logger name="ca.uhn.fhir" level="DEBUG"/> + <Logger name="ca.uhn.fhir" level="WARN"/> <!-- <Logger name="certificate-warning-logger" level="INFO"> <AppenderRef ref="MAIL_CERTIFICATE" /> diff --git a/codex-processes-ap1-docker-test-setup/dic/bpe/cache/Package/README.md b/codex-processes-ap1-docker-test-setup/dic/bpe/cache/Package/README.md new file mode 100644 index 00000000..2cd95e7f --- /dev/null +++ b/codex-processes-ap1-docker-test-setup/dic/bpe/cache/Package/README.md @@ -0,0 +1 @@ +empty directory for FHIR package cache \ No newline at end of file diff --git a/codex-processes-ap1-docker-test-setup/dic/bpe/cache/StructureDefinition/README.md b/codex-processes-ap1-docker-test-setup/dic/bpe/cache/StructureDefinition/README.md new file mode 100644 index 00000000..5f7d118e --- /dev/null +++ b/codex-processes-ap1-docker-test-setup/dic/bpe/cache/StructureDefinition/README.md @@ -0,0 +1 @@ +empty directory for StructureDefinition Snapshots cache \ No newline at end of file diff --git a/codex-processes-ap1-docker-test-setup/dic/bpe/cache/ValueSet/README.md b/codex-processes-ap1-docker-test-setup/dic/bpe/cache/ValueSet/README.md new file mode 100644 index 00000000..f15571d9 --- /dev/null +++ b/codex-processes-ap1-docker-test-setup/dic/bpe/cache/ValueSet/README.md @@ -0,0 +1 @@ +empty directory for expanded ValueSet cache \ No newline at end of file diff --git a/codex-processes-ap1-docker-test-setup/docker-compose.local-dsf-build.yml b/codex-processes-ap1-docker-test-setup/docker-compose.local-dsf-build.yml new file mode 100644 index 00000000..1275bc1d --- /dev/null +++ b/codex-processes-ap1-docker-test-setup/docker-compose.local-dsf-build.yml @@ -0,0 +1,16 @@ +version: '3.8' +services: + dic-fhir: + image: highmed/fhir + dic-bpe: + image: highmed/bpe + + gth-fhir: + image: highmed/fhir + gth-bpe: + image: highmed/bpe + + crr-fhir: + image: highmed/fhir + crr-bpe: + image: highmed/bpe diff --git a/codex-processes-ap1-docker-test-setup/docker-compose.yml b/codex-processes-ap1-docker-test-setup/docker-compose.yml index 58e80e3a..58f1ce48 100644 --- a/codex-processes-ap1-docker-test-setup/docker-compose.yml +++ b/codex-processes-ap1-docker-test-setup/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.8' services: proxy: image: nginx:1.21 - restart: on-failure + restart: "no" ports: - 127.0.0.1:443:443 secrets: @@ -35,9 +35,9 @@ services: db: image: postgres:13 - restart: on-failure + restart: "no" healthcheck: - test: [ "CMD-SHELL", "pg_isready -U liquibase_user -d postgres" ] + test: ["CMD-SHELL", "pg_isready -U liquibase_user -d postgres"] interval: 10s timeout: 5s retries: 5 @@ -67,7 +67,7 @@ services: dic-fhir: image: ghcr.io/highmed/fhir:0.6.0 - restart: on-failure + restart: "no" ports: - 127.0.0.1:5000:5000 secrets: @@ -114,7 +114,7 @@ services: - proxy dic-bpe: image: ghcr.io/highmed/bpe:0.6.0 - restart: on-failure + restart: "no" ports: - 127.0.0.1:5003:5003 secrets: @@ -141,6 +141,9 @@ services: - type: bind source: ./dic/bpe/last_event target: /opt/bpe/last_event + - type: bind + source: ./dic/bpe/cache + target: /opt/bpe/cache environment: TZ: Europe/Berlin EXTRA_JVM_ARGS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5003 @@ -162,6 +165,10 @@ services: DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GTH_IDENTIFIER_VALUE: Test_GTH DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_CRR_PUBLIC_KEY: /run/secrets/codex_crr_public_key.pem DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GECCO_SERVER_BASE_URL: http://dic-fhir-store:8080/fhir + DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GECCO_VALIDATION_PACKAGE_CACHEFOLDER: /opt/bpe/cache/Package/ + DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GECCO_VALIDATION_STRUCTUREDEFINITION_CACHEFOLDER: /opt/bpe/cache/StructureDefinition/ + DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GECCO_VALIDATION_VALUESET_CACHEFOLDER: /opt/bpe/cache/ValueSet/ + DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GECCO_VALIDATION_VALUESET_EXPANSION_SERVER_BASEURL: https://r4.ontoserver.csiro.au/fhir networks: dic-bpe-frontend: dic-bpe-backend: @@ -169,10 +176,10 @@ services: depends_on: - db - dic-fhir - # - dic-fhir-store not defining a dependency here, dic-fhir-store* needs to be started manually + # - dic-fhir-store not defining a dependency here, dic-fhir-store* needs to be started manually dic-fhir-store-hapi: build: ./dic/hapi - restart: on-failure + restart: "no" ports: - 127.0.0.1:8080:8080 environment: @@ -195,7 +202,7 @@ services: gth-fhir: image: ghcr.io/highmed/fhir:0.6.0 - restart: on-failure + restart: "no" ports: - 127.0.0.1:5001:5001 secrets: @@ -242,7 +249,7 @@ services: - proxy gth-bpe: image: ghcr.io/highmed/bpe:0.6.0 - restart: on-failure + restart: "no" ports: - 127.0.0.1:5004:5004 secrets: @@ -299,7 +306,7 @@ services: crr-fhir: image: ghcr.io/highmed/fhir:0.6.0 - restart: on-failure + restart: "no" ports: - 127.0.0.1:5002:5002 secrets: @@ -346,7 +353,7 @@ services: - proxy crr-bpe: image: ghcr.io/highmed/bpe:0.6.0 - restart: on-failure + restart: "no" ports: - 127.0.0.1:5005:5005 secrets: @@ -402,7 +409,7 @@ services: depends_on: - db - crr-fhir - # - crr-fhir-bridge not defining a dependency here, crr-fhir-bridge* needs to be started manually + # - crr-fhir-bridge not defining a dependency here, crr-fhir-bridge* needs to be started manually crr-ehrbase-db: image: ehrbase/ehrbase-postgres networks: @@ -428,7 +435,7 @@ services: SECURITY_AUTHADMINPASSWORD: mySuperAwesomePassword123 SYSTEM_NAME: local.ehrbase.org ADMIN_API_ACTIVE: 'true' -# SERVER_DISABLESTRICTVALIDATION: 'true' + # SERVER_DISABLESTRICTVALIDATION: 'true' TZ: Europe/Berlin depends_on: - crr-ehrbase-db @@ -444,7 +451,7 @@ services: FHIR_BRIDGE_EHRBASE_BASE_URL: http://crr-ehrbase:8080/ehrbase/ FHIR_BRIDGE_FHIR_VALIDATION_OPTIONAL_IDENTIFIER: 'true' TZ: Europe/Berlin -# SPRING_PROFILES_ACTIVE: dev + # SPRING_PROFILES_ACTIVE: dev JAVA_TOOL_OPTIONS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006 depends_on: - crr-ehrbase @@ -547,4 +554,4 @@ networks: volumes: db-data: - name: db-data-codex-dsf-processes \ No newline at end of file + name: db-data-codex-dsf-processes diff --git a/pom.xml b/pom.xml index c32fbac6..0a966a82 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,5 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>de.netzwerk-universitaetsmedizin.codex</groupId> @@ -21,7 +19,7 @@ <main.basedir>${project.basedir}</main.basedir> <hapi.version>5.1.0</hapi.version> - <dsf.version>0.6.0-SNAPSHOT</dsf.version> + <dsf.version>0.7.0-SNAPSHOT</dsf.version> </properties> <description>Business processes for the NUM CODEX project (AP1) as plugins for the HiGHmed Data Sharing Framework.</description> @@ -79,6 +77,11 @@ <artifactId>dsf-fhir-server</artifactId> <version>${dsf.version}</version> </dependency> + <dependency> + <groupId>org.highmed.dsf</groupId> + <artifactId>dsf-tools-documentation-generator</artifactId> + <version>${dsf.version}</version> + </dependency> <dependency> <groupId>ca.uhn.hapi.fhir</groupId> @@ -95,12 +98,12 @@ <dependency> <groupId>de.hs-heilbronn.mi</groupId> <artifactId>log4j2-utils</artifactId> - <version>0.12.0</version> + <version>0.13.0</version> </dependency> <dependency> <groupId>de.hs-heilbronn.mi</groupId> <artifactId>crypto-utils</artifactId> - <version>3.2.0</version> + <version>3.3.0</version> </dependency> <!-- logging --> @@ -110,17 +113,6 @@ <version>1.8.0-beta4</version> </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - <version>2.12.0</version> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-web</artifactId> - <version>5.3.19</version> - </dependency> - <!-- testing --> <dependency> <groupId>junit</groupId> @@ -336,4 +328,4 @@ </build> </profile> </profiles> -</project> +</project> \ No newline at end of file