From 564a931d6ba9422e858a692d73a1dad579f541b2 Mon Sep 17 00:00:00 2001 From: Hauke Hund Date: Tue, 24 May 2022 15:21:41 +0200 Subject: [PATCH 01/29] changed restart policy to "no", DSF version to 0.6.0 restart policy "no" makes it easier to debug issues related to crashing containers --- .../docker-compose.yml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/codex-processes-ap1-docker-test-setup/docker-compose.yml b/codex-processes-ap1-docker-test-setup/docker-compose.yml index 4f88afb7..9c49dccb 100644 --- a/codex-processes-ap1-docker-test-setup/docker-compose.yml +++ b/codex-processes-ap1-docker-test-setup/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.8' services: proxy: image: nginx:1.21 - restart: on-failure + restart: "no" ports: - 127.0.0.1:443:443 secrets: @@ -35,7 +35,7 @@ services: db: image: postgres:13 - restart: on-failure + restart: "no" healthcheck: test: [ "CMD-SHELL", "pg_isready -U liquibase_user -d postgres" ] interval: 10s @@ -66,8 +66,8 @@ services: dic-fhir: - image: ghcr.io/highmed/fhir:0.6.0-RC3 - restart: on-failure + image: ghcr.io/highmed/fhir:0.6.0 + restart: "no" ports: - 127.0.0.1:5000:5000 secrets: @@ -113,8 +113,8 @@ services: - db - proxy dic-bpe: - image: ghcr.io/highmed/bpe:0.6.0-RC3 - restart: on-failure + image: ghcr.io/highmed/bpe:0.6.0 + restart: "no" ports: - 127.0.0.1:5003:5003 secrets: @@ -172,7 +172,7 @@ services: # - dic-fhir-store not defining a dependency here, dic-fhir-store* needs to be started manually dic-fhir-store-hapi: build: ./dic/hapi - restart: on-failure + restart: "no" ports: - 127.0.0.1:8080:8080 environment: @@ -194,8 +194,8 @@ services: gth-fhir: - image: ghcr.io/highmed/fhir:0.6.0-RC3 - restart: on-failure + image: ghcr.io/highmed/fhir:0.6.0 + restart: "no" ports: - 127.0.0.1:5001:5001 secrets: @@ -241,8 +241,8 @@ services: - db - proxy gth-bpe: - image: ghcr.io/highmed/bpe:0.6.0-RC3 - restart: on-failure + image: ghcr.io/highmed/bpe:0.6.0 + restart: "no" ports: - 127.0.0.1:5004:5004 secrets: @@ -298,8 +298,8 @@ services: crr-fhir: - image: ghcr.io/highmed/fhir:0.6.0-RC3 - restart: on-failure + image: ghcr.io/highmed/fhir:0.6.0 + restart: "no" ports: - 127.0.0.1:5002:5002 secrets: @@ -345,8 +345,8 @@ services: - db - proxy crr-bpe: - image: ghcr.io/highmed/bpe:0.6.0-RC3 - restart: on-failure + image: ghcr.io/highmed/bpe:0.6.0 + restart: "no" ports: - 127.0.0.1:5005:5005 secrets: From 978c6bce544d5e33a2f712b8e05bd94ad6a214b8 Mon Sep 17 00:00:00 2001 From: Hauke Hund Date: Tue, 24 May 2022 15:22:23 +0200 Subject: [PATCH 02/29] DSF Version to 0.6.0, added managed version for commons-compress --- pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4cb8fa15..11848572 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ ${project.basedir} 5.1.0 - 0.6.0-SNAPSHOT + 0.6.0 Business processes for the NUM CODEX project (AP1) as plugins for the HiGHmed Data Sharing Framework. @@ -115,6 +115,11 @@ jackson-annotations 2.12.0 + + org.apache.commons + commons-compress + 1.20 + From 5da15174f0ae9bc6b0f1b47f524414b7b856ebd8 Mon Sep 17 00:00:00 2001 From: Hauke Hund Date: Tue, 24 May 2022 15:46:03 +0200 Subject: [PATCH 03/29] initial validation impl., demo data now compliant with gecco 1.0.5 spec This initial process validation task implementation requires 29 additional jars to be added to the BPE as regular (non-process) plugins. Alternatively we could release the process plugin as a tar.gz/zip or we might want to add the needed HAPI dependencies into the DSF BPE itself. This implementation contains some workarounds due to limitations of some classes in the dsf-fhir-validation module. The maven build produces a codex-process-data-transfer-0.5.0-SNAPSHOT.zip containing the codex-process-data-transfer-0.5.0-SNAPSHOT.jar and all dependency needed to start a stand-alone bundle/resource validator. Windows: java -cp codex-process-data-transfer-0.5.0-SNAPSHOT.jar;lib/* de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationMain some-bundle.json some-resource.xml Unix: java -cp codex-process-data-transfer-0.5.0-SNAPSHOT.jar:lib/* de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationMain some-bundle.json some-resource.xml The stand-alone validator outputs debug infos to System.err and the validation result to System.out. For a given single resource a OperationOutcome resource is produced and for a given Bundle, a bundle with added response.outcome per entry with be printed to System.out. The validator validates every json or xml file defined as command line argument. The validator can be configured using a application.properties file within the execution folder or via java system properties, meaning java command line parameters -Dfoo=bar. For configuration properties see classes de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config.ValidationConfig and de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationMain.TestConfig. --- codex-process-data-transfer/pom.xml | 220 +++++++++- .../src/assembly/zip.xml | 26 ++ .../data_transfer/ConstantsDataTransfer.java | 16 + .../DataTransferProcessPluginDefinition.java | 27 +- .../error/ErrorOutputParameterGenerator.java | 59 +++ .../data_transfer/logging/ErrorLogger.java | 17 + .../data_transfer/service/ReadData.java | 22 +- .../data_transfer/service/ValidateData.java | 164 ++++++- .../spring/config/TransferDataConfig.java | 36 +- .../spring/config/ValidationConfig.java | 388 +++++++++++++++++ .../AbstractFhirResourceFileSystemCache.java | 82 ++++ .../validation/AbstractFileSystemCache.java | 153 +++++++ .../validation/BundleValidator.java | 18 + .../validation/BundleValidatorFactory.java | 8 + .../BundleValidatorFactoryImpl.java | 57 +++ .../validation/BundleValidatorImpl.java | 54 +++ .../CodeValidatorForExpandedValueSets.java | 138 ++++++ .../validation/PluginSnapshotGenerator.java | 8 + .../PluginSnapshotGeneratorImpl.java | 69 +++ ...nSnapshotGeneratorWithFileSystemCache.java | 115 +++++ .../PluginSnapshotWithValidationMessages.java | 30 ++ .../validation/PluginWorkerContext.java | 366 ++++++++++++++++ .../validation/ValidationMain.java | 224 ++++++++++ .../validation/ValidationPackage.java | 201 +++++++++ .../validation/ValidationPackageClient.java | 31 ++ .../ValidationPackageClientJersey.java | 85 ++++ ...ationPackageClientWithFileSystemCache.java | 78 ++++ .../ValidationPackageDescriptor.java | 156 +++++++ ...ValidationPackageDescriptorMaintainer.java | 31 ++ .../validation/ValidationPackageEntry.java | 72 +++ .../ValidationPackageIdentifier.java | 78 ++++ .../validation/ValidationPackageManager.java | 79 ++++ .../ValidationPackageManagerImpl.java | 409 ++++++++++++++++++ .../ValidationSupportResources.java | 51 +++ .../ValueSetExpanderWithFileSystemCache.java | 115 +++++ .../validation/ValueSetExpansionClient.java | 19 + .../ValueSetExpansionClientJersey.java | 122 ++++++ ...SetExpansionClientWithFileSystemCache.java | 103 +++++ .../ClosedTypeSlicingRemover.java | 38 ++ ...eLabObservationLab10IdentifierRemover.java | 38 ++ .../StructureDefinitionModifier.java | 9 + .../src/main/resources/bpe/send.bpmn | 22 +- .../num-codex-data-transfer-error-source.xml | 44 ++ .../num-codex-data-transfer-error-type.xml | 30 ++ .../highmed-task-base-0.5.0.xml | 274 ++++++++++++ .../num-codex-extension-error-metadata.xml | 137 ++++++ .../num-codex-task-start-data-receive.xml | 2 + .../num-codex-task-start-data-send.xml | 25 ++ .../num-codex-task-start-data-translate.xml | 2 + .../num-codex-task-start-data-trigger.xml | 2 + .../num-codex-task-stop-data-trigger.xml | 2 + .../num-codex-data-transfer-error-source.xml | 26 ++ .../num-codex-data-transfer-error-type.xml | 26 ++ .../src/main/resources/log4j2.xml | 28 ++ .../service/ValidateDataLearningTest.java | 327 ++++++++++++++ .../fhir/profile/TaskProfileTest.java | 37 +- .../fhir/Bundle/dic_fhir_store_demo_bf.json | 4 +- .../Bundle/dic_fhir_store_demo_bf_create.json | 4 +- .../Bundle/dic_fhir_store_demo_bf_large.json | 65 +-- .../fhir/Bundle/dic_fhir_store_demo_psn.json | 4 +- .../dic_fhir_store_demo_psn_create.json | 8 +- .../Bundle/dic_fhir_store_demo_psn_large.json | 65 +-- .../src/test/resources/log4j2.xml | 2 +- 63 files changed, 5001 insertions(+), 147 deletions(-) create mode 100644 codex-process-data-transfer/src/assembly/zip.xml create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/error/ErrorOutputParameterGenerator.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/logging/ErrorLogger.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFhirResourceFileSystemCache.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFileSystemCache.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidator.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/CodeValidatorForExpandedValueSets.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGenerator.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorImpl.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotWithValidationMessages.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginWorkerContext.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackage.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClient.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientWithFileSystemCache.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptor.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptorMaintainer.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageEntry.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageIdentifier.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationSupportResources.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpanderWithFileSystemCache.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClient.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/StructureDefinitionModifier.java create mode 100644 codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-source.xml create mode 100644 codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-type.xml create mode 100644 codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/highmed-task-base-0.5.0.xml create mode 100644 codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-extension-error-metadata.xml create mode 100644 codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-source.xml create mode 100644 codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-type.xml create mode 100644 codex-process-data-transfer/src/main/resources/log4j2.xml create mode 100644 codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateDataLearningTest.java diff --git a/codex-process-data-transfer/pom.xml b/codex-process-data-transfer/pom.xml index 9c0636c6..b4d61f2c 100644 --- a/codex-process-data-transfer/pom.xml +++ b/codex-process-data-transfer/pom.xml @@ -1,6 +1,4 @@ - + 4.0.0 codex-process-data-transfer @@ -16,31 +14,49 @@ + org.highmed.dsf dsf-bpe-process-base provided + + + org.highmed.dsf + dsf-fhir-validation + provided + + + + org.apache.commons + commons-compress + provided + + + ca.uhn.hapi.fhir hapi-fhir-client - provided de.hs-heilbronn.mi log4j2-utils - test - - - org.highmed.dsf - dsf-fhir-validation - test + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/ValidateDataLearningTest.java + + + org.apache.maven.plugins maven-dependency-plugin @@ -109,6 +125,149 @@ hapi-fhir-client ${hapi.version} + + + + commons-codec + commons-codec + 1.12 + + + ca.uhn.hapi.fhir + hapi-fhir-structures-r5 + ${hapi.version} + + + ca.uhn.hapi.fhir + org.hl7.fhir.r5 + ${hapi.version} + + + ca.uhn.hapi.fhir + hapi-fhir-validation + ${hapi.version} + + + ca.uhn.hapi.fhir + hapi-fhir-converter + ${hapi.version} + + + ca.uhn.hapi.fhir + org.hl7.fhir.convertors + ${hapi.version} + + + net.sf.saxon + Saxon-HE + 9.5.1-5 + + + ca.uhn.hapi.fhir + org.hl7.fhir.validation + ${hapi.version} + + + ca.uhn.hapi.fhir + org.hl7.fhir.dstu2 + ${hapi.version} + + + ca.uhn.hapi.fhir + org.hl7.fhir.dstu2016may + ${hapi.version} + + + ca.uhn.hapi.fhir + org.hl7.fhir.dstu3 + ${hapi.version} + + + xpp3 + xpp3 + 1.1.4c + + + xpp3 + xpp3_xpath + 1.1.4c + + + org.apache.commons + commons-compress + 1.21 + + + org.fhir + ucum + 1.0.2 + + + org.thymeleaf + thymeleaf + 3.0.9.RELEASE + + + ognl + ognl + 3.1.12 + + + org.javassist + javassist + 3.20.0-GA + + + org.attoparser + attoparser + 2.0.4.RELEASE + + + org.unbescape + unbescape + 1.1.5.RELEASE + + + com.github.ben-manes.caffeine + caffeine + 2.7.0 + + + org.checkerframework + checker-qual + 2.6.0 + + + com.google.errorprone + error_prone_annotations + 2.3.3 + + + com.google.code.gson + gson + 2.8.5 + + + commons-logging + commons-logging + 1.2 + + + ca.uhn.hapi.fhir + hapi-fhir-validation-resources-r4 + ${hapi.version} + + + ca.uhn.hapi.fhir + hapi-fhir-validation-resources-r5 + ${hapi.version} + + + org.apache.commons + commons-compress + + + ../codex-processes-ap1-docker-test-setup/dic/bpe/plugin @@ -147,6 +306,29 @@ ../codex-processes-ap1-docker-test-setup/gth/bpe/plugin + + + copy-standalone-dependencies + install + + copy-dependencies + + + ${project.build.directory}/lib + hapi-fhir-base,hapi-fhir-client,hapi-fhir-converter,hapi-fhir-structures-r4,hapi-fhir-structures-r5,hapi-fhir-validation,hapi-fhir-validation-resources-r4, + hapi-fhir-validation-resources-r5,org.hl7.fhir.convertors,org.hl7.fhir.dstu2,org.hl7.fhir.dstu2016may,org.hl7.fhir.dstu3,org.hl7.fhir.r4,org.hl7.fhir.r5,org.hl7.fhir.utilities, + org.hl7.fhir.validation,jackson-annotations,jackson-core,jackson-databind,jackson-module-jaxb-annotations,caffeine,guava, + commons-codec,commons-io,crypto-utils,log4j2-utils, + jakarta.annotation-api,jakarta.ws.rs-api, + commons-compress,commons-lang3,commons-text, + log4j-api,log4j-core,log4j-slf4j-impl,bcpkix-jdk15on,bcprov-jdk15on,bcutil-jdk15on,ucum,jakarta.inject, + jersey-client,jersey-common, + jersey-media-jaxb,jersey-media-json-jackson, + dsf-bpe-process-base,dsf-fhir-rest-adapter,dsf-fhir-validation,dsf-openehr-model,jcl-over-slf4j, + slf4j-api,spring-aop,spring-beans,spring-context,spring-core,spring-expression,spring-jcl,thymeleaf,unbescape,xpp3,xpp3_xpath + + + @@ -199,6 +381,24 @@ + + maven-assembly-plugin + + false + + src/assembly/zip.xml + + + + + zip-assembly + install + + single + + + + \ No newline at end of file diff --git a/codex-process-data-transfer/src/assembly/zip.xml b/codex-process-data-transfer/src/assembly/zip.xml new file mode 100644 index 00000000..f0dadb48 --- /dev/null +++ b/codex-process-data-transfer/src/assembly/zip.xml @@ -0,0 +1,26 @@ + + + zip + + zip + + + + + ${project.build.directory} + + + *.jar + + + + ${project.build.directory}/lib + lib + + *.jar + + + + \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java index 08f72994..964dd65d 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java @@ -64,4 +64,20 @@ public interface ConstantsDataTransfer * dic-source/dic-pseudonym-original */ String PSEUDONYM_PATTERN_STRING = "(?[^/]+)/(?[^/]+)"; + + String HAPI_USER_DATA_SOURCE_ID_ELEMENT = "source-id"; + + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE = "http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-type"; + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE_VALUE_VALIDATION_FAILED = "validation-failed"; + + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE = "http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-source"; + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_MEDIC = "MeDIC"; + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_GTH = "GTH"; + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_FTTP = "fTTP"; + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_CRR = "CRR"; + + String EXTENSION_ERROR_METADATA = "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/error-metadata"; + String EXTENSION_ERROR_METADATA_TYPE = "type"; + String EXTENSION_ERROR_METADATA_SOURCE = "source"; + String EXTENSION_ERROR_METADATA_REFERENCE = "reference"; } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java index cf2b32ad..749e9389 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java @@ -22,6 +22,8 @@ import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.client.GeccoClientFactory; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config.TransferDataConfig; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config.TransferDataSerializerConfig; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config.ValidationConfig; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactory; public class DataTransferProcessPluginDefinition implements ProcessPluginDefinition { @@ -55,7 +57,7 @@ public Stream getBpmnFiles() @Override public Stream> getSpringConfigClasses() { - return Stream.of(TransferDataConfig.class, TransferDataSerializerConfig.class); + return Stream.of(TransferDataConfig.class, TransferDataSerializerConfig.class, ValidationConfig.class); } @Override @@ -68,28 +70,34 @@ public ResourceProvider getResourceProvider(FhirContext fhirContext, ClassLoader var aRec = ActivityDefinitionResource.file("fhir/ActivityDefinition/num-codex-data-receive.xml"); var cD = CodeSystemResource.file("fhir/CodeSystem/num-codex-data-transfer.xml"); + var cDeS = CodeSystemResource.file("fhir/CodeSystem/num-codex-data-transfer-error-source.xml"); + var cDeT = CodeSystemResource.file("fhir/CodeSystem/num-codex-data-transfer-error-type.xml"); var nD = NamingSystemResource.file("fhir/NamingSystem/num-codex-dic-pseudonym-identifier.xml"); var nC = NamingSystemResource.file("fhir/NamingSystem/num-codex-crr-pseudonym-identifier.xml"); var nB = NamingSystemResource.file("fhir/NamingSystem/num-codex-bloom-filter-identifier.xml"); + var sTexErMe = StructureDefinitionResource + .file("fhir/StructureDefinition/num-codex-extension-error-metadata.xml"); + var sTstaDrec = StructureDefinitionResource + .file("fhir/StructureDefinition/num-codex-task-start-data-receive.xml"); + var sTstaDsen = StructureDefinitionResource.file("fhir/StructureDefinition/num-codex-task-start-data-send.xml"); + var sTstaDtra = StructureDefinitionResource + .file("fhir/StructureDefinition/num-codex-task-start-data-translate.xml"); var sTstaDtri = StructureDefinitionResource .file("fhir/StructureDefinition/num-codex-task-start-data-trigger.xml"); var sTstoDtri = StructureDefinitionResource .file("fhir/StructureDefinition/num-codex-task-stop-data-trigger.xml"); - var sTstaDsen = StructureDefinitionResource.file("fhir/StructureDefinition/num-codex-task-start-data-send.xml"); - var sTstaDtra = StructureDefinitionResource - .file("fhir/StructureDefinition/num-codex-task-start-data-translate.xml"); - var sTstaDrec = StructureDefinitionResource - .file("fhir/StructureDefinition/num-codex-task-start-data-receive.xml"); var vD = ValueSetResource.file("fhir/ValueSet/num-codex-data-transfer.xml"); + var vDeS = ValueSetResource.file("fhir/ValueSet/num-codex-data-transfer-error-source.xml"); + var vDeT = ValueSetResource.file("fhir/ValueSet/num-codex-data-transfer-error-type.xml"); Map> resourcesByProcessKeyAndVersion = Map.of( // "wwwnetzwerk-universitaetsmedizinde_dataTrigger/" + VERSION, Arrays.asList(aTri, cD, nD, sTstaDtri, sTstoDtri, vD), // "wwwnetzwerk-universitaetsmedizinde_dataSend/" + VERSION, - Arrays.asList(aSen, cD, nD, nB, sTstaDsen, vD), // + Arrays.asList(aSen, cD, cDeS, cDeT, nD, nB, sTexErMe, sTstaDsen, vD, vDeS, vDeT), // "wwwnetzwerk-universitaetsmedizinde_dataTranslate/" + VERSION, Arrays.asList(aTra, cD, nD, nC, sTstaDtra, vD), // "wwwnetzwerk-universitaetsmedizinde_dataReceive/" + VERSION, @@ -114,5 +122,10 @@ public void onProcessesDeployed(ApplicationContext pluginApplicationContext, Lis { pluginApplicationContext.getBean(FttpClientFactory.class).testConnection(); } + + if (activeProcesses.contains("wwwnetzwerk-universitaetsmedizinde_dataSend")) + { + pluginApplicationContext.getBean(BundleValidatorFactory.class).init(); + } } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/error/ErrorOutputParameterGenerator.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/error/ErrorOutputParameterGenerator.java new file mode 100644 index 00000000..67b82ed8 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/error/ErrorOutputParameterGenerator.java @@ -0,0 +1,59 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.error; + +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_MEDIC; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE_VALUE_VALIDATION_FAILED; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.EXTENSION_ERROR_METADATA; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.EXTENSION_ERROR_METADATA_REFERENCE; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.EXTENSION_ERROR_METADATA_SOURCE; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.EXTENSION_ERROR_METADATA_TYPE; +import static org.highmed.dsf.bpe.ConstantsBase.CODESYSTEM_HIGHMED_BPMN; +import static org.highmed.dsf.bpe.ConstantsBase.CODESYSTEM_HIGHMED_BPMN_VALUE_ERROR; + +import java.util.stream.Stream; + +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Extension; +import org.hl7.fhir.r4.model.IdType; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity; +import org.hl7.fhir.r4.model.OperationOutcome.OperationOutcomeIssueComponent; +import org.hl7.fhir.r4.model.Reference; +import org.hl7.fhir.r4.model.StringType; +import org.hl7.fhir.r4.model.Task.TaskOutputComponent; + +public class ErrorOutputParameterGenerator +{ + public Stream createMeDicValidationError(IdType sourceId, OperationOutcome outcome) + { + return outcome.getIssue().stream() + .filter(i -> IssueSeverity.FATAL.equals(i.getSeverity()) || IssueSeverity.ERROR.equals(i.getSeverity())) + .map(i -> createMeDicValidationError(sourceId, i)); + } + + private TaskOutputComponent createMeDicValidationError(IdType sourceId, OperationOutcomeIssueComponent i) + { + TaskOutputComponent output = new TaskOutputComponent(); + output.getType().getCodingFirstRep().setSystem(CODESYSTEM_HIGHMED_BPMN) + .setCode(CODESYSTEM_HIGHMED_BPMN_VALUE_ERROR); + + Extension metaData = output.addExtension(); + metaData.setUrl(EXTENSION_ERROR_METADATA); + metaData.addExtension().setUrl(EXTENSION_ERROR_METADATA_TYPE) + .setValue(new Coding().setSystem(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE) + .setCode(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE_VALUE_VALIDATION_FAILED)); + metaData.addExtension().setUrl(EXTENSION_ERROR_METADATA_SOURCE) + .setValue(new Coding().setSystem(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE) + .setCode(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_MEDIC)); + + if (sourceId != null) + metaData.addExtension().setUrl(EXTENSION_ERROR_METADATA_REFERENCE) + .setValue(new Reference().setReferenceElement(sourceId)); + + output.setValue(new StringType( + "Validation faild at " + i.getLocation().stream().map(StringType::getValue).findFirst().orElse("?"))); + + return output; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/logging/ErrorLogger.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/logging/ErrorLogger.java new file mode 100644 index 00000000..8dd25985 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/logging/ErrorLogger.java @@ -0,0 +1,17 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.logging; + +import org.hl7.fhir.r4.model.IdType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ErrorLogger +{ + private static final Logger validationLogger = LoggerFactory.getLogger("validation-error-logger"); + + public void logValidationFailed(IdType taskId) + { + validationLogger.debug( + "Validation of FHIR resources faild during execution of data-send process started by Task {}", + taskId.getValue()); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ReadData.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ReadData.java index df6f16b1..3683d397 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ReadData.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ReadData.java @@ -5,6 +5,7 @@ import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_VALUE_EXPORT_FROM; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_VALUE_EXPORT_TO; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.HAPI_USER_DATA_SOURCE_ID_ELEMENT; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.IDENTIFIER_NUM_CODEX_DIC_PSEUDONYM_TYPE_CODE; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.IDENTIFIER_NUM_CODEX_DIC_PSEUDONYM_TYPE_SYSTEM; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.NAMING_SYSTEM_NUM_CODEX_DIC_PSEUDONYM; @@ -36,6 +37,7 @@ import org.hl7.fhir.r4.model.DateTimeType; import org.hl7.fhir.r4.model.DiagnosticReport; import org.hl7.fhir.r4.model.DomainResource; +import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.Identifier; import org.hl7.fhir.r4.model.Immunization; import org.hl7.fhir.r4.model.InstantType; @@ -67,14 +69,17 @@ public class ReadData extends AbstractServiceDelegate private final FhirContext fhirContext; private final GeccoClientFactory geccoClientFactory; + private final String geccoServerBase; public ReadData(FhirWebserviceClientProvider clientProvider, TaskHelper taskHelper, - ReadAccessHelper readAccessHelper, FhirContext fhirContext, GeccoClientFactory geccoClientFactory) + ReadAccessHelper readAccessHelper, FhirContext fhirContext, GeccoClientFactory geccoClientFactory, + String geccoServerBase) { super(clientProvider, taskHelper, readAccessHelper); this.fhirContext = fhirContext; this.geccoClientFactory = geccoClientFactory; + this.geccoServerBase = geccoServerBase; } @Override @@ -84,6 +89,7 @@ public void afterPropertiesSet() throws Exception Objects.requireNonNull(fhirContext, "fhirContext"); Objects.requireNonNull(geccoClientFactory, "geccoClientFactory"); + Objects.requireNonNull(geccoServerBase, "geccoServerBase"); } @Override @@ -155,9 +161,14 @@ protected Bundle toBundle(String pseudonym, Stream resources) List entries = resources.map(r -> { BundleEntryComponent entry = b.addEntry(); + + // storing original resource reference for validation error tracking + entry.setUserData(HAPI_USER_DATA_SOURCE_ID_ELEMENT, getAbsoluteId(r)); + entry.setFullUrl("urn:uuid:" + UUID.randomUUID()); entry.getRequest().setMethod(HTTPVerb.PUT).setUrl(getConditionalUpdateUrl(pseudonym, r)); entry.setResource(setSubjectOrIdentifier(clean(r), pseudonym)); + return entry; }).collect(Collectors.toList()); @@ -165,6 +176,15 @@ protected Bundle toBundle(String pseudonym, Stream resources) return b; } + private IdType getAbsoluteId(DomainResource r) + { + if (r == null) + return null; + + return r.getIdElement().isAbsolute() ? r.getIdElement() + : r.getIdElement().withServerBase(geccoServerBase, r.getResourceType().name()); + } + private DomainResource clean(DomainResource r) { r.setIdElement(null); diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java index 1332fa7c..a79f89a0 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java @@ -1,32 +1,188 @@ package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.BPMN_EXECUTION_VARIABLE_BUNDLE; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE_VALUE_VALIDATION_FAILED; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.HAPI_USER_DATA_SOURCE_ID_ELEMENT; + +import java.util.Objects; + import org.camunda.bpm.engine.delegate.BpmnError; import org.camunda.bpm.engine.delegate.DelegateExecution; import org.highmed.dsf.bpe.delegate.AbstractServiceDelegate; import org.highmed.dsf.fhir.authorization.read.ReadAccessHelper; import org.highmed.dsf.fhir.client.FhirWebserviceClientProvider; import org.highmed.dsf.fhir.task.TaskHelper; +import org.highmed.dsf.fhir.variables.FhirResourceValues; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.IdType; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity; +import org.hl7.fhir.r4.model.OperationOutcome.OperationOutcomeIssueComponent; +import org.hl7.fhir.r4.model.StringType; +import org.hl7.fhir.r4.model.Task; +import org.hl7.fhir.r4.model.Task.TaskStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.error.ErrorOutputParameterGenerator; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.logging.ErrorLogger; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactory; + public class ValidateData extends AbstractServiceDelegate { private static final Logger logger = LoggerFactory.getLogger(ValidateData.class); + private final BundleValidatorFactory bundleValidatorSupplier; + private final ErrorOutputParameterGenerator errorOutputParameterGenerator; + private final ErrorLogger errorLogger; + public ValidateData(FhirWebserviceClientProvider clientProvider, TaskHelper taskHelper, - ReadAccessHelper readAccessHelper) + ReadAccessHelper readAccessHelper, BundleValidatorFactory bundleValidatorSupplier, + ErrorOutputParameterGenerator errorOutputParameterGenerator, ErrorLogger errorLogger) { super(clientProvider, taskHelper, readAccessHelper); + + this.bundleValidatorSupplier = bundleValidatorSupplier; + this.errorOutputParameterGenerator = errorOutputParameterGenerator; + this.errorLogger = errorLogger; + } + + @Override + public void afterPropertiesSet() throws Exception + { + super.afterPropertiesSet(); + + Objects.requireNonNull(bundleValidatorSupplier, "bundleValidatorSupplier"); + Objects.requireNonNull(errorOutputParameterGenerator, "errorOutputParameterGenerator"); + Objects.requireNonNull(errorLogger, "errorLogger"); } @Override protected void doExecute(DelegateExecution execution) throws BpmnError, Exception { - // Bundle bundle = (Bundle) execution.getVariable(BPMN_EXECUTION_VARIABLE_BUNDLE); + Bundle bundle = (Bundle) execution.getVariable(BPMN_EXECUTION_VARIABLE_BUNDLE); + + logger.info("Validating bundle with {} entr{}", bundle.getEntry().size(), + bundle.getEntry().size() == 1 ? "y" : "ies"); + bundle = bundleValidatorSupplier.create().validate(bundle); + + if (bundle.hasEntry()) + { + if (bundle.getEntry().stream().anyMatch(e -> !e.hasResponse() || !e.getResponse().hasOutcome() + || !(e.getResponse().getOutcome() instanceof OperationOutcome))) + { + logger.warn( + "Validation result bundle has entries wihout response.outcome instance of OperationOutcome"); + } + else + { + logValidationDetails(bundle); + + if (bundle.getEntry().stream().map(e -> (OperationOutcome) e.getResponse().getOutcome()) + .flatMap(o -> o.getIssue().stream()).anyMatch(i -> IssueSeverity.FATAL.equals(i.getSeverity()) + || IssueSeverity.ERROR.equals(i.getSeverity()))) + { + logger.error("Validation of transfer bundle failed"); + + addErrorsToTaskAndSetFailed(bundle); + errorLogger.logValidationFailed(getLeadingTaskFromExecutionVariables().getIdElement() + .withServerBase(getFhirWebserviceClientProvider().getLocalBaseUrl(), + getLeadingTaskFromExecutionVariables().getIdElement().getResourceType())); + + throw new BpmnError(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE_VALUE_VALIDATION_FAILED); + } + else + { + removeValidationResultsAndUserData(bundle); + } + } + } + else + { + logger.warn("Validation result bundle has no entries"); + } - // TODO validate against fhir profiles // TODO maybe check only one pseudonym used + } + + private void logValidationDetails(Bundle bundle) + { + bundle.getEntry().stream().filter(e -> e.hasResponse() && e.getResponse().hasOutcome() + && (e.getResponse().getOutcome() instanceof OperationOutcome)).forEach(entry -> + { + IdType sourceId = (IdType) entry.getUserData(HAPI_USER_DATA_SOURCE_ID_ELEMENT); + OperationOutcome outcome = (OperationOutcome) entry.getResponse().getOutcome(); + + outcome.getIssue().forEach(i -> logValidationDetails(sourceId, i)); + }); + } + + private void logValidationDetails(IdType sourceId, OperationOutcomeIssueComponent i) + { + if (i.getSeverity() != null) + { + switch (i.getSeverity()) + { + case FATAL: + case ERROR: + logger.error( + "Validation error for {}{}: {}", sourceId.getValue(), i.getLocation().stream() + .map(StringType::getValue).findFirst().map(l -> " location " + l).orElse(""), + i.getDiagnostics()); + break; + case WARNING: + logger.warn( + "Validation warning for {}{}: {}", sourceId.getValue(), i.getLocation().stream() + .map(StringType::getValue).findFirst().map(l -> " location " + l).orElse(""), + i.getDiagnostics()); + break; + case INFORMATION: + case NULL: + default: + logger.info( + "Validation info for {}{}: {}", sourceId.getValue(), i.getLocation().stream() + .map(StringType::getValue).findFirst().map(l -> " location " + l).orElse(""), + i.getDiagnostics()); + break; + } + } + else + { + logger.info( + "Validation info for {}{}: {}", sourceId.getValue(), i.getLocation().stream() + .map(StringType::getValue).findFirst().map(l -> " location " + l).orElse(""), + i.getDiagnostics()); + } + } + + private void addErrorsToTaskAndSetFailed(Bundle bundle) + { + Task task = getLeadingTaskFromExecutionVariables(); + + task.setStatus(TaskStatus.FAILED); + bundle.getEntry().stream() + .filter(e -> e.hasResponse() && e.getResponse().hasOutcome() + && (e.getResponse().getOutcome() instanceof OperationOutcome) + && ((OperationOutcome) e.getResponse().getOutcome()).getIssue().stream() + .anyMatch(i -> IssueSeverity.FATAL.equals(i.getSeverity()) + || IssueSeverity.ERROR.equals(i.getSeverity()))) + .forEach(entry -> + { + IdType sourceId = (IdType) entry.getUserData(HAPI_USER_DATA_SOURCE_ID_ELEMENT); + OperationOutcome outcome = (OperationOutcome) entry.getResponse().getOutcome(); - logger.warn("Bundle validation not implemented"); + errorOutputParameterGenerator.createMeDicValidationError(sourceId, outcome) + .forEach(task::addOutput); + }); + } + + private void removeValidationResultsAndUserData(Bundle bundle) + { + bundle.getEntry().stream().forEach(e -> + { + e.clearUserData(HAPI_USER_DATA_SOURCE_ID_ELEMENT); + e.setResponse(null); + }); + execution.setVariable(BPMN_EXECUTION_VARIABLE_BUNDLE, FhirResourceValues.create(bundle)); } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java index f3132489..5707f6e1 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java @@ -22,6 +22,8 @@ import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.client.fhir.GeccoFhirClient; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.crypto.CrrKeyProvider; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.crypto.CrrKeyProviderImpl; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.error.ErrorOutputParameterGenerator; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.logging.ErrorLogger; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.message.StartReceiveProcess; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.message.StartSendProcess; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.message.StartTranslateProcess; @@ -44,6 +46,10 @@ import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service.StoreDataForCrr; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service.StoreDataForTransferHub; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service.ValidateData; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactory; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactoryImpl; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageIdentifier; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageManager; @Configuration public class TransferDataConfig @@ -66,6 +72,12 @@ public class TransferDataConfig @Autowired private FhirContext fhirContext; + @Autowired + private ValidationPackageManager validationPackageManager; + + @Autowired + private ValidationPackageIdentifier validationPackageIdentifier; + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.trust.certificates:#{null}}") private String fhirStoreTrustStore; @@ -328,13 +340,33 @@ public HandleNoConsentIdatMerge handleNoConsentIdatMerge() @Bean public ReadData readData() { - return new ReadData(fhirClientProvider, taskHelper, readAccessHelper, fhirContext, geccoClientFactory()); + return new ReadData(fhirClientProvider, taskHelper, readAccessHelper, fhirContext, geccoClientFactory(), + fhirStoreBaseUrl); } @Bean public ValidateData validateData() { - return new ValidateData(fhirClientProvider, taskHelper, readAccessHelper); + return new ValidateData(fhirClientProvider, taskHelper, readAccessHelper, bundleValidatorFactory(), + errorOutputParameterGenerator(), errorLogger()); + } + + @Bean + public ErrorOutputParameterGenerator errorOutputParameterGenerator() + { + return new ErrorOutputParameterGenerator(); + } + + @Bean + public ErrorLogger errorLogger() + { + return new ErrorLogger(); + } + + @Bean + public BundleValidatorFactory bundleValidatorFactory() + { + return new BundleValidatorFactoryImpl(validationPackageManager, validationPackageIdentifier); } @Bean diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java new file mode 100644 index 00000000..82746171 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java @@ -0,0 +1,388 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.List; +import java.util.UUID; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import org.bouncycastle.pkcs.PKCSException; +import org.highmed.dsf.fhir.json.ObjectMapperFactory; +import org.highmed.dsf.fhir.validation.ValueSetExpander; +import org.highmed.dsf.fhir.validation.ValueSetExpanderImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.support.IValidationSupport; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGenerator; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorImpl; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithFileSystemCache; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClient; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClientJersey; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClientWithFileSystemCache; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageIdentifier; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageManager; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageManagerImpl; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpanderWithFileSystemCache; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClient; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClientJersey; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClientWithFileSystemCache; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.StructureDefinitionModifier; +import de.rwh.utils.crypto.CertificateHelper; +import de.rwh.utils.crypto.io.CertificateReader; +import de.rwh.utils.crypto.io.PemIo; + +@Configuration +public class ValidationConfig +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationConfig.class); + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package:de.gecco|1.0.5}") + private String validationPackage; + + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.structureDefinitionModifierClasses:de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover,de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.MiiModuleLabObservationLab10IdentifierRemover}'.trim().split('(,[ ]?)|(\\n)')}") + private List structureDefinitionModifierClasses; + + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.packagesToIgnore:hl7.fhir.r4.core|4.0.1}'.trim().split('(,[ ]?)|(\\n)')}") + private List packagesToIgnore; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.cacheFolder:#{null}}") + private String packageCacheFolder; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.server.baseUrl:https://packages.simplifier.net}") + private String packageServerBaseUrl; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.trust.certificates:#{null}}") + private String packageClientTrustCertificates; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate:#{null}}") + private String packageClientCertificate; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate.private.key:#{null}}") + private String packageClientCertificatePrivateKey; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate.private.key.password:#{null}}") + private char[] packageClientCertificatePrivateKeyPassword; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.basic.username:#{null}}") + private String packageClientBasicAuthUsername; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.basic.password:#{null}}") + private char[] packageClientBasicAuthPassword; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.proxy.schemeHostPort:#{null}}") + private String packageClientProxySchemeHostPort; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.proxy.username:#{null}}") + private String packageClientProxyUsername; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.proxy.password:#{null}}") + private char[] packageClientProxyPassword; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.timeout.connect:10000}") + private int packageClientConnectTimeout; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.timeout.read:300000}") + private int packageClientReadTimeout; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.cacheFolder:#{null}}") + private String valueSetCacheFolder; + + // TODO default should be MII ontology server + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.server.baseUrl:https://r4.ontoserver.csiro.au/fhir}") + private String valueSetExpansionServerBaseUrl; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.trust.certificates:#{null}}") + private String valueSetExpansionClientTrustCertificates; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate:#{null}}") + private String valueSetExpansionClientCertificate; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate.private.key:#{null}}") + private String valueSetExpansionClientCertificatePrivateKey; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate.private.key.password:#{null}}") + private char[] valueSetExpansionClientCertificatePrivateKeyPassword; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.basic.username:#{null}}") + private String valueSetExpansionClientBasicAuthUsername; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.basic.password:#{null}}") + private char[] valueSetExpansionClientBasicAuthPassword; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.proxy.schemeHostPort:#{null}}") + private String valueSetExpansionClientProxySchemeHostPort; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.proxy.username:#{null}}") + private String valueSetExpansionClientProxyUsername; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.proxy.password:#{null}}") + private char[] valueSetExpansionClientProxyPassword; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.timeout.connect:10000}") + private int valueSetExpansionClientConnectTimeout; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.timeout.read:300000}") + private int valueSetExpansionClientReadTimeout; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.structuredefinition.cacheFolder:#{null}}") + private String structureDefinitionCacheFolder; + + @Value("${java.io.tmpdir}") + private String systemTempFolder; + + @Autowired + private FhirContext fhirContext; + + @Bean + public ValidationPackageIdentifier validationPackageIdentifier() + { + if (validationPackage == null || validationPackage.isBlank()) + throw new IllegalArgumentException("Validation package not specified"); + + return ValidationPackageIdentifier.fromString(validationPackage); + } + + @Bean + public ValidationPackageManager validationPackageManager() + { + List structureDefinitionModifiers = structureDefinitionModifierClasses.stream() + .map(this::createStructureDefinitionModifier).collect(Collectors.toList()); + + return new ValidationPackageManagerImpl(validationPackageClient(), valueSetExpansionClient(), objectMapper(), + fhirContext, internalSnapshotGeneratorFactory(), internalValueSetExpanderFactory(), + packagesToIgnore.stream().map(ValidationPackageIdentifier::fromString).collect(Collectors.toList()), + structureDefinitionModifiers); + } + + private StructureDefinitionModifier createStructureDefinitionModifier(String className) + { + try + { + Class modifierClass = Class.forName(className); + if (StructureDefinitionModifier.class.isAssignableFrom(modifierClass)) + return (StructureDefinitionModifier) modifierClass.getConstructor().newInstance(); + else + throw new IllegalArgumentException( + "Class " + className + " not compatible with " + StructureDefinitionModifier.class.getName()); + } + catch (ClassNotFoundException | InstantiationException | IllegalAccessException | InvocationTargetException + | NoSuchMethodException | SecurityException e) + { + throw new RuntimeException(e); + } + } + + private Path cacheFolder(String cacheFolderType, String cacheFolder) + { + try + { + Path cacheFolderPath; + if (cacheFolder != null) + cacheFolderPath = Paths.get(cacheFolder); + else + { + cacheFolderPath = Paths.get(systemTempFolder).resolve("codex_gecco_validation_cache") + .resolve(cacheFolderType); + Files.createDirectories(cacheFolderPath); + logger.debug("Cache folder for typ {} created at {}", cacheFolderType, + cacheFolderPath.toAbsolutePath().toString()); + } + + if (!Files.isWritable(cacheFolderPath)) + throw new IOException("Cache folder for type " + cacheFolderType + " + at " + + cacheFolderPath.toAbsolutePath().toString() + " not writable"); + else + return cacheFolderPath; + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + private Path checkReadable(String file) + { + if (file == null) + return null; + else + { + Path path = Paths.get(file); + + if (!Files.isReadable(path)) + throw new RuntimeException(path.toString() + " not readable"); + + return path; + } + } + + private KeyStore trustStore(String trustStoreType, String trustCertificatesFile) + { + if (trustCertificatesFile == null) + return null; + + Path trustCertificatesPath = checkReadable(trustCertificatesFile); + + try + { + logger.debug("Creating trust-store for {} from {}", trustStoreType, trustCertificatesPath.toString()); + return CertificateReader.allFromCer(trustCertificatesPath); + } + catch (NoSuchAlgorithmException | CertificateException | KeyStoreException | IOException e) + { + throw new RuntimeException(e); + } + } + + private KeyStore keyStore(String keyStoreType, String clientCertificateFile, String clientCertificatePrivateKeyFile, + char[] clientCertificatePrivateKeyPassword, char[] keyStorePassword) + { + if ((clientCertificateFile != null) != (clientCertificatePrivateKeyFile != null)) + throw new IllegalArgumentException(keyStoreType + " certificate or private-key not specified"); + else if (clientCertificateFile == null && clientCertificatePrivateKeyFile == null) + return null; + + Path clientCertificatePath = checkReadable(clientCertificateFile); + Path clientCertificatePrivateKeyPath = checkReadable(clientCertificatePrivateKeyFile); + + try + { + PrivateKey privateKey = PemIo.readPrivateKeyFromPem(clientCertificatePrivateKeyPath, + clientCertificatePrivateKeyPassword); + X509Certificate certificate = PemIo.readX509CertificateFromPem(clientCertificatePath); + + logger.debug("Creating key-store for {} from {} and {} with password {}", clientCertificatePath.toString(), + clientCertificatePrivateKeyPath.toString(), + clientCertificatePrivateKeyPassword != null ? "***" : "null"); + return CertificateHelper.toJksKeyStore(privateKey, new Certificate[] { certificate }, + UUID.randomUUID().toString(), keyStorePassword); + } + catch (NoSuchAlgorithmException | CertificateException | KeyStoreException | IOException | PKCSException e) + { + throw new RuntimeException(e); + } + } + + @Bean + public ValidationPackageClient validationPackageClient() + { + return new ValidationPackageClientWithFileSystemCache(packageCacheFolder(), objectMapper(), + validationPackageClientJersey()); + } + + @Bean + public Path packageCacheFolder() + { + return cacheFolder("Package", packageCacheFolder); + } + + private ValidationPackageClientJersey validationPackageClientJersey() + { + if ((packageClientBasicAuthUsername != null) != (packageClientBasicAuthPassword != null)) + { + throw new IllegalArgumentException( + "Package client basic authentication username or password not specified"); + } + + if ((packageClientProxyUsername != null) != (packageClientProxyPassword != null)) + { + throw new IllegalArgumentException("Package client proxy username or password not specified"); + } + + KeyStore packageClientTrustStore = trustStore("FHIR package client", packageClientTrustCertificates); + char[] packageClientKeyStorePassword = UUID.randomUUID().toString().toCharArray(); + KeyStore packageClientKeyStore = keyStore("FHIR package client", packageClientCertificate, + packageClientCertificatePrivateKey, packageClientCertificatePrivateKeyPassword, + packageClientKeyStorePassword); + + return new ValidationPackageClientJersey(packageServerBaseUrl, packageClientTrustStore, packageClientKeyStore, + packageClientKeyStorePassword, packageClientBasicAuthUsername, packageClientBasicAuthPassword, + packageClientProxySchemeHostPort, packageClientProxyUsername, packageClientProxyPassword, + packageClientConnectTimeout, packageClientReadTimeout); + } + + @Bean + public ValueSetExpansionClient valueSetExpansionClient() + { + return new ValueSetExpansionClientWithFileSystemCache(valueSetCacheFolder(), fhirContext, + valueSetExpansionClientJersey()); + } + + @Bean + public Path valueSetCacheFolder() + { + return cacheFolder("ValueSet", valueSetCacheFolder); + } + + private ValueSetExpansionClient valueSetExpansionClientJersey() + { + if ((valueSetExpansionClientBasicAuthUsername != null) != (valueSetExpansionClientBasicAuthPassword != null)) + { + throw new IllegalArgumentException( + "ValueSet expansion client basic authentication username or password not specified"); + } + + if ((valueSetExpansionClientProxyUsername != null) != (valueSetExpansionClientProxyPassword != null)) + { + throw new IllegalArgumentException("ValueSet expansion client proxy username or password not specified"); + } + + KeyStore valueSetExpansionClientTrustStore = trustStore("ValueSet expansion client", + valueSetExpansionClientTrustCertificates); + char[] valueSetExpansionClientKeyStorePassword = UUID.randomUUID().toString().toCharArray(); + KeyStore valueSetExpansionClientKeyStore = keyStore("ValueSet expansion client", + valueSetExpansionClientCertificate, valueSetExpansionClientCertificatePrivateKey, + valueSetExpansionClientCertificatePrivateKeyPassword, valueSetExpansionClientKeyStorePassword); + + return new ValueSetExpansionClientJersey(valueSetExpansionServerBaseUrl, valueSetExpansionClientTrustStore, + valueSetExpansionClientKeyStore, valueSetExpansionClientKeyStorePassword, + valueSetExpansionClientBasicAuthUsername, valueSetExpansionClientBasicAuthPassword, + valueSetExpansionClientProxySchemeHostPort, valueSetExpansionClientProxyUsername, + valueSetExpansionClientProxyPassword, valueSetExpansionClientConnectTimeout, + valueSetExpansionClientReadTimeout, objectMapper(), fhirContext); + } + + @Bean + public ObjectMapper objectMapper() + { + return ObjectMapperFactory.createObjectMapper(fhirContext); + } + + @Bean + public BiFunction internalSnapshotGeneratorFactory() + { + return (fc, vs) -> new PluginSnapshotGeneratorWithFileSystemCache(structureDefinitionCacheFolder(), fc, + new PluginSnapshotGeneratorImpl(fc, vs)); + } + + @Bean + public Path structureDefinitionCacheFolder() + { + return cacheFolder("StructureDefinition", structureDefinitionCacheFolder); + } + + @Bean + public BiFunction internalValueSetExpanderFactory() + { + return (fc, vs) -> new ValueSetExpanderWithFileSystemCache(valueSetCacheFolder(), fc, + new ValueSetExpanderImpl(fc, vs)); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFhirResourceFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFhirResourceFileSystemCache.java new file mode 100644 index 00000000..0ae7fcfc --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFhirResourceFileSystemCache.java @@ -0,0 +1,82 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Objects; +import java.util.function.Function; + +import org.hl7.fhir.r4.model.Resource; +import org.springframework.beans.factory.InitializingBean; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.model.api.annotation.ResourceDef; +import ca.uhn.fhir.parser.IParser; + +public abstract class AbstractFhirResourceFileSystemCache extends AbstractFileSystemCache + implements InitializingBean +{ + private final Class resourceType; + private final FhirContext fhirContext; + + /** + * For JSON content with gzip compression using the .json.gz file name suffix. + * + * @param cacheFolder + * not null + * @param resourceType + * not null + * @param fhirContext + * not null + * @see AbstractFileSystemCache#FILENAME_SUFFIX + * @see AbstractFileSystemCache#OUT_COMPRESSOR_FACTORY + * @see AbstractFileSystemCache#IN_COMPRESSOR_FACTORY + */ + public AbstractFhirResourceFileSystemCache(Path cacheFolder, Class resourceType, FhirContext fhirContext) + { + super(cacheFolder, AbstractFileSystemCache.FILENAME_SUFFIX, AbstractFileSystemCache.OUT_COMPRESSOR_FACTORY, + AbstractFileSystemCache.IN_COMPRESSOR_FACTORY); + + this.resourceType = resourceType; + this.fhirContext = fhirContext; + } + + public AbstractFhirResourceFileSystemCache(Path cacheFolder, String fileNameSuffix, + FunctionWithIoException outCompressorFactory, + FunctionWithIoException inCompressorFactory, Class resourceType, + FhirContext fhirContext) + { + super(cacheFolder, fileNameSuffix, outCompressorFactory, inCompressorFactory); + + this.resourceType = resourceType; + this.fhirContext = fhirContext; + } + + @Override + public void afterPropertiesSet() throws Exception + { + super.afterPropertiesSet(); + + Objects.requireNonNull(resourceType, "resourceType"); + Objects.requireNonNull(fhirContext, "fhirContext"); + } + + protected IParser getJsonParser() + { + return fhirContext.newJsonParser(); + } + + protected T readResourceFromCache(String url, String version, Function fromResource) throws IOException + { + return readFromCache(url + "|" + version, resourceType.getAnnotation(ResourceDef.class).name(), + reader -> getJsonParser().parseResource(resourceType, reader), fromResource); + } + + protected T writeRsourceToCache(T value, Function toResource, Function toUrl, + Function toVersion) throws IOException + { + return writeToCache(value, r -> toUrl.apply(r) + "|" + toVersion.apply(r), r -> r.getResourceType().name(), + (w, r) -> getJsonParser().encodeResourceToWriter(r, w), toResource); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFileSystemCache.java new file mode 100644 index 00000000..97b60f94 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFileSystemCache.java @@ -0,0 +1,153 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Objects; +import java.util.function.Function; + +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +public abstract class AbstractFileSystemCache implements InitializingBean +{ + public static final String FILENAME_SUFFIX = ".json.gz"; + public static final FunctionWithIoException OUT_COMPRESSOR_FACTORY = GzipCompressorOutputStream::new; + public static final FunctionWithIoException IN_COMPRESSOR_FACTORY = GzipCompressorInputStream::new; + + @FunctionalInterface + public interface FunctionWithIoException + { + R apply(T t) throws IOException; + } + + @FunctionalInterface + public interface BiConsumerWithIoException + { + void accept(T t, U u) throws IOException; + } + + private static final Logger logger = LoggerFactory.getLogger(AbstractFileSystemCache.class); + + private final Path cacheFolder; + private final String filenameSuffix; + private final FunctionWithIoException outCompressorFactory; + private final FunctionWithIoException inCompressorFactory; + + /** + * For JSON content with gzip compression using the .json.gz file name suffix. + * + * @param cacheFolder + * not null + * @see #FILENAME_SUFFIX + * @see #OUT_COMPRESSOR_FACTORY + * @see #IN_COMPRESSOR_FACTORY + */ + public AbstractFileSystemCache(Path cacheFolder) + { + this(cacheFolder, FILENAME_SUFFIX, OUT_COMPRESSOR_FACTORY, IN_COMPRESSOR_FACTORY); + } + + public AbstractFileSystemCache(Path cacheFolder, String filenameSuffix, + FunctionWithIoException outCompressorFactory, + FunctionWithIoException inCompressorFactory) + { + this.cacheFolder = cacheFolder; + this.filenameSuffix = filenameSuffix; + this.outCompressorFactory = outCompressorFactory; + this.inCompressorFactory = inCompressorFactory; + } + + @Override + public void afterPropertiesSet() throws Exception + { + Objects.requireNonNull(cacheFolder, "cacheFolder"); + Objects.requireNonNull(filenameSuffix, "filenameSuffix"); + Objects.requireNonNull(outCompressorFactory, "outCompressorFactory"); + Objects.requireNonNull(inCompressorFactory, "inCompressorFactory"); + + if (!Files.isWritable(cacheFolder)) + throw new IOException("Folder " + cacheFolder.toAbsolutePath().toString() + "not writable"); + } + + private Path cacheFile(String cacheEntryId) + { + cacheEntryId = cacheEntryId.replace("://", "_").replaceAll("/", "_").replace(":", "_").replace("|", "_") + .replace("\\", "_"); + + return cacheFolder.resolve(cacheEntryId + filenameSuffix); + } + + protected final T readFromCache(String cacheEntryId, String cacheEntryType, + FunctionWithIoException decoder) throws IOException + { + return readFromCache(cacheEntryId, cacheEntryType, decoder, Function.identity()); + } + + protected final T readFromCache(String cacheEntryId, String cacheEntryType, + FunctionWithIoException decoder, Function fromResource) throws IOException + { + Path cacheFile = cacheFile(cacheEntryId); + + if (!Files.exists(cacheFile)) + { + logger.debug("Cache file for {} {} does not exist", cacheEntryType, cacheEntryId); + return null; + } + else if (Files.exists(cacheFile) && !Files.isReadable(cacheFile)) + { + logger.error("Cache file for {} {} exist in cache but is not readable", cacheEntryType, cacheEntryId); + return null; + } + + try (InputStream in = Files.newInputStream(cacheFile); + BufferedInputStream bIn = new BufferedInputStream(in); + InputStream cIn = inCompressorFactory.apply(bIn); + InputStreamReader reader = new InputStreamReader(cIn, StandardCharsets.UTF_8)) + { + logger.debug("Reading {} {} from cache at {}", cacheEntryType, cacheEntryId, cacheFile.toString()); + return fromResource.apply(decoder.apply(reader)); + } + } + + protected final T writeToCache(T value, Function toCacheId, Function toCacheEntryType, + BiConsumerWithIoException encoder) throws IOException + { + return writeToCache(value, toCacheId, toCacheEntryType, encoder, Function.identity()); + } + + protected final T writeToCache(T value, Function toCacheId, Function toCacheEntryType, + BiConsumerWithIoException encoder, Function toResource) throws IOException + { + R resource = toResource.apply(value); + String cacheId = toCacheId.apply(resource); + String cacheEntryType = toCacheEntryType.apply(resource); + + Path cacheFile = cacheFile(cacheId); + + try (OutputStream out = Files.newOutputStream(cacheFile, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + BufferedOutputStream bOut = new BufferedOutputStream(out); + OutputStream cOut = outCompressorFactory.apply(bOut); + OutputStreamWriter writer = new OutputStreamWriter(cOut, StandardCharsets.UTF_8)) + { + logger.debug("Wirting {} {} to cache at {}", cacheEntryType, cacheId, cacheFile.toString()); + encoder.accept(writer, resource); + } + + return value; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidator.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidator.java new file mode 100644 index 00000000..b2d88d02 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidator.java @@ -0,0 +1,18 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import org.highmed.dsf.fhir.validation.ResourceValidator; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.OperationOutcome; + +public interface BundleValidator extends ResourceValidator +{ + /** + * Validated all bundle entries with a entry.resource. The validation result will be added as a + * {@link OperationOutcome} resource to the corresponding entry.response.outcome property. + * + * @param bundle + * not null + * @return given bundle with added entry.response.outcome properties + */ + Bundle validate(Bundle bundle); +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java new file mode 100644 index 00000000..4651f8ef --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java @@ -0,0 +1,8 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +public interface BundleValidatorFactory +{ + void init(); + + BundleValidator create(); +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java new file mode 100644 index 00000000..4e8697d0 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java @@ -0,0 +1,57 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.List; +import java.util.Objects; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +import ca.uhn.fhir.context.support.IValidationSupport; + +public class BundleValidatorFactoryImpl implements BundleValidatorFactory, InitializingBean +{ + private static final Logger logger = LoggerFactory.getLogger(BundleValidatorFactoryImpl.class); + + private final ValidationPackageManager validationPackageManager; + private final ValidationPackageIdentifier validationPackageIdentifier; + + private IValidationSupport validationSupport; + + public BundleValidatorFactoryImpl(ValidationPackageManager validationPackageManager, + ValidationPackageIdentifier validationPackageIdentifier) + { + this.validationPackageManager = validationPackageManager; + this.validationPackageIdentifier = validationPackageIdentifier; + } + + @Override + public void afterPropertiesSet() throws Exception + { + Objects.requireNonNull(validationPackageManager, "validationPackageManager"); + Objects.requireNonNull(validationPackageIdentifier, "validationPackageIdentifier"); + } + + @Override + public void init() + { + if (validationSupport != null) + return; + + logger.info("Downloading FHIR validation package {} and dependencies", validationPackageIdentifier.toString()); + List validationPackages = validationPackageManager + .downloadPackageWithDependencies(validationPackageIdentifier); + + logger.info("Expanding ValueSets and generating StructureDefinition snapshots"); + validationSupport = validationPackageManager + .expandValueSetsAndGenerateStructureDefinitionSnapshots(validationPackages); + } + + @Override + public BundleValidator create() + { + init(); + + return validationPackageManager.createBundleValidator(validationSupport); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java new file mode 100644 index 00000000..b7cd991f --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java @@ -0,0 +1,54 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.Objects; + +import org.highmed.dsf.fhir.validation.ResourceValidator; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.Resource; +import org.springframework.beans.factory.InitializingBean; + +import ca.uhn.fhir.validation.ValidationResult; + +public class BundleValidatorImpl implements BundleValidator, InitializingBean +{ + private final ResourceValidator delegate; + + public BundleValidatorImpl(ResourceValidator delegate) + { + this.delegate = delegate; + } + + @Override + public void afterPropertiesSet() throws Exception + { + Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public ValidationResult validate(Resource resource) + { + Objects.requireNonNull(resource, "resource"); + + return delegate.validate(resource); + } + + @Override + public Bundle validate(Bundle bundle) + { + Objects.requireNonNull(bundle, "bundle"); + + bundle.getEntry().stream().forEach(this::validateAndSetOutcome); + return bundle; + } + + private void validateAndSetOutcome(BundleEntryComponent entry) + { + if (entry.hasResource()) + { + ValidationResult validationResult = validate(entry.getResource()); + entry.getResponse().setOutcome((OperationOutcome) validationResult.toOperationOutcome()); + } + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/CodeValidatorForExpandedValueSets.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/CodeValidatorForExpandedValueSets.java new file mode 100644 index 00000000..80ce5bc2 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/CodeValidatorForExpandedValueSets.java @@ -0,0 +1,138 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import static org.apache.commons.lang3.StringUtils.defaultString; +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; + +import java.util.ArrayList; +import java.util.List; + +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.ValueSet; +import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionComponent; +import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent; +import org.hl7.fhir.utilities.validation.ValidationMessage; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.support.ConceptValidationOptions; +import ca.uhn.fhir.context.support.IValidationSupport; +import ca.uhn.fhir.context.support.ValidationSupportContext; +import ca.uhn.fhir.util.VersionIndependentConcept; + +public class CodeValidatorForExpandedValueSets implements IValidationSupport +{ + private final FhirContext fhirContext; + + public CodeValidatorForExpandedValueSets(FhirContext fhirContext) + { + this.fhirContext = fhirContext; + } + + @Override + public FhirContext getFhirContext() + { + return fhirContext; + } + + @Override + public boolean isValueSetSupported(ValidationSupportContext supportContext, String valueSetUrl) + { + return supportContext.getRootValidationSupport().fetchResource(ValueSet.class, valueSetUrl).hasExpansion(); + } + + @Override + public CodeValidationResult validateCodeInValueSet(ValidationSupportContext supportContext, + ConceptValidationOptions options, String codeSystem, String code, String display, IBaseResource valueSet) + { + if (valueSet == null || !(valueSet instanceof ValueSet) || !((ValueSet) valueSet).hasExpansion()) + return new CodeValidationResult().setSeverity(IssueSeverity.ERROR).setMessage("ValueSet not supported"); + + ValueSetExpansionComponent expansion = ((ValueSet) valueSet).getExpansion(); + + return doValidateCodeInValueSet(supportContext, options, codeSystem, code, display, expansion); + } + + public CodeValidationResult doValidateCodeInValueSet(ValidationSupportContext supportContext, + ConceptValidationOptions options, String targetCodeSystem, String targetCode, String targetDisplay, + ValueSetExpansionComponent expansion) + + { + boolean codeSystemCaseSensitive = true; + CodeSystem codeSystem = null; + + if (!options.isInferSystem() && isNotBlank(targetCodeSystem)) + codeSystem = (CodeSystem) supportContext.getRootValidationSupport().fetchCodeSystem(targetCodeSystem); + + List codes = new ArrayList<>(); + flatten(expansion.getContains(), codes); + + String codeSystemName = null; + String codeSystemVersion = null; + String codeSystemContentMode = null; + + if (codeSystem != null) + { + codeSystemCaseSensitive = codeSystem.getCaseSensitive(); + codeSystemName = codeSystem.getName(); + codeSystemVersion = codeSystem.getVersion(); + codeSystemContentMode = codeSystem.getContentElement().getValueAsString(); + } + + for (VersionIndependentConcept nextExpansionCode : codes) + { + boolean codeMatches; + if (codeSystemCaseSensitive) + codeMatches = defaultString(targetCode).equals(nextExpansionCode.getCode()); + else + codeMatches = defaultString(targetCode).equalsIgnoreCase(nextExpansionCode.getCode()); + + if (codeMatches) + { + if (options.isInferSystem() || nextExpansionCode.getSystem().equals(targetCodeSystem)) + { + if (!options.isValidateDisplay() || (isBlank(nextExpansionCode.getDisplay()) + || isBlank(targetDisplay) || nextExpansionCode.getDisplay().equals(targetDisplay))) + { + return new CodeValidationResult().setCode(targetCode).setDisplay(nextExpansionCode.getDisplay()) + .setCodeSystemName(codeSystemName).setCodeSystemVersion(codeSystemVersion); + } + else + { + return new CodeValidationResult().setSeverity(IssueSeverity.ERROR) + .setDisplay(nextExpansionCode.getDisplay()) + .setMessage("Concept Display \"" + targetDisplay + "\" does not match expected \"" + + nextExpansionCode.getDisplay() + "\"") + .setCodeSystemName(codeSystemName).setCodeSystemVersion(codeSystemVersion); + } + } + } + } + + ValidationMessage.IssueSeverity severity; + String message; + if ("fragment".equals(codeSystemContentMode)) + { + severity = ValidationMessage.IssueSeverity.WARNING; + message = "Unknown code in fragment CodeSystem '" + + (isNotBlank(targetCodeSystem) ? targetCodeSystem + "#" : "") + targetCode + "'"; + } + else + { + severity = ValidationMessage.IssueSeverity.ERROR; + message = "Unknown code '" + (isNotBlank(targetCodeSystem) ? targetCodeSystem + "#" : "") + targetCode + + "'"; + } + + return new CodeValidationResult().setSeverityCode(severity.toCode()).setMessage(message); + } + + private void flatten(List components, List concepts) + { + for (ValueSetExpansionContainsComponent next : components) + { + concepts.add(new VersionIndependentConcept(next.getSystem(), next.getCode(), next.getDisplay())); + flatten(next.getContains(), concepts); + } + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGenerator.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGenerator.java new file mode 100644 index 00000000..d1f68006 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGenerator.java @@ -0,0 +1,8 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import org.hl7.fhir.r4.model.StructureDefinition; + +public interface PluginSnapshotGenerator +{ + PluginSnapshotWithValidationMessages generateSnapshot(StructureDefinition differential); +} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorImpl.java new file mode 100644 index 00000000..5295d893 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorImpl.java @@ -0,0 +1,69 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import org.hl7.fhir.r4.conformance.ProfileUtilities; +import org.hl7.fhir.r4.context.IWorkerContext; +import org.hl7.fhir.r4.hapi.ctx.HapiWorkerContext; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.utilities.validation.ValidationMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.support.IValidationSupport; + +public class PluginSnapshotGeneratorImpl implements PluginSnapshotGenerator +{ + private static final Logger logger = LoggerFactory.getLogger(PluginSnapshotGeneratorImpl.class); + + private final IWorkerContext worker; + + public PluginSnapshotGeneratorImpl(FhirContext fhirContext, IValidationSupport validationSupport) + { + worker = createWorker(fhirContext, validationSupport); + } + + protected IWorkerContext createWorker(FhirContext context, IValidationSupport validationSupport) + { + HapiWorkerContext workerContext = new HapiWorkerContext(context, validationSupport); + workerContext.setLocale(context.getLocalizer().getLocale()); + return new PluginWorkerContext(workerContext); + } + + @Override + public PluginSnapshotWithValidationMessages generateSnapshot(StructureDefinition differential) + { + Objects.requireNonNull(differential, "differential"); + + logger.debug("Generating snapshot for StructureDefinition with id {}, url {}, version {}, base {}", + differential.getIdElement().getIdPart(), differential.getUrl(), differential.getVersion(), + differential.getBaseDefinition()); + + StructureDefinition base = worker.fetchResource(StructureDefinition.class, differential.getBaseDefinition()); + + if (base == null) + logger.warn("Base definition with url {} not found", differential.getBaseDefinition()); + + /* ProfileUtilities is not thread safe */ + List messages = new ArrayList<>(); + ProfileUtilities profileUtils = new ProfileUtilities(worker, messages, null); + + profileUtils.generateSnapshot(base, differential, "", "", null); + + if (messages.isEmpty()) + logger.debug("Snapshot generated for StructureDefinition with id {}, url {}, version {}", + differential.getIdElement().getIdPart(), differential.getUrl(), differential.getVersion()); + else + { + logger.warn("Snapshot not generated for StructureDefinition with id {}, url {}, version {}", + differential.getIdElement().getIdPart(), differential.getUrl(), differential.getVersion()); + messages.forEach(m -> logger.warn("Issue while generating snapshot: {} - {} - {}", m.getDisplay(), + m.getLine(), m.getMessage())); + } + + return new PluginSnapshotWithValidationMessages(differential, messages); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java new file mode 100644 index 00000000..2686b8bf --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java @@ -0,0 +1,115 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Objects; + +import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +import ca.uhn.fhir.context.FhirContext; + +public class PluginSnapshotGeneratorWithFileSystemCache + extends AbstractFhirResourceFileSystemCache + implements PluginSnapshotGenerator, InitializingBean +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationPackageClientWithFileSystemCache.class); + + private final PluginSnapshotGenerator delegate; + + /** + * For JSON content with gzip compression using the .json.xz file name suffix. + * + * @param cacheFolder + * not null + * @param fhirContext + * not null + * @param delegate + * not null + * @see AbstractFileSystemCache#FILENAME_SUFFIX + * @see AbstractFileSystemCache#OUT_COMPRESSOR_FACTORY + * @see AbstractFileSystemCache#IN_COMPRESSOR_FACTORY + */ + public PluginSnapshotGeneratorWithFileSystemCache(Path cacheFolder, FhirContext fhirContext, + PluginSnapshotGenerator delegate) + { + super(cacheFolder, StructureDefinition.class, fhirContext); + + this.delegate = delegate; + } + + public PluginSnapshotGeneratorWithFileSystemCache(Path cacheFolder, String fileNameSuffix, + FunctionWithIoException outCompressorFactory, + FunctionWithIoException inCompressorFactory, FhirContext fhirContext, + PluginSnapshotGenerator delegate) + { + super(cacheFolder, fileNameSuffix, outCompressorFactory, inCompressorFactory, StructureDefinition.class, + fhirContext); + + this.delegate = delegate; + } + + @Override + public void afterPropertiesSet() throws Exception + { + super.afterPropertiesSet(); + + Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public PluginSnapshotWithValidationMessages generateSnapshot(StructureDefinition structureDefinition) + { + Objects.requireNonNull(structureDefinition, "differential"); + + if (structureDefinition.hasSnapshot()) + { + logger.debug("StructureDefinition {}|{} has snapshot", structureDefinition.getUrl(), + structureDefinition.getVersion()); + return new PluginSnapshotWithValidationMessages(structureDefinition, Collections.emptyList()); + } + + Objects.requireNonNull(structureDefinition.getUrl(), "structureDefinition.url"); + Objects.requireNonNull(structureDefinition.getVersion(), "structureDefinition.version"); + + try + { + PluginSnapshotWithValidationMessages read = readResourceFromCache(structureDefinition.getUrl(), + structureDefinition.getVersion(), + // needs to return original structureDefinition object with included snapshot + sd -> new PluginSnapshotWithValidationMessages(structureDefinition.setSnapshot(sd.getSnapshot()), + Collections.emptyList())); + if (read != null) + return read; + else + return downloadAndWriteToCache(structureDefinition); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + private PluginSnapshotWithValidationMessages downloadAndWriteToCache(StructureDefinition structureDefinition) + throws IOException + { + PluginSnapshotWithValidationMessages snapshot = delegate.generateSnapshot(structureDefinition); + + if (PublicationStatus.DRAFT.equals(snapshot.getSnapshot().getStatus())) + { + logger.info("Not writing StructureDefinition {}|{} with snapshot and status {} to cache", + snapshot.getSnapshot().getUrl(), snapshot.getSnapshot().getVersion(), + snapshot.getSnapshot().getStatus()); + return snapshot; + } + else + return writeRsourceToCache(snapshot, PluginSnapshotWithValidationMessages::getSnapshot, + StructureDefinition::getUrl, StructureDefinition::getVersion); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotWithValidationMessages.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotWithValidationMessages.java new file mode 100644 index 00000000..b09e382b --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotWithValidationMessages.java @@ -0,0 +1,30 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.utilities.validation.ValidationMessage; + +public class PluginSnapshotWithValidationMessages +{ + private final StructureDefinition snapshot; + private final List messages; + + PluginSnapshotWithValidationMessages(StructureDefinition snapshot, List messages) + { + this.snapshot = Objects.requireNonNull(snapshot, "snapshot"); + this.messages = Objects.requireNonNull(messages, "messages"); + } + + public StructureDefinition getSnapshot() + { + return snapshot; + } + + public List getMessages() + { + return Collections.unmodifiableList(messages); + } +} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginWorkerContext.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginWorkerContext.java new file mode 100644 index 00000000..826787b8 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginWorkerContext.java @@ -0,0 +1,366 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.List; +import java.util.Locale; +import java.util.Set; + +import org.fhir.ucum.UcumService; +import org.hl7.fhir.exceptions.DefinitionException; +import org.hl7.fhir.exceptions.FHIRException; +import org.hl7.fhir.exceptions.TerminologyServiceException; +import org.hl7.fhir.r4.context.IWorkerContext; +import org.hl7.fhir.r4.formats.IParser; +import org.hl7.fhir.r4.formats.ParserType; +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.ConceptMap; +import org.hl7.fhir.r4.model.ElementDefinition.ElementDefinitionBindingComponent; +import org.hl7.fhir.r4.model.MetadataResource; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Resource; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.StructureMap; +import org.hl7.fhir.r4.model.ValueSet; +import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent; +import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome; +import org.hl7.fhir.r4.utils.INarrativeGenerator; +import org.hl7.fhir.r4.utils.IResourceValidator; +import org.hl7.fhir.utilities.TranslationServices; +import org.hl7.fhir.utilities.validation.ValidationOptions; + +public class PluginWorkerContext implements IWorkerContext +{ + private final IWorkerContext delegate; + + public PluginWorkerContext(IWorkerContext delegate) + { + this.delegate = delegate; + } + + @Override + public String getVersion() + { + return delegate.getVersion(); + } + + @Override + public UcumService getUcumService() + { + return delegate.getUcumService(); + } + + @Override + public IParser getParser(ParserType type) + { + return delegate.getParser(type); + } + + @Override + public IParser getParser(String type) + { + return delegate.getParser(type); + } + + @Override + public IParser newJsonParser() + { + return delegate.newJsonParser(); + } + + @Override + public IParser newXmlParser() + { + return delegate.newXmlParser(); + } + + @Override + public INarrativeGenerator getNarrativeGenerator(String prefix, String basePath) + { + return delegate.getNarrativeGenerator(prefix, basePath); + } + + @Override + public IResourceValidator newValidator() throws FHIRException + { + return delegate.newValidator(); + } + + @Override + public T fetchResource(Class class_, String uri) + { + return delegate.fetchResource(class_, uri); + } + + @Override + public T fetchResourceWithException(Class class_, String uri) throws FHIRException + { + return delegate.fetchResourceWithException(class_, uri); + } + + @Override + public Resource fetchResourceById(String type, String uri) + { + return delegate.fetchResourceById(type, uri); + } + + @Override + public boolean hasResource(Class class_, String uri) + { + return delegate.hasResource(class_, uri); + } + + @Override + public void cacheResource(Resource res) throws FHIRException + { + delegate.cacheResource(res); + } + + @Override + public List getResourceNames() + { + return delegate.getResourceNames(); + } + + @Override + public Set getResourceNamesAsSet() + { + return delegate.getResourceNamesAsSet(); + } + + @Override + public List getTypeNames() + { + return delegate.getTypeNames(); + } + + @Override + public List allStructures() + { + return delegate.allStructures(); + } + + @Override + public List getStructures() + { + return delegate.getStructures(); + } + + @Override + public List allConformanceResources() + { + return delegate.allConformanceResources(); + } + + @Override + public void generateSnapshot(StructureDefinition p) throws DefinitionException, FHIRException + { + delegate.generateSnapshot(p); + } + + @Override + public Parameters getExpansionParameters() + { + return delegate.getExpansionParameters(); + } + + @Override + public void setExpansionProfile(Parameters expParameters) + { + delegate.setExpansionProfile(expParameters); + } + + @Override + public CodeSystem fetchCodeSystem(String system) + { + return delegate.fetchCodeSystem(system); + } + + @Override + public boolean supportsSystem(String system) throws TerminologyServiceException + { + return delegate.supportsSystem(system); + } + + @Override + public List findMapsForSource(String url) throws FHIRException + { + return delegate.findMapsForSource(url); + } + + @Override + public ValueSetExpansionOutcome expandVS(ValueSet source, boolean cacheOk, boolean heiarchical) + { + if (source.hasExpansion()) + return new ValueSetExpansionOutcome(source); + else + return new ValueSetExpansionOutcome(null); + } + + @Override + public ValueSetExpansionOutcome expandVS(ElementDefinitionBindingComponent binding, boolean cacheOk, + boolean heiarchical) throws FHIRException + { + return delegate.expandVS(binding, cacheOk, heiarchical); + } + + @Override + + public ValueSetExpansionOutcome expandVS(ConceptSetComponent inc, boolean heirarchical) + throws TerminologyServiceException + { + return delegate.expandVS(inc, heirarchical); + } + + @Override + public Locale getLocale() + { + return delegate.getLocale(); + } + + @Override + public void setLocale(Locale locale) + { + delegate.setLocale(locale); + } + + @Override + public String formatMessage(String theMessage, Object... theMessageArguments) + { + return delegate.formatMessage(theMessage, theMessageArguments); + } + + @Override + public void setValidationMessageLanguage(Locale locale) + { + delegate.setValidationMessageLanguage(locale); + } + + @Override + public ValidationResult validateCode(ValidationOptions options, String system, String code, String display) + { + return delegate.validateCode(options, system, code, display); + } + + @Override + public ValidationResult validateCode(ValidationOptions options, String system, String code, String display, + ValueSet vs) + { + return delegate.validateCode(options, system, code, display, vs); + } + + @Override + public ValidationResult validateCode(ValidationOptions options, String code, ValueSet vs) + { + return delegate.validateCode(options, code, vs); + } + + @Override + public ValidationResult validateCode(ValidationOptions options, Coding code, ValueSet vs) + { + return delegate.validateCode(options, code, vs); + } + + @Override + public ValidationResult validateCode(ValidationOptions options, CodeableConcept code, ValueSet vs) + { + return delegate.validateCode(options, code, vs); + } + + @Override + public ValidationResult validateCode(ValidationOptions options, String system, String code, String display, + ConceptSetComponent vsi) + { + return delegate.validateCode(options, system, code, display, vsi); + } + + @Override + public String getAbbreviation(String name) + { + return delegate.getAbbreviation(name); + } + + @Override + public Set typeTails() + { + return delegate.typeTails(); + } + + @Override + public String oid2Uri(String code) + { + return delegate.oid2Uri(code); + } + + @Override + public boolean hasCache() + { + return delegate.hasCache(); + } + + @Override + public void setLogger(ILoggingService logger) + { + delegate.setLogger(logger); + } + + @Override + public ILoggingService getLogger() + { + return delegate.getLogger(); + } + + @Override + public boolean isNoTerminologyServer() + { + return delegate.isNoTerminologyServer(); + } + + @Override + public TranslationServices translator() + { + return delegate.translator(); + } + + @Override + public List listTransforms() + { + return delegate.listTransforms(); + } + + @Override + public StructureMap getTransform(String url) + { + return delegate.getTransform(url); + } + + @Override + public String getOverrideVersionNs() + { + return delegate.getOverrideVersionNs(); + } + + @Override + public void setOverrideVersionNs(String value) + { + delegate.setOverrideVersionNs(value); + } + + @Override + public StructureDefinition fetchTypeDefinition(String typeName) + { + return delegate.fetchTypeDefinition(typeName); + } + + @Override + public void setUcumService(UcumService ucumService) + { + delegate.setUcumService(ucumService); + } + + @Override + public String getLinkForUrl(String corePath, String s) + { + return delegate.getLinkForUrl(corePath, s); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java new file mode 100644 index 00000000..1406b799 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java @@ -0,0 +1,224 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Locale; +import java.util.Objects; + +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Resource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.i18n.HapiLocalizer; +import ca.uhn.fhir.parser.IParser; +import ca.uhn.fhir.validation.ValidationResult; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config.ValidationConfig; + +public class ValidationMain implements InitializingBean +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationMain.class); + + private static final class FileNameAndResource + { + final String filename; + final Resource resource; + + FileNameAndResource(String filename, Resource resource) + { + this.filename = filename; + this.resource = resource; + } + + String getFilename() + { + return filename; + } + + Resource getResource() + { + return resource; + } + } + + @Configuration + @PropertySource(ignoreResourceNotFound = true, value = "file:application.properties") + public static class TestConfig + { + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.output:JSON}") + private Output output; + + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.output.pretty:true}") + private boolean outputPretty; + + @Autowired + private ValidationPackageManager packageManager; + + @Autowired + private ValidationPackageIdentifier validationPackage; + + @Bean + public FhirContext fhirContext() + { + FhirContext context = FhirContext.forR4(); + HapiLocalizer localizer = new HapiLocalizer() + { + @Override + public Locale getLocale() + { + return Locale.ROOT; + } + }; + context.setLocalizer(localizer); + return context; + } + + @Bean + public ValidationMain validatorMain() + { + return new ValidationMain(fhirContext(), packageManager, validationPackage, output, outputPretty); + } + } + + public static enum Output + { + JSON, XML + } + + public static void main(String[] args) + { + if (args.length == 0) + { + logger.warn("No files to validated specified"); + System.exit(1); + } + + try (AnnotationConfigApplicationContext springContext = new AnnotationConfigApplicationContext(TestConfig.class, + ValidationConfig.class)) + { + springContext.getBean(ValidationMain.class).validate(args); + } + catch (Exception e) + { + logger.error("Unable to create spring context {}: {}", e.getClass().getName(), e.getMessage()); + logger.error("Stack-Trace: ", e); + System.exit(1); + } + } + + private final FhirContext fhirContext; + private final ValidationPackageManager packageManager; + private final ValidationPackageIdentifier validationPackage; + private final Output output; + private final boolean outputPretty; + + public ValidationMain(FhirContext fhirContext, ValidationPackageManager packageManager, + ValidationPackageIdentifier validationPackage, Output output, boolean outputPretty) + { + this.fhirContext = fhirContext; + this.packageManager = packageManager; + this.validationPackage = validationPackage; + this.output = output; + this.outputPretty = outputPretty; + } + + @Override + public void afterPropertiesSet() throws Exception + { + Objects.requireNonNull(fhirContext, "fhirContext"); + Objects.requireNonNull(packageManager, "packageManager"); + Objects.requireNonNull(validationPackage, "validationPackage"); + Objects.requireNonNull(output, "output"); + } + + private void validate(String[] files) + { + logger.info("Using validation package {}", validationPackage); + + BundleValidator validator = packageManager.createBundleValidator(validationPackage.getName(), + validationPackage.getVersion()); + + Arrays.stream(files).map(this::read).filter(r -> r != null).forEach(r -> + { + logger.info("Validating {} from {}", r.getResource().getResourceType().name(), r.getFilename()); + + if (r.getResource() instanceof Bundle) + { + Bundle validationResult = validator.validate((Bundle) r.getResource()); + System.out.println(getOutputParser().encodeResourceToString(validationResult)); + } + else + { + ValidationResult validationResult = validator.validate(r.getResource()); + System.out.println(getOutputParser().encodeResourceToString(validationResult.toOperationOutcome())); + } + }); + } + + private IParser getOutputParser() + { + switch (output) + { + case JSON: + return fhirContext.newJsonParser().setPrettyPrint(outputPretty); + case XML: + return fhirContext.newXmlParser().setPrettyPrint(outputPretty); + default: + throw new IllegalStateException("Output of type " + output + " not supported"); + } + } + + private FileNameAndResource read(String file) + { + if (file.endsWith(".json")) + return tryJson(file); + else if (file.endsWith(".xml")) + return tryXml(file); + else + { + logger.warn("File {} not supported, filename needs to end with .json or .xml", file); + return null; + } + } + + private FileNameAndResource tryJson(String file) + { + try (InputStream in = Files.newInputStream(Paths.get(file))) + { + IBaseResource resource = fhirContext.newJsonParser().parseResource(in); + logger.info("{} read from {}", resource.getClass().getSimpleName(), file); + return new FileNameAndResource(file, (Resource) resource); + } + catch (Exception e) + { + logger.warn("Unable to read " + file + " as JSON, {}: {}", e.getClass().getName(), e.getMessage()); + return null; + } + } + + private FileNameAndResource tryXml(String file) + { + try (InputStream in = Files.newInputStream(Paths.get(file))) + { + IBaseResource resource = fhirContext.newXmlParser().parseResource(in); + logger.info("{} read from {}", resource.getClass().getSimpleName(), file); + return new FileNameAndResource(file, (Resource) resource); + } + catch (Exception e) + { + logger.warn("Unable to read " + file + " as XML, {}: {}", e.getClass().getName(), e.getMessage()); + return null; + } + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackage.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackage.java new file mode 100644 index 00000000..3f14aba8 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackage.java @@ -0,0 +1,201 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.NamingSystem; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.StructureDefinition.StructureDefinitionKind; +import org.hl7.fhir.r4.model.ValueSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +import ca.uhn.fhir.context.FhirContext; + +public class ValidationPackage +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationPackage.class); + + private static final String PACKAGE_JSON_FILENAME = "package/package.json"; + + public static ValidationPackage from(String name, String version, InputStream in) throws IOException + { + try (BufferedInputStream bufferedIn = new BufferedInputStream(in); + GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(bufferedIn); + TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) + { + List entries = new ArrayList<>(); + + ArchiveEntry entry; + while ((entry = tarIn.getNextEntry()) != null) + { + ValidationPackageEntry pEntry = ValidationPackageEntry.from(entry, tarIn); + if (pEntry != null) + entries.add(pEntry); + } + + return new ValidationPackage(name, version, entries); + } + } + + private final String name; + private final String version; + private final List entries = new ArrayList<>(); + + private Map entriesByFileName; + + private ValidationSupportResources resources; + + /** + * @param name + * not null + * @param version + * not null + * @param entries + * may be null + */ + @JsonCreator + public ValidationPackage(@JsonProperty("name") String name, @JsonProperty("version") String version, + @JsonProperty("entries") Collection entries) + { + Objects.requireNonNull(name, "name"); + Objects.requireNonNull(version, "version"); + + this.name = name; + this.version = version; + + if (entries != null) + this.entries.addAll(entries); + } + + @JsonProperty("name") + public String getName() + { + return name; + } + + @JsonProperty("version") + public String getVersion() + { + return version; + } + + @JsonIgnore + public ValidationPackageIdentifier getIdentifier() + { + return new ValidationPackageIdentifier(name, version); + } + + @JsonProperty("entries") + public List getEntries() + { + return Collections.unmodifiableList(entries); + } + + @JsonIgnore + public Map getEntriesByFileName() + { + if (entriesByFileName == null) + entriesByFileName = getEntries().stream().collect(Collectors + .toUnmodifiableMap(ValidationPackageEntry::getFileName, Function.identity(), (e0, e1) -> e1)); + + return entriesByFileName; + } + + @JsonIgnore + public ValidationPackageDescriptor getDescriptor(ObjectMapper mapper) throws IOException + { + ValidationPackageEntry packageJson = getEntriesByFileName().get(PACKAGE_JSON_FILENAME); + return mapper.readValue(packageJson.getContent(), ValidationPackageDescriptor.class); + } + + public void parseResources(FhirContext context) + { + if (resources == null) + { + List codeSystems = new ArrayList<>(); + List namingSystems = new ArrayList<>(); + List structureDefinitions = new ArrayList<>(); + List valueSets = new ArrayList<>(); + + getEntries() + .forEach(doParseResources(context, codeSystems, namingSystems, structureDefinitions, valueSets)); + + resources = new ValidationSupportResources(codeSystems, namingSystems, structureDefinitions, valueSets); + } + } + + private Consumer doParseResources(FhirContext context, List codeSystems, + List namingSystems, List structureDefinitions, List valueSets) + { + return entry -> + { + if ("package/package.json".equals(entry.getFileName()) + || (entry.getFileName() != null && (entry.getFileName().startsWith("package/example") + || entry.getFileName().endsWith(".index.json") || !entry.getFileName().endsWith(".json")))) + { + logger.debug("Ignoring " + entry.getFileName()); + return; + } + + logger.debug("Reading " + entry.getFileName()); + + try + { + IBaseResource resource = context.newJsonParser() + .parseResource(new String(entry.getContent(), StandardCharsets.UTF_8)); + + if (resource instanceof CodeSystem) + codeSystems.add((CodeSystem) resource); + else if (resource instanceof NamingSystem) + namingSystems.add((NamingSystem) resource); + else if (resource instanceof StructureDefinition) + { + if (!StructureDefinitionKind.LOGICAL.equals(((StructureDefinition) resource).getKind())) + structureDefinitions.add((StructureDefinition) resource); + else + logger.debug("Ignoring StructureDefinition with kind = logical"); + } + else if (resource instanceof ValueSet) + valueSets.add((ValueSet) resource); + else + logger.debug("Ignoring resource of type {}", resource.getClass().getName()); + } + catch (Exception e) + { + logger.warn("Ignoring resource with error while parsing, {}: {}", e.getClass().getName(), + e.getMessage()); + } + }; + } + + @JsonIgnore + public ValidationSupportResources getValidationSupportResources() + { + if (resources == null) + throw new IllegalStateException("Resources not parsed"); + + return resources; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClient.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClient.java new file mode 100644 index 00000000..c3cbafa7 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClient.java @@ -0,0 +1,31 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; + +import javax.ws.rs.WebApplicationException; + +public interface ValidationPackageClient +{ + /** + * @param name + * not null + * @param version + * not null + * @return downloaded {@link ValidationPackage}, never null + * @throws IOException + * @throws WebApplicationException + */ + default ValidationPackage download(String name, String version) throws IOException, WebApplicationException + { + return download(new ValidationPackageIdentifier(name, version)); + } + + /** + * @param identifier + * not null + * @return downloaded {@link ValidationPackage}, never null + * @throws IOException + * @throws WebApplicationException + */ + ValidationPackage download(ValidationPackageIdentifier identifier) throws IOException, WebApplicationException; +} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java new file mode 100644 index 00000000..6f32b22b --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java @@ -0,0 +1,85 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyStore; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.SSLContext; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; + +import org.glassfish.jersey.SslConfigurator; +import org.glassfish.jersey.client.ClientProperties; +import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; + +public class ValidationPackageClientJersey implements ValidationPackageClient +{ + private final Client client; + private final String baseUrl; + + public ValidationPackageClientJersey(String baseUrl) + { + this(baseUrl, null, null, null, null, null, null, null, null, 0, 0); + } + + public ValidationPackageClientJersey(String baseUrl, KeyStore trustStore, KeyStore keyStore, + char[] keyStorePassword, String basicAuthUsername, char[] basicAuthPassword, String proxySchemeHostPort, + String proxyUsername, char[] proxyPassword, int connectTimeout, int readTimeout) + { + SSLContext sslContext = null; + if (trustStore != null && keyStore == null && keyStorePassword == null) + sslContext = SslConfigurator.newInstance().trustStore(trustStore).createSSLContext(); + else if (trustStore != null && keyStore != null && keyStorePassword != null) + sslContext = SslConfigurator.newInstance().trustStore(trustStore).keyStore(keyStore) + .keyStorePassword(keyStorePassword).createSSLContext(); + + ClientBuilder builder = ClientBuilder.newBuilder(); + + if (sslContext != null) + builder = builder.sslContext(sslContext); + + if (basicAuthUsername != null && basicAuthPassword != null) + { + HttpAuthenticationFeature basicAuthFeature = HttpAuthenticationFeature.basic(basicAuthUsername, + String.valueOf(basicAuthPassword)); + builder = builder.register(basicAuthFeature); + } + + if (proxySchemeHostPort != null) + { + builder = builder.property(ClientProperties.PROXY_URI, proxySchemeHostPort); + if (proxyUsername != null && proxyPassword != null) + builder = builder.property(ClientProperties.PROXY_USERNAME, proxyUsername) + .property(ClientProperties.PROXY_PASSWORD, String.valueOf(proxyPassword)); + } + + builder = builder.readTimeout(readTimeout, TimeUnit.MILLISECONDS).connectTimeout(connectTimeout, + TimeUnit.MILLISECONDS); + + client = builder.build(); + + this.baseUrl = baseUrl; + } + + private WebTarget getResource() + { + return client.target(baseUrl); + } + + @Override + public ValidationPackage download(ValidationPackageIdentifier identifier) + throws IOException, WebApplicationException + { + Objects.requireNonNull(identifier, "identifier"); + + try (InputStream in = getResource().path(identifier.getName()).path(identifier.getVersion()) + .request("application/tar+gzip").get(InputStream.class)) + { + return ValidationPackage.from(identifier.getName(), identifier.getVersion(), in); + } + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientWithFileSystemCache.java new file mode 100644 index 00000000..85f0615b --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientWithFileSystemCache.java @@ -0,0 +1,78 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Objects; + +import javax.ws.rs.WebApplicationException; + +import org.springframework.beans.factory.InitializingBean; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class ValidationPackageClientWithFileSystemCache extends AbstractFileSystemCache + implements ValidationPackageClient, InitializingBean +{ + private final ObjectMapper mapper; + private final ValidationPackageClient delegate; + + /** + * For JSON content with gzip compression using the .json.xz file name suffix. + * + * @param cacheFolder + * not null + * @param mapper + * not null + * @param delegate + * not null + * @see AbstractFileSystemCache#FILENAME_SUFFIX + * @see AbstractFileSystemCache#OUT_COMPRESSOR_FACTORY + * @see AbstractFileSystemCache#IN_COMPRESSOR_FACTORY + */ + public ValidationPackageClientWithFileSystemCache(Path cacheFolder, ObjectMapper mapper, + ValidationPackageClient delegate) + { + super(cacheFolder); + + this.mapper = mapper; + this.delegate = delegate; + } + + public ValidationPackageClientWithFileSystemCache(Path cacheFolder, String fileNameSuffix, + FunctionWithIoException outCompressorFactory, + FunctionWithIoException inCompressorFactory, ObjectMapper mapper, + ValidationPackageClient delegate) + { + super(cacheFolder, fileNameSuffix, outCompressorFactory, inCompressorFactory); + + this.mapper = mapper; + this.delegate = delegate; + } + + @Override + public void afterPropertiesSet() throws Exception + { + super.afterPropertiesSet(); + + Objects.requireNonNull(mapper, "mapper"); + Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public ValidationPackage download(ValidationPackageIdentifier identifier) + throws IOException, WebApplicationException + { + Objects.requireNonNull(identifier, "identifier"); + + ValidationPackage read = readFromCache(identifier.toString(), "validation package", + r -> mapper.readValue(r, ValidationPackage.class)); + + if (read != null) + return read; + else + return writeToCache(delegate.download(identifier), p -> p.getIdentifier().toString(), + p -> "validation package", mapper::writeValue); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptor.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptor.java new file mode 100644 index 00000000..78bbe34a --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptor.java @@ -0,0 +1,156 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ValidationPackageDescriptor +{ + private final String author; + private final String canonical; + private final Map dependencies = new HashMap<>(); + private final String description; + // fhir-version-list + private final List fhirVersions = new ArrayList<>(); + private final String jurisdiction; + private final List keywords = new ArrayList<>(); + private final String license; + private final List maintainers = new ArrayList<>(); + private final String name; + private final String title; + private final String url; + private final String version; + + @JsonCreator + public ValidationPackageDescriptor(@JsonProperty("author") String author, + @JsonProperty("canonical") String canonical, @JsonProperty("dependencies") Map dependencies, + @JsonProperty("description") String description, + @JsonProperty("fhirVersions") @JsonAlias("fhir-version-list") List fhirVersions, + @JsonProperty("jurisdiction") String jurisdiction, @JsonProperty("keywords") List keywords, + @JsonProperty("license") String license, + @JsonProperty("maintainers") List maintainers, + @JsonProperty("name") String name, @JsonProperty("title") String title, @JsonProperty("url") String url, + @JsonProperty("version") String version) + { + this.author = author; + this.canonical = canonical; + + if (dependencies != null) + this.dependencies.putAll(dependencies); + + this.description = description; + + if (fhirVersions != null) + this.fhirVersions.addAll(fhirVersions); + + this.jurisdiction = jurisdiction; + + if (keywords != null) + this.keywords.addAll(keywords); + + this.license = license; + + if (maintainers != null) + this.maintainers.addAll(maintainers); + + this.name = name; + this.title = title; + this.url = url; + this.version = version; + } + + @JsonProperty("author") + public String getAuthor() + { + return author; + } + + @JsonProperty("canonical") + public String getCanonical() + { + return canonical; + } + + @JsonProperty("dependencies") + public Map getDependencies() + { + return Collections.unmodifiableMap(dependencies); + } + + @JsonIgnore + public List getDependencyIdentifiers() + { + return dependencies.entrySet().stream().map(e -> new ValidationPackageIdentifier(e.getKey(), e.getValue())) + .collect(Collectors.toUnmodifiableList()); + } + + @JsonProperty("description") + public String getDescription() + { + return description; + } + + @JsonProperty("fhirVersions") + public List getFhirVersions() + { + return Collections.unmodifiableList(fhirVersions); + } + + @JsonProperty("jurisdiction") + public String getJurisdiction() + { + return jurisdiction; + } + + @JsonProperty("keywords") + public List getKeywords() + { + return Collections.unmodifiableList(keywords); + } + + @JsonProperty("license") + public String getLicense() + { + return license; + } + + @JsonProperty("maintainers") + public List getMaintainers() + { + return Collections.unmodifiableList(maintainers); + } + + @JsonProperty("name") + public String getName() + { + return name; + } + + @JsonProperty("title") + public String getTitle() + { + return title; + } + + @JsonProperty("url") + public String getUrl() + { + return url; + } + + @JsonProperty("version") + public String getVersion() + { + return version; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptorMaintainer.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptorMaintainer.java new file mode 100644 index 00000000..edd2d971 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageDescriptorMaintainer.java @@ -0,0 +1,31 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ValidationPackageDescriptorMaintainer +{ + private final String email; + private final String name; + + @JsonCreator + public ValidationPackageDescriptorMaintainer(@JsonProperty("email") String email, @JsonProperty("name") String name) + { + this.email = email; + this.name = name; + } + + @JsonProperty("email") + public String getEmail() + { + return email; + } + + @JsonProperty("name") + public String getName() + { + return name; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageEntry.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageEntry.java new file mode 100644 index 00000000..3ec33d73 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageEntry.java @@ -0,0 +1,72 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.io.IOUtils; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class ValidationPackageEntry +{ + /** + * Does not close the input stream. + * + * @param entry + * not null + * @param in + * not null + * @return {@link ValidationPackageEntry} for the given {@link ArchiveEntry} from the given + * {@link TarArchiveInputStream}, null if the given entry can't be read from the input stream. + * @throws IOException + * @see {@link TarArchiveInputStream#canReadEntryData(ArchiveEntry)} + */ + public static ValidationPackageEntry from(ArchiveEntry entry, TarArchiveInputStream in) throws IOException + { + Objects.requireNonNull(entry, "entry"); + Objects.requireNonNull(in, "in"); + + if (!in.canReadEntryData(entry)) + return null; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copy(in, out); + return new ValidationPackageEntry(entry.getName(), entry.getLastModifiedDate(), out.toByteArray()); + } + + private final String fileName; + private final Date lastModified; + private final byte[] content; + + @JsonCreator + public ValidationPackageEntry(@JsonProperty("fileName") String fileName, + @JsonProperty("lastModified") Date lastModified, @JsonProperty("content") byte[] content) + { + this.fileName = fileName; + this.lastModified = lastModified; + this.content = content; + } + + @JsonProperty("fileName") + public String getFileName() + { + return fileName; + } + + @JsonProperty("lastModified") + public Date getLastModified() + { + return lastModified; + } + + @JsonProperty("content") + public byte[] getContent() + { + return content; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageIdentifier.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageIdentifier.java new file mode 100644 index 00000000..eed830ac --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageIdentifier.java @@ -0,0 +1,78 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.Objects; + +public class ValidationPackageIdentifier +{ + public static ValidationPackageIdentifier fromString(String nameAndVersion) + { + String[] split = nameAndVersion.split("\\|"); + + if (split.length != 2) + throw new IllegalArgumentException("Validation package not specified as 'name|version'"); + + return new ValidationPackageIdentifier(split[0], split[1]); + } + + private final String name; + private final String version; + + public ValidationPackageIdentifier(String name, String version) + { + this.name = Objects.requireNonNull(name, "name"); + this.version = Objects.requireNonNull(version, "version"); + } + + public String getName() + { + return name; + } + + public String getVersion() + { + return version; + } + + @Override + public String toString() + { + return name + "|" + version; + } + + @Override + public int hashCode() + { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + ((version == null) ? 0 : version.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ValidationPackageIdentifier other = (ValidationPackageIdentifier) obj; + if (name == null) + { + if (other.name != null) + return false; + } + else if (!name.equals(other.name)) + return false; + if (version == null) + { + if (other.version != null) + return false; + } + else if (!version.equals(other.version)) + return false; + return true; + } +} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java new file mode 100644 index 00000000..df7f1743 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java @@ -0,0 +1,79 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.List; + +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.ValueSet; + +import ca.uhn.fhir.context.support.IValidationSupport; + +public interface ValidationPackageManager +{ + /** + * Downloads the given FHIR package and all its dependencies. + * + * @param name + * not null + * @param version + * not null + * @return unmodifiable list of {@link ValidationPackage}s + */ + default List downloadPackageWithDependencies(String name, String version) + { + return downloadPackageWithDependencies(new ValidationPackageIdentifier(name, version)); + } + + /** + * Downloads the given FHIR package and all its dependencies. + * + * @param name + * not null + * @param version + * not null + * @return unmodifiable list of {@link ValidationPackage}s + */ + List downloadPackageWithDependencies(ValidationPackageIdentifier identifier); + + /** + * Will try to generate snapshots for all {@link StructureDefinition}s and expand all {@link ValueSet}s, before + * returning a {@link IValidationSupport}. + * + * @param validationPackages + * not null + * @return validation support for the validator + */ + IValidationSupport expandValueSetsAndGenerateStructureDefinitionSnapshots( + List validationPackages); + + /** + * @param validationSupport + * not null + * @return {@link BundleValidator} for the given {@link IValidationSupport} + */ + BundleValidator createBundleValidator(IValidationSupport validationSupport); + + /** + * Downloads the given FHIR package and all its dependencies. Will try to generate snapshots for all + * {@link StructureDefinition}s and expand all {@link ValueSet}s, before returning a {@link BundleValidator}. + * + * @param name + * not null + * @param version + * not null + * @return {@link BundleValidator} for the specified FHIR package + */ + default BundleValidator createBundleValidator(String name, String version) + { + return createBundleValidator(new ValidationPackageIdentifier(name, version)); + } + + /** + * Downloads the given FHIR package and all its dependencies. Will try to generate snapshots for all + * {@link StructureDefinition}s and expand all {@link ValueSet}s, before returning a {@link BundleValidator}. + * + * @param identifier + * not null + * @return {@link BundleValidator} for the specified FHIR package + */ + BundleValidator createBundleValidator(ValidationPackageIdentifier identifier); +} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java new file mode 100644 index 00000000..21311664 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java @@ -0,0 +1,409 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; + +import javax.ws.rs.WebApplicationException; + +import org.highmed.dsf.fhir.validation.ResourceValidatorImpl; +import org.highmed.dsf.fhir.validation.ValidationSupportWithCustomResources; +import org.highmed.dsf.fhir.validation.ValueSetExpander; +import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService; +import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport; +import org.hl7.fhir.common.hapi.validation.support.ValidationSupportChain; +import org.hl7.fhir.r4.model.CanonicalType; +import org.hl7.fhir.r4.model.ElementDefinition; +import org.hl7.fhir.r4.model.ElementDefinition.TypeRefComponent; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.ValueSet; +import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent; +import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome; +import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.support.DefaultProfileValidationSupport; +import ca.uhn.fhir.context.support.IValidationSupport; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.MiiModuleLabObservationLab10IdentifierRemover; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.StructureDefinitionModifier; + +public class ValidationPackageManagerImpl implements InitializingBean, ValidationPackageManager +{ + static final Logger logger = LoggerFactory.getLogger(ValidationPackageManagerImpl.class); + + public static final List PACKAGE_IGNORE = List + .of(new ValidationPackageIdentifier("hl7.fhir.r4.core", "4.0.1")); + + // closed type slicings result in error from the snapshot generator + public static final StructureDefinitionModifier CLOSED_TYPE_SLICING_REMOVER = new ClosedTypeSlicingRemover(); + + // mandatory identifier on ObservationLab not compatible with data protection rules with current pseudonymization + public static final StructureDefinitionModifier MII_MODULE_LAB_OBSERVATION_LAB_1_0_IDENTIFIER_REMOVER = new MiiModuleLabObservationLab10IdentifierRemover(); + + private final ValidationPackageClient validationPackageClient; + private final ValueSetExpansionClient valueSetExpansionClient; + + private final ObjectMapper mapper; + private final FhirContext fhirContext; + + private final BiFunction internalSnapshotGeneratorFactory; + private final BiFunction internalValueSetExpanderFactory; + + private final List packagesToIgnore = new ArrayList<>(); + private final List structureDefinitionModifiers = new ArrayList<>(); + + public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, + ValueSetExpansionClient valueSetExpansionClient, ObjectMapper mapper, FhirContext fhirContext, + BiFunction internalSnapshotGeneratorFactory, + BiFunction internalValueSetExpanderFactory) + { + this(validationPackageClient, valueSetExpansionClient, mapper, fhirContext, internalSnapshotGeneratorFactory, + internalValueSetExpanderFactory, PACKAGE_IGNORE, + Arrays.asList(CLOSED_TYPE_SLICING_REMOVER, MII_MODULE_LAB_OBSERVATION_LAB_1_0_IDENTIFIER_REMOVER)); + } + + public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, + ValueSetExpansionClient valueSetExpansionClient, ObjectMapper mapper, FhirContext fhirContext, + BiFunction internalSnapshotGeneratorFactory, + BiFunction internalValueSetExpanderFactory, + Collection packagesToIgnore, + Collection structureDefinitionModifiers) + { + this.validationPackageClient = validationPackageClient; + this.valueSetExpansionClient = valueSetExpansionClient; + this.mapper = mapper; + this.fhirContext = fhirContext; + this.internalSnapshotGeneratorFactory = internalSnapshotGeneratorFactory; + this.internalValueSetExpanderFactory = internalValueSetExpanderFactory; + + if (packagesToIgnore != null) + this.packagesToIgnore.addAll(packagesToIgnore); + + if (structureDefinitionModifiers != null) + this.structureDefinitionModifiers.addAll(structureDefinitionModifiers); + } + + @Override + public void afterPropertiesSet() throws Exception + { + Objects.requireNonNull(validationPackageClient, "validationPackageClient"); + Objects.requireNonNull(valueSetExpansionClient, "valueSetExpansionClient"); + + Objects.requireNonNull(mapper, "mapper"); + Objects.requireNonNull(fhirContext, "fhirContext"); + + Objects.requireNonNull(internalSnapshotGeneratorFactory, "internalSnapshotGeneratorFactory"); + Objects.requireNonNull(internalValueSetExpanderFactory, "internalValueSetExpanderFactory"); + } + + @Override + public List downloadPackageWithDependencies(ValidationPackageIdentifier identifier) + { + Objects.requireNonNull(identifier, "identifier"); + + Map packagesByNameAndVersion = new HashMap<>(); + downloadPackageWithDependencies(identifier, packagesByNameAndVersion); + + return Collections.unmodifiableList(new ArrayList<>(packagesByNameAndVersion.values())); + } + + @Override + public IValidationSupport expandValueSetsAndGenerateStructureDefinitionSnapshots( + List validationPackages) + { + Objects.requireNonNull(validationPackages, "validationPackages"); + + validationPackages.forEach(p -> p.parseResources(fhirContext)); + + List resources = validationPackages.stream() + .map(ValidationPackage::getValidationSupportResources).collect(Collectors.toList()); + + return withSnapshots(resources, withExpandedValueSets(resources)); + } + + @Override + public BundleValidator createBundleValidator(IValidationSupport validationSupport) + { + Objects.requireNonNull(validationSupport, "validationSupport"); + + return new BundleValidatorImpl(new ResourceValidatorImpl(fhirContext, validationSupport)); + } + + @Override + public BundleValidator createBundleValidator(ValidationPackageIdentifier identifier) + { + Objects.requireNonNull(identifier, "identifier"); + + List vPackages = downloadPackageWithDependencies(identifier); + IValidationSupport validationSupport = expandValueSetsAndGenerateStructureDefinitionSnapshots(vPackages); + return createBundleValidator(validationSupport); + } + + private void downloadPackageWithDependencies(ValidationPackageIdentifier identifier, + Map packagesByNameAndVersion) + { + if (packagesByNameAndVersion.containsKey(identifier)) + { + // already downloaded + return; + } + else if (packagesToIgnore.contains(identifier)) + { + logger.debug("Ignoring package {}", identifier.toString()); + return; + } + + ValidationPackage vPackage = downloadAndHandleException(identifier); + packagesByNameAndVersion.put(identifier, vPackage); + + ValidationPackageDescriptor descriptor = getDescriptorAndHandleException(vPackage); + descriptor.getDependencyIdentifiers() + .forEach(i -> downloadPackageWithDependencies(i, packagesByNameAndVersion)); + } + + private ValidationPackage downloadAndHandleException(ValidationPackageIdentifier identifier) + { + try + { + logger.debug("Downloading validation package {}", identifier); + return validationPackageClient.download(identifier); + } + catch (WebApplicationException | IOException e) + { + throw new RuntimeException(e); + } + } + + private ValidationPackageDescriptor getDescriptorAndHandleException(ValidationPackage vPackage) + { + try + { + return vPackage.getDescriptor(mapper); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + private List withExpandedValueSets(List resources) + { + List expandedValueSets = new ArrayList<>(); + ValueSetExpander expander = internalValueSetExpanderFactory.apply(fhirContext, + createSupportChain(fhirContext, resources, Collections.emptyList(), expandedValueSets)); + + resources.stream().flatMap(r -> r.getValueSets().stream()).forEach(v -> + { + logger.debug("Expanding ValueSet {}|{}", v.getUrl(), v.getVersion()); + + // ValueSet uses filter in compose + if (v.hasCompose() && (v.getCompose().hasInclude() || v.getCompose().hasExclude()) + && (v.getCompose().getInclude().stream().anyMatch(ConceptSetComponent::hasFilter) + || v.getCompose().getExclude().stream().anyMatch(ConceptSetComponent::hasFilter))) + { + expandExternal(expandedValueSets, v); + } + else + { + // will try external expansion if internal not successful + expandInternal(expandedValueSets, expander, v); + } + }); + + return expandedValueSets; + } + + private void expandExternal(List expandedValueSets, ValueSet v) + { + try + { + ValueSet expansion = valueSetExpansionClient.expand(v); + expandedValueSets.add(expansion); + } + catch (WebApplicationException e) + { + logger.error("Error while expanding ValueSet {}|{}: {} - {}", v.getUrl(), v.getVersion(), + e.getClass().getName(), e.getMessage()); + getOutcome(e).ifPresent(m -> logger.debug("Expansion error response: {}", m)); + logger.debug("ValueSet with error while expanding: {}", + fhirContext.newJsonParser().encodeResourceToString(v)); + } + catch (Exception e) + { + logger.error("Error while expanding ValueSet {}|{}: {} - {}", v.getUrl(), v.getVersion(), + e.getClass().getName(), e.getMessage()); + logger.debug("ValueSet with error while expanding: {}", + fhirContext.newJsonParser().encodeResourceToString(v)); + } + } + + private void expandInternal(List expandedValueSets, ValueSetExpander expander, ValueSet v) + { + try + { + ValueSetExpansionOutcome expansion = expander.expand(v); + + if (expansion.getError() != null) + logger.warn("Error while expanding ValueSet {}|{}: {}", v.getUrl(), v.getVersion(), + expansion.getError()); + else + expandedValueSets.add(expansion.getValueset()); + } + catch (Exception e) + { + logger.error( + "Error while expanding ValueSet {}|{}: {} - {}, trying to expand via external ontolgy server next", + v.getUrl(), v.getVersion(), e.getClass().getName(), e.getMessage()); + + expandExternal(expandedValueSets, v); + } + } + + private Optional getOutcome(WebApplicationException e) + { + if (e.getResponse().hasEntity()) + { + String response = e.getResponse().readEntity(String.class); + return Optional.of(response); + } + else + return Optional.empty(); + } + + private IValidationSupport withSnapshots(List resources, + List expandedValueSets) + { + Map structureDefinitionsByUrl = resources.stream() + .flatMap(r -> r.getStructureDefinitions().stream()) + .collect(Collectors.toMap(StructureDefinition::getUrl, Function.identity())); + + Map snapshots = new HashMap<>(); + ValidationSupportChain supportChain = createSupportChain(fhirContext, resources, snapshots.values(), + expandedValueSets); + + PluginSnapshotGenerator generator = internalSnapshotGeneratorFactory.apply(fhirContext, supportChain); + + resources.stream().flatMap(r -> r.getStructureDefinitions().stream()) + .filter(s -> s.hasDifferential() && !s.hasSnapshot()) + .forEach(diff -> createSnapshot(structureDefinitionsByUrl, snapshots, generator, diff)); + + return supportChain; + } + + private ValidationSupportChain createSupportChain(FhirContext context, List resources, + Collection snapshots, Collection expandedValueSets) + { + return new ValidationSupportChain(new CodeValidatorForExpandedValueSets(context), + new InMemoryTerminologyServerValidationSupport(context), + new ValidationSupportWithCustomResources(context, snapshots, null, expandedValueSets), + new ValidationSupportChain(resources.stream() + .map(r -> new ValidationSupportWithCustomResources(context, r.getStructureDefinitions(), + r.getCodeSystems(), r.getValueSets())) + .toArray(IValidationSupport[]::new)), + new DefaultProfileValidationSupport(context), new CommonCodeSystemsTerminologyService(context)); + } + + private void createSnapshot(Map structureDefinitionsByUrl, + Map snapshots, PluginSnapshotGenerator generator, StructureDefinition diff) + { + if (snapshots.containsKey(diff.getUrl() + "|" + diff.getVersion())) + return; + + Set dependencies = new HashSet<>(); + Set targetDependencies = new HashSet<>(); + + calculateDependencies(diff, structureDefinitionsByUrl, dependencies, targetDependencies); + + logger.debug("Generating snapshot for {}|{}, base {}, dependencies {}, target-dependencies {}", diff.getUrl(), + diff.getVersion(), diff.getBaseDefinition(), + dependencies.stream().sorted().collect(Collectors.joining(", ", "[", "]")), + targetDependencies.stream().sorted().collect(Collectors.joining(", ", "[", "]"))); + + if (structureDefinitionsByUrl.containsKey(diff.getBaseDefinition())) + createSnapshot(structureDefinitionsByUrl, snapshots, generator, + structureDefinitionsByUrl.get(diff.getBaseDefinition())); + + dependencies.stream().filter(structureDefinitionsByUrl::containsKey).map(structureDefinitionsByUrl::get) + .forEach(s -> createSnapshot(structureDefinitionsByUrl, snapshots, generator, s)); + + targetDependencies.stream().filter(structureDefinitionsByUrl::containsKey).map(structureDefinitionsByUrl::get) + .forEach(s -> createSnapshot(structureDefinitionsByUrl, snapshots, generator, s)); + + try + { + structureDefinitionModifiers.forEach(f -> f.modify(diff)); + + PluginSnapshotWithValidationMessages snapshot = generator.generateSnapshot(diff); + + if (snapshot.getMessages().isEmpty()) + snapshots.put(snapshot.getSnapshot().getUrl() + "|" + snapshot.getSnapshot().getVersion(), + snapshot.getSnapshot()); + else + { + snapshot.getMessages().forEach(m -> + { + if (EnumSet.of(IssueSeverity.FATAL, IssueSeverity.ERROR, IssueSeverity.WARNING) + .contains(m.getLevel())) + logger.warn("{}|{} {}: {}", diff.getUrl(), diff.getVersion(), m.getLevel(), m.toString()); + else + logger.info("{}|{} {}: {}", diff.getUrl(), diff.getVersion(), m.getLevel(), m.toString()); + }); + } + } + catch (Exception e) + { + logger.error("Error while generating snapshot for {}|{}: {} - {}", diff.getUrl(), diff.getVersion(), + e.getClass().getName(), e.getMessage()); + } + } + + private void calculateDependencies(StructureDefinition structureDefinition, + Map structureDefinitionsByUrl, Set dependencies, + Set targetDependencies) + { + for (ElementDefinition element : structureDefinition.getDifferential().getElement()) + { + if (element.getType().stream().filter(t -> !t.getProfile().isEmpty() || !t.getTargetProfile().isEmpty()) + .findAny().isPresent()) + { + for (TypeRefComponent type : element.getType()) + { + if (!type.getProfile().isEmpty()) + { + for (CanonicalType profile : type.getProfile()) + { + dependencies.add(profile.getValue()); + + if (structureDefinitionsByUrl.containsKey(profile.getValue())) + calculateDependencies(structureDefinitionsByUrl.get(profile.getValue()), + structureDefinitionsByUrl, dependencies, targetDependencies); + } + } + + if (!type.getTargetProfile().isEmpty()) + for (CanonicalType targetProfile : type.getTargetProfile()) + targetDependencies.add(targetProfile.getValue()); + } + } + } + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationSupportResources.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationSupportResources.java new file mode 100644 index 00000000..48406a53 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationSupportResources.java @@ -0,0 +1,51 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.NamingSystem; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.ValueSet; + +public class ValidationSupportResources +{ + private final List codeSystems = new ArrayList<>(); + private final List namingSystems = new ArrayList<>(); + private final List structureDefinitions = new ArrayList<>(); + private final List valueSets = new ArrayList<>(); + + public ValidationSupportResources(List codeSystems, List namingSystems, + List structureDefinitions, List valueSets) + { + if (codeSystems != null) + this.codeSystems.addAll(codeSystems); + if (namingSystems != null) + this.namingSystems.addAll(namingSystems); + if (structureDefinitions != null) + this.structureDefinitions.addAll(structureDefinitions); + if (valueSets != null) + this.valueSets.addAll(valueSets); + } + + public List getCodeSystems() + { + return Collections.unmodifiableList(codeSystems); + } + + public List getNamingSystems() + { + return Collections.unmodifiableList(namingSystems); + } + + public List getStructureDefinitions() + { + return Collections.unmodifiableList(structureDefinitions); + } + + public List getValueSets() + { + return Collections.unmodifiableList(valueSets); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpanderWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpanderWithFileSystemCache.java new file mode 100644 index 00000000..b33c1077 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpanderWithFileSystemCache.java @@ -0,0 +1,115 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Objects; + +import org.highmed.dsf.fhir.validation.ValueSetExpander; +import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; +import org.hl7.fhir.r4.model.ValueSet; +import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +import ca.uhn.fhir.context.FhirContext; + +public class ValueSetExpanderWithFileSystemCache + extends AbstractFhirResourceFileSystemCache + implements ValueSetExpander, InitializingBean +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationPackageClientWithFileSystemCache.class); + + private final ValueSetExpander delegate; + + /** + * For JSON content with gzip compression using the .json.xz file name suffix. + * + * @param cacheFolder + * not null + * @param resourceType + * not null + * @param fhirContext + * not null + * @param delegate + * not null + * @see AbstractFileSystemCache#FILENAME_SUFFIX + * @see AbstractFileSystemCache#OUT_COMPRESSOR_FACTORY + * @see AbstractFileSystemCache#IN_COMPRESSOR_FACTORY + */ + public ValueSetExpanderWithFileSystemCache(Path cacheFolder, FhirContext fhirContext, ValueSetExpander delegate) + { + super(cacheFolder, ValueSet.class, fhirContext); + + this.delegate = delegate; + } + + public ValueSetExpanderWithFileSystemCache(Path cacheFolder, String fileNameSuffix, + FunctionWithIoException outCompressorFactory, + FunctionWithIoException inCompressorFactory, FhirContext fhirContext, + ValueSetExpander delegate) + { + super(cacheFolder, fileNameSuffix, outCompressorFactory, inCompressorFactory, ValueSet.class, fhirContext); + + this.delegate = delegate; + } + + @Override + public void afterPropertiesSet() throws Exception + { + super.afterPropertiesSet(); + + Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public ValueSetExpansionOutcome expand(ValueSet valueSet) + { + Objects.requireNonNull(valueSet, "valueSet"); + + if (valueSet.hasExpansion()) + { + logger.debug("ValueSet {}|{} already expanded", valueSet.getUrl(), valueSet.getVersion()); + return new ValueSetExpansionOutcome(valueSet); + } + + Objects.requireNonNull(valueSet.getUrl(), "valueSet.url"); + Objects.requireNonNull(valueSet.getVersion(), "valueSet.version"); + + try + { + // ValueSetExpansionOutcome read = readFromCache(valueSet.getUrl(), valueSet.getVersion(), ValueSet.class, + // ValueSetExpansionOutcome::new); + ValueSetExpansionOutcome read = readResourceFromCache(valueSet.getUrl(), valueSet.getVersion(), + ValueSetExpansionOutcome::new); + + if (read != null) + return read; + else + return downloadAndWriteToCache(valueSet); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + private ValueSetExpansionOutcome downloadAndWriteToCache(ValueSet valueSet) throws IOException + { + ValueSetExpansionOutcome expanded = delegate.expand(valueSet); + + if (PublicationStatus.DRAFT.equals(expanded.getValueset().getStatus())) + { + logger.info("Not writing expanded ValueSet {}|{} with status {} to cache", expanded.getValueset().getUrl(), + expanded.getValueset().getVersion(), expanded.getValueset().getStatus()); + return expanded; + } + else + // return writeToCache(expanded, ValueSetExpansionOutcome::getValueset, ValueSet::getUrl, + // ValueSet::getVersion); + return writeRsourceToCache(expanded, ValueSetExpansionOutcome::getValueset, ValueSet::getUrl, + ValueSet::getVersion); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClient.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClient.java new file mode 100644 index 00000000..29e83b75 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClient.java @@ -0,0 +1,19 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; + +import javax.ws.rs.WebApplicationException; + +import org.hl7.fhir.r4.model.ValueSet; + +public interface ValueSetExpansionClient +{ + /** + * @param valueSet + * not null + * @return expanded {@link ValueSet}, never null + * @throws IOException + * @throws WebApplicationException + */ + ValueSet expand(ValueSet valueSet) throws IOException, WebApplicationException; +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java new file mode 100644 index 00000000..c4ed3925 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java @@ -0,0 +1,122 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.security.KeyStore; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.SSLContext; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; + +import org.glassfish.jersey.SslConfigurator; +import org.glassfish.jersey.client.ClientProperties; +import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; +import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJaxbJsonProvider; +import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJsonProvider; +import org.highmed.dsf.fhir.adapter.OperationOutcomeJsonFhirAdapter; +import org.highmed.dsf.fhir.adapter.OperationOutcomeXmlFhirAdapter; +import org.highmed.dsf.fhir.adapter.ParametersJsonFhirAdapter; +import org.highmed.dsf.fhir.adapter.ParametersXmlFhirAdapter; +import org.highmed.dsf.fhir.adapter.ValueSetJsonFhirAdapter; +import org.highmed.dsf.fhir.adapter.ValueSetXmlFhirAdapter; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.ValueSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.rest.api.Constants; + +public class ValueSetExpansionClientJersey implements ValueSetExpansionClient +{ + private static final Logger logger = LoggerFactory.getLogger(ValueSetExpansionClientJersey.class); + + private final Client client; + private final String baseUrl; + + public ValueSetExpansionClientJersey(String baseUrl, ObjectMapper objectMapper, FhirContext fhirContext) + { + this(baseUrl, null, null, null, null, null, null, null, null, 0, 0, objectMapper, fhirContext); + } + + public ValueSetExpansionClientJersey(String baseUrl, KeyStore trustStore, KeyStore keyStore, + char[] keyStorePassword, String basicAuthUsername, char[] basicAuthPassword, String proxySchemeHostPort, + String proxyUsername, char[] proxyPassword, int connectTimeout, int readTimeout, ObjectMapper objectMapper, + FhirContext fhirContext) + { + SSLContext sslContext = null; + if (trustStore != null && keyStore == null && keyStorePassword == null) + sslContext = SslConfigurator.newInstance().trustStore(trustStore).createSSLContext(); + else if (trustStore != null && keyStore != null && keyStorePassword != null) + sslContext = SslConfigurator.newInstance().trustStore(trustStore).keyStore(keyStore) + .keyStorePassword(keyStorePassword).createSSLContext(); + + ClientBuilder builder = ClientBuilder.newBuilder(); + + if (sslContext != null) + builder = builder.sslContext(sslContext); + + if (basicAuthUsername != null && basicAuthPassword != null) + { + HttpAuthenticationFeature basicAuthFeature = HttpAuthenticationFeature.basic(basicAuthUsername, + String.valueOf(basicAuthPassword)); + builder = builder.register(basicAuthFeature); + } + + if (proxySchemeHostPort != null) + { + builder = builder.property(ClientProperties.PROXY_URI, proxySchemeHostPort); + if (proxyUsername != null && proxyPassword != null) + builder = builder.property(ClientProperties.PROXY_USERNAME, proxyUsername) + .property(ClientProperties.PROXY_PASSWORD, String.valueOf(proxyPassword)); + } + + builder = builder.readTimeout(readTimeout, TimeUnit.MILLISECONDS).connectTimeout(connectTimeout, + TimeUnit.MILLISECONDS); + + if (objectMapper != null) + { + JacksonJaxbJsonProvider p = new JacksonJaxbJsonProvider(JacksonJsonProvider.BASIC_ANNOTATIONS); + p.setMapper(objectMapper); + builder.register(p); + } + + builder.register(new OperationOutcomeJsonFhirAdapter(fhirContext)) + .register(new OperationOutcomeXmlFhirAdapter(fhirContext)) + .register(new ParametersJsonFhirAdapter(fhirContext)) + .register(new ParametersXmlFhirAdapter(fhirContext)).register(new ValueSetJsonFhirAdapter(fhirContext)) + .register(new ValueSetXmlFhirAdapter(fhirContext)); + + client = builder.build(); + + this.baseUrl = baseUrl; + } + + private WebTarget getResource() + { + return client.target(baseUrl); + } + + @Override + public ValueSet expand(ValueSet valueSet) throws WebApplicationException + { + Objects.requireNonNull(valueSet, "valueSet"); + + if (valueSet.hasExpansion()) + { + logger.debug("ValueSet {}|{} already expanded", valueSet.getUrl(), valueSet.getVersion()); + return valueSet; + } + + Parameters parameters = new Parameters(); + parameters.addParameter().setName("valueSet").setResource(valueSet); + + return getResource().path("ValueSet").path("$expand").request(Constants.CT_FHIR_JSON_NEW) + .post(Entity.entity(parameters, Constants.CT_FHIR_JSON_NEW), ValueSet.class); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java new file mode 100644 index 00000000..22a5f810 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java @@ -0,0 +1,103 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Objects; +import java.util.function.Function; + +import javax.ws.rs.WebApplicationException; + +import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; +import org.hl7.fhir.r4.model.ValueSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +import ca.uhn.fhir.context.FhirContext; + +public class ValueSetExpansionClientWithFileSystemCache extends AbstractFhirResourceFileSystemCache + implements ValueSetExpansionClient, InitializingBean +{ + private static final Logger logger = LoggerFactory.getLogger(ValidationPackageClientWithFileSystemCache.class); + + private final ValueSetExpansionClient delegate; + + /** + * For JSON content with gzip compression using the .json.xz file name suffix. + * + * @param cacheFolder + * not null + * @param fhirContext + * not null + * @param delegate + * not null + * @see AbstractFileSystemCache#FILENAME_SUFFIX + * @see AbstractFileSystemCache#OUT_COMPRESSOR_FACTORY + * @see AbstractFileSystemCache#IN_COMPRESSOR_FACTORY + */ + public ValueSetExpansionClientWithFileSystemCache(Path cacheFolder, FhirContext fhirContext, + ValueSetExpansionClient delegate) + { + super(cacheFolder, ValueSet.class, fhirContext); + + this.delegate = delegate; + } + + public ValueSetExpansionClientWithFileSystemCache(Path cacheFolder, String fileNameSuffix, + FunctionWithIoException outCompressorFactory, + FunctionWithIoException inCompressorFactory, FhirContext fhirContext, + ValueSetExpansionClient delegate) + { + super(cacheFolder, fileNameSuffix, outCompressorFactory, inCompressorFactory, ValueSet.class, fhirContext); + + this.delegate = delegate; + } + + @Override + public void afterPropertiesSet() throws Exception + { + super.afterPropertiesSet(); + + Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public ValueSet expand(ValueSet valueSet) throws IOException, WebApplicationException + { + Objects.requireNonNull(valueSet, "valueSet"); + + if (valueSet.hasExpansion()) + { + logger.debug("ValueSet {}|{} already expanded", valueSet.getUrl(), valueSet.getVersion()); + return valueSet; + } + + Objects.requireNonNull(valueSet.getUrl(), "valueSet.url"); + Objects.requireNonNull(valueSet.getVersion(), "valueSet.version"); + + // ValueSet read = readFromCache(valueSet.getUrl(), valueSet.getVersion(), ValueSet.class, Function.identity()); + ValueSet read = readResourceFromCache(valueSet.getUrl(), valueSet.getVersion(), Function.identity()); + + if (read != null) + return read; + else + return downloadAndWriteToCache(valueSet); + } + + private ValueSet downloadAndWriteToCache(ValueSet valueSet) throws IOException + { + ValueSet expanded = delegate.expand(valueSet); + + if (PublicationStatus.DRAFT.equals(expanded.getStatus())) + { + logger.info("Not writing expanded ValueSet {}|{} with status {} to cache", expanded.getUrl(), + expanded.getVersion(), expanded.getStatus()); + return expanded; + } + else + // return writeToCache(expanded, Function.identity(), ValueSet::getUrl, ValueSet::getVersion); + return writeRsourceToCache(expanded, Function.identity(), ValueSet::getUrl, ValueSet::getVersion); + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java new file mode 100644 index 00000000..68a6574c --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java @@ -0,0 +1,38 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition; + +import org.hl7.fhir.r4.model.ElementDefinition; +import org.hl7.fhir.r4.model.ElementDefinition.DiscriminatorType; +import org.hl7.fhir.r4.model.ElementDefinition.ElementDefinitionSlicingComponent; +import org.hl7.fhir.r4.model.ElementDefinition.ElementDefinitionSlicingDiscriminatorComponent; +import org.hl7.fhir.r4.model.ElementDefinition.SlicingRules; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ClosedTypeSlicingRemover implements StructureDefinitionModifier +{ + private static final Logger logger = LoggerFactory.getLogger(ClosedTypeSlicingRemover.class); + + @Override + public StructureDefinition modify(StructureDefinition sd) + { + sd.getDifferential().getElement().stream().filter(ElementDefinition::hasSlicing).forEach(e -> + { + ElementDefinitionSlicingComponent slicing = e.getSlicing(); + if (SlicingRules.OPEN.equals(slicing.getRules()) && slicing.getDiscriminator().size() == 1) + { + ElementDefinitionSlicingDiscriminatorComponent discriminator = slicing.getDiscriminator().get(0); + if (DiscriminatorType.TYPE.equals(discriminator.getType()) && "$this".equals(discriminator.getPath())) + { + logger.warn( + "Removing Type slicing with slicing.rules != closed validation rule with id {} in StructureDefinition {}|{}", + e.getId(), sd.getUrl(), sd.getVersion()); + + e.setSlicing(null); + } + } + }); + + return sd; + } +} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java new file mode 100644 index 00000000..812eba07 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java @@ -0,0 +1,38 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition; + +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.hl7.fhir.r4.model.ElementDefinition; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MiiModuleLabObservationLab10IdentifierRemover implements StructureDefinitionModifier +{ + private static final Logger logger = LoggerFactory.getLogger(MiiModuleLabObservationLab10IdentifierRemover.class); + + @Override + public StructureDefinition modify(StructureDefinition sd) + { + if ("https://www.medizininformatik-initiative.de/fhir/core/modul-labor/StructureDefinition/ObservationLab" + .equals(sd.getUrl()) && "1.0".equals(sd.getVersion())) + { + Predicate toRemove = e -> e.hasPath() + && e.getPath().startsWith("Observation.identifier"); + + List filteredRules = sd.getDifferential().getElement().stream().filter(toRemove.negate()) + .collect(Collectors.toList()); + + logger.warn("Removing validation rules with ids {} in StructureDefinition {}|{}", + sd.getDifferential().getElement().stream().filter(toRemove).map(ElementDefinition::getId) + .collect(Collectors.joining(", ", "[", "]")), + sd.getUrl(), sd.getVersion()); + + sd.getDifferential().setElement(filteredRules); + } + + return sd; + } +} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/StructureDefinitionModifier.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/StructureDefinitionModifier.java new file mode 100644 index 00000000..1312d867 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/StructureDefinitionModifier.java @@ -0,0 +1,9 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition; + +import org.hl7.fhir.r4.model.StructureDefinition; + +@FunctionalInterface +public interface StructureDefinitionModifier +{ + StructureDefinition modify(StructureDefinition sd); +} diff --git a/codex-process-data-transfer/src/main/resources/bpe/send.bpmn b/codex-process-data-transfer/src/main/resources/bpe/send.bpmn index 16d66505..46782498 100644 --- a/codex-process-data-transfer/src/main/resources/bpe/send.bpmn +++ b/codex-process-data-transfer/src/main/resources/bpe/send.bpmn @@ -1,5 +1,5 @@ - + Flow_1km61ly @@ -102,12 +102,21 @@ ${!idatMergeGranted} + + Flow_018z9ct + + + + Flow_018z9ct + + + @@ -184,6 +193,11 @@ + + + + + @@ -238,6 +252,12 @@ + + + + + + diff --git a/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-source.xml b/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-source.xml new file mode 100644 index 00000000..f63dbd87 --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-source.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + <!-- status managed by bpe --> + <status value="unknown" /> + <experimental value="false" /> + <!-- date managed by bpe --> + <date value="#{date}" /> + <publisher value="NUM-CODEX" /> + <description value="CodeSystem with error source values for the NUM-CODEX data-transfer processes" /> + <caseSensitive value="true" /> + <hierarchyMeaning value="grouped-by" /> + <versionNeeded value="false" /> + <content value="complete" /> + <concept> + <code value="MeDIC" /> + <display value="MeDIC" /> + <definition value="Error during process execution at the medical Data Integration Center" /> + </concept> + <concept> + <code value="GTH" /> + <display value="GTH" /> + <definition value="Error during process execution at the GECCO Transfer Hub" /> + </concept> + <concept> + <code value="fTTP" /> + <display value="fTTP" /> + <definition value="Error during process execution at the federated Trusted Third Party" /> + </concept> + <concept> + <code value="CRR" /> + <display value="CRR" /> + <definition value="Error during process execution in the Central Research Repository" /> + </concept> +</CodeSystem> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-type.xml b/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-type.xml new file mode 100644 index 00000000..928a2bf6 --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-type.xml @@ -0,0 +1,30 @@ +<CodeSystem xmlns="http://hl7.org/fhir"> + <meta> + <tag> + <system value="http://highmed.org/fhir/CodeSystem/read-access-tag" /> + <code value="ALL" /> + </tag> + </meta> + <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-type" /> + <!-- version managed by bpe --> + <version value="#{version}" /> + <name value="NumCodexDataTransferErrorType" /> + <title value="NUM-CODEX data-transfer error type" /> + <!-- status managed by bpe --> + <status value="unknown" /> + <experimental value="false" /> + <!-- date managed by bpe --> + <date value="#{date}" /> + <publisher value="NUM-CODEX" /> + <description value="CodeSystem with error type values for the NUM-CODEX data-transfer processes" /> + <caseSensitive value="true" /> + <hierarchyMeaning value="grouped-by" /> + <versionNeeded value="false" /> + <content value="complete" /> + <concept> + <code value="validation-failed" /> + <display value="Validation Failed" /> + <definition value="Error or fatal error during validaton of FHIR resources" /> + </concept> + <!-- TODO add additional error types --> +</CodeSystem> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/highmed-task-base-0.5.0.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/highmed-task-base-0.5.0.xml new file mode 100644 index 00000000..a1f3b4a6 --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/highmed-task-base-0.5.0.xml @@ -0,0 +1,274 @@ +<StructureDefinition xmlns="http://hl7.org/fhir"> + <meta> + <tag> + <system value="http://highmed.org/fhir/CodeSystem/read-access-tag" /> + <code value="ALL" /> + </tag> + </meta> + <url value="http://highmed.org/fhir/StructureDefinition/task-base" /> + <version value="0.5.0" /> + <name value="TaskBase" /> + <status value="active" /> + <experimental value="false" /> + <date value="2021-08-24" /> + <fhirVersion value="4.0.1" /> + <kind value="resource" /> + <abstract value="true" /> + <type value="Task" /> + <baseDefinition value="http://hl7.org/fhir/StructureDefinition/Task" /> + <derivation value="constraint" /> + <differential> + <element id="Task.instantiatesUri"> + <path value="Task.instantiatesUri" /> + <min value="1" /> + </element> + <element id="Task.intent"> + <path value="Task.intent" /> + <fixedCode value="order" /> + </element> + <element id="Task.authoredOn"> + <path value="Task.authoredOn" /> + <min value="1" /> + </element> + <element id="Task.requester"> + <path value="Task.requester" /> + <min value="1" /> + <type> + <code value="Reference" /> + <targetProfile value="http://highmed.org/fhir/StructureDefinition/organization" /> + </type> + </element> + <element id="Task.requester.reference"> + <path value="Task.requester.reference" /> + <max value="0" /> + </element> + <element id="Task.requester.identifier"> + <path value="Task.requester.identifier" /> + <min value="1" /> + </element> + <element id="Task.requester.identifier.system"> + <path value="Task.requester.identifier.system" /> + <min value="1" /> + <fixedUri value="http://highmed.org/sid/organization-identifier" /> + </element> + <element id="Task.requester.identifier.value"> + <path value="Task.requester.identifier.value" /> + <min value="1" /> + </element> + <element id="Task.restriction"> + <path value="Task.restriction" /> + <min value="1" /> + </element> + <element id="Task.restriction.recipient"> + <path value="Task.restriction.recipient" /> + <min value="1" /> + <max value="1" /> + <type> + <code value="Reference" /> + <targetProfile value="http://highmed.org/fhir/StructureDefinition/organization" /> + </type> + </element> + <element id="Task.restriction.recipient.reference"> + <path value="Task.restriction.recipient.reference" /> + <max value="0" /> + </element> + <element id="Task.restriction.recipient.identifier"> + <path value="Task.restriction.recipient.identifier" /> + <min value="1" /> + </element> + <element id="Task.restriction.recipient.identifier.system"> + <path value="Task.restriction.recipient.identifier.system" /> + <min value="1" /> + <fixedUri value="http://highmed.org/sid/organization-identifier" /> + </element> + <element id="Task.restriction.recipient.identifier.value"> + <path value="Task.restriction.recipient.identifier.value" /> + <min value="1" /> + </element> + <element id="Task.input"> + <extension url="http://hl7.org/fhir/StructureDefinition/structuredefinition-explicit-type-name"> + <valueString value="Parameter" /> + </extension> + <path value="Task.input" /> + <slicing> + <discriminator> + <type value="value" /> + <path value="type.coding.system" /> + </discriminator> + <discriminator> + <type value="value" /> + <path value="type.coding.code" /> + </discriminator> + <rules value="openAtEnd" /> + </slicing> + <min value="1" /> + </element> + <element id="Task.input:message-name"> + <extension url="http://hl7.org/fhir/StructureDefinition/structuredefinition-explicit-type-name"> + <valueString value="Parameter" /> + </extension> + <path value="Task.input" /> + <sliceName value="message-name" /> + <min value="1" /> + <max value="1" /> + </element> + <element id="Task.input:message-name.type"> + <path value="Task.input.type" /> + <binding> + <extension url="http://hl7.org/fhir/StructureDefinition/elementdefinition-bindingName"> + <valueString value="TaskInputParameterType" /> + </extension> + <strength value="required" /> + <valueSet value="http://highmed.org/fhir/ValueSet/bpmn-message" /> + </binding> + </element> + <element id="Task.input:message-name.type.coding"> + <path value="Task.input.type.coding" /> + <min value="1" /> + <max value="1" /> + </element> + <element id="Task.input:message-name.type.coding.system"> + <path value="Task.input.type.coding.system" /> + <min value="1" /> + <fixedUri value="http://highmed.org/fhir/CodeSystem/bpmn-message" /> + </element> + <element id="Task.input:message-name.type.coding.code"> + <path value="Task.input.type.coding.code" /> + <min value="1" /> + <fixedCode value="message-name" /> + </element> + <element id="Task.input:message-name.value[x]"> + <path value="Task.input.value[x]" /> + <type> + <code value="string" /> + </type> + </element> + <element id="Task.input:business-key"> + <extension url="http://hl7.org/fhir/StructureDefinition/structuredefinition-explicit-type-name"> + <valueString value="Parameter" /> + </extension> + <path value="Task.input" /> + <sliceName value="business-key" /> + <max value="1" /> + </element> + <element id="Task.input:business-key.type"> + <path value="Task.input.type" /> + <binding> + <extension url="http://hl7.org/fhir/StructureDefinition/elementdefinition-bindingName"> + <valueString value="TaskInputParameterType" /> + </extension> + <strength value="required" /> + <valueSet value="http://highmed.org/fhir/ValueSet/bpmn-message" /> + </binding> + </element> + <element id="Task.input:business-key.type.coding"> + <path value="Task.input.type.coding" /> + <min value="1" /> + <max value="1" /> + </element> + <element id="Task.input:business-key.type.coding.system"> + <path value="Task.input.type.coding.system" /> + <min value="1" /> + <fixedUri value="http://highmed.org/fhir/CodeSystem/bpmn-message" /> + </element> + <element id="Task.input:business-key.type.coding.code"> + <path value="Task.input.type.coding.code" /> + <min value="1" /> + <fixedCode value="business-key" /> + </element> + <element id="Task.input:business-key.value[x]"> + <path value="Task.input.value[x]" /> + <type> + <code value="string" /> + </type> + </element> + <element id="Task.input:correlation-key"> + <extension url="http://hl7.org/fhir/StructureDefinition/structuredefinition-explicit-type-name"> + <valueString value="Parameter" /> + </extension> + <path value="Task.input" /> + <sliceName value="correlation-key" /> + <max value="1" /> + </element> + <element id="Task.input:correlation-key.type"> + <path value="Task.input.type" /> + <binding> + <extension url="http://hl7.org/fhir/StructureDefinition/elementdefinition-bindingName"> + <valueString value="TaskInputParameterType" /> + </extension> + <strength value="required" /> + <valueSet value="http://highmed.org/fhir/ValueSet/bpmn-message" /> + </binding> + </element> + <element id="Task.input:correlation-key.type.coding"> + <path value="Task.input.type.coding" /> + <min value="1" /> + <max value="1" /> + </element> + <element id="Task.input:correlation-key.type.coding.system"> + <path value="Task.input.type.coding.system" /> + <min value="1" /> + <fixedUri value="http://highmed.org/fhir/CodeSystem/bpmn-message" /> + </element> + <element id="Task.input:correlation-key.type.coding.code"> + <path value="Task.input.type.coding.code" /> + <min value="1" /> + <fixedCode value="correlation-key" /> + </element> + <element id="Task.input:correlation-key.value[x]"> + <path value="Task.input.value[x]" /> + <type> + <code value="string" /> + </type> + </element> + <element id="Task.output"> + <path value="Task.output" /> + <slicing> + <discriminator> + <type value="value" /> + <path value="type.coding.system" /> + </discriminator> + <discriminator> + <type value="value" /> + <path value="type.coding.code" /> + </discriminator> + <rules value="openAtEnd" /> + </slicing> + </element> + <element id="Task.output:error"> + <path value="Task.output" /> + <sliceName value="error" /> + </element> + <element id="Task.output:error.type"> + <path value="Task.output.type" /> + <binding> + <extension url="http://hl7.org/fhir/StructureDefinition/elementdefinition-bindingName"> + <valueString value="TaskOutputParameterType" /> + </extension> + <strength value="required" /> + <valueSet value="http://highmed.org/fhir/ValueSet/bpmn-message" /> + </binding> + </element> + <element id="Task.output:error.type.coding"> + <path value="Task.output.type.coding" /> + <min value="1" /> + <max value="1" /> + </element> + <element id="Task.output:error.type.coding.system"> + <path value="Task.output.type.coding.system" /> + <min value="1" /> + <fixedUri value="http://highmed.org/fhir/CodeSystem/bpmn-message" /> + </element> + <element id="Task.output:error.type.coding.code"> + <path value="Task.output.type.coding.code" /> + <min value="1" /> + <fixedCode value="error" /> + </element> + <element id="Task.output:error.value[x]"> + <path value="Task.output.value[x]" /> + <type> + <code value="string" /> + </type> + </element> + </differential> +</StructureDefinition> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-extension-error-metadata.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-extension-error-metadata.xml new file mode 100644 index 00000000..d2377bf2 --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-extension-error-metadata.xml @@ -0,0 +1,137 @@ +<StructureDefinition xmlns="http://hl7.org/fhir"> + <meta> + <tag> + <system value="http://highmed.org/fhir/CodeSystem/read-access-tag" /> + <code value="ALL" /> + </tag> + </meta> + <url value="https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/error-metadata" /> + <!-- version managed by bpe --> + <version value="#{version}" /> + <name value="ErrorMetadata" /> + <!-- status managed by bpe --> + <status value="unknown" /> + <experimental value="false" /> + <!-- date managed by bpe --> + <date value="#{date}" /> + <fhirVersion value="4.0.1" /> + <kind value="complex-type" /> + <abstract value="false" /> + <context> + <type value="element" /> + <expression value="Task.output" /> + </context> + <type value="Extension" /> + <baseDefinition value="http://hl7.org/fhir/StructureDefinition/Extension" /> + <derivation value="constraint" /> + <differential> + <element id="Extension"> + <path value="Extension" /> + <max value="1" /> + </element> + <element id="Extension.extension"> + <path value="Extension.extension" /> + <slicing> + <discriminator> + <type value="value" /> + <path value="url" /> + </discriminator> + <rules value="open" /> + </slicing> + <min value="2" /> + </element> + <element id="Extension.extension:type"> + <path value="Extension.extension" /> + <sliceName value="type" /> + <min value="1" /> + <max value="1" /> + <binding> + <strength value="required" /> + <valueSet value="http://www.netzwerk-universitaetsmedizin.de/fhir/ValueSet/data-transfer-error-type" /> + </binding> + </element> + <element id="Extension.extension:type.url"> + <path value="Extension.extension.url" /> + <fixedUri value="type" /> + </element> + <element id="Extension.extension:type.value[x]"> + <path value="Extension.extension.value[x]" /> + <min value="1" /> + <type> + <code value="Coding" /> + </type> + </element> + <element id="Extension.extension:type.value[x].system"> + <path value="Extension.extension.value[x].system" /> + <min value="1" /> + <fixedUri value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-type" /> + </element> + <element id="Extension.extension:type.value[x].code"> + <path value="Extension.extension.value[x].code" /> + <min value="1" /> + </element> + <element id="Extension.extension:source"> + <path value="Extension.extension" /> + <sliceName value="source" /> + <min value="1" /> + <max value="1" /> + <binding> + <strength value="required" /> + <valueSet value="http://www.netzwerk-universitaetsmedizin.de/fhir/ValueSet/data-transfer-error-source" /> + </binding> + </element> + <element id="Extension.extension:source.url"> + <path value="Extension.extension.url" /> + <fixedUri value="source" /> + </element> + <element id="Extension.extension:source.value[x]"> + <path value="Extension.extension.value[x]" /> + <min value="1" /> + <type> + <code value="Coding" /> + </type> + </element> + <element id="Extension.extension:source.value[x].system"> + <path value="Extension.extension.value[x].system" /> + <min value="1" /> + <fixedUri value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-source" /> + </element> + <element id="Extension.extension:source.value[x].code"> + <path value="Extension.extension.value[x].code" /> + <min value="1" /> + </element> + <element id="Extension.extension:reference"> + <path value="Extension.extension" /> + <sliceName value="reference" /> + <min value="0" /> + <max value="1" /> + </element> + <element id="Extension.extension:reference.url"> + <path value="Extension.extension.url" /> + <fixedUri value="reference" /> + </element> + <element id="Extension.extension:reference.value[x]"> + <path value="Extension.extension.value[x]" /> + <min value="1" /> + <type> + <code value="Reference" /> + </type> + </element> + <element id="Extension.extension:reference.value[x].reference"> + <path value="Extension.extension.value[x].reference" /> + <min value="1" /> + </element> + <element id="Extension.extension:reference.value[x].identifier"> + <path value="Extension.extension.value[x].identifier" /> + <max value="0" /> + </element> + <element id="Extension.url"> + <path value="Extension.url" /> + <fixedUri value="https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/error-metadata" /> + </element> + <element id="Extension.value[x]"> + <path value="Extension.value[x]" /> + <max value="0" /> + </element> + </differential> +</StructureDefinition> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-receive.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-receive.xml index dd0227a0..bad90d7d 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-receive.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-receive.xml @@ -6,8 +6,10 @@ </tag> </meta> <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/task-start-data-receive" /> + <!-- version managed by bpe --> <version value="#{version}" /> <name value="TaskStartDataReceive" /> + <!-- status managed by bpe --> <status value="unknown" /> <experimental value="false" /> <!-- date managed by bpe --> diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-send.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-send.xml index 679aac08..03d12449 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-send.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-send.xml @@ -144,5 +144,30 @@ <code value="instant" /> </type> </element> + <element id="Task.output:error"> + <path value="Task.output" /> + <sliceName value="error" /> + </element> + <element id="Task.output:error.extension"> + <path value="Task.output.extension" /> + <slicing> + <discriminator> + <type value="value" /> + <path value="url" /> + </discriminator> + <rules value="open" /> + </slicing> + <min value="0" /> + </element> + <element id="Task.output:error.extension:error-metadata"> + <path value="Task.output.extension" /> + <sliceName value="error-metadata" /> + <min value="0" /> + <type> + <code value="Extension" /> + <profile value="https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/error-metadata" /> + </type> + <isModifier value="false" /> + </element> </differential> </StructureDefinition> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-translate.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-translate.xml index 9b05f064..34c6941a 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-translate.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-translate.xml @@ -6,8 +6,10 @@ </tag> </meta> <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/task-start-data-translate" /> + <!-- version managed by bpe --> <version value="#{version}" /> <name value="TaskStartDataTranslate" /> + <!-- status managed by bpe --> <status value="unknown" /> <experimental value="false" /> <!-- date managed by bpe --> diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-trigger.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-trigger.xml index 8912cfec..dae32234 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-trigger.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-start-data-trigger.xml @@ -6,8 +6,10 @@ </tag> </meta> <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/task-start-data-trigger" /> + <!-- version managed by bpe --> <version value="#{version}" /> <name value="TaskStartDataTrigger" /> + <!-- status managed by bpe --> <status value="unknown" /> <experimental value="false" /> <!-- date managed by bpe --> diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-stop-data-trigger.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-stop-data-trigger.xml index 28d49323..6c8a2b77 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-stop-data-trigger.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-task-stop-data-trigger.xml @@ -6,8 +6,10 @@ </tag> </meta> <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/task-stop-data-trigger" /> + <!-- version managed by bpe --> <version value="#{version}" /> <name value="TaskStopDataTrigger" /> + <!-- status managed by bpe --> <status value="unknown" /> <experimental value="false" /> <!-- date managed by bpe --> diff --git a/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-source.xml b/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-source.xml new file mode 100644 index 00000000..c7574a44 --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-source.xml @@ -0,0 +1,26 @@ +<ValueSet xmlns="http://hl7.org/fhir"> + <meta> + <tag> + <system value="http://highmed.org/fhir/CodeSystem/read-access-tag" /> + <code value="ALL" /> + </tag> + </meta> + <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/ValueSet/data-transfer-error-source"/> + <!-- version managed by bpe --> + <version value="#{version}" /> + <name value="NumCodexDataTransferErrorSource"/> + <title value="NUM-CODEX data-transfer error source"/> + <!-- status managed by bpe --> + <status value="unknown" /> + <experimental value="false"/> + <!-- date managed by bpe --> + <date value="#{date}"/> + <publisher value="NUM-CODEX"/> + <description value="CodeSystem with with error source values for the NUM-CODEX data-transfer processes"/> + <immutable value="true"/> + <compose> + <include> + <system value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-source"/> + </include> + </compose> +</ValueSet> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-type.xml b/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-type.xml new file mode 100644 index 00000000..3788fc1e --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-type.xml @@ -0,0 +1,26 @@ +<ValueSet xmlns="http://hl7.org/fhir"> + <meta> + <tag> + <system value="http://highmed.org/fhir/CodeSystem/read-access-tag" /> + <code value="ALL" /> + </tag> + </meta> + <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/ValueSet/data-transfer-error-type"/> + <!-- version managed by bpe --> + <version value="#{version}" /> + <name value="NumCodexDataTransferErrorType"/> + <title value="NUM-CODEX data-transfer error type"/> + <!-- status managed by bpe --> + <status value="unknown" /> + <experimental value="false"/> + <!-- date managed by bpe --> + <date value="#{date}"/> + <publisher value="NUM-CODEX"/> + <description value="CodeSystem with with error type values for the NUM-CODEX data-transfer processes"/> + <immutable value="true"/> + <compose> + <include> + <system value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-type"/> + </include> + </compose> +</ValueSet> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/log4j2.xml b/codex-process-data-transfer/src/main/resources/log4j2.xml new file mode 100644 index 00000000..cdab37f7 --- /dev/null +++ b/codex-process-data-transfer/src/main/resources/log4j2.xml @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Config for ValidationMain class --> +<Configuration status="INFO" monitorInterval="30" verbose="false"> + + <Appenders> + <Console name="CONSOLE" target="SYSTEM_ERR"> + <PatternLayout pattern="%p\t%t - %C{1}.%M(%L) | %m%n" /> + </Console> + </Appenders> + + <Loggers> + <Logger name="de.rwh" level="TRACE" /> + <Logger name="org.highmed" level="TRACE" /> + <Logger name="de.netzwerk_universitaetsmedizin" level="TRACE" /> + <Logger name="org.apache" level="WARN" /> + <Logger name="org.springframework" level="WARN" /> + <Logger name="jndi" level="WARN" /> + <Logger name="org.eclipse.jetty" level="INFO" /> + <Logger name="com.sun.jersey" level="WARN" /> + <Logger name="liquibase" level="WARN" /> + <Logger name="ca.uhn.hl7v2" level="WARN" /> + <Logger name="ca.uhn.fhir" level="WARN" /> + + <Root level="WARN"> + <AppenderRef ref="CONSOLE" /> + </Root> + </Loggers> +</Configuration> \ No newline at end of file diff --git a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateDataLearningTest.java b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateDataLearningTest.java new file mode 100644 index 00000000..ea91e49a --- /dev/null +++ b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateDataLearningTest.java @@ -0,0 +1,327 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.highmed.dsf.fhir.json.ObjectMapperFactory; +import org.highmed.dsf.fhir.validation.ValidationSupportWithCustomResources; +import org.highmed.dsf.fhir.validation.ValueSetExpanderImpl; +import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService; +import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport; +import org.hl7.fhir.common.hapi.validation.support.ValidationSupportChain; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; +import org.hl7.fhir.r4.model.CanonicalType; +import org.hl7.fhir.r4.model.ElementDefinition; +import org.hl7.fhir.r4.model.ElementDefinition.TypeRefComponent; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.StringType; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.support.DefaultProfileValidationSupport; +import ca.uhn.fhir.context.support.IValidationSupport; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidator; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGenerator; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorImpl; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithFileSystemCache; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotWithValidationMessages; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackage; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClient; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClientJersey; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClientWithFileSystemCache; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageDescriptor; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageManager; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageManagerImpl; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpanderWithFileSystemCache; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClient; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClientJersey; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClientWithFileSystemCache; + +public class ValidateDataLearningTest +{ + private static final Logger logger = LoggerFactory.getLogger(ValidateDataLearningTest.class); + + private static final Path cacheFolder = Paths.get("target"); + private static final FhirContext fhirContext = FhirContext.forR4(); + private static final ObjectMapper mapper = ObjectMapperFactory.createObjectMapper(fhirContext); + + @Test + public void testDownloadTagGz() throws Exception + { + ValidationPackageClient client = new ValidationPackageClientJersey("https://packages.simplifier.net"); + + ValidationPackage validationPackage = client.download("de.gecco", "1.0.5"); + + validationPackage.getEntries().forEach(e -> + { + if ("package/package.json".equals(e.getFileName())) + logger.debug(new String(e.getContent(), StandardCharsets.UTF_8)); + }); + + ValidationPackageDescriptor descriptor = validationPackage.getDescriptor(mapper); + logger.debug(descriptor.getName() + "/" + descriptor.getVersion() + ":"); + descriptor.getDependencies().forEach((k, v) -> logger.debug("\t" + k + "/" + v)); + } + + @Test + public void testDownloadWithDependencies() throws Exception + { + ValidationPackageClient validationPackageClient = new ValidationPackageClientJersey( + "https://packages.simplifier.net"); + ValidationPackageClient validationPackageClientWithCache = new ValidationPackageClientWithFileSystemCache( + cacheFolder, mapper, validationPackageClient); + + ValueSetExpansionClient valueSetExpansionClient = new ValueSetExpansionClientJersey( + "https://r4.ontoserver.csiro.au/fhir", mapper, fhirContext); + ValueSetExpansionClient valueSetExpansionClientWithCache = new ValueSetExpansionClientWithFileSystemCache( + cacheFolder, fhirContext, valueSetExpansionClient); + + ValidationPackageManager manager = new ValidationPackageManagerImpl(validationPackageClientWithCache, + valueSetExpansionClientWithCache, mapper, fhirContext, PluginSnapshotGeneratorImpl::new, + ValueSetExpanderImpl::new); + + List<ValidationPackage> validationPackages = manager.downloadPackageWithDependencies("de.gecco", "1.0.5"); + validationPackages.forEach(p -> + { + logger.debug(p.getName() + "/" + p.getVersion()); + p.parseResources(fhirContext); + }); + } + + @Test + public void testValidate() throws Exception + { + ValidationPackageClient validationPackageClient = new ValidationPackageClientJersey( + "https://packages.simplifier.net"); + ValidationPackageClient validationPackageClientWithCache = new ValidationPackageClientWithFileSystemCache( + cacheFolder, mapper, validationPackageClient); + ValueSetExpansionClient valueSetExpansionClient = new ValueSetExpansionClientJersey( + "https://r4.ontoserver.csiro.au/fhir", mapper, fhirContext); + ValueSetExpansionClient valueSetExpansionClientWithCache = new ValueSetExpansionClientWithFileSystemCache( + cacheFolder, fhirContext, valueSetExpansionClient); + ValidationPackageManager manager = new ValidationPackageManagerImpl(validationPackageClientWithCache, + valueSetExpansionClientWithCache, mapper, fhirContext, + (fc, vs) -> new PluginSnapshotGeneratorWithFileSystemCache(cacheFolder, fc, + new PluginSnapshotGeneratorImpl(fc, vs)), + (fc, vs) -> new ValueSetExpanderWithFileSystemCache(cacheFolder, fc, new ValueSetExpanderImpl(fc, vs))); + + BundleValidator validator = manager.createBundleValidator("de.gecco", "1.0.5"); + + logger.debug("---------- executing validation tests ----------"); + + Path bundleFolder = Paths.get("src/test/resources/fhir/Bundle"); + String[] bundles = { "dic_fhir_store_demo_bf_large.json", "dic_fhir_store_demo_bf.json", + "dic_fhir_store_demo_psn_large.json", "dic_fhir_store_demo_psn.json" }; + Arrays.stream(bundles).map(bundleFolder::resolve).forEach(validateWith(validator)); + } + + private Consumer<Path> validateWith(BundleValidator validator) + { + return file -> + { + logger.debug("----- {} -----", file.toString()); + + Bundle bundle; + try + { + byte[] bundleData = Files.readAllBytes(file); + bundle = fhirContext.newJsonParser().parseResource(Bundle.class, + new String(bundleData, StandardCharsets.UTF_8)); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + + Bundle validationResult = validator.validate(bundle); + + logger.info("Validation result bundle: {}", + fhirContext.newJsonParser().encodeResourceToString(validationResult)); + + for (int i = 0; i < validationResult.getEntry().size(); i++) + { + BundleEntryComponent entry = validationResult.getEntry().get(i); + OperationOutcome outcome = (OperationOutcome) entry.getResponse().getOutcome(); + + final int index = i; + outcome.getIssue().forEach(issue -> + { + if (OperationOutcome.IssueSeverity.FATAL.equals(issue.getSeverity())) + logger.error( + "Bundle index {} fatal validation error ({}): {}", index, issue.getLocation().stream() + .map(StringType::getValue).collect(Collectors.joining(", ")), + issue.getDiagnostics()); + else if (OperationOutcome.IssueSeverity.ERROR.equals(issue.getSeverity())) + logger.error( + "Bundle index {} validation error ({}): {}", index, issue.getLocation().stream() + .map(StringType::getValue).collect(Collectors.joining(", ")), + issue.getDiagnostics()); + else if (OperationOutcome.IssueSeverity.WARNING.equals(issue.getSeverity())) + logger.warn( + "Bundle index {} validation warning ({}): {}", index, issue.getLocation().stream() + .map(StringType::getValue).collect(Collectors.joining(", ")), + issue.getDiagnostics()); + else if (issue.hasLocation()) + logger.info( + "Bundle index {} validation info ({}): {}", index, issue.getLocation().stream() + .map(StringType::getValue).collect(Collectors.joining(", ")), + issue.getDiagnostics()); + else + logger.info("Bundle index {} validation info: {}", index, issue.getDiagnostics()); + }); + } + }; + } + + @Test + public void testGenerateSnapshots() throws Exception + { + ValidationPackageClient validationPackageClient = new ValidationPackageClientJersey( + "https://packages.simplifier.net"); + ValidationPackageClient validationPackageClientWithCache = new ValidationPackageClientWithFileSystemCache( + cacheFolder, mapper, validationPackageClient); + ValueSetExpansionClient valueSetExpansionClient = new ValueSetExpansionClientJersey( + "https://r4.ontoserver.csiro.au/fhir", mapper, fhirContext); + ValueSetExpansionClient valueSetExpansionClientWithCache = new ValueSetExpansionClientWithFileSystemCache( + cacheFolder, fhirContext, valueSetExpansionClient); + ValidationPackageManager manager = new ValidationPackageManagerImpl(validationPackageClientWithCache, + valueSetExpansionClientWithCache, mapper, fhirContext, PluginSnapshotGeneratorImpl::new, + ValueSetExpanderImpl::new); + + List<ValidationPackage> validationPackages = manager.downloadPackageWithDependencies("de.gecco", "1.0.5"); + validationPackages.forEach(p -> + { + logger.debug(p.getName() + "/" + p.getVersion()); + p.parseResources(fhirContext); + }); + + validationPackages.stream().flatMap(p -> p.getValidationSupportResources().getStructureDefinitions().stream()) + .sorted(Comparator.comparing(StructureDefinition::getUrl) + .thenComparing(Comparator.comparing(StructureDefinition::getVersion))) + .forEach(s -> logger.debug(s.getUrl() + " " + s.getVersion())); + + StructureDefinition miiRef = validationPackages.stream() + .flatMap(p -> p.getValidationSupportResources().getStructureDefinitions().stream()) + .filter(s -> "https://www.medizininformatik-initiative.de/fhir/core/StructureDefinition/MII-Reference" + .equals(s.getUrl())) + .findFirst().get(); + + PluginSnapshotGenerator sGen = new PluginSnapshotGeneratorImpl(fhirContext, new ValidationSupportChain( + new InMemoryTerminologyServerValidationSupport(fhirContext), + new ValidationSupportChain(validationPackages.stream() + .map(ValidationPackage::getValidationSupportResources) + .map(r -> new ValidationSupportWithCustomResources(fhirContext, r.getStructureDefinitions(), + r.getCodeSystems(), r.getValueSets())) + .toArray(IValidationSupport[]::new)), + new DefaultProfileValidationSupport(fhirContext), + new CommonCodeSystemsTerminologyService(fhirContext))); + + PluginSnapshotWithValidationMessages result = sGen.generateSnapshot(miiRef); + + result.getMessages().forEach(m -> + { + if (IssueSeverity.ERROR.equals(m.getLevel()) || IssueSeverity.FATAL.equals(m.getLevel())) + logger.error("Error while generating snapshot for {}|{}: {}", result.getSnapshot().getUrl(), + result.getSnapshot().getVersion(), m.toString()); + else if (IssueSeverity.WARNING.equals(m.getLevel())) + logger.warn("Warning while generating snapshot for {}|{}: {}", result.getSnapshot().getUrl(), + result.getSnapshot().getVersion(), m.toString()); + else + logger.info("Info while generating snapshot for {}|{}: {}", result.getSnapshot().getUrl(), + result.getSnapshot().getVersion(), m.toString()); + }); + + Map<String, StructureDefinition> sDefByUrl = validationPackages.stream() + .flatMap(p -> p.getValidationSupportResources().getStructureDefinitions().stream()) + .collect(Collectors.toMap(StructureDefinition::getUrl, Function.identity())); + + validationPackages.stream().flatMap(p -> p.getValidationSupportResources().getStructureDefinitions().stream()) + .forEach(s -> + { + + logger.info("StructureDefinition {}|{}:", s.getUrl(), s.getVersion()); + printTree(s, sDefByUrl); + logger.debug(""); + }); + } + + private void printTree(StructureDefinition def, Map<String, StructureDefinition> structureDefinitionsByUrl) + { + logger.debug(""); + + Set<String> profileDependencies = new HashSet<>(); + Set<String> targetProfileDependencies = new HashSet<>(); + printTree(def.getUrl(), def, structureDefinitionsByUrl, "", profileDependencies, targetProfileDependencies); + + if (!profileDependencies.isEmpty()) + { + logger.debug(""); + logger.debug(" Profile-Dependencies:"); + profileDependencies.stream().sorted().forEach(url -> logger.debug(" " + url)); + } + if (!targetProfileDependencies.isEmpty()) + { + logger.debug(""); + logger.debug(" TargetProfile-Dependencies:"); + targetProfileDependencies.stream().sorted().forEach(url -> logger.debug(" " + url)); + } + } + + private void printTree(String k, StructureDefinition def, + Map<String, StructureDefinition> structureDefinitionsByUrl, String indentation, + Set<String> profileDependencies, Set<String> targetProfileDependencies) + { + logger.debug(indentation + "Profile: " + k); + for (ElementDefinition element : def.getDifferential().getElement()) + { + if (element.getType().stream().filter(t -> !t.getProfile().isEmpty() || !t.getTargetProfile().isEmpty()) + .findAny().isPresent()) + { + logger.debug(indentation + " Element: " + element.getId() + " (Path: " + element.getPath() + ")"); + for (TypeRefComponent type : element.getType()) + { + if (!type.getProfile().isEmpty()) + { + for (CanonicalType profile : type.getProfile()) + { + profileDependencies.add(profile.getValue()); + + if (structureDefinitionsByUrl.containsKey(profile.getValue())) + printTree(profile.getValue(), structureDefinitionsByUrl.get(profile.getValue()), + structureDefinitionsByUrl, indentation + " ", profileDependencies, + targetProfileDependencies); + else + logger.debug(indentation + " Profile: " + profile.getValue() + " ?"); + } + } + if (!type.getTargetProfile().isEmpty()) + { + for (CanonicalType targetProfile : type.getTargetProfile()) + { + targetProfileDependencies.add(targetProfile.getValue()); + logger.debug(indentation + " TargetProfile: " + targetProfile.getValue()); + } + } + } + } + } + } +} diff --git a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/fhir/profile/TaskProfileTest.java b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/fhir/profile/TaskProfileTest.java index 0817ccd2..4e4f1424 100644 --- a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/fhir/profile/TaskProfileTest.java +++ b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/fhir/profile/TaskProfileTest.java @@ -41,6 +41,8 @@ import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.Identifier; import org.hl7.fhir.r4.model.InstantType; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity; import org.hl7.fhir.r4.model.Reference; import org.hl7.fhir.r4.model.ResourceType; import org.hl7.fhir.r4.model.StringType; @@ -55,6 +57,7 @@ import ca.uhn.fhir.model.api.TemporalPrecisionEnum; import ca.uhn.fhir.validation.ResultSeverityEnum; import ca.uhn.fhir.validation.ValidationResult; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.error.ErrorOutputParameterGenerator; public class TaskProfileTest { @@ -62,13 +65,16 @@ public class TaskProfileTest @ClassRule public static final ValidationSupportRule validationRule = new ValidationSupportRule(VERSION, DATE, - Arrays.asList("highmed-task-base-0.5.0.xml", "num-codex-task-start-data-receive.xml", - "num-codex-task-start-data-send.xml", "num-codex-task-start-data-translate.xml", - "num-codex-task-start-data-trigger.xml", "num-codex-task-stop-data-trigger.xml"), + Arrays.asList("highmed-task-base-0.5.0.xml", "num-codex-extension-error-metadata.xml", + "num-codex-task-start-data-receive.xml", "num-codex-task-start-data-send.xml", + "num-codex-task-start-data-translate.xml", "num-codex-task-start-data-trigger.xml", + "num-codex-task-stop-data-trigger.xml"), Arrays.asList("highmed-read-access-tag-0.5.0.xml", "highmed-bpmn-message-0.5.0.xml", - "num-codex-data-transfer.xml"), + "num-codex-data-transfer.xml", "num-codex-data-transfer-error-source.xml", + "num-codex-data-transfer-error-type.xml"), Arrays.asList("highmed-read-access-tag-0.5.0.xml", "highmed-bpmn-message-0.5.0.xml", - "num-codex-data-transfer.xml")); + "num-codex-data-transfer.xml", "num-codex-data-transfer-error-source.xml", + "num-codex-data-transfer-error-type.xml")); private ResourceValidator resourceValidator = new ResourceValidatorImpl(validationRule.getFhirContext(), validationRule.getValidationSupport()); @@ -221,6 +227,27 @@ public void testTaskStartDataSendValidWithExportFrom() throws Exception || ResultSeverityEnum.FATAL.equals(m.getSeverity())).count()); } + @Test + public void testTaskStartDataSendValidWithValidationError() throws Exception + { + Task task = createValidTaskStartDataSendWithIdentifierReference(); + task.setStatus(TaskStatus.FAILED); + + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(IssueSeverity.ERROR).addLocation("Patient.identifier[0].system"); + new ErrorOutputParameterGenerator() + .createMeDicValidationError(new IdType("http://gecco.fhir.server/fhir", "Patient", "42", null), outcome) + .forEach(task::addOutput); + + logTask(task); + + ValidationResult result = resourceValidator.validate(task); + ValidationSupportRule.logValidationMessages(logger, result); + + assertEquals(0, result.getMessages().stream().filter(m -> ResultSeverityEnum.ERROR.equals(m.getSeverity()) + || ResultSeverityEnum.FATAL.equals(m.getSeverity())).count()); + } + private Task createValidTaskStartDataSendWithIdentifierReference() { Task task = new Task(); diff --git a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf.json b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf.json index 96e277b1..e9050085 100644 --- a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf.json +++ b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf.json @@ -16,8 +16,8 @@ "url": "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/ethnic-group", "valueCoding": { "system": "http://snomed.info/sct", - "code": "186019001", - "display": "Other ethnic, mixed origin" + "code": "26242008", + "display": "Mixed (qualifier value)" } }, { diff --git a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_create.json b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_create.json index b561f235..34a695d9 100644 --- a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_create.json +++ b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_create.json @@ -16,8 +16,8 @@ "url": "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/ethnic-group", "valueCoding": { "system": "http://snomed.info/sct", - "code": "186019001", - "display": "Other ethnic, mixed origin" + "code": "26242008", + "display": "Mixed (qualifier value)" } }, { diff --git a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_large.json b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_large.json index 0d46defe..a54fc226 100644 --- a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_large.json +++ b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_bf_large.json @@ -21,8 +21,8 @@ "url": "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/ethnic-group", "valueCoding": { "system": "http://snomed.info/sct", - "code": "186019001", - "display": "Other ethnic, mixed origin" + "code": "26242008", + "display": "Mixed (qualifier value)" } }, { @@ -91,10 +91,6 @@ { "system":"http://loinc.org", "code":"8691-8" - }, - { - "system":"http://snomed.info/sct", - "code":"443846001" } ], "text":"History of Travel" @@ -907,18 +903,6 @@ "reference":"urn:uuid:e6230cf4-4022-486c-8a4a-000000000001" }, "effectiveDateTime":"2021-02-17T01:01:00.000+01:00", - "interpretation":[ - { - "coding":[ - { - "system":"http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", - "code":"L", - "display":"low" - } - ], - "text":"Below low normal" - } - ], "bodySite":{ "coding":[ { @@ -941,11 +925,6 @@ "system":"http://snomed.info/sct", "code":"271649006", "display":"Systolic blood pressure" - }, - { - "system":"http://acme.org/devices/clinical-codes", - "code":"bp-s", - "display":"Systolic Blood pressure" } ] }, @@ -954,19 +933,7 @@ "unit":"mmHg", "system":"http://unitsofmeasure.org", "code":"mm[Hg]" - }, - "interpretation":[ - { - "coding":[ - { - "system":"http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", - "code":"N", - "display":"normal" - } - ], - "text":"Normal" - } - ] + } }, { "code":{ @@ -975,27 +942,20 @@ "system":"http://loinc.org", "code":"8462-4", "display":"Diastolic blood pressure" + }, + { + "system": "http://snomed.info/sct", + "code": "271650006", + "display": "Diastolic blood pressure" } ] }, "valueQuantity":{ - "value":107, + "value":72, "unit":"mmHg", "system":"http://unitsofmeasure.org", "code":"mm[Hg]" - }, - "interpretation":[ - { - "coding":[ - { - "system":"http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", - "code":"L", - "display":"low" - } - ], - "text":"Below low normal" - } - ] + } } ] }, @@ -1090,9 +1050,8 @@ "code":{ "coding":[ { - "code":"01", - "display":"Is the patient in the intensive care unit?", - "system":"https://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/ecrf-parameter-codes" + "system": "http://loinc.org", + "code": "95420-6" } ] }, diff --git a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn.json b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn.json index 37921866..3a0165fc 100644 --- a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn.json +++ b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn.json @@ -16,8 +16,8 @@ "url": "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/ethnic-group", "valueCoding": { "system": "http://snomed.info/sct", - "code": "186019001", - "display": "Other ethnic, mixed origin" + "code": "26242008", + "display": "Mixed (qualifier value)" } }, { diff --git a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_create.json b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_create.json index 9cd04ad1..274a6ab7 100644 --- a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_create.json +++ b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_create.json @@ -16,8 +16,8 @@ "url": "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/ethnic-group", "valueCoding": { "system": "http://snomed.info/sct", - "code": "186019001", - "display": "Other ethnic, mixed origin" + "code": "26242008", + "display": "Mixed (qualifier value)" } }, { @@ -107,8 +107,8 @@ "coding": [ { "system": "http://snomed.info/sct", - "code": "413839001", - "display": "Chronic lung disease" + "code": "13645005", + "display": "Chronic obstructive lung disease (disorder)" } ] }, diff --git a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_large.json b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_large.json index 6a0d7be9..c967e484 100644 --- a/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_large.json +++ b/codex-process-data-transfer/src/test/resources/fhir/Bundle/dic_fhir_store_demo_psn_large.json @@ -21,8 +21,8 @@ "url": "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/ethnic-group", "valueCoding": { "system": "http://snomed.info/sct", - "code": "186019001", - "display": "Other ethnic, mixed origin" + "code": "26242008", + "display": "Mixed (qualifier value)" } }, { @@ -91,10 +91,6 @@ { "system":"http://loinc.org", "code":"8691-8" - }, - { - "system":"http://snomed.info/sct", - "code":"443846001" } ], "text":"History of Travel" @@ -907,18 +903,6 @@ "reference":"urn:uuid:e6230cf4-4022-486c-8a4a-000000000001" }, "effectiveDateTime":"2021-02-17T01:01:00.000+01:00", - "interpretation":[ - { - "coding":[ - { - "system":"http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", - "code":"L", - "display":"low" - } - ], - "text":"Below low normal" - } - ], "bodySite":{ "coding":[ { @@ -941,11 +925,6 @@ "system":"http://snomed.info/sct", "code":"271649006", "display":"Systolic blood pressure" - }, - { - "system":"http://acme.org/devices/clinical-codes", - "code":"bp-s", - "display":"Systolic Blood pressure" } ] }, @@ -954,19 +933,7 @@ "unit":"mmHg", "system":"http://unitsofmeasure.org", "code":"mm[Hg]" - }, - "interpretation":[ - { - "coding":[ - { - "system":"http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", - "code":"N", - "display":"normal" - } - ], - "text":"Normal" - } - ] + } }, { "code":{ @@ -975,27 +942,20 @@ "system":"http://loinc.org", "code":"8462-4", "display":"Diastolic blood pressure" + }, + { + "system": "http://snomed.info/sct", + "code": "271650006", + "display": "Diastolic blood pressure" } ] }, "valueQuantity":{ - "value":107, + "value":72, "unit":"mmHg", "system":"http://unitsofmeasure.org", "code":"mm[Hg]" - }, - "interpretation":[ - { - "coding":[ - { - "system":"http://terminology.hl7.org/CodeSystem/v3-ObservationInterpretation", - "code":"L", - "display":"low" - } - ], - "text":"Below low normal" - } - ] + } } ] }, @@ -1090,9 +1050,8 @@ "code":{ "coding":[ { - "code":"01", - "display":"Is the patient in the intensive care unit?", - "system":"https://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/ecrf-parameter-codes" + "system": "http://loinc.org", + "code": "95420-6" } ] }, diff --git a/codex-process-data-transfer/src/test/resources/log4j2.xml b/codex-process-data-transfer/src/test/resources/log4j2.xml index 736faa29..73f8205a 100644 --- a/codex-process-data-transfer/src/test/resources/log4j2.xml +++ b/codex-process-data-transfer/src/test/resources/log4j2.xml @@ -51,7 +51,7 @@ <Logger name="com.sun.jersey" level="WARN"/> <Logger name="liquibase" level="WARN"/> <Logger name="ca.uhn.hl7v2" level="WARN"/> - <Logger name="ca.uhn.fhir" level="DEBUG"/> + <Logger name="ca.uhn.fhir" level="WARN"/> <!-- <Logger name="certificate-warning-logger" level="INFO"> <AppenderRef ref="MAIL_CERTIFICATE" /> From 7f16f6b9d2030e4b0e2cabd82e5b5a5d26728e35 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Wed, 25 May 2022 16:19:35 +0200 Subject: [PATCH 04/29] more dependencies added to the lib folder of the stand-alone validator --- codex-process-data-transfer/pom.xml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/codex-process-data-transfer/pom.xml b/codex-process-data-transfer/pom.xml index b4d61f2c..16ce3370 100644 --- a/codex-process-data-transfer/pom.xml +++ b/codex-process-data-transfer/pom.xml @@ -318,12 +318,9 @@ <includeArtifactIds>hapi-fhir-base,hapi-fhir-client,hapi-fhir-converter,hapi-fhir-structures-r4,hapi-fhir-structures-r5,hapi-fhir-validation,hapi-fhir-validation-resources-r4, hapi-fhir-validation-resources-r5,org.hl7.fhir.convertors,org.hl7.fhir.dstu2,org.hl7.fhir.dstu2016may,org.hl7.fhir.dstu3,org.hl7.fhir.r4,org.hl7.fhir.r5,org.hl7.fhir.utilities, org.hl7.fhir.validation,jackson-annotations,jackson-core,jackson-databind,jackson-module-jaxb-annotations,caffeine,guava, - commons-codec,commons-io,crypto-utils,log4j2-utils, - jakarta.annotation-api,jakarta.ws.rs-api, - commons-compress,commons-lang3,commons-text, - log4j-api,log4j-core,log4j-slf4j-impl,bcpkix-jdk15on,bcprov-jdk15on,bcutil-jdk15on,ucum,jakarta.inject, - jersey-client,jersey-common, - jersey-media-jaxb,jersey-media-json-jackson, + commons-codec,commons-io,crypto-utils,log4j2-utils,jakarta.annotation-api,jakarta.ws.rs-api,jakarta.xml.bind-api,commons-compress,commons-lang3,commons-text, + log4j-api,log4j-core,log4j-slf4j-impl,bcpkix-jdk15on,bcprov-jdk15on,bcutil-jdk15on,ucum,hk2-api,hk2-locator,hk2-utils,aopalliance-repackaged, + jakarta.inject,jersey-client,jersey-common,jersey-entity-filtering,jersey-hk2,jersey-media-jaxb,jersey-media-json-jackson, dsf-bpe-process-base,dsf-fhir-rest-adapter,dsf-fhir-validation,dsf-openehr-model,jcl-over-slf4j, slf4j-api,spring-aop,spring-beans,spring-context,spring-core,spring-expression,spring-jcl,thymeleaf,unbescape,xpp3,xpp3_xpath </includeArtifactIds> From be89ddbca7a2b355581935a5c7b0a9f3e660e092 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Wed, 25 May 2022 21:17:24 +0200 Subject: [PATCH 05/29] simplified logging pattern --- .../src/main/resources/log4j2.xml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/codex-process-data-transfer/src/main/resources/log4j2.xml b/codex-process-data-transfer/src/main/resources/log4j2.xml index cdab37f7..ac70ba75 100644 --- a/codex-process-data-transfer/src/main/resources/log4j2.xml +++ b/codex-process-data-transfer/src/main/resources/log4j2.xml @@ -4,22 +4,12 @@ <Appenders> <Console name="CONSOLE" target="SYSTEM_ERR"> - <PatternLayout pattern="%p\t%t - %C{1}.%M(%L) | %m%n" /> + <PatternLayout pattern="%p\t| %m%n" /> </Console> </Appenders> <Loggers> - <Logger name="de.rwh" level="TRACE" /> - <Logger name="org.highmed" level="TRACE" /> - <Logger name="de.netzwerk_universitaetsmedizin" level="TRACE" /> - <Logger name="org.apache" level="WARN" /> - <Logger name="org.springframework" level="WARN" /> - <Logger name="jndi" level="WARN" /> - <Logger name="org.eclipse.jetty" level="INFO" /> - <Logger name="com.sun.jersey" level="WARN" /> - <Logger name="liquibase" level="WARN" /> - <Logger name="ca.uhn.hl7v2" level="WARN" /> - <Logger name="ca.uhn.fhir" level="WARN" /> + <Logger name="de.netzwerk_universitaetsmedizin" level="DEBUG" /> <Root level="WARN"> <AppenderRef ref="CONSOLE" /> From 9ae6786ad2cb8da5e7a96e495757e308f112ae30 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Wed, 25 May 2022 21:17:41 +0200 Subject: [PATCH 06/29] more dependencies added to the lib folder of the stand-alone validator --- codex-process-data-transfer/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codex-process-data-transfer/pom.xml b/codex-process-data-transfer/pom.xml index 16ce3370..1b2f0ead 100644 --- a/codex-process-data-transfer/pom.xml +++ b/codex-process-data-transfer/pom.xml @@ -318,9 +318,9 @@ <includeArtifactIds>hapi-fhir-base,hapi-fhir-client,hapi-fhir-converter,hapi-fhir-structures-r4,hapi-fhir-structures-r5,hapi-fhir-validation,hapi-fhir-validation-resources-r4, hapi-fhir-validation-resources-r5,org.hl7.fhir.convertors,org.hl7.fhir.dstu2,org.hl7.fhir.dstu2016may,org.hl7.fhir.dstu3,org.hl7.fhir.r4,org.hl7.fhir.r5,org.hl7.fhir.utilities, org.hl7.fhir.validation,jackson-annotations,jackson-core,jackson-databind,jackson-module-jaxb-annotations,caffeine,guava, - commons-codec,commons-io,crypto-utils,log4j2-utils,jakarta.annotation-api,jakarta.ws.rs-api,jakarta.xml.bind-api,commons-compress,commons-lang3,commons-text, - log4j-api,log4j-core,log4j-slf4j-impl,bcpkix-jdk15on,bcprov-jdk15on,bcutil-jdk15on,ucum,hk2-api,hk2-locator,hk2-utils,aopalliance-repackaged, - jakarta.inject,jersey-client,jersey-common,jersey-entity-filtering,jersey-hk2,jersey-media-jaxb,jersey-media-json-jackson, + commons-codec,commons-io,crypto-utils,log4j2-utils,jakarta.activation,jakarta.annotation-api,jakarta.ws.rs-api,jakarta.xml.bind-api,commons-compress,commons-lang3,commons-text, + httpclient,httpcore,log4j-api,log4j-core,log4j-slf4j-impl,bcpkix-jdk15on,bcprov-jdk15on,bcutil-jdk15on,ucum,hk2-api,hk2-locator,hk2-utils,osgi-resource-locator, + aopalliance-repackaged,jakarta.inject,jersey-apache-connector,jersey-client,jersey-common,jersey-entity-filtering,jersey-hk2,jersey-media-jaxb,jersey-media-json-jackson, dsf-bpe-process-base,dsf-fhir-rest-adapter,dsf-fhir-validation,dsf-openehr-model,jcl-over-slf4j, slf4j-api,spring-aop,spring-beans,spring-context,spring-core,spring-expression,spring-jcl,thymeleaf,unbescape,xpp3,xpp3_xpath </includeArtifactIds> From 1666aa94cb26568edf0b4da9e743615c7ad23425 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Wed, 25 May 2022 21:19:26 +0200 Subject: [PATCH 07/29] fixed client init code for trust store defined but no key store defined clients only initialize a trust store only ssl context, if key store and key store password are null. --- .../data_transfer/spring/config/ValidationConfig.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java index 82746171..05c8fec5 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java @@ -314,9 +314,9 @@ private ValidationPackageClientJersey validationPackageClientJersey() packageClientKeyStorePassword); return new ValidationPackageClientJersey(packageServerBaseUrl, packageClientTrustStore, packageClientKeyStore, - packageClientKeyStorePassword, packageClientBasicAuthUsername, packageClientBasicAuthPassword, - packageClientProxySchemeHostPort, packageClientProxyUsername, packageClientProxyPassword, - packageClientConnectTimeout, packageClientReadTimeout); + packageClientKeyStore == null ? null : packageClientKeyStorePassword, packageClientBasicAuthUsername, + packageClientBasicAuthPassword, packageClientProxySchemeHostPort, packageClientProxyUsername, + packageClientProxyPassword, packageClientConnectTimeout, packageClientReadTimeout); } @Bean @@ -353,7 +353,8 @@ private ValueSetExpansionClient valueSetExpansionClientJersey() valueSetExpansionClientCertificatePrivateKeyPassword, valueSetExpansionClientKeyStorePassword); return new ValueSetExpansionClientJersey(valueSetExpansionServerBaseUrl, valueSetExpansionClientTrustStore, - valueSetExpansionClientKeyStore, valueSetExpansionClientKeyStorePassword, + valueSetExpansionClientKeyStore, + valueSetExpansionClientKeyStore == null ? null : valueSetExpansionClientKeyStorePassword, valueSetExpansionClientBasicAuthUsername, valueSetExpansionClientBasicAuthPassword, valueSetExpansionClientProxySchemeHostPort, valueSetExpansionClientProxyUsername, valueSetExpansionClientProxyPassword, valueSetExpansionClientConnectTimeout, From 5178615819f34121687ff4c9006360a998829fd6 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Wed, 25 May 2022 21:22:56 +0200 Subject: [PATCH 08/29] fixed forward proxy support, added ontology server connection test Jersey based clients not use the ApacheConnectorProvider. Standard connector provider has no support for forwarding proxies. Added a ontology server connection test to the ValueSetExpansionClient and usage of to ValidatorMain. --- .../validation/ValidationMain.java | 61 ++++++++++++++++--- .../ValidationPackageClientJersey.java | 15 ++--- .../validation/ValueSetExpansionClient.java | 3 + .../ValueSetExpansionClientJersey.java | 28 ++++++--- ...SetExpansionClientWithFileSystemCache.java | 7 +++ 5 files changed, 92 insertions(+), 22 deletions(-) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java index 1406b799..54a6197d 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java @@ -6,9 +6,11 @@ import java.util.Arrays; import java.util.Locale; import java.util.Objects; +import java.util.stream.Stream; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.CapabilityStatement; import org.hl7.fhir.r4.model.Resource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,6 +21,8 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.EnumerablePropertySource; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.i18n.HapiLocalizer; @@ -68,6 +72,12 @@ public static class TestConfig @Autowired private ValidationPackageIdentifier validationPackage; + @Autowired + private ValueSetExpansionClient valueSetExpansionClient; + + @Autowired + private ConfigurableEnvironment environment; + @Bean public FhirContext fhirContext() { @@ -87,7 +97,8 @@ public Locale getLocale() @Bean public ValidationMain validatorMain() { - return new ValidationMain(fhirContext(), packageManager, validationPackage, output, outputPretty); + return new ValidationMain(environment, fhirContext(), packageManager, validationPackage, output, + outputPretty, valueSetExpansionClient); } } @@ -107,44 +118,70 @@ public static void main(String[] args) try (AnnotationConfigApplicationContext springContext = new AnnotationConfigApplicationContext(TestConfig.class, ValidationConfig.class)) { - springContext.getBean(ValidationMain.class).validate(args); + ValidationMain main = springContext.getBean(ValidationMain.class); + + main.testOntologyServerConnection(); + main.validate(args); } catch (Exception e) { - logger.error("Unable to create spring context {}: {}", e.getClass().getName(), e.getMessage()); - logger.error("Stack-Trace: ", e); + logger.error("", e); System.exit(1); } } + private final ConfigurableEnvironment environment; private final FhirContext fhirContext; private final ValidationPackageManager packageManager; private final ValidationPackageIdentifier validationPackage; private final Output output; private final boolean outputPretty; + private final ValueSetExpansionClient valueSetExpansionClient; - public ValidationMain(FhirContext fhirContext, ValidationPackageManager packageManager, - ValidationPackageIdentifier validationPackage, Output output, boolean outputPretty) + public ValidationMain(ConfigurableEnvironment environment, FhirContext fhirContext, + ValidationPackageManager packageManager, ValidationPackageIdentifier validationPackage, Output output, + boolean outputPretty, ValueSetExpansionClient valueSetExpansionClient) { + this.environment = environment; this.fhirContext = fhirContext; this.packageManager = packageManager; this.validationPackage = validationPackage; this.output = output; this.outputPretty = outputPretty; + this.valueSetExpansionClient = valueSetExpansionClient; } @Override public void afterPropertiesSet() throws Exception { + Objects.requireNonNull(environment, "environment"); Objects.requireNonNull(fhirContext, "fhirContext"); Objects.requireNonNull(packageManager, "packageManager"); Objects.requireNonNull(validationPackage, "validationPackage"); Objects.requireNonNull(output, "output"); + Objects.requireNonNull(valueSetExpansionClient, "valueSetExpansionClient"); + } + + public void testOntologyServerConnection() + { + logger.info("Testing connection to ontology server"); + try + { + CapabilityStatement metadata = valueSetExpansionClient.getMetadata(); + logger.info("Connection test OK: {} - {}", metadata.getSoftware().getName(), + metadata.getSoftware().getVersion()); + } + catch (Exception e) + { + logger.info("Connection test failed: {}", e.getMessage()); + throw e; + } } - private void validate(String[] files) + public void validate(String[] files) { logger.info("Using validation package {}", validationPackage); + getAllNumProperties().forEach(c -> logger.debug("Config: {}", c)); BundleValidator validator = packageManager.createBundleValidator(validationPackage.getName(), validationPackage.getVersion()); @@ -166,6 +203,16 @@ private void validate(String[] files) }); } + private Stream<String> getAllNumProperties() + { + return environment.getPropertySources().stream().filter(p -> p instanceof EnumerablePropertySource<?>) + .map(p -> (EnumerablePropertySource<?>) p) + .flatMap(p -> Arrays.stream(p.getPropertyNames()) + .filter(n -> n.startsWith("de.netzwerk.universitaetsmedizin")) + .map(k -> new String[] { k, Objects.toString(p.getProperty(k)) })) + .map(e -> e[0].contains("password") ? new String[] { e[0], "*****" } : e).map(e -> e[0] + ": " + e[1]); + } + private IParser getOutputParser() { switch (output) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java index 6f32b22b..576a77d1 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java @@ -13,6 +13,8 @@ import javax.ws.rs.client.WebTarget; import org.glassfish.jersey.SslConfigurator; +import org.glassfish.jersey.apache.connector.ApacheConnectorProvider; +import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.client.ClientProperties; import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; @@ -49,13 +51,12 @@ else if (trustStore != null && keyStore != null && keyStorePassword != null) builder = builder.register(basicAuthFeature); } - if (proxySchemeHostPort != null) - { - builder = builder.property(ClientProperties.PROXY_URI, proxySchemeHostPort); - if (proxyUsername != null && proxyPassword != null) - builder = builder.property(ClientProperties.PROXY_USERNAME, proxyUsername) - .property(ClientProperties.PROXY_PASSWORD, String.valueOf(proxyPassword)); - } + ClientConfig config = new ClientConfig(); + config.connectorProvider(new ApacheConnectorProvider()); + config.property(ClientProperties.PROXY_URI, proxySchemeHostPort); + config.property(ClientProperties.PROXY_USERNAME, proxyUsername); + config.property(ClientProperties.PROXY_PASSWORD, proxyPassword == null ? null : String.valueOf(proxyPassword)); + builder = builder.withConfig(config); builder = builder.readTimeout(readTimeout, TimeUnit.MILLISECONDS).connectTimeout(connectTimeout, TimeUnit.MILLISECONDS); diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClient.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClient.java index 29e83b75..391cbfda 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClient.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClient.java @@ -4,6 +4,7 @@ import javax.ws.rs.WebApplicationException; +import org.hl7.fhir.r4.model.CapabilityStatement; import org.hl7.fhir.r4.model.ValueSet; public interface ValueSetExpansionClient @@ -16,4 +17,6 @@ public interface ValueSetExpansionClient * @throws WebApplicationException */ ValueSet expand(ValueSet valueSet) throws IOException, WebApplicationException; + + CapabilityStatement getMetadata() throws WebApplicationException; } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java index c4ed3925..6a987524 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java @@ -12,16 +12,21 @@ import javax.ws.rs.client.WebTarget; import org.glassfish.jersey.SslConfigurator; +import org.glassfish.jersey.apache.connector.ApacheConnectorProvider; +import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.client.ClientProperties; import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJaxbJsonProvider; import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJsonProvider; +import org.highmed.dsf.fhir.adapter.CapabilityStatementJsonFhirAdapter; +import org.highmed.dsf.fhir.adapter.CapabilityStatementXmlFhirAdapter; import org.highmed.dsf.fhir.adapter.OperationOutcomeJsonFhirAdapter; import org.highmed.dsf.fhir.adapter.OperationOutcomeXmlFhirAdapter; import org.highmed.dsf.fhir.adapter.ParametersJsonFhirAdapter; import org.highmed.dsf.fhir.adapter.ParametersXmlFhirAdapter; import org.highmed.dsf.fhir.adapter.ValueSetJsonFhirAdapter; import org.highmed.dsf.fhir.adapter.ValueSetXmlFhirAdapter; +import org.hl7.fhir.r4.model.CapabilityStatement; import org.hl7.fhir.r4.model.Parameters; import org.hl7.fhir.r4.model.ValueSet; import org.slf4j.Logger; @@ -68,13 +73,12 @@ else if (trustStore != null && keyStore != null && keyStorePassword != null) builder = builder.register(basicAuthFeature); } - if (proxySchemeHostPort != null) - { - builder = builder.property(ClientProperties.PROXY_URI, proxySchemeHostPort); - if (proxyUsername != null && proxyPassword != null) - builder = builder.property(ClientProperties.PROXY_USERNAME, proxyUsername) - .property(ClientProperties.PROXY_PASSWORD, String.valueOf(proxyPassword)); - } + ClientConfig config = new ClientConfig(); + config.connectorProvider(new ApacheConnectorProvider()); + config.property(ClientProperties.PROXY_URI, proxySchemeHostPort); + config.property(ClientProperties.PROXY_USERNAME, proxyUsername); + config.property(ClientProperties.PROXY_PASSWORD, proxyPassword == null ? null : String.valueOf(proxyPassword)); + builder = builder.withConfig(config); builder = builder.readTimeout(readTimeout, TimeUnit.MILLISECONDS).connectTimeout(connectTimeout, TimeUnit.MILLISECONDS); @@ -86,7 +90,9 @@ else if (trustStore != null && keyStore != null && keyStorePassword != null) builder.register(p); } - builder.register(new OperationOutcomeJsonFhirAdapter(fhirContext)) + builder.register(new CapabilityStatementJsonFhirAdapter(fhirContext)) + .register(new CapabilityStatementXmlFhirAdapter(fhirContext)) + .register(new OperationOutcomeJsonFhirAdapter(fhirContext)) .register(new OperationOutcomeXmlFhirAdapter(fhirContext)) .register(new ParametersJsonFhirAdapter(fhirContext)) .register(new ParametersXmlFhirAdapter(fhirContext)).register(new ValueSetJsonFhirAdapter(fhirContext)) @@ -119,4 +125,10 @@ public ValueSet expand(ValueSet valueSet) throws WebApplicationException return getResource().path("ValueSet").path("$expand").request(Constants.CT_FHIR_JSON_NEW) .post(Entity.entity(parameters, Constants.CT_FHIR_JSON_NEW), ValueSet.class); } + + @Override + public CapabilityStatement getMetadata() throws WebApplicationException + { + return getResource().path("metadata").request(Constants.CT_FHIR_JSON_NEW).get(CapabilityStatement.class); + } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java index 22a5f810..3a4811d6 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java @@ -9,6 +9,7 @@ import javax.ws.rs.WebApplicationException; +import org.hl7.fhir.r4.model.CapabilityStatement; import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; import org.hl7.fhir.r4.model.ValueSet; import org.slf4j.Logger; @@ -100,4 +101,10 @@ private ValueSet downloadAndWriteToCache(ValueSet valueSet) throws IOException // return writeToCache(expanded, Function.identity(), ValueSet::getUrl, ValueSet::getVersion); return writeRsourceToCache(expanded, Function.identity(), ValueSet::getUrl, ValueSet::getVersion); } + + @Override + public CapabilityStatement getMetadata() throws WebApplicationException + { + return delegate.getMetadata(); + } } From d8ec31718e58931179feec78049798b387a88730 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Wed, 25 May 2022 22:17:38 +0200 Subject: [PATCH 09/29] adds profile modifier to fix GECCO radiology-procedures missing min=0 The GECCO 1.0.5 radiology-procedures profile needs a min=0 config in the rule Procedure.code.coding:dicom in order for the HAPI Snapshot Generator to generate the correct StructureDefinition snapshot. --- .../spring/config/ValidationConfig.java | 6 +++- ...adiologyProceduresCodingSliceMinFixer.java | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/GeccoRadiologyProceduresCodingSliceMinFixer.java diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java index 05c8fec5..6dd76347 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java @@ -58,7 +58,11 @@ public class ValidationConfig @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package:de.gecco|1.0.5}") private String validationPackage; - @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.structureDefinitionModifierClasses:de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover,de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.MiiModuleLabObservationLab10IdentifierRemover}'.trim().split('(,[ ]?)|(\\n)')}") + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.structureDefinitionModifierClasses:" + + "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover," + + "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.MiiModuleLabObservationLab10IdentifierRemover," + + "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.GeccoRadiologyProceduresCodingSliceMinFixer" + + "}'.trim().split('(,[ ]?)|(\\n)')}") private List<String> structureDefinitionModifierClasses; @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.packagesToIgnore:hl7.fhir.r4.core|4.0.1}'.trim().split('(,[ ]?)|(\\n)')}") diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/GeccoRadiologyProceduresCodingSliceMinFixer.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/GeccoRadiologyProceduresCodingSliceMinFixer.java new file mode 100644 index 00000000..01673c83 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/GeccoRadiologyProceduresCodingSliceMinFixer.java @@ -0,0 +1,29 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition; + +import org.hl7.fhir.r4.model.StructureDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GeccoRadiologyProceduresCodingSliceMinFixer implements StructureDefinitionModifier +{ + private static final Logger logger = LoggerFactory.getLogger(GeccoRadiologyProceduresCodingSliceMinFixer.class); + + @Override + public StructureDefinition modify(StructureDefinition sd) + { + if ("https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/radiology-procedures" + .equals(sd.getUrl()) && "1.0.5".equals(sd.getVersion())) + { + sd.getDifferential().getElement().stream().filter( + e -> "Procedure.code.coding".equals(e.getPath()) && e.hasMax() && e.hasSliceName() && !e.hasMin()) + .forEach(e -> + { + logger.warn("Adding min=0 to rule with id {} in StructureDefinition {}|{}", e.getId(), + sd.getUrl(), sd.getVersion()); + e.setMin(0); + }); + } + + return sd; + } +} From d47efa8f2a5128abd5d72efad229bfef98bb850a Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Fri, 27 May 2022 09:56:20 +0200 Subject: [PATCH 10/29] Added "dicom" fix to ValidationPackageManagerImpl constructor The contructor of ValidationPackageManagerImpl used in the ValidateDataLearningTest was missing the "dicom" fix, aka GeccoRadiologyProceduresCodingSliceMinFixer --- .../validation/ValidationPackageManagerImpl.java | 8 ++++---- .../structure_definition/ClosedTypeSlicingRemover.java | 3 +++ .../GeccoRadiologyProceduresCodingSliceMinFixer.java | 3 +++ .../MiiModuleLabObservationLab10IdentifierRemover.java | 3 +++ 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java index 21311664..3d913885 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java @@ -43,6 +43,7 @@ import ca.uhn.fhir.context.support.DefaultProfileValidationSupport; import ca.uhn.fhir.context.support.IValidationSupport; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.GeccoRadiologyProceduresCodingSliceMinFixer; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.MiiModuleLabObservationLab10IdentifierRemover; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.StructureDefinitionModifier; @@ -53,11 +54,9 @@ public class ValidationPackageManagerImpl implements InitializingBean, Validatio public static final List<ValidationPackageIdentifier> PACKAGE_IGNORE = List .of(new ValidationPackageIdentifier("hl7.fhir.r4.core", "4.0.1")); - // closed type slicings result in error from the snapshot generator public static final StructureDefinitionModifier CLOSED_TYPE_SLICING_REMOVER = new ClosedTypeSlicingRemover(); - - // mandatory identifier on ObservationLab not compatible with data protection rules with current pseudonymization public static final StructureDefinitionModifier MII_MODULE_LAB_OBSERVATION_LAB_1_0_IDENTIFIER_REMOVER = new MiiModuleLabObservationLab10IdentifierRemover(); + public static final StructureDefinitionModifier GECCO_RADIOLOGY_PROCEDURES_CODING_SLICE_MIN_FIXER = new GeccoRadiologyProceduresCodingSliceMinFixer(); private final ValidationPackageClient validationPackageClient; private final ValueSetExpansionClient valueSetExpansionClient; @@ -78,7 +77,8 @@ public ValidationPackageManagerImpl(ValidationPackageClient validationPackageCli { this(validationPackageClient, valueSetExpansionClient, mapper, fhirContext, internalSnapshotGeneratorFactory, internalValueSetExpanderFactory, PACKAGE_IGNORE, - Arrays.asList(CLOSED_TYPE_SLICING_REMOVER, MII_MODULE_LAB_OBSERVATION_LAB_1_0_IDENTIFIER_REMOVER)); + Arrays.asList(CLOSED_TYPE_SLICING_REMOVER, MII_MODULE_LAB_OBSERVATION_LAB_1_0_IDENTIFIER_REMOVER, + GECCO_RADIOLOGY_PROCEDURES_CODING_SLICE_MIN_FIXER)); } public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java index 68a6574c..c3df1e98 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java @@ -9,6 +9,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Closed type slicings result in error from the snapshot generator. + */ public class ClosedTypeSlicingRemover implements StructureDefinitionModifier { private static final Logger logger = LoggerFactory.getLogger(ClosedTypeSlicingRemover.class); diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/GeccoRadiologyProceduresCodingSliceMinFixer.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/GeccoRadiologyProceduresCodingSliceMinFixer.java index 01673c83..165ff58f 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/GeccoRadiologyProceduresCodingSliceMinFixer.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/GeccoRadiologyProceduresCodingSliceMinFixer.java @@ -4,6 +4,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * HAPI snapshot generator adds min=1, if no min value specified in the parent StructureDefinition. + */ public class GeccoRadiologyProceduresCodingSliceMinFixer implements StructureDefinitionModifier { private static final Logger logger = LoggerFactory.getLogger(GeccoRadiologyProceduresCodingSliceMinFixer.class); diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java index 812eba07..1edfa7e6 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java @@ -9,6 +9,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Mandatory identifier on ObservationLab not compatible with data protection rules with current pseudonymization. + */ public class MiiModuleLabObservationLab10IdentifierRemover implements StructureDefinitionModifier { private static final Logger logger = LoggerFactory.getLogger(MiiModuleLabObservationLab10IdentifierRemover.class); From 3f863a000b476a7a6c05e41e9aefb50e27a72873 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Fri, 27 May 2022 21:28:58 +0200 Subject: [PATCH 11/29] removed inadvertently committed highmed task base profile --- .../highmed-task-base-0.5.0.xml | 274 ------------------ 1 file changed, 274 deletions(-) delete mode 100644 codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/highmed-task-base-0.5.0.xml diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/highmed-task-base-0.5.0.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/highmed-task-base-0.5.0.xml deleted file mode 100644 index a1f3b4a6..00000000 --- a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/highmed-task-base-0.5.0.xml +++ /dev/null @@ -1,274 +0,0 @@ -<StructureDefinition xmlns="http://hl7.org/fhir"> - <meta> - <tag> - <system value="http://highmed.org/fhir/CodeSystem/read-access-tag" /> - <code value="ALL" /> - </tag> - </meta> - <url value="http://highmed.org/fhir/StructureDefinition/task-base" /> - <version value="0.5.0" /> - <name value="TaskBase" /> - <status value="active" /> - <experimental value="false" /> - <date value="2021-08-24" /> - <fhirVersion value="4.0.1" /> - <kind value="resource" /> - <abstract value="true" /> - <type value="Task" /> - <baseDefinition value="http://hl7.org/fhir/StructureDefinition/Task" /> - <derivation value="constraint" /> - <differential> - <element id="Task.instantiatesUri"> - <path value="Task.instantiatesUri" /> - <min value="1" /> - </element> - <element id="Task.intent"> - <path value="Task.intent" /> - <fixedCode value="order" /> - </element> - <element id="Task.authoredOn"> - <path value="Task.authoredOn" /> - <min value="1" /> - </element> - <element id="Task.requester"> - <path value="Task.requester" /> - <min value="1" /> - <type> - <code value="Reference" /> - <targetProfile value="http://highmed.org/fhir/StructureDefinition/organization" /> - </type> - </element> - <element id="Task.requester.reference"> - <path value="Task.requester.reference" /> - <max value="0" /> - </element> - <element id="Task.requester.identifier"> - <path value="Task.requester.identifier" /> - <min value="1" /> - </element> - <element id="Task.requester.identifier.system"> - <path value="Task.requester.identifier.system" /> - <min value="1" /> - <fixedUri value="http://highmed.org/sid/organization-identifier" /> - </element> - <element id="Task.requester.identifier.value"> - <path value="Task.requester.identifier.value" /> - <min value="1" /> - </element> - <element id="Task.restriction"> - <path value="Task.restriction" /> - <min value="1" /> - </element> - <element id="Task.restriction.recipient"> - <path value="Task.restriction.recipient" /> - <min value="1" /> - <max value="1" /> - <type> - <code value="Reference" /> - <targetProfile value="http://highmed.org/fhir/StructureDefinition/organization" /> - </type> - </element> - <element id="Task.restriction.recipient.reference"> - <path value="Task.restriction.recipient.reference" /> - <max value="0" /> - </element> - <element id="Task.restriction.recipient.identifier"> - <path value="Task.restriction.recipient.identifier" /> - <min value="1" /> - </element> - <element id="Task.restriction.recipient.identifier.system"> - <path value="Task.restriction.recipient.identifier.system" /> - <min value="1" /> - <fixedUri value="http://highmed.org/sid/organization-identifier" /> - </element> - <element id="Task.restriction.recipient.identifier.value"> - <path value="Task.restriction.recipient.identifier.value" /> - <min value="1" /> - </element> - <element id="Task.input"> - <extension url="http://hl7.org/fhir/StructureDefinition/structuredefinition-explicit-type-name"> - <valueString value="Parameter" /> - </extension> - <path value="Task.input" /> - <slicing> - <discriminator> - <type value="value" /> - <path value="type.coding.system" /> - </discriminator> - <discriminator> - <type value="value" /> - <path value="type.coding.code" /> - </discriminator> - <rules value="openAtEnd" /> - </slicing> - <min value="1" /> - </element> - <element id="Task.input:message-name"> - <extension url="http://hl7.org/fhir/StructureDefinition/structuredefinition-explicit-type-name"> - <valueString value="Parameter" /> - </extension> - <path value="Task.input" /> - <sliceName value="message-name" /> - <min value="1" /> - <max value="1" /> - </element> - <element id="Task.input:message-name.type"> - <path value="Task.input.type" /> - <binding> - <extension url="http://hl7.org/fhir/StructureDefinition/elementdefinition-bindingName"> - <valueString value="TaskInputParameterType" /> - </extension> - <strength value="required" /> - <valueSet value="http://highmed.org/fhir/ValueSet/bpmn-message" /> - </binding> - </element> - <element id="Task.input:message-name.type.coding"> - <path value="Task.input.type.coding" /> - <min value="1" /> - <max value="1" /> - </element> - <element id="Task.input:message-name.type.coding.system"> - <path value="Task.input.type.coding.system" /> - <min value="1" /> - <fixedUri value="http://highmed.org/fhir/CodeSystem/bpmn-message" /> - </element> - <element id="Task.input:message-name.type.coding.code"> - <path value="Task.input.type.coding.code" /> - <min value="1" /> - <fixedCode value="message-name" /> - </element> - <element id="Task.input:message-name.value[x]"> - <path value="Task.input.value[x]" /> - <type> - <code value="string" /> - </type> - </element> - <element id="Task.input:business-key"> - <extension url="http://hl7.org/fhir/StructureDefinition/structuredefinition-explicit-type-name"> - <valueString value="Parameter" /> - </extension> - <path value="Task.input" /> - <sliceName value="business-key" /> - <max value="1" /> - </element> - <element id="Task.input:business-key.type"> - <path value="Task.input.type" /> - <binding> - <extension url="http://hl7.org/fhir/StructureDefinition/elementdefinition-bindingName"> - <valueString value="TaskInputParameterType" /> - </extension> - <strength value="required" /> - <valueSet value="http://highmed.org/fhir/ValueSet/bpmn-message" /> - </binding> - </element> - <element id="Task.input:business-key.type.coding"> - <path value="Task.input.type.coding" /> - <min value="1" /> - <max value="1" /> - </element> - <element id="Task.input:business-key.type.coding.system"> - <path value="Task.input.type.coding.system" /> - <min value="1" /> - <fixedUri value="http://highmed.org/fhir/CodeSystem/bpmn-message" /> - </element> - <element id="Task.input:business-key.type.coding.code"> - <path value="Task.input.type.coding.code" /> - <min value="1" /> - <fixedCode value="business-key" /> - </element> - <element id="Task.input:business-key.value[x]"> - <path value="Task.input.value[x]" /> - <type> - <code value="string" /> - </type> - </element> - <element id="Task.input:correlation-key"> - <extension url="http://hl7.org/fhir/StructureDefinition/structuredefinition-explicit-type-name"> - <valueString value="Parameter" /> - </extension> - <path value="Task.input" /> - <sliceName value="correlation-key" /> - <max value="1" /> - </element> - <element id="Task.input:correlation-key.type"> - <path value="Task.input.type" /> - <binding> - <extension url="http://hl7.org/fhir/StructureDefinition/elementdefinition-bindingName"> - <valueString value="TaskInputParameterType" /> - </extension> - <strength value="required" /> - <valueSet value="http://highmed.org/fhir/ValueSet/bpmn-message" /> - </binding> - </element> - <element id="Task.input:correlation-key.type.coding"> - <path value="Task.input.type.coding" /> - <min value="1" /> - <max value="1" /> - </element> - <element id="Task.input:correlation-key.type.coding.system"> - <path value="Task.input.type.coding.system" /> - <min value="1" /> - <fixedUri value="http://highmed.org/fhir/CodeSystem/bpmn-message" /> - </element> - <element id="Task.input:correlation-key.type.coding.code"> - <path value="Task.input.type.coding.code" /> - <min value="1" /> - <fixedCode value="correlation-key" /> - </element> - <element id="Task.input:correlation-key.value[x]"> - <path value="Task.input.value[x]" /> - <type> - <code value="string" /> - </type> - </element> - <element id="Task.output"> - <path value="Task.output" /> - <slicing> - <discriminator> - <type value="value" /> - <path value="type.coding.system" /> - </discriminator> - <discriminator> - <type value="value" /> - <path value="type.coding.code" /> - </discriminator> - <rules value="openAtEnd" /> - </slicing> - </element> - <element id="Task.output:error"> - <path value="Task.output" /> - <sliceName value="error" /> - </element> - <element id="Task.output:error.type"> - <path value="Task.output.type" /> - <binding> - <extension url="http://hl7.org/fhir/StructureDefinition/elementdefinition-bindingName"> - <valueString value="TaskOutputParameterType" /> - </extension> - <strength value="required" /> - <valueSet value="http://highmed.org/fhir/ValueSet/bpmn-message" /> - </binding> - </element> - <element id="Task.output:error.type.coding"> - <path value="Task.output.type.coding" /> - <min value="1" /> - <max value="1" /> - </element> - <element id="Task.output:error.type.coding.system"> - <path value="Task.output.type.coding.system" /> - <min value="1" /> - <fixedUri value="http://highmed.org/fhir/CodeSystem/bpmn-message" /> - </element> - <element id="Task.output:error.type.coding.code"> - <path value="Task.output.type.coding.code" /> - <min value="1" /> - <fixedCode value="error" /> - </element> - <element id="Task.output:error.value[x]"> - <path value="Task.output.value[x]" /> - <type> - <code value="string" /> - </type> - </element> - </differential> -</StructureDefinition> \ No newline at end of file From 60362f5cd7719a672fc868a50f351c9b6c41701e Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Mon, 6 Jun 2022 18:08:51 +0200 Subject: [PATCH 12/29] =?UTF-8?q?terminology=20server=20K=C3=B6ln=20now=20?= =?UTF-8?q?default,=20more=20debugging=20output=20if=20enabled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 7 +- codex-process-data-transfer/pom.xml | 22 +++-- .../DataTransferProcessPluginDefinition.java | 14 +++- .../data_transfer/service/ValidateData.java | 72 +++++++++------- .../spring/config/ValidationConfig.java | 84 ++++++++++++++----- .../validation/BundleValidatorFactory.java | 12 ++- .../BundleValidatorFactoryImpl.java | 7 +- .../validation/ValidationMain.java | 27 ++---- .../ValidationPackageClientJersey.java | 19 ++++- .../ValueSetExpansionClientJersey.java | 24 ++++-- pom.xml | 16 ++-- 11 files changed, 205 insertions(+), 99 deletions(-) diff --git a/.gitignore b/.gitignore index 6fcbdf64..80d27460 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,9 @@ codex-processes-ap1-docker-test-setup/**/fhir/log/*.log codex-processes-ap1-docker-test-setup/**/fhir/log/*.log.gz codex-processes-ap1-docker-test-setup/secrets/*.pem -codex-processes-ap1-docker-test-setup/.env \ No newline at end of file +codex-processes-ap1-docker-test-setup/.env + +### +# codex-process-data-transfer ignores +### +codex-process-data-transfer/application.properties \ No newline at end of file diff --git a/codex-process-data-transfer/pom.xml b/codex-process-data-transfer/pom.xml index 1b2f0ead..6324d01e 100644 --- a/codex-process-data-transfer/pom.xml +++ b/codex-process-data-transfer/pom.xml @@ -32,18 +32,24 @@ <artifactId>commons-compress</artifactId> <scope>provided</scope> </dependency> + <!-- TODO add de.hs-heilbronn.mi:log4j2-utils to dsf-bpe-process-base's dependencies --> + <dependency> + <groupId>de.hs-heilbronn.mi</groupId> + <artifactId>log4j2-utils</artifactId> + <scope>provided</scope> + </dependency> + <!-- TODO add org.slf4j:jul-to-slf4j to dsf-bpe-process-base's dependencies --> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>jul-to-slf4j</artifactId> + <scope>provided</scope> + </dependency> <!-- must be added as regular DSF plugins --> <dependency> <groupId>ca.uhn.hapi.fhir</groupId> <artifactId>hapi-fhir-client</artifactId> </dependency> - - <dependency> - <groupId>de.hs-heilbronn.mi</groupId> - <artifactId>log4j2-utils</artifactId> - <!-- <scope>test</scope> --> - </dependency> </dependencies> <build> @@ -125,7 +131,7 @@ <artifactId>hapi-fhir-client</artifactId> <version>${hapi.version}</version> </artifactItem> - + <!-- TODO vv remove when part of dsf-bpe-process-base dependencies --> <artifactItem> <groupId>commons-codec</groupId> @@ -267,7 +273,7 @@ <artifactId>commons-compress</artifactId> </artifactItem> <!-- TODO ^^ remove when part of dsf-bpe-process-base dependencies --> - + </artifactItems> <outputDirectory>../codex-processes-ap1-docker-test-setup/dic/bpe/plugin</outputDirectory> </configuration> diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java index 749e9389..17a6ef26 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java @@ -14,6 +14,8 @@ import org.highmed.dsf.fhir.resources.ResourceProvider; import org.highmed.dsf.fhir.resources.StructureDefinitionResource; import org.highmed.dsf.fhir.resources.ValueSetResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; import org.springframework.core.env.PropertyResolver; @@ -27,6 +29,8 @@ public class DataTransferProcessPluginDefinition implements ProcessPluginDefinition { + private static final Logger logger = LoggerFactory.getLogger(DataTransferProcessPluginDefinition.class); + public static final String VERSION = "0.5.0"; public static final LocalDate DATE = LocalDate.of(2021, 9, 6); @@ -125,7 +129,15 @@ public void onProcessesDeployed(ApplicationContext pluginApplicationContext, Lis if (activeProcesses.contains("wwwnetzwerk-universitaetsmedizinde_dataSend")) { - pluginApplicationContext.getBean(BundleValidatorFactory.class).init(); + boolean testOk = pluginApplicationContext.getBean(ValidationConfig.class) + .testConnectionToTerminologyServer(); + + if (testOk) + pluginApplicationContext.getBean(BundleValidatorFactory.class).init(); + else + logger.warn( + "Due to an error while testing the connection to the terminology server the {} can not be initialized, this will lead to the validation of bundles beeing skipped.", + BundleValidatorFactory.class.getSimpleName()); } } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java index a79f89a0..94e629d0 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java @@ -60,47 +60,57 @@ public void afterPropertiesSet() throws Exception @Override protected void doExecute(DelegateExecution execution) throws BpmnError, Exception { - Bundle bundle = (Bundle) execution.getVariable(BPMN_EXECUTION_VARIABLE_BUNDLE); - - logger.info("Validating bundle with {} entr{}", bundle.getEntry().size(), - bundle.getEntry().size() == 1 ? "y" : "ies"); - bundle = bundleValidatorSupplier.create().validate(bundle); - - if (bundle.hasEntry()) + bundleValidatorSupplier.create().ifPresentOrElse(validator -> { - if (bundle.getEntry().stream().anyMatch(e -> !e.hasResponse() || !e.getResponse().hasOutcome() - || !(e.getResponse().getOutcome() instanceof OperationOutcome))) - { - logger.warn( - "Validation result bundle has entries wihout response.outcome instance of OperationOutcome"); - } - else - { - logValidationDetails(bundle); + Bundle bundle = (Bundle) execution.getVariable(BPMN_EXECUTION_VARIABLE_BUNDLE); - if (bundle.getEntry().stream().map(e -> (OperationOutcome) e.getResponse().getOutcome()) - .flatMap(o -> o.getIssue().stream()).anyMatch(i -> IssueSeverity.FATAL.equals(i.getSeverity()) - || IssueSeverity.ERROR.equals(i.getSeverity()))) - { - logger.error("Validation of transfer bundle failed"); + logger.info("Validating bundle with {} entr{}", bundle.getEntry().size(), + bundle.getEntry().size() == 1 ? "y" : "ies"); - addErrorsToTaskAndSetFailed(bundle); - errorLogger.logValidationFailed(getLeadingTaskFromExecutionVariables().getIdElement() - .withServerBase(getFhirWebserviceClientProvider().getLocalBaseUrl(), - getLeadingTaskFromExecutionVariables().getIdElement().getResourceType())); + bundle = validator.validate(bundle); - throw new BpmnError(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE_VALUE_VALIDATION_FAILED); + if (bundle.hasEntry()) + { + if (bundle.getEntry().stream().anyMatch(e -> !e.hasResponse() || !e.getResponse().hasOutcome() + || !(e.getResponse().getOutcome() instanceof OperationOutcome))) + { + logger.warn( + "Validation result bundle has entries wihout response.outcome instance of OperationOutcome"); } else { - removeValidationResultsAndUserData(bundle); + logValidationDetails(bundle); + + if (bundle.getEntry().stream().map(e -> (OperationOutcome) e.getResponse().getOutcome()) + .flatMap(o -> o.getIssue().stream()) + .anyMatch(i -> IssueSeverity.FATAL.equals(i.getSeverity()) + || IssueSeverity.ERROR.equals(i.getSeverity()))) + { + logger.error("Validation of transfer bundle failed"); + + addErrorsToTaskAndSetFailed(bundle); + errorLogger.logValidationFailed(getLeadingTaskFromExecutionVariables().getIdElement() + .withServerBase(getFhirWebserviceClientProvider().getLocalBaseUrl(), + getLeadingTaskFromExecutionVariables().getIdElement().getResourceType())); + + throw new BpmnError(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE_VALUE_VALIDATION_FAILED); + } + else + { + removeValidationResultsAndUserData(bundle); + } } } - } - else + else + { + logger.warn("Validation result bundle has no entries"); + } + }, () -> { - logger.warn("Validation result bundle has no entries"); - } + logger.warn( + "{} not initialized, skipping validation. This is likley due to an error during startup of the process plugin.", + BundleValidatorFactory.class.getSimpleName()); + }); // TODO maybe check only one pseudonym used } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java index 6dd76347..f70a8a80 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java @@ -17,10 +17,13 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; +import javax.ws.rs.WebApplicationException; + import org.bouncycastle.pkcs.PKCSException; import org.highmed.dsf.fhir.json.ObjectMapperFactory; import org.highmed.dsf.fhir.validation.ValueSetExpander; import org.highmed.dsf.fhir.validation.ValueSetExpanderImpl; +import org.hl7.fhir.r4.model.CapabilityStatement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -107,11 +110,14 @@ public class ValidationConfig @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.timeout.read:300000}") private int packageClientReadTimeout; + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.verbose:false}") + private boolean packageClientVerbose; + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.cacheFolder:#{null}}") private String valueSetCacheFolder; // TODO default should be MII ontology server - @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.server.baseUrl:https://r4.ontoserver.csiro.au/fhir}") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.server.baseUrl:https://terminology-highmed.medic.medfak.uni-koeln.de/fhir}") private String valueSetExpansionServerBaseUrl; @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.trust.certificates:#{null}}") @@ -147,6 +153,9 @@ public class ValidationConfig @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.timeout.read:300000}") private int valueSetExpansionClientReadTimeout; + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.verbose:false}") + private boolean valueSetExpansionClientVerbose; + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.structuredefinition.cacheFolder:#{null}}") private String structureDefinitionCacheFolder; @@ -195,6 +204,26 @@ private StructureDefinitionModifier createStructureDefinitionModifier(String cla } } + @Bean + public BiFunction<FhirContext, IValidationSupport, PluginSnapshotGenerator> internalSnapshotGeneratorFactory() + { + return (fc, vs) -> new PluginSnapshotGeneratorWithFileSystemCache(structureDefinitionCacheFolder(), fc, + new PluginSnapshotGeneratorImpl(fc, vs)); + } + + @Bean + public Path structureDefinitionCacheFolder() + { + return cacheFolder("StructureDefinition", structureDefinitionCacheFolder); + } + + @Bean + public BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory() + { + return (fc, vs) -> new ValueSetExpanderWithFileSystemCache(valueSetCacheFolder(), fc, + new ValueSetExpanderImpl(fc, vs)); + } + private Path cacheFolder(String cacheFolderType, String cacheFolder) { try @@ -273,8 +302,8 @@ else if (clientCertificateFile == null && clientCertificatePrivateKeyFile == nul clientCertificatePrivateKeyPassword); X509Certificate certificate = PemIo.readX509CertificateFromPem(clientCertificatePath); - logger.debug("Creating key-store for {} from {} and {} with password {}", clientCertificatePath.toString(), - clientCertificatePrivateKeyPath.toString(), + logger.debug("Creating key-store for {} from {} and {} with password {}", keyStoreType, + clientCertificatePath.toString(), clientCertificatePrivateKeyPath.toString(), clientCertificatePrivateKeyPassword != null ? "***" : "null"); return CertificateHelper.toJksKeyStore(privateKey, new Certificate[] { certificate }, UUID.randomUUID().toString(), keyStorePassword); @@ -320,7 +349,8 @@ private ValidationPackageClientJersey validationPackageClientJersey() return new ValidationPackageClientJersey(packageServerBaseUrl, packageClientTrustStore, packageClientKeyStore, packageClientKeyStore == null ? null : packageClientKeyStorePassword, packageClientBasicAuthUsername, packageClientBasicAuthPassword, packageClientProxySchemeHostPort, packageClientProxyUsername, - packageClientProxyPassword, packageClientConnectTimeout, packageClientReadTimeout); + packageClientProxyPassword, packageClientConnectTimeout, packageClientReadTimeout, + packageClientVerbose); } @Bean @@ -362,7 +392,7 @@ private ValueSetExpansionClient valueSetExpansionClientJersey() valueSetExpansionClientBasicAuthUsername, valueSetExpansionClientBasicAuthPassword, valueSetExpansionClientProxySchemeHostPort, valueSetExpansionClientProxyUsername, valueSetExpansionClientProxyPassword, valueSetExpansionClientConnectTimeout, - valueSetExpansionClientReadTimeout, objectMapper(), fhirContext); + valueSetExpansionClientReadTimeout, valueSetExpansionClientVerbose, objectMapper(), fhirContext); } @Bean @@ -371,23 +401,37 @@ public ObjectMapper objectMapper() return ObjectMapperFactory.createObjectMapper(fhirContext); } - @Bean - public BiFunction<FhirContext, IValidationSupport, PluginSnapshotGenerator> internalSnapshotGeneratorFactory() + public boolean testConnectionToTerminologyServer() { - return (fc, vs) -> new PluginSnapshotGeneratorWithFileSystemCache(structureDefinitionCacheFolder(), fc, - new PluginSnapshotGeneratorImpl(fc, vs)); - } + logger.info( + "Testing connection to terminology server with {trustStorePath: {}, certificatePath: {}, privateKeyPath: {}, privateKeyPassword: {}," + + " basicAuthUsername {}, basicAuthPassword {}, serverBase: {}, proxyUrl {}, proxyUsername, proxyPassword {}}", + valueSetExpansionClientTrustCertificates, valueSetExpansionClientCertificate, + valueSetExpansionClientCertificatePrivateKey, + valueSetExpansionClientCertificatePrivateKeyPassword != null ? "***" : "null", + valueSetExpansionClientBasicAuthUsername, + valueSetExpansionClientBasicAuthPassword != null ? "***" : "null", + valueSetExpansionClientProxySchemeHostPort, valueSetExpansionClientProxySchemeHostPort, + valueSetExpansionClientProxyUsername, valueSetExpansionClientProxyPassword != null ? "***" : "null"); - @Bean - public Path structureDefinitionCacheFolder() - { - return cacheFolder("StructureDefinition", structureDefinitionCacheFolder); - } + try + { + CapabilityStatement metadata = valueSetExpansionClientJersey().getMetadata(); + logger.info("Connection test OK: {} - {}", metadata.getSoftware().getName(), + metadata.getSoftware().getVersion()); + return true; + } + catch (Exception e) + { + if (e instanceof WebApplicationException) + { + String response = ((WebApplicationException) e).getResponse().readEntity(String.class); + logger.error("Connection test failed: {} - {}", e.getMessage(), response); + } + else + logger.error("Connection test failed: {}", e.getMessage()); - @Bean - public BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory() - { - return (fc, vs) -> new ValueSetExpanderWithFileSystemCache(valueSetCacheFolder(), fc, - new ValueSetExpanderImpl(fc, vs)); + return false; + } } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java index 4651f8ef..7f2fb912 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java @@ -1,8 +1,18 @@ package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; +import java.util.Optional; + public interface BundleValidatorFactory { + /** + * Initializes the {@link BundleValidatorFactory} by downloading all necessary FHIR implementation guides, expanding + * ValueSets and generating StructureDefinition snapshots. + */ void init(); - BundleValidator create(); + /** + * @return {@link Optional#empty()} if this {@link BundleValidatorFactory} was not initialized + * @see BundleValidatorFactory#init() + */ + Optional<BundleValidator> create(); } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java index 4e8697d0..c424d9cb 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java @@ -2,6 +2,7 @@ import java.util.List; import java.util.Objects; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,10 +49,8 @@ public void init() } @Override - public BundleValidator create() + public Optional<BundleValidator> create() { - init(); - - return validationPackageManager.createBundleValidator(validationSupport); + return Optional.ofNullable(validationSupport).map(validationPackageManager::createBundleValidator); } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java index 54a6197d..b7d3807b 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java @@ -10,7 +10,6 @@ import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r4.model.Bundle; -import org.hl7.fhir.r4.model.CapabilityStatement; import org.hl7.fhir.r4.model.Resource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -118,10 +117,14 @@ public static void main(String[] args) try (AnnotationConfigApplicationContext springContext = new AnnotationConfigApplicationContext(TestConfig.class, ValidationConfig.class)) { - ValidationMain main = springContext.getBean(ValidationMain.class); + ValidationConfig config = springContext.getBean(ValidationConfig.class); + boolean testOk = config.testConnectionToTerminologyServer(); - main.testOntologyServerConnection(); - main.validate(args); + if (testOk) + { + ValidationMain main = springContext.getBean(ValidationMain.class); + main.validate(args); + } } catch (Exception e) { @@ -162,22 +165,6 @@ public void afterPropertiesSet() throws Exception Objects.requireNonNull(valueSetExpansionClient, "valueSetExpansionClient"); } - public void testOntologyServerConnection() - { - logger.info("Testing connection to ontology server"); - try - { - CapabilityStatement metadata = valueSetExpansionClient.getMetadata(); - logger.info("Connection test OK: {} - {}", metadata.getSoftware().getName(), - metadata.getSoftware().getVersion()); - } - catch (Exception e) - { - logger.info("Connection test failed: {}", e.getMessage()); - throw e; - } - } - public void validate(String[] files) { logger.info("Using validation package {}", validationPackage); diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java index 576a77d1..fc0f53c6 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageClientJersey.java @@ -5,6 +5,7 @@ import java.security.KeyStore; import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; import javax.net.ssl.SSLContext; import javax.ws.rs.WebApplicationException; @@ -17,24 +18,32 @@ import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.client.ClientProperties; import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; +import org.glassfish.jersey.logging.LoggingFeature; +import org.glassfish.jersey.logging.LoggingFeature.Verbosity; public class ValidationPackageClientJersey implements ValidationPackageClient { + private static final java.util.logging.Logger requestDebugLogger = java.util.logging.Logger + .getLogger(ValueSetExpansionClientJersey.class.getName()); + private final Client client; private final String baseUrl; public ValidationPackageClientJersey(String baseUrl) { - this(baseUrl, null, null, null, null, null, null, null, null, 0, 0); + this(baseUrl, null, null, null, null, null, null, null, null, 0, 0, false); } public ValidationPackageClientJersey(String baseUrl, KeyStore trustStore, KeyStore keyStore, char[] keyStorePassword, String basicAuthUsername, char[] basicAuthPassword, String proxySchemeHostPort, - String proxyUsername, char[] proxyPassword, int connectTimeout, int readTimeout) + String proxyUsername, char[] proxyPassword, int connectTimeout, int readTimeout, boolean logRequests) { SSLContext sslContext = null; if (trustStore != null && keyStore == null && keyStorePassword == null) sslContext = SslConfigurator.newInstance().trustStore(trustStore).createSSLContext(); + if (trustStore == null && keyStore != null && keyStorePassword != null) + sslContext = SslConfigurator.newInstance().keyStore(keyStore).keyStorePassword(keyStorePassword) + .createSSLContext(); else if (trustStore != null && keyStore != null && keyStorePassword != null) sslContext = SslConfigurator.newInstance().trustStore(trustStore).keyStore(keyStore) .keyStorePassword(keyStorePassword).createSSLContext(); @@ -61,6 +70,12 @@ else if (trustStore != null && keyStore != null && keyStorePassword != null) builder = builder.readTimeout(readTimeout, TimeUnit.MILLISECONDS).connectTimeout(connectTimeout, TimeUnit.MILLISECONDS); + if (logRequests) + { + builder = builder.register(new LoggingFeature(requestDebugLogger, Level.FINE, Verbosity.PAYLOAD_ANY, + LoggingFeature.DEFAULT_MAX_ENTITY_SIZE)); + } + client = builder.build(); this.baseUrl = baseUrl; diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java index 6a987524..ca76021c 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientJersey.java @@ -3,6 +3,7 @@ import java.security.KeyStore; import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; import javax.net.ssl.SSLContext; import javax.ws.rs.WebApplicationException; @@ -18,6 +19,8 @@ import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJaxbJsonProvider; import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJsonProvider; +import org.glassfish.jersey.logging.LoggingFeature; +import org.glassfish.jersey.logging.LoggingFeature.Verbosity; import org.highmed.dsf.fhir.adapter.CapabilityStatementJsonFhirAdapter; import org.highmed.dsf.fhir.adapter.CapabilityStatementXmlFhirAdapter; import org.highmed.dsf.fhir.adapter.OperationOutcomeJsonFhirAdapter; @@ -40,23 +43,28 @@ public class ValueSetExpansionClientJersey implements ValueSetExpansionClient { private static final Logger logger = LoggerFactory.getLogger(ValueSetExpansionClientJersey.class); + private static final java.util.logging.Logger requestDebugLogger = java.util.logging.Logger + .getLogger(ValueSetExpansionClientJersey.class.getName()); private final Client client; private final String baseUrl; public ValueSetExpansionClientJersey(String baseUrl, ObjectMapper objectMapper, FhirContext fhirContext) { - this(baseUrl, null, null, null, null, null, null, null, null, 0, 0, objectMapper, fhirContext); + this(baseUrl, null, null, null, null, null, null, null, null, 0, 0, false, objectMapper, fhirContext); } public ValueSetExpansionClientJersey(String baseUrl, KeyStore trustStore, KeyStore keyStore, char[] keyStorePassword, String basicAuthUsername, char[] basicAuthPassword, String proxySchemeHostPort, - String proxyUsername, char[] proxyPassword, int connectTimeout, int readTimeout, ObjectMapper objectMapper, - FhirContext fhirContext) + String proxyUsername, char[] proxyPassword, int connectTimeout, int readTimeout, boolean logRequests, + ObjectMapper objectMapper, FhirContext fhirContext) { SSLContext sslContext = null; if (trustStore != null && keyStore == null && keyStorePassword == null) sslContext = SslConfigurator.newInstance().trustStore(trustStore).createSSLContext(); + else if (trustStore == null && keyStore != null && keyStorePassword != null) + sslContext = SslConfigurator.newInstance().keyStore(keyStore).keyStorePassword(keyStorePassword) + .createSSLContext(); else if (trustStore != null && keyStore != null && keyStorePassword != null) sslContext = SslConfigurator.newInstance().trustStore(trustStore).keyStore(keyStore) .keyStorePassword(keyStorePassword).createSSLContext(); @@ -87,10 +95,10 @@ else if (trustStore != null && keyStore != null && keyStorePassword != null) { JacksonJaxbJsonProvider p = new JacksonJaxbJsonProvider(JacksonJsonProvider.BASIC_ANNOTATIONS); p.setMapper(objectMapper); - builder.register(p); + builder = builder.register(p); } - builder.register(new CapabilityStatementJsonFhirAdapter(fhirContext)) + builder = builder.register(new CapabilityStatementJsonFhirAdapter(fhirContext)) .register(new CapabilityStatementXmlFhirAdapter(fhirContext)) .register(new OperationOutcomeJsonFhirAdapter(fhirContext)) .register(new OperationOutcomeXmlFhirAdapter(fhirContext)) @@ -98,6 +106,12 @@ else if (trustStore != null && keyStore != null && keyStorePassword != null) .register(new ParametersXmlFhirAdapter(fhirContext)).register(new ValueSetJsonFhirAdapter(fhirContext)) .register(new ValueSetXmlFhirAdapter(fhirContext)); + if (logRequests) + { + builder = builder.register(new LoggingFeature(requestDebugLogger, Level.FINE, Verbosity.PAYLOAD_ANY, + LoggingFeature.DEFAULT_MAX_ENTITY_SIZE)); + } + client = builder.build(); this.baseUrl = baseUrl; diff --git a/pom.xml b/pom.xml index 11848572..8239d721 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,5 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>de.netzwerk-universitaetsmedizin.codex</groupId> @@ -22,6 +20,7 @@ <main.basedir>${project.basedir}</main.basedir> <hapi.version>5.1.0</hapi.version> <dsf.version>0.6.0</dsf.version> + <slf4j.version>1.8.0-beta4</slf4j.version> </properties> <description>Business processes for the NUM CODEX project (AP1) as plugins for the HiGHmed Data Sharing Framework.</description> @@ -107,7 +106,12 @@ <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> - <version>1.8.0-beta4</version> + <version>${slf4j.version}</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>jul-to-slf4j</artifactId> + <version>${slf4j.version}</version> </dependency> <dependency> @@ -115,7 +119,7 @@ <artifactId>jackson-annotations</artifactId> <version>2.12.0</version> </dependency> - <dependency> + <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> <version>1.20</version> @@ -332,4 +336,4 @@ </build> </profile> </profiles> -</project> +</project> \ No newline at end of file From dc834362289d180c3498a660825e06f4aa6e7493 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Sat, 11 Jun 2022 21:07:15 +0200 Subject: [PATCH 13/29] renamed error CodeSystem/ValueSet, removed comments, changed log level Removed "-type" suffix from the error code CodeSystem/ValueSet. Removed not needed TODO comments. Changed log level from error to warning for first try ValueSet expansion errors (errors resulting in trying to expand via external terminology server next). Changed external terminology server for the test setup. --- .../processes/data_transfer/ConstantsDataTransfer.java | 4 ++-- .../DataTransferProcessPluginDefinition.java | 6 +++--- .../error/ErrorOutputParameterGenerator.java | 8 ++++---- .../processes/data_transfer/service/ValidateData.java | 6 ++---- .../data_transfer/spring/config/ValidationConfig.java | 1 - .../validation/ValidationPackageManagerImpl.java | 2 +- ...rror-type.xml => num-codex-data-transfer-error.xml} | 10 +++++----- .../num-codex-extension-error-metadata.xml | 4 ++-- ...rror-type.xml => num-codex-data-transfer-error.xml} | 10 +++++----- .../codex/processes/fhir/profile/TaskProfileTest.java | 4 ++-- .../docker-compose.yml | 1 + 11 files changed, 27 insertions(+), 29 deletions(-) rename codex-process-data-transfer/src/main/resources/fhir/CodeSystem/{num-codex-data-transfer-error-type.xml => num-codex-data-transfer-error.xml} (74%) rename codex-process-data-transfer/src/main/resources/fhir/ValueSet/{num-codex-data-transfer-error-type.xml => num-codex-data-transfer-error.xml} (69%) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java index 964dd65d..7511f1ac 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java @@ -67,8 +67,8 @@ public interface ConstantsDataTransfer String HAPI_USER_DATA_SOURCE_ID_ELEMENT = "source-id"; - String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE = "http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-type"; - String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE_VALUE_VALIDATION_FAILED = "validation-failed"; + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR = "http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error"; + String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED = "validation-failed"; String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE = "http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-source"; String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_MEDIC = "MeDIC"; diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java index 17a6ef26..85f1bada 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java @@ -75,7 +75,7 @@ public ResourceProvider getResourceProvider(FhirContext fhirContext, ClassLoader var cD = CodeSystemResource.file("fhir/CodeSystem/num-codex-data-transfer.xml"); var cDeS = CodeSystemResource.file("fhir/CodeSystem/num-codex-data-transfer-error-source.xml"); - var cDeT = CodeSystemResource.file("fhir/CodeSystem/num-codex-data-transfer-error-type.xml"); + var cDe = CodeSystemResource.file("fhir/CodeSystem/num-codex-data-transfer-error.xml"); var nD = NamingSystemResource.file("fhir/NamingSystem/num-codex-dic-pseudonym-identifier.xml"); var nC = NamingSystemResource.file("fhir/NamingSystem/num-codex-crr-pseudonym-identifier.xml"); @@ -95,13 +95,13 @@ public ResourceProvider getResourceProvider(FhirContext fhirContext, ClassLoader var vD = ValueSetResource.file("fhir/ValueSet/num-codex-data-transfer.xml"); var vDeS = ValueSetResource.file("fhir/ValueSet/num-codex-data-transfer-error-source.xml"); - var vDeT = ValueSetResource.file("fhir/ValueSet/num-codex-data-transfer-error-type.xml"); + var vDe = ValueSetResource.file("fhir/ValueSet/num-codex-data-transfer-error.xml"); Map<String, List<AbstractResource>> resourcesByProcessKeyAndVersion = Map.of( // "wwwnetzwerk-universitaetsmedizinde_dataTrigger/" + VERSION, Arrays.asList(aTri, cD, nD, sTstaDtri, sTstoDtri, vD), // "wwwnetzwerk-universitaetsmedizinde_dataSend/" + VERSION, - Arrays.asList(aSen, cD, cDeS, cDeT, nD, nB, sTexErMe, sTstaDsen, vD, vDeS, vDeT), // + Arrays.asList(aSen, cD, cDeS, cDe, nD, nB, sTexErMe, sTstaDsen, vD, vDeS, vDe), // "wwwnetzwerk-universitaetsmedizinde_dataTranslate/" + VERSION, Arrays.asList(aTra, cD, nD, nC, sTstaDtra, vD), // "wwwnetzwerk-universitaetsmedizinde_dataReceive/" + VERSION, diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/error/ErrorOutputParameterGenerator.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/error/ErrorOutputParameterGenerator.java index 67b82ed8..ac481a8a 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/error/ErrorOutputParameterGenerator.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/error/ErrorOutputParameterGenerator.java @@ -1,9 +1,9 @@ package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.error; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_MEDIC; -import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE; -import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE_VALUE_VALIDATION_FAILED; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.EXTENSION_ERROR_METADATA; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.EXTENSION_ERROR_METADATA_REFERENCE; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.EXTENSION_ERROR_METADATA_SOURCE; @@ -41,8 +41,8 @@ private TaskOutputComponent createMeDicValidationError(IdType sourceId, Operatio Extension metaData = output.addExtension(); metaData.setUrl(EXTENSION_ERROR_METADATA); metaData.addExtension().setUrl(EXTENSION_ERROR_METADATA_TYPE) - .setValue(new Coding().setSystem(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE) - .setCode(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE_VALUE_VALIDATION_FAILED)); + .setValue(new Coding().setSystem(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR) + .setCode(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED)); metaData.addExtension().setUrl(EXTENSION_ERROR_METADATA_SOURCE) .setValue(new Coding().setSystem(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE) .setCode(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_SOURCE_VALUE_MEDIC)); diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java index 94e629d0..ea4123c8 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java @@ -1,7 +1,7 @@ package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.BPMN_EXECUTION_VARIABLE_BUNDLE; -import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE_VALUE_VALIDATION_FAILED; +import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED; import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.HAPI_USER_DATA_SOURCE_ID_ELEMENT; import java.util.Objects; @@ -93,7 +93,7 @@ protected void doExecute(DelegateExecution execution) throws BpmnError, Exceptio .withServerBase(getFhirWebserviceClientProvider().getLocalBaseUrl(), getLeadingTaskFromExecutionVariables().getIdElement().getResourceType())); - throw new BpmnError(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_TYPE_VALUE_VALIDATION_FAILED); + throw new BpmnError(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED); } else { @@ -111,8 +111,6 @@ protected void doExecute(DelegateExecution execution) throws BpmnError, Exceptio "{} not initialized, skipping validation. This is likley due to an error during startup of the process plugin.", BundleValidatorFactory.class.getSimpleName()); }); - - // TODO maybe check only one pseudonym used } private void logValidationDetails(Bundle bundle) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java index f70a8a80..08f78079 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java @@ -116,7 +116,6 @@ public class ValidationConfig @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.cacheFolder:#{null}}") private String valueSetCacheFolder; - // TODO default should be MII ontology server @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.server.baseUrl:https://terminology-highmed.medic.medfak.uni-koeln.de/fhir}") private String valueSetExpansionServerBaseUrl; diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java index 3d913885..73ff05b9 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java @@ -270,7 +270,7 @@ private void expandInternal(List<ValueSet> expandedValueSets, ValueSetExpander e } catch (Exception e) { - logger.error( + logger.warn( "Error while expanding ValueSet {}|{}: {} - {}, trying to expand via external ontolgy server next", v.getUrl(), v.getVersion(), e.getClass().getName(), e.getMessage()); diff --git a/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-type.xml b/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error.xml similarity index 74% rename from codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-type.xml rename to codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error.xml index 928a2bf6..0fce6339 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error-type.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/CodeSystem/num-codex-data-transfer-error.xml @@ -5,18 +5,18 @@ <code value="ALL" /> </tag> </meta> - <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-type" /> + <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error" /> <!-- version managed by bpe --> <version value="#{version}" /> - <name value="NumCodexDataTransferErrorType" /> - <title value="NUM-CODEX data-transfer error type" /> + <name value="NumCodexDataTransferError" /> + <title value="NUM-CODEX data-transfer error" /> <!-- status managed by bpe --> <status value="unknown" /> <experimental value="false" /> <!-- date managed by bpe --> <date value="#{date}" /> <publisher value="NUM-CODEX" /> - <description value="CodeSystem with error type values for the NUM-CODEX data-transfer processes" /> + <description value="CodeSystem with error codes for the NUM-CODEX data-transfer processes" /> <caseSensitive value="true" /> <hierarchyMeaning value="grouped-by" /> <versionNeeded value="false" /> @@ -26,5 +26,5 @@ <display value="Validation Failed" /> <definition value="Error or fatal error during validaton of FHIR resources" /> </concept> - <!-- TODO add additional error types --> + <!-- TODO add additional error codes --> </CodeSystem> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-extension-error-metadata.xml b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-extension-error-metadata.xml index d2377bf2..f4e1f9a4 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-extension-error-metadata.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/StructureDefinition/num-codex-extension-error-metadata.xml @@ -47,7 +47,7 @@ <max value="1" /> <binding> <strength value="required" /> - <valueSet value="http://www.netzwerk-universitaetsmedizin.de/fhir/ValueSet/data-transfer-error-type" /> + <valueSet value="http://www.netzwerk-universitaetsmedizin.de/fhir/ValueSet/data-transfer-error" /> </binding> </element> <element id="Extension.extension:type.url"> @@ -64,7 +64,7 @@ <element id="Extension.extension:type.value[x].system"> <path value="Extension.extension.value[x].system" /> <min value="1" /> - <fixedUri value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-type" /> + <fixedUri value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error" /> </element> <element id="Extension.extension:type.value[x].code"> <path value="Extension.extension.value[x].code" /> diff --git a/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-type.xml b/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error.xml similarity index 69% rename from codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-type.xml rename to codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error.xml index 3788fc1e..c78602f2 100644 --- a/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error-type.xml +++ b/codex-process-data-transfer/src/main/resources/fhir/ValueSet/num-codex-data-transfer-error.xml @@ -5,22 +5,22 @@ <code value="ALL" /> </tag> </meta> - <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/ValueSet/data-transfer-error-type"/> + <url value="http://www.netzwerk-universitaetsmedizin.de/fhir/ValueSet/data-transfer-error"/> <!-- version managed by bpe --> <version value="#{version}" /> - <name value="NumCodexDataTransferErrorType"/> - <title value="NUM-CODEX data-transfer error type"/> + <name value="NumCodexDataTransferError"/> + <title value="NUM-CODEX data-transfer error"/> <!-- status managed by bpe --> <status value="unknown" /> <experimental value="false"/> <!-- date managed by bpe --> <date value="#{date}"/> <publisher value="NUM-CODEX"/> - <description value="CodeSystem with with error type values for the NUM-CODEX data-transfer processes"/> + <description value="CodeSystem with with error codes for the NUM-CODEX data-transfer processes"/> <immutable value="true"/> <compose> <include> - <system value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error-type"/> + <system value="http://www.netzwerk-universitaetsmedizin.de/fhir/CodeSystem/data-transfer-error"/> </include> </compose> </ValueSet> \ No newline at end of file diff --git a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/fhir/profile/TaskProfileTest.java b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/fhir/profile/TaskProfileTest.java index 4e4f1424..93e957ca 100644 --- a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/fhir/profile/TaskProfileTest.java +++ b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/fhir/profile/TaskProfileTest.java @@ -71,10 +71,10 @@ public class TaskProfileTest "num-codex-task-stop-data-trigger.xml"), Arrays.asList("highmed-read-access-tag-0.5.0.xml", "highmed-bpmn-message-0.5.0.xml", "num-codex-data-transfer.xml", "num-codex-data-transfer-error-source.xml", - "num-codex-data-transfer-error-type.xml"), + "num-codex-data-transfer-error.xml"), Arrays.asList("highmed-read-access-tag-0.5.0.xml", "highmed-bpmn-message-0.5.0.xml", "num-codex-data-transfer.xml", "num-codex-data-transfer-error-source.xml", - "num-codex-data-transfer-error-type.xml")); + "num-codex-data-transfer-error.xml")); private ResourceValidator resourceValidator = new ResourceValidatorImpl(validationRule.getFhirContext(), validationRule.getValidationSupport()); diff --git a/codex-processes-ap1-docker-test-setup/docker-compose.yml b/codex-processes-ap1-docker-test-setup/docker-compose.yml index f854c3e1..ba6a71b1 100644 --- a/codex-processes-ap1-docker-test-setup/docker-compose.yml +++ b/codex-processes-ap1-docker-test-setup/docker-compose.yml @@ -162,6 +162,7 @@ services: DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GTH_IDENTIFIER_VALUE: Test_GTH DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_CRR_PUBLIC_KEY: /run/secrets/codex_crr_public_key.pem DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GECCO_SERVER_BASE_URL: http://dic-fhir-store:8080/fhir + DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GECCO_VALIDATION_VALUESET_EXPANSION_SERVER_BASEURL: https://r4.ontoserver.csiro.au/fhir networks: dic-bpe-frontend: dic-bpe-backend: From e9a8c81f83fa3b67fe6d6d1e958e9302498cc05e Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Sun, 12 Jun 2022 21:11:28 +0200 Subject: [PATCH 14/29] environment variable documentation, some code cleanup --- codex-process-data-transfer/pom.xml | 33 ++++++ .../spring/config/TransferDataConfig.java | 112 ++++++++++++++++-- .../spring/config/ValidationConfig.java | 74 ++++++++---- .../validation/ValidationMain.java | 2 +- .../ValidationPackageManagerImpl.java | 12 +- pom.xml | 5 + 6 files changed, 202 insertions(+), 36 deletions(-) diff --git a/codex-process-data-transfer/pom.xml b/codex-process-data-transfer/pom.xml index a90ef979..1a55cdff 100644 --- a/codex-process-data-transfer/pom.xml +++ b/codex-process-data-transfer/pom.xml @@ -20,6 +20,10 @@ <artifactId>dsf-bpe-process-base</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.highmed.dsf</groupId> + <artifactId>dsf-tools-documentation-generator</artifactId> + </dependency> <!-- TODO add org.springframework:spring-web to dsf-bpe-process-base's dependencies --> <dependency> @@ -415,6 +419,35 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>exec</goal> + </goals> + <phase>prepare-package</phase> + </execution> + </executions> + <configuration> + <executable>java</executable> + <arguments> + <argument>-classpath</argument> + <classpath/> + <argument> + org.highmed.dsf.tools.generator.DocumentationGenerator + </argument> + <argument> + de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.spring.config + </argument> + </arguments> + <includeProjectDependencies>true</includeProjectDependencies> + <addResourcesToClasspath>true</addResourcesToClasspath> + <classpathScope>compile</classpathScope> + <workingDirectory>${project.basedir}</workingDirectory> + </configuration> + </plugin> </plugins> </build> </project> \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java index 5707f6e1..726440b8 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java @@ -10,6 +10,7 @@ import org.highmed.dsf.fhir.organization.EndpointProvider; import org.highmed.dsf.fhir.organization.OrganizationProvider; import org.highmed.dsf.fhir.task.TaskHelper; +import org.highmed.dsf.tools.generator.ProcessDocumentation; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -78,126 +79,221 @@ public class TransferDataConfig @Autowired private ValidationPackageIdentifier validationPackageIdentifier; + @ProcessDocumentation(description = "PEM encoded file with trusted certificates to validate the server-certificate of the GECCO FHIR server", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, recommendation = "Use docker secret file to configure", example = "/run/secrets/gecco_fhir_server_ca.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.trust.certificates:#{null}}") private String fhirStoreTrustStore; + @ProcessDocumentation(description = "PEM encoded file with client-certificate, if GECCO FHIR server requires mutual TLS authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, recommendation = "Use docker secret file to configure", example = "/run/secrets/gecco_fhir_server_client_certificate.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.certificate:#{null}}") private String fhirStoreCertificate; + @ProcessDocumentation(description = "PEM encoded file with private-key for the client-certificate defined via `de.netzwerk.universitaetsmedizin.codex.gecco.server.certificate`", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, recommendation = "Use docker secret file to configure", example = "/run/secrets/gecco_fhir_server_client_certificate_private_key.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.private.key:#{null}}") private String fhirStorePrivateKey; + @ProcessDocumentation(description = "Password to decrypt the private-key defined via `de.netzwerk.universitaetsmedizin.codex.gecco.server.private.key`", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/gecco_fhir_server_client_certificate_private_key.pem.password") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.private.key.password:#{null}}") private char[] fhirStorePrivateKeyPassword; + @ProcessDocumentation(description = "Base URL of the GECCO FHIR server", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, example = "http://foo.bar/fhir") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.base.url:#{null}}") private String fhirStoreBaseUrl; + @ProcessDocumentation(description = "Basic authentication username to authenticate against the GECCO FHIR server, set if the server requests authentication using basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.basicauth.username:#{null}}") private String fhirStoreUsername; + @ProcessDocumentation(description = "Basic authentication password to authenticate against the GECCO FHIR server, set if the server requests authentication using basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/gecco_fhir_server_basicauth.password") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.basicauth.password:#{null}}") private String fhirStorePassword; + @ProcessDocumentation(description = "Bearer token to authenticate against the GECCO FHIR server, set if the server requests authentication using bearer token, cannot be set using docker secrets", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.bearer.token:#{null}}") private String fhirStoreBearerToken; + @ProcessDocumentation(description = "Connection timeout in milliseconds used when accessing the GECCO FHIR server, time until a connection needs to be established before aborting", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.timeout.connect:10000}") private int fhirStoreConnectTimeout; - @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.timeout.socket:10000}") - private int fhirStoreSocketTimeout; - + @ProcessDocumentation(description = "Connection request timeout in milliseconds used when requesting a connection from the connection manager while accessing the GECCO FHIR server, time until a connection needs to be established before aborting", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.timeout.connection.request:10000}") private int fhirStoreConnectionRequestTimeout; + @ProcessDocumentation(description = "Maximum period of inactivity in milliseconds between two consecutive data packets from the GECCO FHIR server, time until the server needs to send a data packet before aborting", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.timeout.socket:10000}") + private int fhirStoreSocketTimeout; + + @ProcessDocumentation(description = "GECCO FHIR Server client implementation class", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.client:de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.client.fhir.FhirBridgeClient}") private String fhirStoreClientClass; + @ProcessDocumentation(description = "To enable verbose logging of requests and replies to the GECCO FHIR server set to `true`", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.client.hapi.verbose:false}") private boolean fhirStoreHapiClientVerbose; + @ProcessDocumentation(description = "Forwarding proxy server url, set if the GECCO FHIR server can only be reached via a proxy server", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, example = "http://proxy.foo:8080") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.proxy.url:#{null}}") private String fhirStoreProxyUrl; + @ProcessDocumentation(description = "Forwarding proxy server basic authentication username, set if the GECCO FHIR server can only be reached via a proxy server that requires basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataReceive" }) @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.proxy.username:#{null}}") private String fhirStoreProxyUsername; + @ProcessDocumentation(description = "Forwarding proxy server basic authentication password, set if the GECCO FHIR server can only be reached via a proxy server that requires basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataReceive" }, recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/gecco_fhir_server_proxy_basicauth.password") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.proxy.password:#{null}}") private String fhirStoreProxyPassword; + @ProcessDocumentation(description = "To enable the use of logical references instead of chained parameters (`patient:identifier` instead of `patient.identifier`) while searching for Patients in the GECCO FHIR server set to `true`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataReceive") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.use.chained.parameter.not.logical.reference:true}") private boolean fhirStoreUseChainedParameterNotLogicalReference; + @ProcessDocumentation(description = "Location of a FHIR batch bundle used to override the internally provided bundle used to search for GECCO FHIR ressources", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.server.search.bundle.override:#{null}}") private String fhirStoreSearchBundleOverride; + @ProcessDocumentation(description = "Location of the CRR public-key e.g. [crr_public-key-pre-prod.pem](https://keys.num-codex.de/crr_public-key-pre-prod.pem) used to RSA encrypt FHIR bundles for the central repository", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Ask central repository management for the correct public key regarding the test and production environments", example = "/run/secrets/crr_public-key-pre-prod.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.crr.public.key:#{null}}") private String crrPublicKeyFile; + @ProcessDocumentation(description = "Location of the CRR private-key used to RSA decrypt FHIR bundles for the central repository", processNames = "wwwnetzwerk-universitaetsmedizinde_dataReceive", example = "/run/secrets/crr_private-key-pre-prod.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.crr.private.key:#{null}}") private String crrPrivateKeyFile; + @ProcessDocumentation(description = "DSF organization identifier of the GECCO Transfer Hub", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gth.identifier.value:hs-heilbronn.de}") private String geccoTransferHubIdentifierValue; + @ProcessDocumentation(description = "DSF organization identifier of the central research repository", processNames = "wwwnetzwerk-universitaetsmedizinde_dataTranslate") @Value("${de.netzwerk.universitaetsmedizin.codex.crr.identifier.value:num-codex.de}") private String crrIdentifierValue; - @Value("#{'${de.netzwerk.universitaetsmedizin.codex.consent.granted.oids.mdat.transfer:2.16.840.1.113883.3.1937.777.24.5.3.8,2.16.840.1.113883.3.1937.777.24.5.3.9,2.16.840.1.113883.3.1937.777.24.5.3.33,2.16.840.1.113883.3.1937.777.24.5.3.34}'.split(',')}") + @ProcessDocumentation(description = "List of expected consent provision permit codes for exporting GECCO resources", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.consent.granted.oids.mdat.transfer:" + + "2.16.840.1.113883.3.1937.777.24.5.3.8," + "2.16.840.1.113883.3.1937.777.24.5.3.9," + + "2.16.840.1.113883.3.1937.777.24.5.3.33," + "2.16.840.1.113883.3.1937.777.24.5.3.34" + + "}'.trim().split('(,[ ]?)|(\\n)')}") private List<String> mdatTransferGrantedOids; - @Value("#{'${de.netzwerk.universitaetsmedizin.codex.consent.granted.oids.idat.merge:2.16.840.1.113883.3.1937.777.24.5.3.4}'.split(',')}") + @ProcessDocumentation(description = "List of expected consent provision permit codes for requesting a pseudonym based on a bloom filter from the fTTP", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.consent.granted.oids.idat.merge:" + + "2.16.840.1.113883.3.1937.777.24.5.3.4}'.trim().split('(,[ ]?)|(\\n)')}") private List<String> idatMergeGrantedOids; + @ProcessDocumentation(description = "PEM encoded file with trusted certificates to validate the server-certificate of the fTTP server", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Use docker secret file to configure", example = "/run/secrets/fttp_server_ca.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.trust.certificates:#{null}}") private String fttpTrustStore; + @ProcessDocumentation(description = "PEM encoded file with client-certificate used to authenticated against fTTP server", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Use docker secret file to configure", example = "/run/secrets/fttp_server_client_certificate.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.certificate:#{null}}") private String fttpCertificate; + @ProcessDocumentation(description = "PEM encoded file with private-key for the client-certificate defined via `de.netzwerk.universitaetsmedizin.codex.fttp.certificate`", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Use docker secret file to configure", example = "/run/secrets/fttp_server_client_certificate_private_key.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.private.key:#{null}}") private String fttpPrivateKey; + @ProcessDocumentation(description = "Password to decrypt the private-key defined via `de.netzwerk.universitaetsmedizin.codex.fttp.private.key`", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/fttp_server_client_certificate_private_key.pem.password") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.private.key.password:#{null}}") private char[] fttpPrivateKeyPassword; + @ProcessDocumentation(description = "Connection timeout in milliseconds used when accessing the fTTP server, time until a connection needs to be established before aborting", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.timeout.connect:10000}") private int fttpConnectTimeout; - @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.timeout.socket:10000}") - private int fttpSocketTimeout; - + @ProcessDocumentation(description = "Connection request timeout in milliseconds used when requesting a connection from the connection manager while accessing the fTTP server, time until a connection needs to be established before aborting", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.timeout.connection.request:10000}") private int fttpConnectionRequestTimeout; + @ProcessDocumentation(description = "Maximum period of inactivity in milliseconds between two consecutive data packets from the fTTP server, time until the server needs to send a data packet before aborting", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) + @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.timeout.socket:10000}") + private int fttpSocketTimeout; + + @ProcessDocumentation(description = "Basic authentication username to authenticate against the fTTP server, set if the server requests authentication using basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.basicauth.username:#{null}}") private String fttpBasicAuthUsername; + @ProcessDocumentation(description = "Basic authentication password to authenticate against the fTTP server, set if the server requests authentication using basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/fttp_server_basicauth.password") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.basicauth.password:#{null}}") private String fttpBasicAuthPassword; + @ProcessDocumentation(description = "TODO", processNames = { "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Specify if you are using the send process to request pseudonyms from the fTTP") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.server.base.url:#{null}}") private String fttpServerBase; + @ProcessDocumentation(description = "Your organizations API key provided by the fTTP, the fTTP API key can not be defined via docker secret file and needs to be defined directly via the environment variable", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Specify if you are using the send process to request pseudonyms from the fTTP") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.api.key:#{null}}") private String fttpApiKey; + @ProcessDocumentation(description = "Study identifier specified by the fTTP", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.study:num}") private String fttpStudy; + @ProcessDocumentation(description = "Pseudonymization domain target identifier specified by the fTTP", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, example = "dic_heidelberg", recommendation = "Specify if you are using the send process to request pseudonyms from the fTTP") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.target:codex}") private String fttpTarget; + @ProcessDocumentation(description = "To enable verbose logging of requests and replies to the fTTP server set to `true`", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.client.hapi.verbose:false}") private boolean fttpHapiClientVerbose; + @ProcessDocumentation(description = "Forwarding proxy server url, set if the fTTP server can only be reached via a proxy server", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, example = "http://proxy.foo:8080") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.proxy.url:#{null}}") private String fttpProxyUrl; + @ProcessDocumentation(description = "Forwarding proxy server basic authentication username, set if the fTTPserver can only be reached via a proxy server that requires basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }) @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.proxy.username:#{null}}") private String fttpProxyUsername; + @ProcessDocumentation(description = "Forwarding proxy server basic authentication password, set if the fTTP server can only be reached via a proxy server that requires basic authentication", processNames = { + "wwwnetzwerk-universitaetsmedizinde_dataSend", + "wwwnetzwerk-universitaetsmedizinde_dataTranslate" }, recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/fttp_server_proxy_basicauth.password") @Value("${de.netzwerk.universitaetsmedizin.codex.fttp.proxy.password:#{null}}") private String fttpProxyPassword; diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java index 08f78079..3fb8ae18 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java @@ -13,6 +13,7 @@ import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.List; +import java.util.Objects; import java.util.UUID; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -23,6 +24,7 @@ import org.highmed.dsf.fhir.json.ObjectMapperFactory; import org.highmed.dsf.fhir.validation.ValueSetExpander; import org.highmed.dsf.fhir.validation.ValueSetExpanderImpl; +import org.highmed.dsf.tools.generator.ProcessDocumentation; import org.hl7.fhir.r4.model.CapabilityStatement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,104 +60,136 @@ public class ValidationConfig { private static final Logger logger = LoggerFactory.getLogger(ValidationConfig.class); + @ProcessDocumentation(description = "FHIR implementation guide package used to validated resources, specify as `name|version`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package:de.gecco|1.0.5}") private String validationPackage; - @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.structureDefinitionModifierClasses:" - + "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover," - + "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.MiiModuleLabObservationLab10IdentifierRemover," - + "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.GeccoRadiologyProceduresCodingSliceMinFixer" - + "}'.trim().split('(,[ ]?)|(\\n)')}") - private List<String> structureDefinitionModifierClasses; - - @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.packagesToIgnore:hl7.fhir.r4.core|4.0.1}'.trim().split('(,[ ]?)|(\\n)')}") - private List<String> packagesToIgnore; + @ProcessDocumentation(description = "FHIR implementation guide packages that do not need to be downloaded, list with `name|version` values", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.noDownload:hl7.fhir.r4.core|4.0.1}'.trim().split('(,[ ]?)|(\\n)')}") + private List<String> noDownloadPackages; - @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.cacheFolder:#{null}}") + @ProcessDocumentation(description = "Folder for storing downloaded FHIR implementation guide packages", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.cacheFolder:${java.io.tmpdir}/codex_gecco_validation_cache/Package}") private String packageCacheFolder; + @ProcessDocumentation(description = "Base The base address of the FHIR repository containing GECCO data.URL of the FHIR implementation guide package server to download validation packages from", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.server.baseUrl:https://packages.simplifier.net}") private String packageServerBaseUrl; + @ProcessDocumentation(description = "PEM encoded file with trusted certificates to validate the server-certificate of the FHIR implementation guide package server, uses the JVMs trust store if not specified", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure", example = "/run/secrets/validation_package_server_ca.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.trust.certificates:#{null}}") private String packageClientTrustCertificates; + @ProcessDocumentation(description = "PEM encoded file with client-certificate, if the FHIR implementation guide package server requires mutual TLS authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure", example = "/run/secrets/validation_package_server_client_certificate.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate:#{null}}") private String packageClientCertificate; + @ProcessDocumentation(description = "PEM encoded file with private-key for the client-certificate defined via `de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure", example = "/run/secrets/validation_package_server_client_certificate_private_key.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate.private.key:#{null}}") private String packageClientCertificatePrivateKey; + @ProcessDocumentation(description = "Password to decrypt the private-key defined via `de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate.private.key`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/validation_package_server_client_certificate_private_key.pem.password") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.certificate.private.key.password:#{null}}") private char[] packageClientCertificatePrivateKeyPassword; + @ProcessDocumentation(description = "Basic authentication username to authenticate against the FHIR implementation guide package server, set if the server requests authentication using basic authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.basic.username:#{null}}") private String packageClientBasicAuthUsername; + @ProcessDocumentation(description = "Basic authentication password to authenticate against the FHIR implementation guide package server, set if the server requests authentication using basic authentication ", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/validation_package_server_basicauth.password") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.authentication.basic.password:#{null}}") private char[] packageClientBasicAuthPassword; + @ProcessDocumentation(description = "Forwarding proxy server url, set if the FHIR implementation guide package server can only be reached via a proxy server", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", example = "http://proxy.foo:8080") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.proxy.schemeHostPort:#{null}}") private String packageClientProxySchemeHostPort; + @ProcessDocumentation(description = "Forwarding proxy server basic authentication username, set if the FHIR implementation guide package server can only be reached via a proxy server that requires basic authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.proxy.username:#{null}}") private String packageClientProxyUsername; + @ProcessDocumentation(description = "Forwarding proxy server basic authentication password, set if the FHIR implementation guide package server can only be reached via a proxy server that requires basic authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/validation_package_server_proxy_basicauth.password") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.proxy.password:#{null}}") private char[] packageClientProxyPassword; + @ProcessDocumentation(description = "Connection timeout in milliseconds used when accessing the FHIR implementation guide package server, time until a connection needs to be established before aborting", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.timeout.connect:10000}") private int packageClientConnectTimeout; + @ProcessDocumentation(description = "Read timeout in milliseconds used when accessing the FHIR implementation guide package server, time until the server needs to send a reply before aborting", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.timeout.read:300000}") private int packageClientReadTimeout; + @ProcessDocumentation(description = "To enable verbose logging of requests and replies to the FHIR implementation guide package server set to `true`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.verbose:false}") private boolean packageClientVerbose; - @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.cacheFolder:#{null}}") + @ProcessDocumentation(description = "Folder for storing expanded ValueSets", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.cacheFolder:${java.io.tmpdir}/codex_gecco_validation_cache/ValueSet}") private String valueSetCacheFolder; + @ProcessDocumentation(description = "Base URL of the terminology server used to expand FHIR ValueSets", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Specify a local terminology server to improve ValueSet expansion speed") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.server.baseUrl:https://terminology-highmed.medic.medfak.uni-koeln.de/fhir}") private String valueSetExpansionServerBaseUrl; + @ProcessDocumentation(description = "PEM encoded file with trusted certificates to validate the server-certificate of the terminology server, uses the JVMs trust store if not specified", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure", example = "/run/secrets/terminology_server_ca.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.trust.certificates:#{null}}") private String valueSetExpansionClientTrustCertificates; + @ProcessDocumentation(description = "PEM encoded file with client-certificate, if the terminology server requires mutual TLS authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure", example = "/run/secrets/terminology_server_client_certificate.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate:#{null}}") private String valueSetExpansionClientCertificate; + @ProcessDocumentation(description = "PEM encoded file with private-key for the client-certificate defined via `de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure", example = "/run/secrets/terminology_server_client_certificate_private_key.pem") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate.private.key:#{null}}") private String valueSetExpansionClientCertificatePrivateKey; + @ProcessDocumentation(description = "Password to decrypt the private-key defined via `de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate.private.key`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/terminology_server_client_certificate_private_key.pem.password") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.certificate.private.key.password:#{null}}") private char[] valueSetExpansionClientCertificatePrivateKeyPassword; + @ProcessDocumentation(description = "Basic authentication username to authenticate against the terminology server, set if the server requests authentication using basic authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.basic.username:#{null}}") private String valueSetExpansionClientBasicAuthUsername; + @ProcessDocumentation(description = "Basic authentication password to authenticate against the terminology server, set if the server requests authentication using basic authentication ", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/terminology_server_basicauth.password") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.authentication.basic.password:#{null}}") private char[] valueSetExpansionClientBasicAuthPassword; + @ProcessDocumentation(description = "Forwarding proxy server url, set if the terminology server can only be reached via a proxy server", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", example = "http://proxy.foo:8080") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.proxy.schemeHostPort:#{null}}") private String valueSetExpansionClientProxySchemeHostPort; + @ProcessDocumentation(description = "Forwarding proxy server basic authentication username, set if the terminology server can only be reached via a proxy server that requires basic authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.proxy.username:#{null}}") private String valueSetExpansionClientProxyUsername; + @ProcessDocumentation(description = "Forwarding proxy server basic authentication password, set if the terminology server can only be reached via a proxy server that requires basic authentication", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend", recommendation = "Use docker secret file to configure by using `${env_variable}_FILE`", example = "/run/secrets/terminology_server_proxy_basicauth.password") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.proxy.password:#{null}}") private char[] valueSetExpansionClientProxyPassword; + @ProcessDocumentation(description = "Connection timeout in milliseconds used when accessing the terminology server, time until a connection needs to be established before aborting", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.timeout.connect:10000}") private int valueSetExpansionClientConnectTimeout; + @ProcessDocumentation(description = "Read timeout in milliseconds used when accessing the terminology server, time until the server needs to send a reply before aborting", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.timeout.read:300000}") private int valueSetExpansionClientReadTimeout; + @ProcessDocumentation(description = "To enable verbose logging of requests and replies to the terminology server set to `true`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.expansion.client.verbose:false}") private boolean valueSetExpansionClientVerbose; - @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.structuredefinition.cacheFolder:#{null}}") + @ProcessDocumentation(description = "List of StructureDefinition modifier classes, modifiers are executed before atempting to generate a StructureDefinition snapshot", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.structuredefinition.modifierClasses:" + + "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover," + + "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.MiiModuleLabObservationLab10IdentifierRemover," + + "de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.GeccoRadiologyProceduresCodingSliceMinFixer" + + "}'.trim().split('(,[ ]?)|(\\n)')}") + private List<String> structureDefinitionModifierClasses; + + @ProcessDocumentation(description = "Folder for storing generated StructureDefinition snapshots", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.structuredefinition.cacheFolder:${java.io.tmpdir}/codex_gecco_validation_cache/StructureDefinition}") private String structureDefinitionCacheFolder; @Value("${java.io.tmpdir}") @@ -181,7 +215,7 @@ public ValidationPackageManager validationPackageManager() return new ValidationPackageManagerImpl(validationPackageClient(), valueSetExpansionClient(), objectMapper(), fhirContext, internalSnapshotGeneratorFactory(), internalValueSetExpanderFactory(), - packagesToIgnore.stream().map(ValidationPackageIdentifier::fromString).collect(Collectors.toList()), + noDownloadPackages.stream().map(ValidationPackageIdentifier::fromString).collect(Collectors.toList()), structureDefinitionModifiers); } @@ -225,17 +259,15 @@ public BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalVal private Path cacheFolder(String cacheFolderType, String cacheFolder) { + Objects.requireNonNull(cacheFolder, "cacheFolder"); + Path cacheFolderPath = Paths.get(cacheFolder); + try { - Path cacheFolderPath; - if (cacheFolder != null) - cacheFolderPath = Paths.get(cacheFolder); - else + if (cacheFolderPath.startsWith(systemTempFolder)) { - cacheFolderPath = Paths.get(systemTempFolder).resolve("codex_gecco_validation_cache") - .resolve(cacheFolderType); Files.createDirectories(cacheFolderPath); - logger.debug("Cache folder for typ {} created at {}", cacheFolderType, + logger.debug("Cache folder for type {} created at {}", cacheFolderType, cacheFolderPath.toAbsolutePath().toString()); } @@ -415,7 +447,7 @@ public boolean testConnectionToTerminologyServer() try { - CapabilityStatement metadata = valueSetExpansionClientJersey().getMetadata(); + CapabilityStatement metadata = valueSetExpansionClient().getMetadata(); logger.info("Connection test OK: {} - {}", metadata.getSoftware().getName(), metadata.getSoftware().getVersion()); return true; diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java index b7d3807b..ec110767 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationMain.java @@ -197,7 +197,7 @@ private Stream<String> getAllNumProperties() .flatMap(p -> Arrays.stream(p.getPropertyNames()) .filter(n -> n.startsWith("de.netzwerk.universitaetsmedizin")) .map(k -> new String[] { k, Objects.toString(p.getProperty(k)) })) - .map(e -> e[0].contains("password") ? new String[] { e[0], "*****" } : e).map(e -> e[0] + ": " + e[1]); + .map(e -> e[0].contains("password") ? new String[] { e[0], "***" } : e).map(e -> e[0] + ": " + e[1]); } private IParser getOutputParser() diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java index 73ff05b9..6f37bc81 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java @@ -67,7 +67,7 @@ public class ValidationPackageManagerImpl implements InitializingBean, Validatio private final BiFunction<FhirContext, IValidationSupport, PluginSnapshotGenerator> internalSnapshotGeneratorFactory; private final BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory; - private final List<ValidationPackageIdentifier> packagesToIgnore = new ArrayList<>(); + private final List<ValidationPackageIdentifier> noDownloadPackages = new ArrayList<>(); private final List<StructureDefinitionModifier> structureDefinitionModifiers = new ArrayList<>(); public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, @@ -85,7 +85,7 @@ public ValidationPackageManagerImpl(ValidationPackageClient validationPackageCli ValueSetExpansionClient valueSetExpansionClient, ObjectMapper mapper, FhirContext fhirContext, BiFunction<FhirContext, IValidationSupport, PluginSnapshotGenerator> internalSnapshotGeneratorFactory, BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory, - Collection<ValidationPackageIdentifier> packagesToIgnore, + Collection<ValidationPackageIdentifier> noDownloadPackages, Collection<? extends StructureDefinitionModifier> structureDefinitionModifiers) { this.validationPackageClient = validationPackageClient; @@ -95,8 +95,8 @@ public ValidationPackageManagerImpl(ValidationPackageClient validationPackageCli this.internalSnapshotGeneratorFactory = internalSnapshotGeneratorFactory; this.internalValueSetExpanderFactory = internalValueSetExpanderFactory; - if (packagesToIgnore != null) - this.packagesToIgnore.addAll(packagesToIgnore); + if (noDownloadPackages != null) + this.noDownloadPackages.addAll(noDownloadPackages); if (structureDefinitionModifiers != null) this.structureDefinitionModifiers.addAll(structureDefinitionModifiers); @@ -166,9 +166,9 @@ private void downloadPackageWithDependencies(ValidationPackageIdentifier identif // already downloaded return; } - else if (packagesToIgnore.contains(identifier)) + else if (noDownloadPackages.contains(identifier)) { - logger.debug("Ignoring package {}", identifier.toString()); + logger.debug("Not downloading package {}", identifier.toString()); return; } diff --git a/pom.xml b/pom.xml index a89b98a5..c4bae9cf 100644 --- a/pom.xml +++ b/pom.xml @@ -78,6 +78,11 @@ <artifactId>dsf-fhir-server</artifactId> <version>${dsf.version}</version> </dependency> + <dependency> + <groupId>org.highmed.dsf</groupId> + <artifactId>dsf-tools-documentation-generator</artifactId> + <version>${dsf.version}</version> + </dependency> <dependency> <groupId>ca.uhn.hapi.fhir</groupId> From 2197f925af5dae4e2ef0b42387b367f805ae8172 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Tue, 14 Jun 2022 01:11:23 +0200 Subject: [PATCH 15/29] added docker-compose file to override dsf images with local builds To start the dic-fhir service with a locally build DSF image use: docker-compose -f docker-compose.yml -f docker-compose.local-dsf-build.yml up dic-fhir --- .../docker-compose.local-dsf-build.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 codex-processes-ap1-docker-test-setup/docker-compose.local-dsf-build.yml diff --git a/codex-processes-ap1-docker-test-setup/docker-compose.local-dsf-build.yml b/codex-processes-ap1-docker-test-setup/docker-compose.local-dsf-build.yml new file mode 100644 index 00000000..a2608f54 --- /dev/null +++ b/codex-processes-ap1-docker-test-setup/docker-compose.local-dsf-build.yml @@ -0,0 +1,16 @@ +version: '3.8' +services: + dic-fhir: + image: highmed/fhir + dic-bpe: + image: highmed/bpe + + gth-fhir: + image: highmed/fhir + gth-bpe: + image: highmed/bpe + + crr-fhir: + image: highmed/fhir + crr-bpe: + image: highmed/bpe \ No newline at end of file From 4cdaffd4cefe3a0c933dbf272e32d6d8a086ccb9 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Tue, 14 Jun 2022 01:14:15 +0200 Subject: [PATCH 16/29] switched to DSF 0.7.0-SNAPSHOT, remove not needed code and dependencies These changes depend on highmed/highmed-dsf#354 being merged. --- codex-process-data-transfer/pom.xml | 174 ------------------ .../spring/config/ValidationConfig.java | 4 +- .../validation/PluginSnapshotGenerator.java | 8 - .../PluginSnapshotGeneratorImpl.java | 52 +----- ...nSnapshotGeneratorWithFileSystemCache.java | 33 ++-- .../PluginSnapshotWithValidationMessages.java | 30 --- .../ValidationPackageManagerImpl.java | 14 +- .../service/ValidateDataLearningTest.java | 8 +- pom.xml | 2 +- 9 files changed, 39 insertions(+), 286 deletions(-) delete mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGenerator.java delete mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotWithValidationMessages.java diff --git a/codex-process-data-transfer/pom.xml b/codex-process-data-transfer/pom.xml index 1a55cdff..679f59d7 100644 --- a/codex-process-data-transfer/pom.xml +++ b/codex-process-data-transfer/pom.xml @@ -25,37 +25,6 @@ <artifactId>dsf-tools-documentation-generator</artifactId> </dependency> - <!-- TODO add org.springframework:spring-web to dsf-bpe-process-base's dependencies --> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-web</artifactId> - <scope>provided</scope> - </dependency> - <!-- TODO add org.highmed.dsf:dsf-fhir-validation to dsf-bpe-process-base's dependencies --> - <dependency> - <groupId>org.highmed.dsf</groupId> - <artifactId>dsf-fhir-validation</artifactId> - <scope>provided</scope> - </dependency> - <!-- TODO add org.apache.commons:commons-compress to dsf-bpe-process-base's dependencies --> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - <scope>provided</scope> - </dependency> - <!-- TODO add de.hs-heilbronn.mi:log4j2-utils to dsf-bpe-process-base's dependencies --> - <dependency> - <groupId>de.hs-heilbronn.mi</groupId> - <artifactId>log4j2-utils</artifactId> - <scope>provided</scope> - </dependency> - <!-- TODO add org.slf4j:jul-to-slf4j to dsf-bpe-process-base's dependencies --> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jul-to-slf4j</artifactId> - <scope>provided</scope> - </dependency> - <!-- must be added as regular DSF plugins --> <dependency> <groupId>ca.uhn.hapi.fhir</groupId> @@ -148,149 +117,6 @@ <artifactId>hapi-fhir-client</artifactId> <version>${hapi.version}</version> </artifactItem> - - <!-- TODO vv remove when part of dsf-bpe-process-base dependencies --> - <artifactItem> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - <version>1.12</version> - </artifactItem> - <artifactItem> - <groupId>ca.uhn.hapi.fhir</groupId> - <artifactId>hapi-fhir-structures-r5</artifactId> - <version>${hapi.version}</version> - </artifactItem> - <artifactItem> - <groupId>ca.uhn.hapi.fhir</groupId> - <artifactId>org.hl7.fhir.r5</artifactId> - <version>${hapi.version}</version> - </artifactItem> - <artifactItem> - <groupId>ca.uhn.hapi.fhir</groupId> - <artifactId>hapi-fhir-validation</artifactId> - <version>${hapi.version}</version> - </artifactItem> - <artifactItem> - <groupId>ca.uhn.hapi.fhir</groupId> - <artifactId>hapi-fhir-converter</artifactId> - <version>${hapi.version}</version> - </artifactItem> - <artifactItem> - <groupId>ca.uhn.hapi.fhir</groupId> - <artifactId>org.hl7.fhir.convertors</artifactId> - <version>${hapi.version}</version> - </artifactItem> - <artifactItem> - <groupId>net.sf.saxon</groupId> - <artifactId>Saxon-HE</artifactId> - <version>9.5.1-5</version> - </artifactItem> - <artifactItem> - <groupId>ca.uhn.hapi.fhir</groupId> - <artifactId>org.hl7.fhir.validation</artifactId> - <version>${hapi.version}</version> - </artifactItem> - <artifactItem> - <groupId>ca.uhn.hapi.fhir</groupId> - <artifactId>org.hl7.fhir.dstu2</artifactId> - <version>${hapi.version}</version> - </artifactItem> - <artifactItem> - <groupId>ca.uhn.hapi.fhir</groupId> - <artifactId>org.hl7.fhir.dstu2016may</artifactId> - <version>${hapi.version}</version> - </artifactItem> - <artifactItem> - <groupId>ca.uhn.hapi.fhir</groupId> - <artifactId>org.hl7.fhir.dstu3</artifactId> - <version>${hapi.version}</version> - </artifactItem> - <artifactItem> - <groupId>xpp3</groupId> - <artifactId>xpp3</artifactId> - <version>1.1.4c</version> - </artifactItem> - <artifactItem> - <groupId>xpp3</groupId> - <artifactId>xpp3_xpath</artifactId> - <version>1.1.4c</version> - </artifactItem> - <artifactItem> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - <version>1.21</version> - </artifactItem> - <artifactItem> - <groupId>org.fhir</groupId> - <artifactId>ucum</artifactId> - <version>1.0.2</version> - </artifactItem> - <artifactItem> - <groupId>org.thymeleaf</groupId> - <artifactId>thymeleaf</artifactId> - <version>3.0.9.RELEASE</version> - </artifactItem> - <artifactItem> - <groupId>ognl</groupId> - <artifactId>ognl</artifactId> - <version>3.1.12</version> - </artifactItem> - <artifactItem> - <groupId>org.javassist</groupId> - <artifactId>javassist</artifactId> - <version>3.20.0-GA</version> - </artifactItem> - <artifactItem> - <groupId>org.attoparser</groupId> - <artifactId>attoparser</artifactId> - <version>2.0.4.RELEASE</version> - </artifactItem> - <artifactItem> - <groupId>org.unbescape</groupId> - <artifactId>unbescape</artifactId> - <version>1.1.5.RELEASE</version> - </artifactItem> - <artifactItem> - <groupId>com.github.ben-manes.caffeine</groupId> - <artifactId>caffeine</artifactId> - <version>2.7.0</version> - </artifactItem> - <artifactItem> - <groupId>org.checkerframework</groupId> - <artifactId>checker-qual</artifactId> - <version>2.6.0</version> - </artifactItem> - <artifactItem> - <groupId>com.google.errorprone</groupId> - <artifactId>error_prone_annotations</artifactId> - <version>2.3.3</version> - </artifactItem> - <artifactItem> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - <version>2.8.5</version> - </artifactItem> - <artifactItem> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - <version>1.2</version> - </artifactItem> - <artifactItem> - <groupId>ca.uhn.hapi.fhir</groupId> - <artifactId>hapi-fhir-validation-resources-r4</artifactId> - <version>${hapi.version}</version> - </artifactItem> - <artifactItem> - <groupId>ca.uhn.hapi.fhir</groupId> - <artifactId>hapi-fhir-validation-resources-r5</artifactId> - <version>${hapi.version}</version> - </artifactItem> - <artifactItem> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - </artifactItem> - <!-- TODO ^^ remove when part of dsf-bpe-process-base dependencies --> - </artifactItems> <outputDirectory>../codex-processes-ap1-docker-test-setup/dic/bpe/plugin</outputDirectory> </configuration> diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java index 3fb8ae18..d3eabd3c 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java @@ -22,6 +22,7 @@ import org.bouncycastle.pkcs.PKCSException; import org.highmed.dsf.fhir.json.ObjectMapperFactory; +import org.highmed.dsf.fhir.validation.SnapshotGenerator; import org.highmed.dsf.fhir.validation.ValueSetExpander; import org.highmed.dsf.fhir.validation.ValueSetExpanderImpl; import org.highmed.dsf.tools.generator.ProcessDocumentation; @@ -37,7 +38,6 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.support.IValidationSupport; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGenerator; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorImpl; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithFileSystemCache; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClient; @@ -238,7 +238,7 @@ private StructureDefinitionModifier createStructureDefinitionModifier(String cla } @Bean - public BiFunction<FhirContext, IValidationSupport, PluginSnapshotGenerator> internalSnapshotGeneratorFactory() + public BiFunction<FhirContext, IValidationSupport, SnapshotGenerator> internalSnapshotGeneratorFactory() { return (fc, vs) -> new PluginSnapshotGeneratorWithFileSystemCache(structureDefinitionCacheFolder(), fc, new PluginSnapshotGeneratorImpl(fc, vs)); diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGenerator.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGenerator.java deleted file mode 100644 index d1f68006..00000000 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGenerator.java +++ /dev/null @@ -1,8 +0,0 @@ -package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; - -import org.hl7.fhir.r4.model.StructureDefinition; - -public interface PluginSnapshotGenerator -{ - PluginSnapshotWithValidationMessages generateSnapshot(StructureDefinition differential); -} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorImpl.java index 5295d893..72ef5470 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorImpl.java @@ -1,29 +1,17 @@ package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -import org.hl7.fhir.r4.conformance.ProfileUtilities; +import org.highmed.dsf.fhir.validation.SnapshotGeneratorImpl; import org.hl7.fhir.r4.context.IWorkerContext; import org.hl7.fhir.r4.hapi.ctx.HapiWorkerContext; -import org.hl7.fhir.r4.model.StructureDefinition; -import org.hl7.fhir.utilities.validation.ValidationMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.support.IValidationSupport; -public class PluginSnapshotGeneratorImpl implements PluginSnapshotGenerator +public class PluginSnapshotGeneratorImpl extends SnapshotGeneratorImpl { - private static final Logger logger = LoggerFactory.getLogger(PluginSnapshotGeneratorImpl.class); - - private final IWorkerContext worker; - public PluginSnapshotGeneratorImpl(FhirContext fhirContext, IValidationSupport validationSupport) { - worker = createWorker(fhirContext, validationSupport); + super(fhirContext, validationSupport); } protected IWorkerContext createWorker(FhirContext context, IValidationSupport validationSupport) @@ -32,38 +20,4 @@ protected IWorkerContext createWorker(FhirContext context, IValidationSupport va workerContext.setLocale(context.getLocalizer().getLocale()); return new PluginWorkerContext(workerContext); } - - @Override - public PluginSnapshotWithValidationMessages generateSnapshot(StructureDefinition differential) - { - Objects.requireNonNull(differential, "differential"); - - logger.debug("Generating snapshot for StructureDefinition with id {}, url {}, version {}, base {}", - differential.getIdElement().getIdPart(), differential.getUrl(), differential.getVersion(), - differential.getBaseDefinition()); - - StructureDefinition base = worker.fetchResource(StructureDefinition.class, differential.getBaseDefinition()); - - if (base == null) - logger.warn("Base definition with url {} not found", differential.getBaseDefinition()); - - /* ProfileUtilities is not thread safe */ - List<ValidationMessage> messages = new ArrayList<>(); - ProfileUtilities profileUtils = new ProfileUtilities(worker, messages, null); - - profileUtils.generateSnapshot(base, differential, "", "", null); - - if (messages.isEmpty()) - logger.debug("Snapshot generated for StructureDefinition with id {}, url {}, version {}", - differential.getIdElement().getIdPart(), differential.getUrl(), differential.getVersion()); - else - { - logger.warn("Snapshot not generated for StructureDefinition with id {}, url {}, version {}", - differential.getIdElement().getIdPart(), differential.getUrl(), differential.getVersion()); - messages.forEach(m -> logger.warn("Issue while generating snapshot: {} - {} - {}", m.getDisplay(), - m.getLine(), m.getMessage())); - } - - return new PluginSnapshotWithValidationMessages(differential, messages); - } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java index 2686b8bf..dc3ce969 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java @@ -7,6 +7,8 @@ import java.util.Collections; import java.util.Objects; +import org.highmed.dsf.fhir.validation.SnapshotGenerator; +import org.highmed.dsf.fhir.validation.SnapshotGenerator.SnapshotWithValidationMessages; import org.hl7.fhir.r4.model.Enumerations.PublicationStatus; import org.hl7.fhir.r4.model.StructureDefinition; import org.slf4j.Logger; @@ -16,12 +18,12 @@ import ca.uhn.fhir.context.FhirContext; public class PluginSnapshotGeneratorWithFileSystemCache - extends AbstractFhirResourceFileSystemCache<PluginSnapshotWithValidationMessages, StructureDefinition> - implements PluginSnapshotGenerator, InitializingBean + extends AbstractFhirResourceFileSystemCache<SnapshotWithValidationMessages, StructureDefinition> + implements SnapshotGenerator, InitializingBean { private static final Logger logger = LoggerFactory.getLogger(ValidationPackageClientWithFileSystemCache.class); - private final PluginSnapshotGenerator delegate; + private final SnapshotGenerator delegate; /** * For JSON content with gzip compression using the <code>.json.xz</code> file name suffix. @@ -37,7 +39,7 @@ public class PluginSnapshotGeneratorWithFileSystemCache * @see AbstractFileSystemCache#IN_COMPRESSOR_FACTORY */ public PluginSnapshotGeneratorWithFileSystemCache(Path cacheFolder, FhirContext fhirContext, - PluginSnapshotGenerator delegate) + SnapshotGenerator delegate) { super(cacheFolder, StructureDefinition.class, fhirContext); @@ -47,7 +49,7 @@ public PluginSnapshotGeneratorWithFileSystemCache(Path cacheFolder, FhirContext public PluginSnapshotGeneratorWithFileSystemCache(Path cacheFolder, String fileNameSuffix, FunctionWithIoException<OutputStream, OutputStream> outCompressorFactory, FunctionWithIoException<InputStream, InputStream> inCompressorFactory, FhirContext fhirContext, - PluginSnapshotGenerator delegate) + SnapshotGenerator delegate) { super(cacheFolder, fileNameSuffix, outCompressorFactory, inCompressorFactory, StructureDefinition.class, fhirContext); @@ -64,7 +66,7 @@ public void afterPropertiesSet() throws Exception } @Override - public PluginSnapshotWithValidationMessages generateSnapshot(StructureDefinition structureDefinition) + public SnapshotWithValidationMessages generateSnapshot(StructureDefinition structureDefinition) { Objects.requireNonNull(structureDefinition, "differential"); @@ -72,7 +74,7 @@ public PluginSnapshotWithValidationMessages generateSnapshot(StructureDefinition { logger.debug("StructureDefinition {}|{} has snapshot", structureDefinition.getUrl(), structureDefinition.getVersion()); - return new PluginSnapshotWithValidationMessages(structureDefinition, Collections.emptyList()); + return new SnapshotWithValidationMessages(structureDefinition, Collections.emptyList()); } Objects.requireNonNull(structureDefinition.getUrl(), "structureDefinition.url"); @@ -80,10 +82,10 @@ public PluginSnapshotWithValidationMessages generateSnapshot(StructureDefinition try { - PluginSnapshotWithValidationMessages read = readResourceFromCache(structureDefinition.getUrl(), + SnapshotWithValidationMessages read = readResourceFromCache(structureDefinition.getUrl(), structureDefinition.getVersion(), // needs to return original structureDefinition object with included snapshot - sd -> new PluginSnapshotWithValidationMessages(structureDefinition.setSnapshot(sd.getSnapshot()), + sd -> new SnapshotWithValidationMessages(structureDefinition.setSnapshot(sd.getSnapshot()), Collections.emptyList())); if (read != null) return read; @@ -96,10 +98,10 @@ public PluginSnapshotWithValidationMessages generateSnapshot(StructureDefinition } } - private PluginSnapshotWithValidationMessages downloadAndWriteToCache(StructureDefinition structureDefinition) + private SnapshotWithValidationMessages downloadAndWriteToCache(StructureDefinition structureDefinition) throws IOException { - PluginSnapshotWithValidationMessages snapshot = delegate.generateSnapshot(structureDefinition); + SnapshotWithValidationMessages snapshot = delegate.generateSnapshot(structureDefinition); if (PublicationStatus.DRAFT.equals(snapshot.getSnapshot().getStatus())) { @@ -109,7 +111,14 @@ private PluginSnapshotWithValidationMessages downloadAndWriteToCache(StructureDe return snapshot; } else - return writeRsourceToCache(snapshot, PluginSnapshotWithValidationMessages::getSnapshot, + return writeRsourceToCache(snapshot, SnapshotWithValidationMessages::getSnapshot, StructureDefinition::getUrl, StructureDefinition::getVersion); } + + @Override + public SnapshotWithValidationMessages generateSnapshot(StructureDefinition differential, + String baseAbsoluteUrlPrefix) + { + throw new UnsupportedOperationException("not implemented"); + } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotWithValidationMessages.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotWithValidationMessages.java deleted file mode 100644 index b09e382b..00000000 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotWithValidationMessages.java +++ /dev/null @@ -1,30 +0,0 @@ -package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; - -import java.util.Collections; -import java.util.List; -import java.util.Objects; - -import org.hl7.fhir.r4.model.StructureDefinition; -import org.hl7.fhir.utilities.validation.ValidationMessage; - -public class PluginSnapshotWithValidationMessages -{ - private final StructureDefinition snapshot; - private final List<ValidationMessage> messages; - - PluginSnapshotWithValidationMessages(StructureDefinition snapshot, List<ValidationMessage> messages) - { - this.snapshot = Objects.requireNonNull(snapshot, "snapshot"); - this.messages = Objects.requireNonNull(messages, "messages"); - } - - public StructureDefinition getSnapshot() - { - return snapshot; - } - - public List<ValidationMessage> getMessages() - { - return Collections.unmodifiableList(messages); - } -} \ No newline at end of file diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java index 6f37bc81..34b5f8ba 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java @@ -20,6 +20,8 @@ import javax.ws.rs.WebApplicationException; import org.highmed.dsf.fhir.validation.ResourceValidatorImpl; +import org.highmed.dsf.fhir.validation.SnapshotGenerator; +import org.highmed.dsf.fhir.validation.SnapshotGenerator.SnapshotWithValidationMessages; import org.highmed.dsf.fhir.validation.ValidationSupportWithCustomResources; import org.highmed.dsf.fhir.validation.ValueSetExpander; import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService; @@ -64,7 +66,7 @@ public class ValidationPackageManagerImpl implements InitializingBean, Validatio private final ObjectMapper mapper; private final FhirContext fhirContext; - private final BiFunction<FhirContext, IValidationSupport, PluginSnapshotGenerator> internalSnapshotGeneratorFactory; + private final BiFunction<FhirContext, IValidationSupport, SnapshotGenerator> internalSnapshotGeneratorFactory; private final BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory; private final List<ValidationPackageIdentifier> noDownloadPackages = new ArrayList<>(); @@ -72,7 +74,7 @@ public class ValidationPackageManagerImpl implements InitializingBean, Validatio public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, ValueSetExpansionClient valueSetExpansionClient, ObjectMapper mapper, FhirContext fhirContext, - BiFunction<FhirContext, IValidationSupport, PluginSnapshotGenerator> internalSnapshotGeneratorFactory, + BiFunction<FhirContext, IValidationSupport, SnapshotGenerator> internalSnapshotGeneratorFactory, BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory) { this(validationPackageClient, valueSetExpansionClient, mapper, fhirContext, internalSnapshotGeneratorFactory, @@ -83,7 +85,7 @@ public ValidationPackageManagerImpl(ValidationPackageClient validationPackageCli public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, ValueSetExpansionClient valueSetExpansionClient, ObjectMapper mapper, FhirContext fhirContext, - BiFunction<FhirContext, IValidationSupport, PluginSnapshotGenerator> internalSnapshotGeneratorFactory, + BiFunction<FhirContext, IValidationSupport, SnapshotGenerator> internalSnapshotGeneratorFactory, BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory, Collection<ValidationPackageIdentifier> noDownloadPackages, Collection<? extends StructureDefinitionModifier> structureDefinitionModifiers) @@ -300,7 +302,7 @@ private IValidationSupport withSnapshots(List<ValidationSupportResources> resour ValidationSupportChain supportChain = createSupportChain(fhirContext, resources, snapshots.values(), expandedValueSets); - PluginSnapshotGenerator generator = internalSnapshotGeneratorFactory.apply(fhirContext, supportChain); + SnapshotGenerator generator = internalSnapshotGeneratorFactory.apply(fhirContext, supportChain); resources.stream().flatMap(r -> r.getStructureDefinitions().stream()) .filter(s -> s.hasDifferential() && !s.hasSnapshot()) @@ -323,7 +325,7 @@ private ValidationSupportChain createSupportChain(FhirContext context, List<Vali } private void createSnapshot(Map<String, StructureDefinition> structureDefinitionsByUrl, - Map<String, StructureDefinition> snapshots, PluginSnapshotGenerator generator, StructureDefinition diff) + Map<String, StructureDefinition> snapshots, SnapshotGenerator generator, StructureDefinition diff) { if (snapshots.containsKey(diff.getUrl() + "|" + diff.getVersion())) return; @@ -352,7 +354,7 @@ private void createSnapshot(Map<String, StructureDefinition> structureDefinition { structureDefinitionModifiers.forEach(f -> f.modify(diff)); - PluginSnapshotWithValidationMessages snapshot = generator.generateSnapshot(diff); + SnapshotWithValidationMessages snapshot = generator.generateSnapshot(diff); if (snapshot.getMessages().isEmpty()) snapshots.put(snapshot.getSnapshot().getUrl() + "|" + snapshot.getSnapshot().getVersion(), diff --git a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateDataLearningTest.java b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateDataLearningTest.java index ea91e49a..acbfce0f 100644 --- a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateDataLearningTest.java +++ b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateDataLearningTest.java @@ -15,6 +15,8 @@ import java.util.stream.Collectors; import org.highmed.dsf.fhir.json.ObjectMapperFactory; +import org.highmed.dsf.fhir.validation.SnapshotGenerator; +import org.highmed.dsf.fhir.validation.SnapshotGenerator.SnapshotWithValidationMessages; import org.highmed.dsf.fhir.validation.ValidationSupportWithCustomResources; import org.highmed.dsf.fhir.validation.ValueSetExpanderImpl; import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService; @@ -39,10 +41,8 @@ import ca.uhn.fhir.context.support.DefaultProfileValidationSupport; import ca.uhn.fhir.context.support.IValidationSupport; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidator; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGenerator; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorImpl; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithFileSystemCache; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotWithValidationMessages; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackage; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClient; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClientJersey; @@ -224,7 +224,7 @@ public void testGenerateSnapshots() throws Exception .equals(s.getUrl())) .findFirst().get(); - PluginSnapshotGenerator sGen = new PluginSnapshotGeneratorImpl(fhirContext, new ValidationSupportChain( + SnapshotGenerator sGen = new PluginSnapshotGeneratorImpl(fhirContext, new ValidationSupportChain( new InMemoryTerminologyServerValidationSupport(fhirContext), new ValidationSupportChain(validationPackages.stream() .map(ValidationPackage::getValidationSupportResources) @@ -234,7 +234,7 @@ public void testGenerateSnapshots() throws Exception new DefaultProfileValidationSupport(fhirContext), new CommonCodeSystemsTerminologyService(fhirContext))); - PluginSnapshotWithValidationMessages result = sGen.generateSnapshot(miiRef); + SnapshotWithValidationMessages result = sGen.generateSnapshot(miiRef); result.getMessages().forEach(m -> { diff --git a/pom.xml b/pom.xml index c4bae9cf..1247484b 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ <main.basedir>${project.basedir}</main.basedir> <hapi.version>5.1.0</hapi.version> - <dsf.version>0.6.0</dsf.version> + <dsf.version>0.7.0-SNAPSHOT</dsf.version> <slf4j.version>1.8.0-beta4</slf4j.version> </properties> From eda676772e010df2eba8f2348094271ba0ce9942 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Tue, 14 Jun 2022 17:04:13 +0200 Subject: [PATCH 17/29] removed not needed dependency management entries, some cleanup Increased version numbers of log42-utils and crypto-utils dependencies to the latest versions. --- codex-process-data-transfer/pom.xml | 2 +- pom.xml | 28 +++------------------------- 2 files changed, 4 insertions(+), 26 deletions(-) diff --git a/codex-process-data-transfer/pom.xml b/codex-process-data-transfer/pom.xml index 679f59d7..fc82d444 100644 --- a/codex-process-data-transfer/pom.xml +++ b/codex-process-data-transfer/pom.xml @@ -25,7 +25,7 @@ <artifactId>dsf-tools-documentation-generator</artifactId> </dependency> - <!-- must be added as regular DSF plugins --> + <!-- must be added as regular DSF plugin --> <dependency> <groupId>ca.uhn.hapi.fhir</groupId> <artifactId>hapi-fhir-client</artifactId> diff --git a/pom.xml b/pom.xml index 1247484b..0a966a82 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,6 @@ <main.basedir>${project.basedir}</main.basedir> <hapi.version>5.1.0</hapi.version> <dsf.version>0.7.0-SNAPSHOT</dsf.version> - <slf4j.version>1.8.0-beta4</slf4j.version> </properties> <description>Business processes for the NUM CODEX project (AP1) as plugins for the HiGHmed Data Sharing Framework.</description> @@ -99,40 +98,19 @@ <dependency> <groupId>de.hs-heilbronn.mi</groupId> <artifactId>log4j2-utils</artifactId> - <version>0.12.0</version> + <version>0.13.0</version> </dependency> <dependency> <groupId>de.hs-heilbronn.mi</groupId> <artifactId>crypto-utils</artifactId> - <version>3.2.0</version> + <version>3.3.0</version> </dependency> <!-- logging --> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> - <version>${slf4j.version}</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jul-to-slf4j</artifactId> - <version>${slf4j.version}</version> - </dependency> - - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - <version>2.12.0</version> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - <version>1.20</version> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-web</artifactId> - <version>5.3.19</version> + <version>1.8.0-beta4</version> </dependency> <!-- testing --> From dc93f4f92d0b980c2a212e1a596e1cc7c508e4d9 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Thu, 16 Jun 2022 00:39:09 +0200 Subject: [PATCH 18/29] log fixes, code cleanup, new SnapshotGeneration wrapper for profile mods --- .../spring/config/ValidationConfig.java | 13 ++-- ...nSnapshotGeneratorWithFileSystemCache.java | 4 +- .../PluginSnapshotGeneratorWithModifiers.java | 71 +++++++++++++++++++ .../ValidationPackageManagerImpl.java | 26 ++----- ...SetExpansionClientWithFileSystemCache.java | 6 +- .../ClosedTypeSlicingRemover.java | 2 +- ...eLabObservationLab10IdentifierRemover.java | 2 +- .../ValidateDataLearningTest.java | 19 +---- 8 files changed, 91 insertions(+), 52 deletions(-) create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithModifiers.java rename codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/{service => validation}/ValidateDataLearningTest.java (89%) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java index d3eabd3c..c1103e6b 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java @@ -40,6 +40,7 @@ import ca.uhn.fhir.context.support.IValidationSupport; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorImpl; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithFileSystemCache; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithModifiers; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClient; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClientJersey; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClientWithFileSystemCache; @@ -210,13 +211,9 @@ public ValidationPackageIdentifier validationPackageIdentifier() @Bean public ValidationPackageManager validationPackageManager() { - List<StructureDefinitionModifier> structureDefinitionModifiers = structureDefinitionModifierClasses.stream() - .map(this::createStructureDefinitionModifier).collect(Collectors.toList()); - return new ValidationPackageManagerImpl(validationPackageClient(), valueSetExpansionClient(), objectMapper(), fhirContext, internalSnapshotGeneratorFactory(), internalValueSetExpanderFactory(), - noDownloadPackages.stream().map(ValidationPackageIdentifier::fromString).collect(Collectors.toList()), - structureDefinitionModifiers); + noDownloadPackages.stream().map(ValidationPackageIdentifier::fromString).collect(Collectors.toList())); } private StructureDefinitionModifier createStructureDefinitionModifier(String className) @@ -240,8 +237,12 @@ private StructureDefinitionModifier createStructureDefinitionModifier(String cla @Bean public BiFunction<FhirContext, IValidationSupport, SnapshotGenerator> internalSnapshotGeneratorFactory() { + List<StructureDefinitionModifier> structureDefinitionModifiers = structureDefinitionModifierClasses.stream() + .map(this::createStructureDefinitionModifier).collect(Collectors.toList()); + return (fc, vs) -> new PluginSnapshotGeneratorWithFileSystemCache(structureDefinitionCacheFolder(), fc, - new PluginSnapshotGeneratorImpl(fc, vs)); + new PluginSnapshotGeneratorWithModifiers(new PluginSnapshotGeneratorImpl(fc, vs), + structureDefinitionModifiers)); } @Bean diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java index dc3ce969..5feca93d 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java @@ -90,7 +90,7 @@ public SnapshotWithValidationMessages generateSnapshot(StructureDefinition struc if (read != null) return read; else - return downloadAndWriteToCache(structureDefinition); + return generateSnapshotAndWriteToCache(structureDefinition); } catch (IOException e) { @@ -98,7 +98,7 @@ public SnapshotWithValidationMessages generateSnapshot(StructureDefinition struc } } - private SnapshotWithValidationMessages downloadAndWriteToCache(StructureDefinition structureDefinition) + private SnapshotWithValidationMessages generateSnapshotAndWriteToCache(StructureDefinition structureDefinition) throws IOException { SnapshotWithValidationMessages snapshot = delegate.generateSnapshot(structureDefinition); diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithModifiers.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithModifiers.java new file mode 100644 index 00000000..f1c48bc5 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithModifiers.java @@ -0,0 +1,71 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Objects; + +import org.highmed.dsf.fhir.validation.SnapshotGenerator; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.springframework.beans.factory.InitializingBean; + +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.GeccoRadiologyProceduresCodingSliceMinFixer; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.MiiModuleLabObservationLab10IdentifierRemover; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.StructureDefinitionModifier; + +public class PluginSnapshotGeneratorWithModifiers implements SnapshotGenerator, InitializingBean +{ + public static final StructureDefinitionModifier CLOSED_TYPE_SLICING_REMOVER = new ClosedTypeSlicingRemover(); + public static final StructureDefinitionModifier MII_MODULE_LAB_OBSERVATION_LAB_1_0_IDENTIFIER_REMOVER = new MiiModuleLabObservationLab10IdentifierRemover(); + public static final StructureDefinitionModifier GECCO_RADIOLOGY_PROCEDURES_CODING_SLICE_MIN_FIXER = new GeccoRadiologyProceduresCodingSliceMinFixer(); + + private final SnapshotGenerator delegate; + private final List<StructureDefinitionModifier> structureDefinitionModifiers = new ArrayList<>(); + + public PluginSnapshotGeneratorWithModifiers(SnapshotGenerator delegate) + { + this(delegate, Arrays.asList(CLOSED_TYPE_SLICING_REMOVER, MII_MODULE_LAB_OBSERVATION_LAB_1_0_IDENTIFIER_REMOVER, + GECCO_RADIOLOGY_PROCEDURES_CODING_SLICE_MIN_FIXER)); + } + + public PluginSnapshotGeneratorWithModifiers(SnapshotGenerator delegate, + Collection<? extends StructureDefinitionModifier> structureDefinitionModifiers) + { + this.delegate = delegate; + + if (structureDefinitionModifiers != null) + this.structureDefinitionModifiers.addAll(structureDefinitionModifiers); + } + + @Override + public void afterPropertiesSet() throws Exception + { + Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public SnapshotWithValidationMessages generateSnapshot(StructureDefinition differential) + { + return delegate.generateSnapshot(modify(differential)); + } + + @Override + public SnapshotWithValidationMessages generateSnapshot(StructureDefinition differential, + String baseAbsoluteUrlPrefix) + { + return delegate.generateSnapshot(modify(differential), baseAbsoluteUrlPrefix); + } + + private StructureDefinition modify(StructureDefinition differential) + { + if (differential == null) + return null; + + for (StructureDefinitionModifier mod : structureDefinitionModifiers) + differential = mod.modify(differential); + + return differential; + } +} diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java index 34b5f8ba..ab17380d 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -44,22 +43,14 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.support.DefaultProfileValidationSupport; import ca.uhn.fhir.context.support.IValidationSupport; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.ClosedTypeSlicingRemover; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.GeccoRadiologyProceduresCodingSliceMinFixer; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.MiiModuleLabObservationLab10IdentifierRemover; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.structure_definition.StructureDefinitionModifier; public class ValidationPackageManagerImpl implements InitializingBean, ValidationPackageManager { - static final Logger logger = LoggerFactory.getLogger(ValidationPackageManagerImpl.class); + private static final Logger logger = LoggerFactory.getLogger(ValidationPackageManagerImpl.class); public static final List<ValidationPackageIdentifier> PACKAGE_IGNORE = List .of(new ValidationPackageIdentifier("hl7.fhir.r4.core", "4.0.1")); - public static final StructureDefinitionModifier CLOSED_TYPE_SLICING_REMOVER = new ClosedTypeSlicingRemover(); - public static final StructureDefinitionModifier MII_MODULE_LAB_OBSERVATION_LAB_1_0_IDENTIFIER_REMOVER = new MiiModuleLabObservationLab10IdentifierRemover(); - public static final StructureDefinitionModifier GECCO_RADIOLOGY_PROCEDURES_CODING_SLICE_MIN_FIXER = new GeccoRadiologyProceduresCodingSliceMinFixer(); - private final ValidationPackageClient validationPackageClient; private final ValueSetExpansionClient valueSetExpansionClient; @@ -70,7 +61,6 @@ public class ValidationPackageManagerImpl implements InitializingBean, Validatio private final BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory; private final List<ValidationPackageIdentifier> noDownloadPackages = new ArrayList<>(); - private final List<StructureDefinitionModifier> structureDefinitionModifiers = new ArrayList<>(); public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, ValueSetExpansionClient valueSetExpansionClient, ObjectMapper mapper, FhirContext fhirContext, @@ -78,17 +68,14 @@ public ValidationPackageManagerImpl(ValidationPackageClient validationPackageCli BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory) { this(validationPackageClient, valueSetExpansionClient, mapper, fhirContext, internalSnapshotGeneratorFactory, - internalValueSetExpanderFactory, PACKAGE_IGNORE, - Arrays.asList(CLOSED_TYPE_SLICING_REMOVER, MII_MODULE_LAB_OBSERVATION_LAB_1_0_IDENTIFIER_REMOVER, - GECCO_RADIOLOGY_PROCEDURES_CODING_SLICE_MIN_FIXER)); + internalValueSetExpanderFactory, PACKAGE_IGNORE); } public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, ValueSetExpansionClient valueSetExpansionClient, ObjectMapper mapper, FhirContext fhirContext, BiFunction<FhirContext, IValidationSupport, SnapshotGenerator> internalSnapshotGeneratorFactory, BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory, - Collection<ValidationPackageIdentifier> noDownloadPackages, - Collection<? extends StructureDefinitionModifier> structureDefinitionModifiers) + Collection<ValidationPackageIdentifier> noDownloadPackages) { this.validationPackageClient = validationPackageClient; this.valueSetExpansionClient = valueSetExpansionClient; @@ -99,9 +86,6 @@ public ValidationPackageManagerImpl(ValidationPackageClient validationPackageCli if (noDownloadPackages != null) this.noDownloadPackages.addAll(noDownloadPackages); - - if (structureDefinitionModifiers != null) - this.structureDefinitionModifiers.addAll(structureDefinitionModifiers); } @Override @@ -273,7 +257,7 @@ private void expandInternal(List<ValueSet> expandedValueSets, ValueSetExpander e catch (Exception e) { logger.warn( - "Error while expanding ValueSet {}|{}: {} - {}, trying to expand via external ontolgy server next", + "Error while expanding ValueSet {}|{}: {} - {}, trying to expand via external terminology server next", v.getUrl(), v.getVersion(), e.getClass().getName(), e.getMessage()); expandExternal(expandedValueSets, v); @@ -352,8 +336,6 @@ private void createSnapshot(Map<String, StructureDefinition> structureDefinition try { - structureDefinitionModifiers.forEach(f -> f.modify(diff)); - SnapshotWithValidationMessages snapshot = generator.generateSnapshot(diff); if (snapshot.getMessages().isEmpty()) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java index 3a4811d6..9f74d38b 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValueSetExpansionClientWithFileSystemCache.java @@ -78,16 +78,15 @@ public ValueSet expand(ValueSet valueSet) throws IOException, WebApplicationExce Objects.requireNonNull(valueSet.getUrl(), "valueSet.url"); Objects.requireNonNull(valueSet.getVersion(), "valueSet.version"); - // ValueSet read = readFromCache(valueSet.getUrl(), valueSet.getVersion(), ValueSet.class, Function.identity()); ValueSet read = readResourceFromCache(valueSet.getUrl(), valueSet.getVersion(), Function.identity()); if (read != null) return read; else - return downloadAndWriteToCache(valueSet); + return expandAndWriteToCache(valueSet); } - private ValueSet downloadAndWriteToCache(ValueSet valueSet) throws IOException + private ValueSet expandAndWriteToCache(ValueSet valueSet) throws IOException { ValueSet expanded = delegate.expand(valueSet); @@ -98,7 +97,6 @@ private ValueSet downloadAndWriteToCache(ValueSet valueSet) throws IOException return expanded; } else - // return writeToCache(expanded, Function.identity(), ValueSet::getUrl, ValueSet::getVersion); return writeRsourceToCache(expanded, Function.identity(), ValueSet::getUrl, ValueSet::getVersion); } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java index c3df1e98..7e7a957a 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/ClosedTypeSlicingRemover.java @@ -28,7 +28,7 @@ public StructureDefinition modify(StructureDefinition sd) if (DiscriminatorType.TYPE.equals(discriminator.getType()) && "$this".equals(discriminator.getPath())) { logger.warn( - "Removing Type slicing with slicing.rules != closed validation rule with id {} in StructureDefinition {}|{}", + "Removing Type slicing with slicing.rules != closed from validation rule with id {} in StructureDefinition {}|{}", e.getId(), sd.getUrl(), sd.getVersion()); e.setSlicing(null); diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java index 1edfa7e6..d505280a 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/structure_definition/MiiModuleLabObservationLab10IdentifierRemover.java @@ -28,7 +28,7 @@ public StructureDefinition modify(StructureDefinition sd) List<ElementDefinition> filteredRules = sd.getDifferential().getElement().stream().filter(toRemove.negate()) .collect(Collectors.toList()); - logger.warn("Removing validation rules with ids {} in StructureDefinition {}|{}", + logger.warn("Removing validation rules with ids {} from StructureDefinition {}|{}", sd.getDifferential().getElement().stream().filter(toRemove).map(ElementDefinition::getId) .collect(Collectors.joining(", ", "[", "]")), sd.getUrl(), sd.getVersion()); diff --git a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateDataLearningTest.java b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidateDataLearningTest.java similarity index 89% rename from codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateDataLearningTest.java rename to codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidateDataLearningTest.java index acbfce0f..e4113415 100644 --- a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateDataLearningTest.java +++ b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidateDataLearningTest.java @@ -1,4 +1,4 @@ -package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service; +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -40,20 +40,6 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.support.DefaultProfileValidationSupport; import ca.uhn.fhir.context.support.IValidationSupport; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidator; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorImpl; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithFileSystemCache; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackage; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClient; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClientJersey; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageClientWithFileSystemCache; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageDescriptor; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageManager; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageManagerImpl; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpanderWithFileSystemCache; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClient; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClientJersey; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValueSetExpansionClientWithFileSystemCache; public class ValidateDataLearningTest { @@ -234,6 +220,8 @@ public void testGenerateSnapshots() throws Exception new DefaultProfileValidationSupport(fhirContext), new CommonCodeSystemsTerminologyService(fhirContext))); + sGen = new PluginSnapshotGeneratorWithModifiers(sGen); + SnapshotWithValidationMessages result = sGen.generateSnapshot(miiRef); result.getMessages().forEach(m -> @@ -256,7 +244,6 @@ else if (IssueSeverity.WARNING.equals(m.getLevel())) validationPackages.stream().flatMap(p -> p.getValidationSupportResources().getStructureDefinitions().stream()) .forEach(s -> { - logger.info("StructureDefinition {}|{}:", s.getUrl(), s.getVersion()); printTree(s, sDefByUrl); logger.debug(""); From ab3d581e24272fa8c796252daf246fb9d4123964 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Thu, 16 Jun 2022 00:41:18 +0200 Subject: [PATCH 19/29] added Package, StructureDefinition, ValueSet cache to test setup --- .gitignore | 2 ++ .../dic/bpe/cache/Package/README.md | 1 + .../dic/bpe/cache/StructureDefinition/README.md | 1 + .../dic/bpe/cache/ValueSet/README.md | 1 + codex-processes-ap1-docker-test-setup/docker-compose.yml | 6 ++++++ 5 files changed, 11 insertions(+) create mode 100644 codex-processes-ap1-docker-test-setup/dic/bpe/cache/Package/README.md create mode 100644 codex-processes-ap1-docker-test-setup/dic/bpe/cache/StructureDefinition/README.md create mode 100644 codex-processes-ap1-docker-test-setup/dic/bpe/cache/ValueSet/README.md diff --git a/.gitignore b/.gitignore index 80d27460..461addf8 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,8 @@ codex-processes-ap1-docker-test-setup/**/bpe/last_event/time.file codex-processes-ap1-docker-test-setup/**/bpe/plugin/*.jar codex-processes-ap1-docker-test-setup/**/bpe/process/*.jar +codex-processes-ap1-docker-test-setup/dic/bpe/cache/**/*.json.gz + codex-processes-ap1-docker-test-setup/**/fhir/conf/bundle.xml codex-processes-ap1-docker-test-setup/**/fhir/log/*.log codex-processes-ap1-docker-test-setup/**/fhir/log/*.log.gz diff --git a/codex-processes-ap1-docker-test-setup/dic/bpe/cache/Package/README.md b/codex-processes-ap1-docker-test-setup/dic/bpe/cache/Package/README.md new file mode 100644 index 00000000..2cd95e7f --- /dev/null +++ b/codex-processes-ap1-docker-test-setup/dic/bpe/cache/Package/README.md @@ -0,0 +1 @@ +empty directory for FHIR package cache \ No newline at end of file diff --git a/codex-processes-ap1-docker-test-setup/dic/bpe/cache/StructureDefinition/README.md b/codex-processes-ap1-docker-test-setup/dic/bpe/cache/StructureDefinition/README.md new file mode 100644 index 00000000..5f7d118e --- /dev/null +++ b/codex-processes-ap1-docker-test-setup/dic/bpe/cache/StructureDefinition/README.md @@ -0,0 +1 @@ +empty directory for StructureDefinition Snapshots cache \ No newline at end of file diff --git a/codex-processes-ap1-docker-test-setup/dic/bpe/cache/ValueSet/README.md b/codex-processes-ap1-docker-test-setup/dic/bpe/cache/ValueSet/README.md new file mode 100644 index 00000000..f15571d9 --- /dev/null +++ b/codex-processes-ap1-docker-test-setup/dic/bpe/cache/ValueSet/README.md @@ -0,0 +1 @@ +empty directory for expanded ValueSet cache \ No newline at end of file diff --git a/codex-processes-ap1-docker-test-setup/docker-compose.yml b/codex-processes-ap1-docker-test-setup/docker-compose.yml index ba6a71b1..85451bf4 100644 --- a/codex-processes-ap1-docker-test-setup/docker-compose.yml +++ b/codex-processes-ap1-docker-test-setup/docker-compose.yml @@ -141,6 +141,9 @@ services: - type: bind source: ./dic/bpe/last_event target: /opt/bpe/last_event + - type: bind + source: ./dic/bpe/cache + target: /opt/bpe/cache environment: TZ: Europe/Berlin EXTRA_JVM_ARGS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5003 @@ -162,6 +165,9 @@ services: DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GTH_IDENTIFIER_VALUE: Test_GTH DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_CRR_PUBLIC_KEY: /run/secrets/codex_crr_public_key.pem DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GECCO_SERVER_BASE_URL: http://dic-fhir-store:8080/fhir + DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GECCO_VALIDATION_PACKAGE_CACHEFOLDER: /opt/bpe/cache/Package/ + DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GECCO_VALIDATION_STRUCTUREDEFINITION_CACHEFOLDER: /opt/bpe/cache/StructureDefinition/ + DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GECCO_VALIDATION_VALUESET_CACHEFOLDER: /opt/bpe/cache/ValueSet/ DE_NETZWERK_UNIVERSITAETSMEDIZIN_CODEX_GECCO_VALIDATION_VALUESET_EXPANSION_SERVER_BASEURL: https://r4.ontoserver.csiro.au/fhir networks: dic-bpe-frontend: From 18bea38efe8a79d89695e8d5a280e941bc5b7ae2 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Thu, 16 Jun 2022 04:28:04 +0200 Subject: [PATCH 20/29] reworked StructureDefinition snapshot generation and ValueSet expansion Snapshots are only generate for StructureDefinitions from the root validation package and its dependencies. ValueSet are only expanded if they are part of binding rules from these StructureDefinitions. The binding strength can be configured. --- .../spring/config/ValidationConfig.java | 15 +- .../BundleValidatorFactoryImpl.java | 5 +- .../validation/ValidationPackageManager.java | 27 +- .../ValidationPackageManagerImpl.java | 191 ++++++-------- .../ValidationPackageWithDepedencies.java | 232 ++++++++++++++++++ .../validation/ValidateDataLearningTest.java | 57 ++--- 6 files changed, 364 insertions(+), 163 deletions(-) create mode 100644 codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageWithDepedencies.java diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java index c1103e6b..b827118c 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java @@ -12,6 +12,7 @@ import java.security.cert.Certificate; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.util.EnumSet; import java.util.List; import java.util.Objects; import java.util.UUID; @@ -27,6 +28,7 @@ import org.highmed.dsf.fhir.validation.ValueSetExpanderImpl; import org.highmed.dsf.tools.generator.ProcessDocumentation; import org.hl7.fhir.r4.model.CapabilityStatement; +import org.hl7.fhir.r4.model.Enumerations.BindingStrength; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -125,6 +127,10 @@ public class ValidationConfig @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package.client.verbose:false}") private boolean packageClientVerbose; + @ProcessDocumentation(description = "ValueSets found in the StructureDefinitions with the specified binding strength will be expanded", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("#{'${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.bindingStrength:required,extensible,preferred,example}'.trim().split('(,[ ]?)|(\\n)')}") + private List<String> valueSetExpansionBindingStrengths; + @ProcessDocumentation(description = "Folder for storing expanded ValueSets", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.valueset.cacheFolder:${java.io.tmpdir}/codex_gecco_validation_cache/ValueSet}") private String valueSetCacheFolder; @@ -211,9 +217,14 @@ public ValidationPackageIdentifier validationPackageIdentifier() @Bean public ValidationPackageManager validationPackageManager() { + List<ValidationPackageIdentifier> noDownload = noDownloadPackages.stream() + .map(ValidationPackageIdentifier::fromString).collect(Collectors.toList()); + EnumSet<BindingStrength> bindingStrengths = EnumSet.copyOf( + valueSetExpansionBindingStrengths.stream().map(BindingStrength::fromCode).collect(Collectors.toList())); + return new ValidationPackageManagerImpl(validationPackageClient(), valueSetExpansionClient(), objectMapper(), - fhirContext, internalSnapshotGeneratorFactory(), internalValueSetExpanderFactory(), - noDownloadPackages.stream().map(ValidationPackageIdentifier::fromString).collect(Collectors.toList())); + fhirContext, internalSnapshotGeneratorFactory(), internalValueSetExpanderFactory(), noDownload, + bindingStrengths); } private StructureDefinitionModifier createStructureDefinitionModifier(String className) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java index c424d9cb..f7d664ed 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java @@ -1,6 +1,5 @@ package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; -import java.util.List; import java.util.Objects; import java.util.Optional; @@ -40,12 +39,12 @@ public void init() return; logger.info("Downloading FHIR validation package {} and dependencies", validationPackageIdentifier.toString()); - List<ValidationPackage> validationPackages = validationPackageManager + ValidationPackageWithDepedencies packageWithDependencies = validationPackageManager .downloadPackageWithDependencies(validationPackageIdentifier); logger.info("Expanding ValueSets and generating StructureDefinition snapshots"); validationSupport = validationPackageManager - .expandValueSetsAndGenerateStructureDefinitionSnapshots(validationPackages); + .expandValueSetsAndGenerateStructureDefinitionSnapshots(packageWithDependencies); } @Override diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java index df7f1743..0d289cda 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java @@ -1,7 +1,6 @@ package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; -import java.util.List; - +import org.hl7.fhir.r4.model.Enumerations.BindingStrength; import org.hl7.fhir.r4.model.StructureDefinition; import org.hl7.fhir.r4.model.ValueSet; @@ -18,7 +17,7 @@ public interface ValidationPackageManager * not <code>null</code> * @return unmodifiable list of {@link ValidationPackage}s */ - default List<ValidationPackage> downloadPackageWithDependencies(String name, String version) + default ValidationPackageWithDepedencies downloadPackageWithDependencies(String name, String version) { return downloadPackageWithDependencies(new ValidationPackageIdentifier(name, version)); } @@ -32,18 +31,20 @@ default List<ValidationPackage> downloadPackageWithDependencies(String name, Str * not <code>null</code> * @return unmodifiable list of {@link ValidationPackage}s */ - List<ValidationPackage> downloadPackageWithDependencies(ValidationPackageIdentifier identifier); + ValidationPackageWithDepedencies downloadPackageWithDependencies(ValidationPackageIdentifier identifier); /** - * Will try to generate snapshots for all {@link StructureDefinition}s and expand all {@link ValueSet}s, before - * returning a {@link IValidationSupport}. + * Will try to generate snapshots for all {@link StructureDefinition}s of the root package and its dependencies, + * will try to expand all {@link ValueSet}s with binding strength {@link BindingStrength#EXTENSIBLE}, + * {@link BindingStrength#PREFERRED} or {@link BindingStrength#REQUIRED} used by the {@link StructureDefinition} of + * the root package or their dependencies, before returning a {@link IValidationSupport}. * - * @param validationPackages + * @param packageWithDependencies * not <code>null</code> * @return validation support for the validator */ IValidationSupport expandValueSetsAndGenerateStructureDefinitionSnapshots( - List<ValidationPackage> validationPackages); + ValidationPackageWithDepedencies packageWithDependencies); /** * @param validationSupport @@ -54,7 +55,10 @@ IValidationSupport expandValueSetsAndGenerateStructureDefinitionSnapshots( /** * Downloads the given FHIR package and all its dependencies. Will try to generate snapshots for all - * {@link StructureDefinition}s and expand all {@link ValueSet}s, before returning a {@link BundleValidator}. + * {@link StructureDefinition}s of the specified (root) package and its dependencies, will try to expand all + * {@link ValueSet}s with binding strength {@link BindingStrength#EXTENSIBLE}, {@link BindingStrength#PREFERRED} or + * {@link BindingStrength#REQUIRED} used by the {@link StructureDefinition} of the specified (root) package or their + * dependencies, before returning a {@link IValidationSupport}. * * @param name * not <code>null</code> @@ -69,7 +73,10 @@ default BundleValidator createBundleValidator(String name, String version) /** * Downloads the given FHIR package and all its dependencies. Will try to generate snapshots for all - * {@link StructureDefinition}s and expand all {@link ValueSet}s, before returning a {@link BundleValidator}. + * {@link StructureDefinition}s of the specified (root) package and its dependencies, will try to expand all + * {@link ValueSet}s with binding strength {@link BindingStrength#EXTENSIBLE}, {@link BindingStrength#PREFERRED} or + * {@link BindingStrength#REQUIRED} used by the {@link StructureDefinition} of the specified (root) package or their + * dependencies, before returning a {@link IValidationSupport}. * * @param identifier * not <code>null</code> diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java index ab17380d..f49d96e2 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java @@ -6,14 +6,11 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.function.BiFunction; -import java.util.function.Function; import java.util.stream.Collectors; import javax.ws.rs.WebApplicationException; @@ -26,11 +23,9 @@ import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService; import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport; import org.hl7.fhir.common.hapi.validation.support.ValidationSupportChain; -import org.hl7.fhir.r4.model.CanonicalType; -import org.hl7.fhir.r4.model.ElementDefinition; -import org.hl7.fhir.r4.model.ElementDefinition.TypeRefComponent; import org.hl7.fhir.r4.model.StructureDefinition; import org.hl7.fhir.r4.model.ValueSet; +import org.hl7.fhir.r4.model.Enumerations.BindingStrength; import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent; import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; @@ -48,9 +43,11 @@ public class ValidationPackageManagerImpl implements InitializingBean, Validatio { private static final Logger logger = LoggerFactory.getLogger(ValidationPackageManagerImpl.class); - public static final List<ValidationPackageIdentifier> PACKAGE_IGNORE = List + public static final List<ValidationPackageIdentifier> NO_PACKAGE_DOWNLOAD = List .of(new ValidationPackageIdentifier("hl7.fhir.r4.core", "4.0.1")); + public static final EnumSet<BindingStrength> VALUE_SET_BINDING_STRENGTHS = EnumSet.allOf(BindingStrength.class); + private final ValidationPackageClient validationPackageClient; private final ValueSetExpansionClient valueSetExpansionClient; @@ -61,6 +58,7 @@ public class ValidationPackageManagerImpl implements InitializingBean, Validatio private final BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory; private final List<ValidationPackageIdentifier> noDownloadPackages = new ArrayList<>(); + private final EnumSet<BindingStrength> valueSetBindingStrengths; public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, ValueSetExpansionClient valueSetExpansionClient, ObjectMapper mapper, FhirContext fhirContext, @@ -68,14 +66,15 @@ public ValidationPackageManagerImpl(ValidationPackageClient validationPackageCli BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory) { this(validationPackageClient, valueSetExpansionClient, mapper, fhirContext, internalSnapshotGeneratorFactory, - internalValueSetExpanderFactory, PACKAGE_IGNORE); + internalValueSetExpanderFactory, NO_PACKAGE_DOWNLOAD, VALUE_SET_BINDING_STRENGTHS); } public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, ValueSetExpansionClient valueSetExpansionClient, ObjectMapper mapper, FhirContext fhirContext, BiFunction<FhirContext, IValidationSupport, SnapshotGenerator> internalSnapshotGeneratorFactory, BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory, - Collection<ValidationPackageIdentifier> noDownloadPackages) + Collection<ValidationPackageIdentifier> noDownloadPackages, + EnumSet<BindingStrength> valueSetBindingStrengths) { this.validationPackageClient = validationPackageClient; this.valueSetExpansionClient = valueSetExpansionClient; @@ -86,6 +85,8 @@ public ValidationPackageManagerImpl(ValidationPackageClient validationPackageCli if (noDownloadPackages != null) this.noDownloadPackages.addAll(noDownloadPackages); + + this.valueSetBindingStrengths = valueSetBindingStrengths; } @Override @@ -102,28 +103,25 @@ public void afterPropertiesSet() throws Exception } @Override - public List<ValidationPackage> downloadPackageWithDependencies(ValidationPackageIdentifier identifier) + public ValidationPackageWithDepedencies downloadPackageWithDependencies(ValidationPackageIdentifier identifier) { Objects.requireNonNull(identifier, "identifier"); Map<ValidationPackageIdentifier, ValidationPackage> packagesByNameAndVersion = new HashMap<>(); downloadPackageWithDependencies(identifier, packagesByNameAndVersion); - return Collections.unmodifiableList(new ArrayList<>(packagesByNameAndVersion.values())); + return ValidationPackageWithDepedencies.from(packagesByNameAndVersion, identifier); } @Override public IValidationSupport expandValueSetsAndGenerateStructureDefinitionSnapshots( - List<ValidationPackage> validationPackages) + ValidationPackageWithDepedencies packageWithDependencies) { - Objects.requireNonNull(validationPackages, "validationPackages"); + Objects.requireNonNull(packageWithDependencies, "packageWithDependencies"); - validationPackages.forEach(p -> p.parseResources(fhirContext)); + packageWithDependencies.parseResources(fhirContext); - List<ValidationSupportResources> resources = validationPackages.stream() - .map(ValidationPackage::getValidationSupportResources).collect(Collectors.toList()); - - return withSnapshots(resources, withExpandedValueSets(resources)); + return withSnapshots(packageWithDependencies, withExpandedValueSets(packageWithDependencies)); } @Override @@ -139,8 +137,9 @@ public BundleValidator createBundleValidator(ValidationPackageIdentifier identif { Objects.requireNonNull(identifier, "identifier"); - List<ValidationPackage> vPackages = downloadPackageWithDependencies(identifier); - IValidationSupport validationSupport = expandValueSetsAndGenerateStructureDefinitionSnapshots(vPackages); + ValidationPackageWithDepedencies packageWithDependencies = downloadPackageWithDependencies(identifier); + IValidationSupport validationSupport = expandValueSetsAndGenerateStructureDefinitionSnapshots( + packageWithDependencies); return createBundleValidator(validationSupport); } @@ -191,13 +190,13 @@ private ValidationPackageDescriptor getDescriptorAndHandleException(ValidationPa } } - private List<ValueSet> withExpandedValueSets(List<ValidationSupportResources> resources) + private List<ValueSet> withExpandedValueSets(ValidationPackageWithDepedencies packageWithDependencies) { List<ValueSet> expandedValueSets = new ArrayList<>(); ValueSetExpander expander = internalValueSetExpanderFactory.apply(fhirContext, - createSupportChain(fhirContext, resources, Collections.emptyList(), expandedValueSets)); + createSupportChain(fhirContext, packageWithDependencies, Collections.emptyList(), expandedValueSets)); - resources.stream().flatMap(r -> r.getValueSets().stream()).forEach(v -> + packageWithDependencies.getValueSetsIncludingDependencies(valueSetBindingStrengths).forEach(v -> { logger.debug("Expanding ValueSet {}|{}", v.getUrl(), v.getVersion()); @@ -275,119 +274,85 @@ private Optional<String> getOutcome(WebApplicationException e) return Optional.empty(); } - private IValidationSupport withSnapshots(List<ValidationSupportResources> resources, + private IValidationSupport withSnapshots(ValidationPackageWithDepedencies packageWithDependencies, List<ValueSet> expandedValueSets) { - Map<String, StructureDefinition> structureDefinitionsByUrl = resources.stream() - .flatMap(r -> r.getStructureDefinitions().stream()) - .collect(Collectors.toMap(StructureDefinition::getUrl, Function.identity())); - Map<String, StructureDefinition> snapshots = new HashMap<>(); - ValidationSupportChain supportChain = createSupportChain(fhirContext, resources, snapshots.values(), - expandedValueSets); + ValidationSupportChain supportChain = createSupportChain(fhirContext, packageWithDependencies, + snapshots.values(), expandedValueSets); SnapshotGenerator generator = internalSnapshotGeneratorFactory.apply(fhirContext, supportChain); - resources.stream().flatMap(r -> r.getStructureDefinitions().stream()) + packageWithDependencies.getValidationSupportResources().getStructureDefinitions().stream() .filter(s -> s.hasDifferential() && !s.hasSnapshot()) - .forEach(diff -> createSnapshot(structureDefinitionsByUrl, snapshots, generator, diff)); + .forEach(diff -> createSnapshot(packageWithDependencies, snapshots, generator, diff)); return supportChain; } - private ValidationSupportChain createSupportChain(FhirContext context, List<ValidationSupportResources> resources, - Collection<? extends StructureDefinition> snapshots, Collection<? extends ValueSet> expandedValueSets) - { - return new ValidationSupportChain(new CodeValidatorForExpandedValueSets(context), - new InMemoryTerminologyServerValidationSupport(context), - new ValidationSupportWithCustomResources(context, snapshots, null, expandedValueSets), - new ValidationSupportChain(resources.stream() - .map(r -> new ValidationSupportWithCustomResources(context, r.getStructureDefinitions(), - r.getCodeSystems(), r.getValueSets())) - .toArray(IValidationSupport[]::new)), - new DefaultProfileValidationSupport(context), new CommonCodeSystemsTerminologyService(context)); - } - - private void createSnapshot(Map<String, StructureDefinition> structureDefinitionsByUrl, + private void createSnapshot(ValidationPackageWithDepedencies packageWithDependencies, Map<String, StructureDefinition> snapshots, SnapshotGenerator generator, StructureDefinition diff) { if (snapshots.containsKey(diff.getUrl() + "|" + diff.getVersion())) return; - Set<String> dependencies = new HashSet<>(); - Set<String> targetDependencies = new HashSet<>(); - - calculateDependencies(diff, structureDefinitionsByUrl, dependencies, targetDependencies); + List<StructureDefinition> definitions = new ArrayList<>(); + definitions.addAll(packageWithDependencies.getStructureDefinitionDependencies(diff)); + definitions.add(diff); - logger.debug("Generating snapshot for {}|{}, base {}, dependencies {}, target-dependencies {}", diff.getUrl(), - diff.getVersion(), diff.getBaseDefinition(), - dependencies.stream().sorted().collect(Collectors.joining(", ", "[", "]")), - targetDependencies.stream().sorted().collect(Collectors.joining(", ", "[", "]"))); - - if (structureDefinitionsByUrl.containsKey(diff.getBaseDefinition())) - createSnapshot(structureDefinitionsByUrl, snapshots, generator, - structureDefinitionsByUrl.get(diff.getBaseDefinition())); - - dependencies.stream().filter(structureDefinitionsByUrl::containsKey).map(structureDefinitionsByUrl::get) - .forEach(s -> createSnapshot(structureDefinitionsByUrl, snapshots, generator, s)); - - targetDependencies.stream().filter(structureDefinitionsByUrl::containsKey).map(structureDefinitionsByUrl::get) - .forEach(s -> createSnapshot(structureDefinitionsByUrl, snapshots, generator, s)); - - try - { - SnapshotWithValidationMessages snapshot = generator.generateSnapshot(diff); - - if (snapshot.getMessages().isEmpty()) - snapshots.put(snapshot.getSnapshot().getUrl() + "|" + snapshot.getSnapshot().getVersion(), - snapshot.getSnapshot()); - else - { - snapshot.getMessages().forEach(m -> - { - if (EnumSet.of(IssueSeverity.FATAL, IssueSeverity.ERROR, IssueSeverity.WARNING) - .contains(m.getLevel())) - logger.warn("{}|{} {}: {}", diff.getUrl(), diff.getVersion(), m.getLevel(), m.toString()); - else - logger.info("{}|{} {}: {}", diff.getUrl(), diff.getVersion(), m.getLevel(), m.toString()); - }); - } - } - catch (Exception e) - { - logger.error("Error while generating snapshot for {}|{}: {} - {}", diff.getUrl(), diff.getVersion(), - e.getClass().getName(), e.getMessage()); - } - } + logger.debug("Generating snapshot for {}|{}, base {}, dependencies {}", diff.getUrl(), diff.getVersion(), + diff.getBaseDefinition(), + definitions.stream() + .filter(sd -> !sd.equals(diff) && !sd.getUrl().equals(diff.getBaseDefinition()) + && !(sd.getUrl() + "|" + sd.getVersion()).equals(diff.getBaseDefinition())) + .map(sd -> sd.getUrl() + "|" + sd.getVersion()).sorted() + .collect(Collectors.joining(", ", "[", "]"))); - private void calculateDependencies(StructureDefinition structureDefinition, - Map<String, StructureDefinition> structureDefinitionsByUrl, Set<String> dependencies, - Set<String> targetDependencies) - { - for (ElementDefinition element : structureDefinition.getDifferential().getElement()) - { - if (element.getType().stream().filter(t -> !t.getProfile().isEmpty() || !t.getTargetProfile().isEmpty()) - .findAny().isPresent()) - { - for (TypeRefComponent type : element.getType()) + definitions.stream().filter(sd -> sd.hasDifferential() && !sd.hasSnapshot() + && !snapshots.containsKey(sd.getUrl() + "|" + sd.getVersion())).forEach(sd -> { - if (!type.getProfile().isEmpty()) + try { - for (CanonicalType profile : type.getProfile()) - { - dependencies.add(profile.getValue()); + SnapshotWithValidationMessages snapshot = generator.generateSnapshot(sd); - if (structureDefinitionsByUrl.containsKey(profile.getValue())) - calculateDependencies(structureDefinitionsByUrl.get(profile.getValue()), - structureDefinitionsByUrl, dependencies, targetDependencies); + if (!snapshot.getMessages().stream().anyMatch( + m -> EnumSet.of(IssueSeverity.FATAL, IssueSeverity.ERROR, IssueSeverity.WARNING) + .contains(m.getLevel()))) + { + snapshots.put(snapshot.getSnapshot().getUrl() + "|" + snapshot.getSnapshot().getVersion(), + snapshot.getSnapshot()); + } + else + { + snapshot.getMessages().forEach(m -> + { + if (EnumSet.of(IssueSeverity.FATAL, IssueSeverity.ERROR, IssueSeverity.WARNING) + .contains(m.getLevel())) + logger.warn("{}|{} {}: {}", diff.getUrl(), diff.getVersion(), m.getLevel(), + m.toString()); + else + logger.info("{}|{} {}: {}", diff.getUrl(), diff.getVersion(), m.getLevel(), + m.toString()); + }); } } + catch (Exception e) + { + logger.error("Error while generating snapshot for {}|{}: {} - {}", diff.getUrl(), + diff.getVersion(), e.getClass().getName(), e.getMessage()); + } + }); + } - if (!type.getTargetProfile().isEmpty()) - for (CanonicalType targetProfile : type.getTargetProfile()) - targetDependencies.add(targetProfile.getValue()); - } - } - } + private ValidationSupportChain createSupportChain(FhirContext context, + ValidationPackageWithDepedencies packageWithDependencies, + Collection<? extends StructureDefinition> snapshots, Collection<? extends ValueSet> expandedValueSets) + { + return new ValidationSupportChain(new CodeValidatorForExpandedValueSets(context), + new InMemoryTerminologyServerValidationSupport(context), + new ValidationSupportWithCustomResources(context, snapshots, null, expandedValueSets), + new ValidationSupportWithCustomResources(context, packageWithDependencies.getAllStructureDefinitions(), + packageWithDependencies.getAllCodeSystems(), packageWithDependencies.getAllValueSets()), + new DefaultProfileValidationSupport(context), new CommonCodeSystemsTerminologyService(context)); } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageWithDepedencies.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageWithDepedencies.java new file mode 100644 index 00000000..6a46d1e5 --- /dev/null +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageWithDepedencies.java @@ -0,0 +1,232 @@ +package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.ElementDefinition; +import org.hl7.fhir.r4.model.MetadataResource; +import org.hl7.fhir.r4.model.NamingSystem; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.UriType; +import org.hl7.fhir.r4.model.ValueSet; +import org.hl7.fhir.r4.model.ElementDefinition.ElementDefinitionBindingComponent; +import org.hl7.fhir.r4.model.Enumerations.BindingStrength; +import org.hl7.fhir.r4.model.StructureDefinition.StructureDefinitionDifferentialComponent; + +import ca.uhn.fhir.context.FhirContext; + +public class ValidationPackageWithDepedencies extends ValidationPackage +{ + public static ValidationPackageWithDepedencies from( + Map<ValidationPackageIdentifier, ValidationPackage> packagesByNameAndVersion, + ValidationPackageIdentifier rootPackageIdentifier) + { + Objects.requireNonNull(packagesByNameAndVersion, "packagesByNameAndVersion"); + Objects.requireNonNull(rootPackageIdentifier, "rootPackageIdentifier"); + + ValidationPackage rootPackage = packagesByNameAndVersion.get(rootPackageIdentifier); + if (rootPackage == null) + throw new IllegalArgumentException("root package not part of given map"); + + List<ValidationPackage> packages = packagesByNameAndVersion.entrySet().stream() + .filter(e -> !rootPackageIdentifier.equals(e.getKey())).map(Entry::getValue) + .collect(Collectors.toList()); + + return new ValidationPackageWithDepedencies(rootPackage, packages); + } + + private final List<ValidationPackage> dependencies = new ArrayList<>(); + + private Map<String, List<StructureDefinition>> structureDefinitionsByUrl; + private Map<String, StructureDefinition> structureDefinitionsByUrlAndVersion; + + public ValidationPackageWithDepedencies(ValidationPackage validationPackage, List<ValidationPackage> dependencies) + { + super(validationPackage.getName(), validationPackage.getVersion(), validationPackage.getEntries()); + + if (dependencies != null) + this.dependencies.addAll(dependencies); + } + + public List<ValidationPackage> getDependencies() + { + return Collections.unmodifiableList(dependencies); + } + + @Override + public void parseResources(FhirContext context) + { + super.parseResources(context); + + getDependencies().forEach(p -> p.parseResources(context)); + } + + private <R extends MetadataResource> List<R> getAll(Function<ValidationSupportResources, List<R>> accessor) + { + return Stream.concat(Stream.of(this), getDependencies().stream()) + .map(ValidationPackage::getValidationSupportResources).map(accessor).flatMap(List::stream) + .collect(Collectors.toList()); + } + + public List<CodeSystem> getAllCodeSystems() + { + return getAll(ValidationSupportResources::getCodeSystems); + } + + public List<NamingSystem> getAllNamingSystems() + { + return getAll(ValidationSupportResources::getNamingSystems); + } + + public List<StructureDefinition> getAllStructureDefinitions() + { + return getAll(ValidationSupportResources::getStructureDefinitions); + } + + public List<ValueSet> getAllValueSets() + { + return getAll(ValidationSupportResources::getValueSets); + } + + public ValidationSupportResources getAllValidationSupportResources() + { + return new ValidationSupportResources(getAllCodeSystems(), getAllNamingSystems(), getAllStructureDefinitions(), + getAllValueSets()); + } + + private Map<String, List<StructureDefinition>> getStructureDefinitionsByUrl() + { + if (structureDefinitionsByUrl == null) + structureDefinitionsByUrl = getAllStructureDefinitions().stream().filter(StructureDefinition::hasUrl) + .collect(Collectors.toMap(StructureDefinition::getUrl, Collections::singletonList, (sd1, sd2) -> + { + List<StructureDefinition> sds = new ArrayList<>(); + sds.addAll(sd1); + sds.addAll(sd2); + return sds; + })); + + return structureDefinitionsByUrl; + } + + private Map<String, StructureDefinition> getStructureDefinitionsByUrlAndVersion() + { + if (structureDefinitionsByUrlAndVersion == null) + structureDefinitionsByUrlAndVersion = getAllStructureDefinitions().stream() + .filter(StructureDefinition::hasUrl).filter(StructureDefinition::hasVersion) + .collect(Collectors.toMap(s -> s.getUrl() + "|" + s.getVersion(), Function.identity())); + + return structureDefinitionsByUrlAndVersion; + } + + public List<StructureDefinition> getStructureDefinitionDependencies(StructureDefinition structureDefinition) + { + return doGetDependencies(structureDefinition, new HashSet<>()); + } + + private List<StructureDefinition> doGetDependencies(StructureDefinition structureDefinition, Set<String> visited) + { + if (visited.contains(structureDefinition.getUrl()) + || visited.contains(structureDefinition.getUrl() + "|" + structureDefinition.getVersion())) + return Collections.emptyList(); + else + { + visited.add(structureDefinition.getUrl()); + visited.add(structureDefinition.getUrl() + "|" + structureDefinition.getVersion()); + } + + List<StructureDefinition> dependencies = new ArrayList<>(); + Set<StructureDefinition> baseDefinitions = getStructureDefinitionsByUrl( + structureDefinition.getBaseDefinition()); + + baseDefinitions.forEach(sd -> dependencies.addAll(doGetDependencies(sd, visited))); + dependencies.addAll(baseDefinitions); + + structureDefinition.getDifferential().getElement().stream().forEach(e -> + { + if (e.hasPath() && "Extension.url".equals(e.getPath()) && e.hasFixed() && e.getFixed() instanceof UriType) + { + UriType t = (UriType) e.getFixed(); + Set<StructureDefinition> extensions = getStructureDefinitionsByUrl(t.getValue()); + extensions.forEach(sd -> dependencies.addAll(doGetDependencies(sd, visited))); + dependencies.addAll(extensions); + } + + if (e.hasType()) + { + e.getType().forEach(t -> + { + if (t.hasProfile()) + { + t.getProfile().forEach(p -> + { + Set<StructureDefinition> profiles = getStructureDefinitionsByUrl(p.getValue()); + profiles.forEach(sd -> dependencies.addAll(doGetDependencies(sd, visited))); + dependencies.addAll(profiles); + }); + } + + if (t.hasTargetProfile()) + { + t.getTargetProfile().forEach(p -> + { + Set<StructureDefinition> targetProfiles = getStructureDefinitionsByUrl(p.getValue()); + targetProfiles.forEach(sd -> dependencies.addAll(doGetDependencies(sd, visited))); + dependencies.addAll(targetProfiles); + }); + } + }); + } + }); + + return dependencies; + } + + private Set<StructureDefinition> getStructureDefinitionsByUrl(String sdUrl) + { + Set<StructureDefinition> sds = new HashSet<>(); + + List<StructureDefinition> byUrl = getStructureDefinitionsByUrl().get(sdUrl); + if (byUrl != null) + sds.addAll(byUrl); + + StructureDefinition byUrlAndVersion = getStructureDefinitionsByUrlAndVersion().get(sdUrl); + if (byUrlAndVersion != null) + sds.add(byUrlAndVersion); + + return sds; + } + + private Set<String> findValueSetsWithBindingStrength(Stream<StructureDefinition> sds, + EnumSet<BindingStrength> bindingStrengths) + { + return sds.filter(StructureDefinition::hasDifferential).map(StructureDefinition::getDifferential) + .filter(StructureDefinitionDifferentialComponent::hasElement) + .map(StructureDefinitionDifferentialComponent::getElement).flatMap(List::stream) + .filter(ElementDefinition::hasBinding).map(ElementDefinition::getBinding) + .filter(b -> bindingStrengths.contains(b.getStrength())) + .map(ElementDefinitionBindingComponent::getValueSet).collect(Collectors.toSet()); + } + + public List<ValueSet> getValueSetsIncludingDependencies(EnumSet<BindingStrength> bindingStrengths) + { + Stream<StructureDefinition> sds = getValidationSupportResources().getStructureDefinitions().stream() + .flatMap(sd -> Stream.concat(Stream.of(sd), getStructureDefinitionDependencies(sd).stream())) + .distinct(); + + Set<String> neededValueSets = findValueSetsWithBindingStrength(sds, bindingStrengths); + return getAllValueSets().stream().filter(vs -> neededValueSets.contains(vs.getUrl()) + || neededValueSets.contains(vs.getUrl() + "|" + vs.getVersion())).collect(Collectors.toList()); + } +} diff --git a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidateDataLearningTest.java b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidateDataLearningTest.java index e4113415..f7e8ef74 100644 --- a/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidateDataLearningTest.java +++ b/codex-process-data-transfer/src/test/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidateDataLearningTest.java @@ -7,7 +7,6 @@ import java.util.Arrays; import java.util.Comparator; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Consumer; @@ -39,7 +38,6 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.support.DefaultProfileValidationSupport; -import ca.uhn.fhir.context.support.IValidationSupport; public class ValidateDataLearningTest { @@ -84,12 +82,9 @@ public void testDownloadWithDependencies() throws Exception valueSetExpansionClientWithCache, mapper, fhirContext, PluginSnapshotGeneratorImpl::new, ValueSetExpanderImpl::new); - List<ValidationPackage> validationPackages = manager.downloadPackageWithDependencies("de.gecco", "1.0.5"); - validationPackages.forEach(p -> - { - logger.debug(p.getName() + "/" + p.getVersion()); - p.parseResources(fhirContext); - }); + ValidationPackageWithDepedencies packageWithDependencies = manager.downloadPackageWithDependencies("de.gecco", + "1.0.5"); + packageWithDependencies.parseResources(fhirContext); } @Test @@ -192,33 +187,27 @@ public void testGenerateSnapshots() throws Exception valueSetExpansionClientWithCache, mapper, fhirContext, PluginSnapshotGeneratorImpl::new, ValueSetExpanderImpl::new); - List<ValidationPackage> validationPackages = manager.downloadPackageWithDependencies("de.gecco", "1.0.5"); - validationPackages.forEach(p -> - { - logger.debug(p.getName() + "/" + p.getVersion()); - p.parseResources(fhirContext); - }); + ValidationPackageWithDepedencies packageWithDependencies = manager.downloadPackageWithDependencies("de.gecco", + "1.0.5"); + packageWithDependencies.parseResources(fhirContext); - validationPackages.stream().flatMap(p -> p.getValidationSupportResources().getStructureDefinitions().stream()) + packageWithDependencies.getAllStructureDefinitions().stream() .sorted(Comparator.comparing(StructureDefinition::getUrl) .thenComparing(Comparator.comparing(StructureDefinition::getVersion))) .forEach(s -> logger.debug(s.getUrl() + " " + s.getVersion())); - StructureDefinition miiRef = validationPackages.stream() - .flatMap(p -> p.getValidationSupportResources().getStructureDefinitions().stream()) + StructureDefinition miiRef = packageWithDependencies.getAllStructureDefinitions().stream() .filter(s -> "https://www.medizininformatik-initiative.de/fhir/core/StructureDefinition/MII-Reference" .equals(s.getUrl())) .findFirst().get(); - SnapshotGenerator sGen = new PluginSnapshotGeneratorImpl(fhirContext, new ValidationSupportChain( - new InMemoryTerminologyServerValidationSupport(fhirContext), - new ValidationSupportChain(validationPackages.stream() - .map(ValidationPackage::getValidationSupportResources) - .map(r -> new ValidationSupportWithCustomResources(fhirContext, r.getStructureDefinitions(), - r.getCodeSystems(), r.getValueSets())) - .toArray(IValidationSupport[]::new)), - new DefaultProfileValidationSupport(fhirContext), - new CommonCodeSystemsTerminologyService(fhirContext))); + SnapshotGenerator sGen = new PluginSnapshotGeneratorImpl(fhirContext, + new ValidationSupportChain(new InMemoryTerminologyServerValidationSupport(fhirContext), + new ValidationSupportWithCustomResources(fhirContext, + packageWithDependencies.getAllStructureDefinitions(), + packageWithDependencies.getAllCodeSystems(), packageWithDependencies.getAllValueSets()), + new DefaultProfileValidationSupport(fhirContext), + new CommonCodeSystemsTerminologyService(fhirContext))); sGen = new PluginSnapshotGeneratorWithModifiers(sGen); @@ -237,17 +226,15 @@ else if (IssueSeverity.WARNING.equals(m.getLevel())) result.getSnapshot().getVersion(), m.toString()); }); - Map<String, StructureDefinition> sDefByUrl = validationPackages.stream() - .flatMap(p -> p.getValidationSupportResources().getStructureDefinitions().stream()) + Map<String, StructureDefinition> sDefByUrl = packageWithDependencies.getAllStructureDefinitions().stream() .collect(Collectors.toMap(StructureDefinition::getUrl, Function.identity())); - validationPackages.stream().flatMap(p -> p.getValidationSupportResources().getStructureDefinitions().stream()) - .forEach(s -> - { - logger.info("StructureDefinition {}|{}:", s.getUrl(), s.getVersion()); - printTree(s, sDefByUrl); - logger.debug(""); - }); + packageWithDependencies.getAllStructureDefinitions().forEach(s -> + { + logger.info("StructureDefinition {}|{}:", s.getUrl(), s.getVersion()); + printTree(s, sDefByUrl); + logger.debug(""); + }); } private void printTree(StructureDefinition def, Map<String, StructureDefinition> structureDefinitionsByUrl) From 5b0ab5a386b50d76962cca9cd92f746a4f50725d Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Thu, 16 Jun 2022 04:30:00 +0200 Subject: [PATCH 21/29] imports sorted --- .../validation/ValidationPackageManagerImpl.java | 2 +- .../validation/ValidationPackageWithDepedencies.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java index f49d96e2..96dd71ea 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java @@ -23,9 +23,9 @@ import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService; import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport; import org.hl7.fhir.common.hapi.validation.support.ValidationSupportChain; +import org.hl7.fhir.r4.model.Enumerations.BindingStrength; import org.hl7.fhir.r4.model.StructureDefinition; import org.hl7.fhir.r4.model.ValueSet; -import org.hl7.fhir.r4.model.Enumerations.BindingStrength; import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent; import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageWithDepedencies.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageWithDepedencies.java index 6a46d1e5..765b7dbc 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageWithDepedencies.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageWithDepedencies.java @@ -15,14 +15,14 @@ import org.hl7.fhir.r4.model.CodeSystem; import org.hl7.fhir.r4.model.ElementDefinition; +import org.hl7.fhir.r4.model.ElementDefinition.ElementDefinitionBindingComponent; +import org.hl7.fhir.r4.model.Enumerations.BindingStrength; import org.hl7.fhir.r4.model.MetadataResource; import org.hl7.fhir.r4.model.NamingSystem; import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.StructureDefinition.StructureDefinitionDifferentialComponent; import org.hl7.fhir.r4.model.UriType; import org.hl7.fhir.r4.model.ValueSet; -import org.hl7.fhir.r4.model.ElementDefinition.ElementDefinitionBindingComponent; -import org.hl7.fhir.r4.model.Enumerations.BindingStrength; -import org.hl7.fhir.r4.model.StructureDefinition.StructureDefinitionDifferentialComponent; import ca.uhn.fhir.context.FhirContext; From 5f7d8b53c5e10ae53339559b034e25907b63374c Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Fri, 17 Jun 2022 01:08:38 +0200 Subject: [PATCH 22/29] modified/fixed log messages --- .../data_transfer/DataTransferProcessPluginDefinition.java | 2 +- .../codex/processes/data_transfer/service/ValidateData.java | 2 +- .../data_transfer/validation/AbstractFileSystemCache.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java index 85f1bada..5959137c 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java @@ -136,7 +136,7 @@ public void onProcessesDeployed(ApplicationContext pluginApplicationContext, Lis pluginApplicationContext.getBean(BundleValidatorFactory.class).init(); else logger.warn( - "Due to an error while testing the connection to the terminology server the {} can not be initialized, this will lead to the validation of bundles beeing skipped.", + "Due to an error while testing the connection to the terminology server {} was not initialized, validation of bundles will be skipped.", BundleValidatorFactory.class.getSimpleName()); } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java index ea4123c8..b81fb2cf 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java @@ -108,7 +108,7 @@ protected void doExecute(DelegateExecution execution) throws BpmnError, Exceptio }, () -> { logger.warn( - "{} not initialized, skipping validation. This is likley due to an error during startup of the process plugin.", + "{} not initialized, skipping validation. This is likley due to an error during startup of the process plugin", BundleValidatorFactory.class.getSimpleName()); }); } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFileSystemCache.java index 97b60f94..9a234665 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFileSystemCache.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/AbstractFileSystemCache.java @@ -144,7 +144,7 @@ protected final <R> T writeToCache(T value, Function<R, String> toCacheId, Funct OutputStream cOut = outCompressorFactory.apply(bOut); OutputStreamWriter writer = new OutputStreamWriter(cOut, StandardCharsets.UTF_8)) { - logger.debug("Wirting {} {} to cache at {}", cacheEntryType, cacheId, cacheFile.toString()); + logger.debug("Writing {} {} to cache at {}", cacheEntryType, cacheId, cacheFile.toString()); encoder.accept(writer, resource); } From 0bb89758af1ab88f9050093917f8c160fd4cd528 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Fri, 17 Jun 2022 01:18:02 +0200 Subject: [PATCH 23/29] added special case for snapshot generator result without actual snapshot --- .../PluginSnapshotGeneratorWithFileSystemCache.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java index 5feca93d..bebd67a1 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/PluginSnapshotGeneratorWithFileSystemCache.java @@ -110,6 +110,12 @@ private SnapshotWithValidationMessages generateSnapshotAndWriteToCache(Structure snapshot.getSnapshot().getStatus()); return snapshot; } + else if (!snapshot.getSnapshot().hasSnapshot()) + { + logger.info("Not writing StructureDefinition {}|{} without snapshot to cache", + snapshot.getSnapshot().getUrl(), snapshot.getSnapshot().getVersion()); + return snapshot; + } else return writeRsourceToCache(snapshot, SnapshotWithValidationMessages::getSnapshot, StructureDefinition::getUrl, StructureDefinition::getVersion); From 2eb3a38d27db7a42a8213123171e10d27327b128 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Fri, 17 Jun 2022 01:33:10 +0200 Subject: [PATCH 24/29] snapshots now added to validator if errors occurred during generation change log level for errors during internal ValueSet expansion --- .../ValidationPackageManagerImpl.java | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java index 96dd71ea..a1717e64 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java @@ -255,7 +255,7 @@ private void expandInternal(List<ValueSet> expandedValueSets, ValueSetExpander e } catch (Exception e) { - logger.warn( + logger.info( "Error while expanding ValueSet {}|{}: {} - {}, trying to expand via external terminology server next", v.getUrl(), v.getVersion(), e.getClass().getName(), e.getMessage()); @@ -315,26 +315,24 @@ private void createSnapshot(ValidationPackageWithDepedencies packageWithDependen { SnapshotWithValidationMessages snapshot = generator.generateSnapshot(sd); - if (!snapshot.getMessages().stream().anyMatch( - m -> EnumSet.of(IssueSeverity.FATAL, IssueSeverity.ERROR, IssueSeverity.WARNING) - .contains(m.getLevel()))) - { + if (snapshot.getSnapshot().hasSnapshot()) snapshots.put(snapshot.getSnapshot().getUrl() + "|" + snapshot.getSnapshot().getVersion(), snapshot.getSnapshot()); - } else + logger.error( + "Error while generating snapshot for {}|{}: Not snaphsot returned from generator", + diff.getUrl(), diff.getVersion()); + + snapshot.getMessages().forEach(m -> { - snapshot.getMessages().forEach(m -> - { - if (EnumSet.of(IssueSeverity.FATAL, IssueSeverity.ERROR, IssueSeverity.WARNING) - .contains(m.getLevel())) - logger.warn("{}|{} {}: {}", diff.getUrl(), diff.getVersion(), m.getLevel(), - m.toString()); - else - logger.info("{}|{} {}: {}", diff.getUrl(), diff.getVersion(), m.getLevel(), - m.toString()); - }); - } + if (EnumSet.of(IssueSeverity.FATAL, IssueSeverity.ERROR, IssueSeverity.WARNING) + .contains(m.getLevel())) + logger.warn("{}|{} {}: {}", diff.getUrl(), diff.getVersion(), m.getLevel(), + m.toString()); + else + logger.info("{}|{} {}: {}", diff.getUrl(), diff.getVersion(), m.getLevel(), + m.toString()); + }); } catch (Exception e) { From 64da39dc7ec6d1fe5ff2c4a5250d6380cf6d9f37 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Sat, 18 Jun 2022 01:59:13 +0200 Subject: [PATCH 25/29] fixed log message --- .../data_transfer/spring/config/ValidationConfig.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java index b827118c..1cbeecce 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java @@ -453,9 +453,9 @@ public boolean testConnectionToTerminologyServer() valueSetExpansionClientCertificatePrivateKey, valueSetExpansionClientCertificatePrivateKeyPassword != null ? "***" : "null", valueSetExpansionClientBasicAuthUsername, - valueSetExpansionClientBasicAuthPassword != null ? "***" : "null", - valueSetExpansionClientProxySchemeHostPort, valueSetExpansionClientProxySchemeHostPort, - valueSetExpansionClientProxyUsername, valueSetExpansionClientProxyPassword != null ? "***" : "null"); + valueSetExpansionClientBasicAuthPassword != null ? "***" : "null", valueSetExpansionServerBaseUrl, + valueSetExpansionClientProxySchemeHostPort, valueSetExpansionClientProxyUsername, + valueSetExpansionClientProxyPassword != null ? "***" : "null"); try { From ca34abc5efccce79eab693c1aa3f6fdc7d7ddf8c Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Sat, 18 Jun 2022 02:04:32 +0200 Subject: [PATCH 26/29] code cleanup/refactoring, NPE fix at ReadData when server not configured Added getServerBase Method to GeccoClient interface and implementing classes. Removed geccoServerBase field from ReadData service, since it may not be configured (when testing without GECCO FHIR Server or at the GTH), resulting in a NullPointerException. --- .../data_transfer/client/GeccoClient.java | 2 ++ .../client/GeccoClientFactory.java | 23 ++++++++++++++----- .../data_transfer/client/GeccoClientImpl.java | 18 ++++++++++----- .../client/fhir/AbstractFhirClient.java | 5 ++-- .../data_transfer/service/ReadData.java | 5 +--- 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClient.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClient.java index 8964267c..a5778b15 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClient.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClient.java @@ -8,6 +8,8 @@ public interface GeccoClient { + String getServerBase(); + FhirContext getFhirContext(); void testConnection(); diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientFactory.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientFactory.java index 3f97407d..1bb0c0fa 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientFactory.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientFactory.java @@ -39,21 +39,27 @@ private static final class GeccoClientStub implements GeccoClient } @Override - public void testConnection() + public String getServerBase() { - logger.warn("Stub implementation, no connection test performed"); + return null; } @Override - public GeccoFhirClient getFhirClient() + public FhirContext getFhirContext() { - return new GeccoFhirClientStub(this); + return fhirContext; } @Override - public FhirContext getFhirContext() + public void testConnection() { - return fhirContext; + logger.warn("Stub implementation, no connection test performed"); + } + + @Override + public GeccoFhirClient getFhirClient() + { + return new GeccoFhirClientStub(this); } @Override @@ -141,6 +147,11 @@ public GeccoClientFactory(Path trustStorePath, Path certificatePath, Path privat this.useChainedParameterNotLogicalReference = useChainedParameterNotLogicalReference; } + public String getServerBase() + { + return geccoServerBase; + } + public void testConnection() { try diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientImpl.java index 40962fba..b8eccfb7 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/GeccoClientImpl.java @@ -127,6 +127,18 @@ private void configureLoggingInterceptor(IGenericClient client) } } + @Override + public String getServerBase() + { + return geccoServerBase; + } + + @Override + public FhirContext getFhirContext() + { + return fhirContext; + } + @Override public void testConnection() { @@ -154,12 +166,6 @@ public GeccoFhirClient getFhirClient() } } - @Override - public FhirContext getFhirContext() - { - return fhirContext; - } - @Override public Path getSearchBundleOverride() { diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/fhir/AbstractFhirClient.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/fhir/AbstractFhirClient.java index 7f11e8cc..5bfe464d 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/fhir/AbstractFhirClient.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/fhir/AbstractFhirClient.java @@ -233,8 +233,9 @@ private Optional<PatientReference> getIdentifierPatientReference(Patient patient private PatientReference getAbsoluteUrlPatientReference(Patient patient) { IdType idElement = patient.getIdElement(); - return PatientReference.from(new IdType(geccoClient.getGenericFhirClient().getServerBase(), - idElement.getResourceType(), idElement.getIdPart(), null).getValue()); + return PatientReference + .from(new IdType(geccoClient.getServerBase(), idElement.getResourceType(), idElement.getIdPart(), null) + .getValue()); } private Stream<Patient> getPatients(Bundle bundle) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ReadData.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ReadData.java index 4a6a1601..b416c717 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ReadData.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ReadData.java @@ -70,7 +70,6 @@ public class ReadData extends AbstractServiceDelegate private final FhirContext fhirContext; private final GeccoClientFactory geccoClientFactory; - private final String geccoServerBase; public ReadData(FhirWebserviceClientProvider clientProvider, TaskHelper taskHelper, ReadAccessHelper readAccessHelper, FhirContext fhirContext, GeccoClientFactory geccoClientFactory, @@ -80,7 +79,6 @@ public ReadData(FhirWebserviceClientProvider clientProvider, TaskHelper taskHelp this.fhirContext = fhirContext; this.geccoClientFactory = geccoClientFactory; - this.geccoServerBase = geccoServerBase; } @Override @@ -90,7 +88,6 @@ public void afterPropertiesSet() throws Exception Objects.requireNonNull(fhirContext, "fhirContext"); Objects.requireNonNull(geccoClientFactory, "geccoClientFactory"); - Objects.requireNonNull(geccoServerBase, "geccoServerBase"); } @Override @@ -183,7 +180,7 @@ private IdType getAbsoluteId(DomainResource r) return null; return r.getIdElement().isAbsolute() ? r.getIdElement() - : r.getIdElement().withServerBase(geccoServerBase, r.getResourceType().name()); + : r.getIdElement().withServerBase(geccoClientFactory.getServerBase(), r.getResourceType().name()); } private DomainResource clean(DomainResource r) From 0793ab50a3cdf4c53d90f2fa6931e15c4afb17ca Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Sat, 18 Jun 2022 17:06:13 +0200 Subject: [PATCH 27/29] fixed yml lint warnings, added docker-compose.override.yml to .gitignore The file codex-processes-ap1-docker-test-setup/docker-compose.override.yml has been added to .gitignore and can now be used to defined local overrides. See https://docs.docker.com/compose/extends on how to use override files. --- .gitignore | 1 + .../docker-compose.local-dsf-build.yml | 2 +- .../docker-compose.yml | 12 ++++++------ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 461addf8..a0300533 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ codex-processes-ap1-docker-test-setup/**/fhir/log/*.log.gz codex-processes-ap1-docker-test-setup/secrets/*.pem codex-processes-ap1-docker-test-setup/.env +codex-processes-ap1-docker-test-setup/docker-compose.override.yml ### # codex-process-data-transfer ignores diff --git a/codex-processes-ap1-docker-test-setup/docker-compose.local-dsf-build.yml b/codex-processes-ap1-docker-test-setup/docker-compose.local-dsf-build.yml index a2608f54..1275bc1d 100644 --- a/codex-processes-ap1-docker-test-setup/docker-compose.local-dsf-build.yml +++ b/codex-processes-ap1-docker-test-setup/docker-compose.local-dsf-build.yml @@ -13,4 +13,4 @@ services: crr-fhir: image: highmed/fhir crr-bpe: - image: highmed/bpe \ No newline at end of file + image: highmed/bpe diff --git a/codex-processes-ap1-docker-test-setup/docker-compose.yml b/codex-processes-ap1-docker-test-setup/docker-compose.yml index 85451bf4..58f1ce48 100644 --- a/codex-processes-ap1-docker-test-setup/docker-compose.yml +++ b/codex-processes-ap1-docker-test-setup/docker-compose.yml @@ -37,7 +37,7 @@ services: image: postgres:13 restart: "no" healthcheck: - test: [ "CMD-SHELL", "pg_isready -U liquibase_user -d postgres" ] + test: ["CMD-SHELL", "pg_isready -U liquibase_user -d postgres"] interval: 10s timeout: 5s retries: 5 @@ -176,7 +176,7 @@ services: depends_on: - db - dic-fhir - # - dic-fhir-store not defining a dependency here, dic-fhir-store* needs to be started manually + # - dic-fhir-store not defining a dependency here, dic-fhir-store* needs to be started manually dic-fhir-store-hapi: build: ./dic/hapi restart: "no" @@ -409,7 +409,7 @@ services: depends_on: - db - crr-fhir - # - crr-fhir-bridge not defining a dependency here, crr-fhir-bridge* needs to be started manually + # - crr-fhir-bridge not defining a dependency here, crr-fhir-bridge* needs to be started manually crr-ehrbase-db: image: ehrbase/ehrbase-postgres networks: @@ -435,7 +435,7 @@ services: SECURITY_AUTHADMINPASSWORD: mySuperAwesomePassword123 SYSTEM_NAME: local.ehrbase.org ADMIN_API_ACTIVE: 'true' -# SERVER_DISABLESTRICTVALIDATION: 'true' + # SERVER_DISABLESTRICTVALIDATION: 'true' TZ: Europe/Berlin depends_on: - crr-ehrbase-db @@ -451,7 +451,7 @@ services: FHIR_BRIDGE_EHRBASE_BASE_URL: http://crr-ehrbase:8080/ehrbase/ FHIR_BRIDGE_FHIR_VALIDATION_OPTIONAL_IDENTIFIER: 'true' TZ: Europe/Berlin -# SPRING_PROFILES_ACTIVE: dev + # SPRING_PROFILES_ACTIVE: dev JAVA_TOOL_OPTIONS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006 depends_on: - crr-ehrbase @@ -554,4 +554,4 @@ networks: volumes: db-data: - name: db-data-codex-dsf-processes \ No newline at end of file + name: db-data-codex-dsf-processes From 76c208a2b84d3598cfd8e952cc7fe461f594c31f Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Sat, 18 Jun 2022 20:44:43 +0200 Subject: [PATCH 28/29] New log message with # of found pat in find new data step of trigger --- .../codex/processes/data_transfer/service/FindNewData.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/FindNewData.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/FindNewData.java index bea445af..414b00c5 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/FindNewData.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/FindNewData.java @@ -133,6 +133,11 @@ protected PatientReferenceList searchForPatientReferencesWithNewData(DateWithPre GeccoFhirClient fhirClient = geccoClientFactory.getGeccoClient().getFhirClient(); - return fhirClient.getPatientReferencesWithNewData(exportFrom, exportTo); + PatientReferenceList references = fhirClient.getPatientReferencesWithNewData(exportFrom, exportTo); + + logger.info("Found {} patient{} with changes to transport", references.getReferences().size(), + references.getReferences().size() != 1 ? "s" : ""); + + return references; } } From 7bc60c106be060b16c1f533dd26711652cec8061 Mon Sep 17 00:00:00 2001 From: Hauke Hund <hauke.hund@hs-heilbronn.de> Date: Sat, 18 Jun 2022 20:50:50 +0200 Subject: [PATCH 29/29] env variable to disable validation, error when profile not supported Added new environment variable to disable FHIR validation. Added new functionality to test if all resources are declaring at least on supported profile. A validation error is raised if no supported profile is defined. Profiles are supported if they are declared in the root validation package (currently de.gecco | 1.0.5) or are dependencies of the StructureDefinitions. Only profile with abstract = false and kind = resource are supported as claimed profiles by resources beeing validated. --- .../data_transfer/service/ValidateData.java | 29 ++++++-- .../spring/config/TransferDataConfig.java | 16 +--- .../spring/config/ValidationConfig.java | 26 +++++-- .../validation/BundleValidatorFactory.java | 5 ++ .../BundleValidatorFactoryImpl.java | 20 ++++- .../validation/BundleValidatorImpl.java | 74 ++++++++++++++++--- .../validation/ValidationPackageManager.java | 8 +- .../ValidationPackageManagerImpl.java | 18 +++-- 8 files changed, 148 insertions(+), 48 deletions(-) diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java index b81fb2cf..9f9af31f 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java @@ -14,6 +14,8 @@ import org.highmed.dsf.fhir.task.TaskHelper; import org.highmed.dsf.fhir.variables.FhirResourceValues; import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; +import org.hl7.fhir.r4.model.Bundle.BundleEntryResponseComponent; import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.OperationOutcome; import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity; @@ -60,6 +62,12 @@ public void afterPropertiesSet() throws Exception @Override protected void doExecute(DelegateExecution execution) throws BpmnError, Exception { + if (!bundleValidatorSupplier.isEnabled()) + { + logger.warn("Validation disabled, skipping validation. Modify configuration to enable validation"); + return; + } + bundleValidatorSupplier.create().ifPresentOrElse(validator -> { Bundle bundle = (Bundle) execution.getVariable(BPMN_EXECUTION_VARIABLE_BUNDLE); @@ -81,19 +89,28 @@ protected void doExecute(DelegateExecution execution) throws BpmnError, Exceptio { logValidationDetails(bundle); - if (bundle.getEntry().stream().map(e -> (OperationOutcome) e.getResponse().getOutcome()) - .flatMap(o -> o.getIssue().stream()) - .anyMatch(i -> IssueSeverity.FATAL.equals(i.getSeverity()) - || IssueSeverity.ERROR.equals(i.getSeverity()))) + long resourcesWithErrorCount = bundle.getEntry().stream().filter(BundleEntryComponent::hasResponse) + .map(BundleEntryComponent::getResponse).filter(BundleEntryResponseComponent::hasOutcome) + .map(BundleEntryResponseComponent::getOutcome).filter(r -> r instanceof OperationOutcome) + .map(o -> (OperationOutcome) o).map( + o -> o.getIssue().stream() + .anyMatch(i -> IssueSeverity.FATAL.equals(i.getSeverity()) + || IssueSeverity.ERROR.equals(i.getSeverity()))) + .filter(b -> b).count(); + + if (resourcesWithErrorCount > 0) { - logger.error("Validation of transfer bundle failed"); + logger.error("Validation of transfer bundle failed, {} resource{} with error", + resourcesWithErrorCount, resourcesWithErrorCount != 1 ? "s" : ""); addErrorsToTaskAndSetFailed(bundle); errorLogger.logValidationFailed(getLeadingTaskFromExecutionVariables().getIdElement() .withServerBase(getFhirWebserviceClientProvider().getLocalBaseUrl(), getLeadingTaskFromExecutionVariables().getIdElement().getResourceType())); - throw new BpmnError(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED); + throw new BpmnError(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED, + "Validation of transfer bundle failed, " + resourcesWithErrorCount + " resource" + + (resourcesWithErrorCount != 1 ? "s" : "") + " with error"); } else { diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java index 726440b8..ecd2b221 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java @@ -48,9 +48,6 @@ import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service.StoreDataForTransferHub; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service.ValidateData; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactory; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactoryImpl; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageIdentifier; -import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageManager; @Configuration public class TransferDataConfig @@ -74,10 +71,7 @@ public class TransferDataConfig private FhirContext fhirContext; @Autowired - private ValidationPackageManager validationPackageManager; - - @Autowired - private ValidationPackageIdentifier validationPackageIdentifier; + private BundleValidatorFactory bundleValidatorFactory; @ProcessDocumentation(description = "PEM encoded file with trusted certificates to validate the server-certificate of the GECCO FHIR server", processNames = { "wwwnetzwerk-universitaetsmedizinde_dataSend", @@ -443,7 +437,7 @@ public ReadData readData() @Bean public ValidateData validateData() { - return new ValidateData(fhirClientProvider, taskHelper, readAccessHelper, bundleValidatorFactory(), + return new ValidateData(fhirClientProvider, taskHelper, readAccessHelper, bundleValidatorFactory, errorOutputParameterGenerator(), errorLogger()); } @@ -459,12 +453,6 @@ public ErrorLogger errorLogger() return new ErrorLogger(); } - @Bean - public BundleValidatorFactory bundleValidatorFactory() - { - return new BundleValidatorFactoryImpl(validationPackageManager, validationPackageIdentifier); - } - @Bean public EncryptData encryptData() { diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java index 1cbeecce..2a8dff7a 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java @@ -40,6 +40,8 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.support.IValidationSupport; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactory; +import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactoryImpl; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorImpl; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithFileSystemCache; import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithModifiers; @@ -63,6 +65,10 @@ public class ValidationConfig { private static final Logger logger = LoggerFactory.getLogger(ValidationConfig.class); + @ProcessDocumentation(description = "Enables/disables local FHIR validation, set to `false` to skip validation", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") + @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation:true}") + private boolean validationEnabled; + @ProcessDocumentation(description = "FHIR implementation guide package used to validated resources, specify as `name|version`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend") @Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package:de.gecco|1.0.5}") private String validationPackage; @@ -222,9 +228,9 @@ public ValidationPackageManager validationPackageManager() EnumSet<BindingStrength> bindingStrengths = EnumSet.copyOf( valueSetExpansionBindingStrengths.stream().map(BindingStrength::fromCode).collect(Collectors.toList())); - return new ValidationPackageManagerImpl(validationPackageClient(), valueSetExpansionClient(), objectMapper(), - fhirContext, internalSnapshotGeneratorFactory(), internalValueSetExpanderFactory(), noDownload, - bindingStrengths); + return new ValidationPackageManagerImpl(validationPackageClient(), valueSetExpansionClient(), + validationObjectMapper(), fhirContext, internalSnapshotGeneratorFactory(), + internalValueSetExpanderFactory(), noDownload, bindingStrengths); } private StructureDefinitionModifier createStructureDefinitionModifier(String className) @@ -360,7 +366,7 @@ else if (clientCertificateFile == null && clientCertificatePrivateKeyFile == nul @Bean public ValidationPackageClient validationPackageClient() { - return new ValidationPackageClientWithFileSystemCache(packageCacheFolder(), objectMapper(), + return new ValidationPackageClientWithFileSystemCache(packageCacheFolder(), validationObjectMapper(), validationPackageClientJersey()); } @@ -435,11 +441,12 @@ private ValueSetExpansionClient valueSetExpansionClientJersey() valueSetExpansionClientBasicAuthUsername, valueSetExpansionClientBasicAuthPassword, valueSetExpansionClientProxySchemeHostPort, valueSetExpansionClientProxyUsername, valueSetExpansionClientProxyPassword, valueSetExpansionClientConnectTimeout, - valueSetExpansionClientReadTimeout, valueSetExpansionClientVerbose, objectMapper(), fhirContext); + valueSetExpansionClientReadTimeout, valueSetExpansionClientVerbose, validationObjectMapper(), + fhirContext); } @Bean - public ObjectMapper objectMapper() + public ObjectMapper validationObjectMapper() { return ObjectMapperFactory.createObjectMapper(fhirContext); } @@ -477,4 +484,11 @@ public boolean testConnectionToTerminologyServer() return false; } } + + @Bean + public BundleValidatorFactory bundleValidatorFactory() + { + return new BundleValidatorFactoryImpl(validationEnabled, validationPackageManager(), + validationPackageIdentifier()); + } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java index 7f2fb912..275db6d6 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java @@ -4,6 +4,11 @@ public interface BundleValidatorFactory { + /** + * @return <code>true</code> if validation is enabled + */ + boolean isEnabled(); + /** * Initializes the {@link BundleValidatorFactory} by downloading all necessary FHIR implementation guides, expanding * ValueSets and generating StructureDefinition snapshots. diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java index f7d664ed..2bbe86d4 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java @@ -13,14 +13,17 @@ public class BundleValidatorFactoryImpl implements BundleValidatorFactory, Initi { private static final Logger logger = LoggerFactory.getLogger(BundleValidatorFactoryImpl.class); + private final boolean validationEnabled; private final ValidationPackageManager validationPackageManager; private final ValidationPackageIdentifier validationPackageIdentifier; private IValidationSupport validationSupport; + private ValidationPackageWithDepedencies packageWithDependencies; - public BundleValidatorFactoryImpl(ValidationPackageManager validationPackageManager, + public BundleValidatorFactoryImpl(boolean validationEnabled, ValidationPackageManager validationPackageManager, ValidationPackageIdentifier validationPackageIdentifier) { + this.validationEnabled = validationEnabled; this.validationPackageManager = validationPackageManager; this.validationPackageIdentifier = validationPackageIdentifier; } @@ -32,6 +35,12 @@ public void afterPropertiesSet() throws Exception Objects.requireNonNull(validationPackageIdentifier, "validationPackageIdentifier"); } + @Override + public boolean isEnabled() + { + return validationEnabled; + } + @Override public void init() { @@ -39,8 +48,7 @@ public void init() return; logger.info("Downloading FHIR validation package {} and dependencies", validationPackageIdentifier.toString()); - ValidationPackageWithDepedencies packageWithDependencies = validationPackageManager - .downloadPackageWithDependencies(validationPackageIdentifier); + packageWithDependencies = validationPackageManager.downloadPackageWithDependencies(validationPackageIdentifier); logger.info("Expanding ValueSets and generating StructureDefinition snapshots"); validationSupport = validationPackageManager @@ -50,6 +58,10 @@ public void init() @Override public Optional<BundleValidator> create() { - return Optional.ofNullable(validationSupport).map(validationPackageManager::createBundleValidator); + if (validationPackageManager == null || validationSupport == null) + return Optional.empty(); + else + return Optional + .of(validationPackageManager.createBundleValidator(validationSupport, packageWithDependencies)); } } diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java index b7cd991f..f4c05730 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java @@ -1,29 +1,55 @@ package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation; +import java.util.Collections; +import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.highmed.dsf.fhir.validation.ResourceValidator; import org.hl7.fhir.r4.model.Bundle; import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; +import org.hl7.fhir.r4.model.CanonicalType; import org.hl7.fhir.r4.model.OperationOutcome; import org.hl7.fhir.r4.model.Resource; -import org.springframework.beans.factory.InitializingBean; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.StructureDefinition.StructureDefinitionKind; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.validation.ResultSeverityEnum; +import ca.uhn.fhir.validation.SingleValidationMessage; import ca.uhn.fhir.validation.ValidationResult; -public class BundleValidatorImpl implements BundleValidator, InitializingBean +public class BundleValidatorImpl implements BundleValidator { + private static final Logger logger = LoggerFactory.getLogger(BundleValidatorImpl.class); + + private final FhirContext fhirContext; private final ResourceValidator delegate; + private final Set<String> expectedStructureDefinitionUrls; + private final Set<String> expectedStructureDefinitionUrlsWithVersion; - public BundleValidatorImpl(ResourceValidator delegate) + public BundleValidatorImpl(FhirContext fhirContext, ValidationPackageWithDepedencies packageWithDependencies, + ResourceValidator delegate) { - this.delegate = delegate; - } + this.fhirContext = Objects.requireNonNull(fhirContext, "fhirContext"); - @Override - public void afterPropertiesSet() throws Exception - { - Objects.requireNonNull(delegate, "delegate"); + Set<StructureDefinition> sds = Objects.requireNonNull(packageWithDependencies, "packageWithDependencies") + .getValidationSupportResources().getStructureDefinitions().stream() + .flatMap(sd -> Stream.concat(Stream.of(sd), + packageWithDependencies.getStructureDefinitionDependencies(sd).stream())) + .filter(sd -> StructureDefinitionKind.RESOURCE.equals(sd.getKind())) + .filter(sd -> !sd.hasAbstract() || !sd.getAbstract()).filter(StructureDefinition::hasUrl) + .collect(Collectors.toSet()); + + expectedStructureDefinitionUrls = sds.stream().map(StructureDefinition::getUrl).collect(Collectors.toSet()); + expectedStructureDefinitionUrlsWithVersion = sds.stream().filter(StructureDefinition::hasVersion) + .map(sd -> sd.getUrl() + "|" + sd.getVersion()).collect(Collectors.toSet()); + + this.delegate = Objects.requireNonNull(delegate, "delegate"); } @Override @@ -31,7 +57,35 @@ public ValidationResult validate(Resource resource) { Objects.requireNonNull(resource, "resource"); - return delegate.validate(resource); + Set<String> profiles = resource.getMeta().getProfile().stream().map(CanonicalType::getValue) + .collect(Collectors.toSet()); + + if (!Collections.disjoint(profiles, expectedStructureDefinitionUrls) + || !Collections.disjoint(profiles, expectedStructureDefinitionUrlsWithVersion)) + { + // at least one supported profile claimed + return delegate.validate(resource); + } + else + { + SingleValidationMessage message = new SingleValidationMessage(); + message.setLocationString(resource.getResourceType().name() + ".meta.profile"); + + String messageText; + if (profiles.isEmpty()) + messageText = "No supported profile claimed"; + else + messageText = "No supported profile claimed, profile" + (profiles.size() == 1 ? "" : "s") + " " + + profiles.stream().sorted().collect(Collectors.joining(", ", "[", "]")) + " not supported"; + + message.setMessage(messageText); + message.setSeverity(ResultSeverityEnum.ERROR); + + logger.debug("Supported profiles {}", expectedStructureDefinitionUrlsWithVersion.stream().sorted() + .collect(Collectors.joining(", ", "[", "]"))); + + return new ValidationResult(fhirContext, List.of(message)); + } } @Override diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java index 0d289cda..d402e5fe 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java @@ -49,9 +49,13 @@ IValidationSupport expandValueSetsAndGenerateStructureDefinitionSnapshots( /** * @param validationSupport * not <code>null</code> - * @return {@link BundleValidator} for the given {@link IValidationSupport} + * @param packageWithDependencies + * not <code>null</code> + * @return {@link BundleValidator} for the given {@link IValidationSupport} and + * {@link ValidationPackageWithDepedencies} */ - BundleValidator createBundleValidator(IValidationSupport validationSupport); + BundleValidator createBundleValidator(IValidationSupport validationSupport, + ValidationPackageWithDepedencies packageWithDependencies); /** * Downloads the given FHIR package and all its dependencies. Will try to generate snapshots for all diff --git a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java index a1717e64..59b07950 100644 --- a/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java +++ b/codex-process-data-transfer/src/main/java/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManagerImpl.java @@ -43,10 +43,11 @@ public class ValidationPackageManagerImpl implements InitializingBean, Validatio { private static final Logger logger = LoggerFactory.getLogger(ValidationPackageManagerImpl.class); - public static final List<ValidationPackageIdentifier> NO_PACKAGE_DOWNLOAD = List + public static final List<ValidationPackageIdentifier> DEFAULT_NO_PACKAGE_DOWNLOAD_LIST = List .of(new ValidationPackageIdentifier("hl7.fhir.r4.core", "4.0.1")); - public static final EnumSet<BindingStrength> VALUE_SET_BINDING_STRENGTHS = EnumSet.allOf(BindingStrength.class); + public static final EnumSet<BindingStrength> DEFAULT_VALUE_SET_BINDING_STRENGTHS = EnumSet + .allOf(BindingStrength.class); private final ValidationPackageClient validationPackageClient; private final ValueSetExpansionClient valueSetExpansionClient; @@ -66,7 +67,7 @@ public ValidationPackageManagerImpl(ValidationPackageClient validationPackageCli BiFunction<FhirContext, IValidationSupport, ValueSetExpander> internalValueSetExpanderFactory) { this(validationPackageClient, valueSetExpansionClient, mapper, fhirContext, internalSnapshotGeneratorFactory, - internalValueSetExpanderFactory, NO_PACKAGE_DOWNLOAD, VALUE_SET_BINDING_STRENGTHS); + internalValueSetExpanderFactory, DEFAULT_NO_PACKAGE_DOWNLOAD_LIST, DEFAULT_VALUE_SET_BINDING_STRENGTHS); } public ValidationPackageManagerImpl(ValidationPackageClient validationPackageClient, @@ -125,11 +126,16 @@ public IValidationSupport expandValueSetsAndGenerateStructureDefinitionSnapshots } @Override - public BundleValidator createBundleValidator(IValidationSupport validationSupport) + public BundleValidator createBundleValidator(IValidationSupport validationSupport, + ValidationPackageWithDepedencies packageWithDependencies) { Objects.requireNonNull(validationSupport, "validationSupport"); + Objects.requireNonNull(packageWithDependencies, "packageWithDependencies"); + + BundleValidatorImpl validator = new BundleValidatorImpl(fhirContext, packageWithDependencies, + new ResourceValidatorImpl(fhirContext, validationSupport)); - return new BundleValidatorImpl(new ResourceValidatorImpl(fhirContext, validationSupport)); + return validator; } @Override @@ -140,7 +146,7 @@ public BundleValidator createBundleValidator(ValidationPackageIdentifier identif ValidationPackageWithDepedencies packageWithDependencies = downloadPackageWithDependencies(identifier); IValidationSupport validationSupport = expandValueSetsAndGenerateStructureDefinitionSnapshots( packageWithDependencies); - return createBundleValidator(validationSupport); + return createBundleValidator(validationSupport, packageWithDependencies); } private void downloadPackageWithDependencies(ValidationPackageIdentifier identifier,