env variable to disable validation, error when profile not supported

Added new environment variable to disable FHIR validation. Added new functionality to test if all resources are declaring at least on supported profile. A validation error is raised if no supported profile is defined. Profiles are supported if they are declared in the root validation package (currently de.gecco | 1.0.5) or are dependencies of the StructureDefinitions. Only profile with abstract = false and kind = resource are supported as claimed profiles by resources beeing validated.
num-codex · Jun 18, 2022 · 7bc60c1 · 7bc60c1
1 parent 76c208a
commit 7bc60c1
Show file tree

Hide file tree

Showing 8 changed files with 148 additions and 48 deletions.
diff --git a/.../de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java b/.../de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/ValidateData.java
@@ -14,6 +14,8 @@
 import org.highmed.dsf.fhir.task.TaskHelper;
 import org.highmed.dsf.fhir.variables.FhirResourceValues;
 import org.hl7.fhir.r4.model.Bundle;
+import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent;
+import org.hl7.fhir.r4.model.Bundle.BundleEntryResponseComponent;
 import org.hl7.fhir.r4.model.IdType;
 import org.hl7.fhir.r4.model.OperationOutcome;
 import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity;
@@ -60,6 +62,12 @@ public void afterPropertiesSet() throws Exception
 	@Override
 	protected void doExecute(DelegateExecution execution) throws BpmnError, Exception
 	{
+		if (!bundleValidatorSupplier.isEnabled())
+		{
+			logger.warn("Validation disabled, skipping validation. Modify configuration to enable validation");
+			return;
+		}
+
 		bundleValidatorSupplier.create().ifPresentOrElse(validator ->
 		{
 			Bundle bundle = (Bundle) execution.getVariable(BPMN_EXECUTION_VARIABLE_BUNDLE);
@@ -81,19 +89,28 @@ protected void doExecute(DelegateExecution execution) throws BpmnError, Exceptio
 				{
 					logValidationDetails(bundle);
 
-					if (bundle.getEntry().stream().map(e -> (OperationOutcome) e.getResponse().getOutcome())
-							.flatMap(o -> o.getIssue().stream())
-							.anyMatch(i -> IssueSeverity.FATAL.equals(i.getSeverity())
-									|| IssueSeverity.ERROR.equals(i.getSeverity())))
+					long resourcesWithErrorCount = bundle.getEntry().stream().filter(BundleEntryComponent::hasResponse)
+							.map(BundleEntryComponent::getResponse).filter(BundleEntryResponseComponent::hasOutcome)
+							.map(BundleEntryResponseComponent::getOutcome).filter(r -> r instanceof OperationOutcome)
+							.map(o -> (OperationOutcome) o).map(
+									o -> o.getIssue().stream()
+											.anyMatch(i -> IssueSeverity.FATAL.equals(i.getSeverity())
+													|| IssueSeverity.ERROR.equals(i.getSeverity())))
+							.filter(b -> b).count();
+
+					if (resourcesWithErrorCount > 0)
 					{
-						logger.error("Validation of transfer bundle failed");
+						logger.error("Validation of transfer bundle failed, {} resource{} with error",
+								resourcesWithErrorCount, resourcesWithErrorCount != 1 ? "s" : "");
 
 						addErrorsToTaskAndSetFailed(bundle);
 						errorLogger.logValidationFailed(getLeadingTaskFromExecutionVariables().getIdElement()
 								.withServerBase(getFhirWebserviceClientProvider().getLocalBaseUrl(),
 										getLeadingTaskFromExecutionVariables().getIdElement().getResourceType()));
 
-						throw new BpmnError(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED);
+						throw new BpmnError(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED,
+								"Validation of transfer bundle failed, " + resourcesWithErrorCount + " resource"
+										+ (resourcesWithErrorCount != 1 ? "s" : "") + " with error");
 					}
 					else
 					{

diff --git a/..._universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java b/..._universitaetsmedizin/codex/processes/data_transfer/spring/config/TransferDataConfig.java
@@ -48,9 +48,6 @@
 import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service.StoreDataForTransferHub;
 import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.service.ValidateData;
 import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactory;
-import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactoryImpl;
-import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageIdentifier;
-import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.ValidationPackageManager;
 
 @Configuration
 public class TransferDataConfig
@@ -74,10 +71,7 @@ public class TransferDataConfig
 	private FhirContext fhirContext;
 
 	@Autowired
-	private ValidationPackageManager validationPackageManager;
-
-	@Autowired
-	private ValidationPackageIdentifier validationPackageIdentifier;
+	private BundleValidatorFactory bundleValidatorFactory;
 
 	@ProcessDocumentation(description = "PEM encoded file with trusted certificates to validate the server-certificate of the GECCO FHIR server", processNames = {
 			"wwwnetzwerk-universitaetsmedizinde_dataSend",
@@ -443,7 +437,7 @@ public ReadData readData()
 	@Bean
 	public ValidateData validateData()
 	{
-		return new ValidateData(fhirClientProvider, taskHelper, readAccessHelper, bundleValidatorFactory(),
+		return new ValidateData(fhirClientProvider, taskHelper, readAccessHelper, bundleValidatorFactory,
 				errorOutputParameterGenerator(), errorLogger());
 	}
 
@@ -459,12 +453,6 @@ public ErrorLogger errorLogger()
 		return new ErrorLogger();
 	}
 
-	@Bean
-	public BundleValidatorFactory bundleValidatorFactory()
-	{
-		return new BundleValidatorFactoryImpl(validationPackageManager, validationPackageIdentifier);
-	}
-
 	@Bean
 	public EncryptData encryptData()
 	{

diff --git a/...rk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java b/...rk_universitaetsmedizin/codex/processes/data_transfer/spring/config/ValidationConfig.java
@@ -40,6 +40,8 @@
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.support.IValidationSupport;
+import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactory;
+import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.BundleValidatorFactoryImpl;
 import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorImpl;
 import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithFileSystemCache;
 import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation.PluginSnapshotGeneratorWithModifiers;
@@ -63,6 +65,10 @@ public class ValidationConfig
 {
 	private static final Logger logger = LoggerFactory.getLogger(ValidationConfig.class);
 
+	@ProcessDocumentation(description = "Enables/disables local FHIR validation, set to `false` to skip validation", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend")
+	@Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation:true}")
+	private boolean validationEnabled;
+
 	@ProcessDocumentation(description = "FHIR implementation guide package used to validated resources, specify as `name|version`", processNames = "wwwnetzwerk-universitaetsmedizinde_dataSend")
 	@Value("${de.netzwerk.universitaetsmedizin.codex.gecco.validation.package:de.gecco|1.0.5}")
 	private String validationPackage;
@@ -222,9 +228,9 @@ public ValidationPackageManager validationPackageManager()
 		EnumSet<BindingStrength> bindingStrengths = EnumSet.copyOf(
 				valueSetExpansionBindingStrengths.stream().map(BindingStrength::fromCode).collect(Collectors.toList()));
 
-		return new ValidationPackageManagerImpl(validationPackageClient(), valueSetExpansionClient(), objectMapper(),
-				fhirContext, internalSnapshotGeneratorFactory(), internalValueSetExpanderFactory(), noDownload,
-				bindingStrengths);
+		return new ValidationPackageManagerImpl(validationPackageClient(), valueSetExpansionClient(),
+				validationObjectMapper(), fhirContext, internalSnapshotGeneratorFactory(),
+				internalValueSetExpanderFactory(), noDownload, bindingStrengths);
 	}
 
 	private StructureDefinitionModifier createStructureDefinitionModifier(String className)
@@ -360,7 +366,7 @@ else if (clientCertificateFile == null && clientCertificatePrivateKeyFile == nul
 	@Bean
 	public ValidationPackageClient validationPackageClient()
 	{
-		return new ValidationPackageClientWithFileSystemCache(packageCacheFolder(), objectMapper(),
+		return new ValidationPackageClientWithFileSystemCache(packageCacheFolder(), validationObjectMapper(),
 				validationPackageClientJersey());
 	}
 
@@ -435,11 +441,12 @@ private ValueSetExpansionClient valueSetExpansionClientJersey()
 				valueSetExpansionClientBasicAuthUsername, valueSetExpansionClientBasicAuthPassword,
 				valueSetExpansionClientProxySchemeHostPort, valueSetExpansionClientProxyUsername,
 				valueSetExpansionClientProxyPassword, valueSetExpansionClientConnectTimeout,
-				valueSetExpansionClientReadTimeout, valueSetExpansionClientVerbose, objectMapper(), fhirContext);
+				valueSetExpansionClientReadTimeout, valueSetExpansionClientVerbose, validationObjectMapper(),
+				fhirContext);
 	}
 
 	@Bean
-	public ObjectMapper objectMapper()
+	public ObjectMapper validationObjectMapper()
 	{
 		return ObjectMapperFactory.createObjectMapper(fhirContext);
 	}
@@ -477,4 +484,11 @@ public boolean testConnectionToTerminologyServer()
 			return false;
 		}
 	}
+
+	@Bean
+	public BundleValidatorFactory bundleValidatorFactory()
+	{
+		return new BundleValidatorFactoryImpl(validationEnabled, validationPackageManager(),
+				validationPackageIdentifier());
+	}
 }
diff --git a/...universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java b/...universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactory.java
@@ -4,6 +4,11 @@
 
 public interface BundleValidatorFactory
 {
+	/**
+	 * @return <code>true</code> if validation is enabled
+	 */
+	boolean isEnabled();
+
 	/**
 	 * Initializes the {@link BundleValidatorFactory} by downloading all necessary FHIR implementation guides, expanding
 	 * ValueSets and generating StructureDefinition snapshots.

diff --git a/...ersitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java b/...ersitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorFactoryImpl.java
@@ -13,14 +13,17 @@ public class BundleValidatorFactoryImpl implements BundleValidatorFactory, Initi
 {
 	private static final Logger logger = LoggerFactory.getLogger(BundleValidatorFactoryImpl.class);
 
+	private final boolean validationEnabled;
 	private final ValidationPackageManager validationPackageManager;
 	private final ValidationPackageIdentifier validationPackageIdentifier;
 
 	private IValidationSupport validationSupport;
+	private ValidationPackageWithDepedencies packageWithDependencies;
 
-	public BundleValidatorFactoryImpl(ValidationPackageManager validationPackageManager,
+	public BundleValidatorFactoryImpl(boolean validationEnabled, ValidationPackageManager validationPackageManager,
 			ValidationPackageIdentifier validationPackageIdentifier)
 	{
+		this.validationEnabled = validationEnabled;
 		this.validationPackageManager = validationPackageManager;
 		this.validationPackageIdentifier = validationPackageIdentifier;
 	}
@@ -32,15 +35,20 @@ public void afterPropertiesSet() throws Exception
 		Objects.requireNonNull(validationPackageIdentifier, "validationPackageIdentifier");
 	}
 
+	@Override
+	public boolean isEnabled()
+	{
+		return validationEnabled;
+	}
+
 	@Override
 	public void init()
 	{
 		if (validationSupport != null)
 			return;
 
 		logger.info("Downloading FHIR validation package {} and dependencies", validationPackageIdentifier.toString());
-		ValidationPackageWithDepedencies packageWithDependencies = validationPackageManager
-				.downloadPackageWithDependencies(validationPackageIdentifier);
+		packageWithDependencies = validationPackageManager.downloadPackageWithDependencies(validationPackageIdentifier);
 
 		logger.info("Expanding ValueSets and generating StructureDefinition snapshots");
 		validationSupport = validationPackageManager
@@ -50,6 +58,10 @@ public void init()
 	@Override
 	public Optional<BundleValidator> create()
 	{
-		return Optional.ofNullable(validationSupport).map(validationPackageManager::createBundleValidator);
+		if (validationPackageManager == null || validationSupport == null)
+			return Optional.empty();
+		else
+			return Optional
+					.of(validationPackageManager.createBundleValidator(validationSupport, packageWithDependencies));
 	}
 }
diff --git a/...rk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java b/...rk_universitaetsmedizin/codex/processes/data_transfer/validation/BundleValidatorImpl.java
@@ -1,37 +1,91 @@
 package de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.validation;
 
+import java.util.Collections;
+import java.util.List;
 import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.highmed.dsf.fhir.validation.ResourceValidator;
 import org.hl7.fhir.r4.model.Bundle;
 import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent;
+import org.hl7.fhir.r4.model.CanonicalType;
 import org.hl7.fhir.r4.model.OperationOutcome;
 import org.hl7.fhir.r4.model.Resource;
-import org.springframework.beans.factory.InitializingBean;
+import org.hl7.fhir.r4.model.StructureDefinition;
+import org.hl7.fhir.r4.model.StructureDefinition.StructureDefinitionKind;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.validation.ResultSeverityEnum;
+import ca.uhn.fhir.validation.SingleValidationMessage;
 import ca.uhn.fhir.validation.ValidationResult;
 
-public class BundleValidatorImpl implements BundleValidator, InitializingBean
+public class BundleValidatorImpl implements BundleValidator
 {
+	private static final Logger logger = LoggerFactory.getLogger(BundleValidatorImpl.class);
+
+	private final FhirContext fhirContext;
 	private final ResourceValidator delegate;
+	private final Set<String> expectedStructureDefinitionUrls;
+	private final Set<String> expectedStructureDefinitionUrlsWithVersion;
 
-	public BundleValidatorImpl(ResourceValidator delegate)
+	public BundleValidatorImpl(FhirContext fhirContext, ValidationPackageWithDepedencies packageWithDependencies,
+			ResourceValidator delegate)
 	{
-		this.delegate = delegate;
-	}
+		this.fhirContext = Objects.requireNonNull(fhirContext, "fhirContext");
 
-	@Override
-	public void afterPropertiesSet() throws Exception
-	{
-		Objects.requireNonNull(delegate, "delegate");
+		Set<StructureDefinition> sds = Objects.requireNonNull(packageWithDependencies, "packageWithDependencies")
+				.getValidationSupportResources().getStructureDefinitions().stream()
+				.flatMap(sd -> Stream.concat(Stream.of(sd),
+						packageWithDependencies.getStructureDefinitionDependencies(sd).stream()))
+				.filter(sd -> StructureDefinitionKind.RESOURCE.equals(sd.getKind()))
+				.filter(sd -> !sd.hasAbstract() || !sd.getAbstract()).filter(StructureDefinition::hasUrl)
+				.collect(Collectors.toSet());
+
+		expectedStructureDefinitionUrls = sds.stream().map(StructureDefinition::getUrl).collect(Collectors.toSet());
+		expectedStructureDefinitionUrlsWithVersion = sds.stream().filter(StructureDefinition::hasVersion)
+				.map(sd -> sd.getUrl() + "|" + sd.getVersion()).collect(Collectors.toSet());
+
+		this.delegate = Objects.requireNonNull(delegate, "delegate");
 	}
 
 	@Override
 	public ValidationResult validate(Resource resource)
 	{
 		Objects.requireNonNull(resource, "resource");
 
-		return delegate.validate(resource);
+		Set<String> profiles = resource.getMeta().getProfile().stream().map(CanonicalType::getValue)
+				.collect(Collectors.toSet());
+
+		if (!Collections.disjoint(profiles, expectedStructureDefinitionUrls)
+				|| !Collections.disjoint(profiles, expectedStructureDefinitionUrlsWithVersion))
+		{
+			// at least one supported profile claimed
+			return delegate.validate(resource);
+		}
+		else
+		{
+			SingleValidationMessage message = new SingleValidationMessage();
+			message.setLocationString(resource.getResourceType().name() + ".meta.profile");
+
+			String messageText;
+			if (profiles.isEmpty())
+				messageText = "No supported profile claimed";
+			else
+				messageText = "No supported profile claimed, profile" + (profiles.size() == 1 ? "" : "s") + " "
+						+ profiles.stream().sorted().collect(Collectors.joining(", ", "[", "]")) + " not supported";
+
+			message.setMessage(messageText);
+			message.setSeverity(ResultSeverityEnum.ERROR);
+
+			logger.debug("Supported profiles {}", expectedStructureDefinitionUrlsWithVersion.stream().sorted()
+					.collect(Collectors.joining(", ", "[", "]")));
+
+			return new ValidationResult(fhirContext, List.of(message));
+		}
 	}
 
 	@Override

diff --git a/...iversitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java b/...iversitaetsmedizin/codex/processes/data_transfer/validation/ValidationPackageManager.java
@@ -49,9 +49,13 @@ IValidationSupport expandValueSetsAndGenerateStructureDefinitionSnapshots(
 	/**
 	 * @param validationSupport
 	 *            not <code>null</code>
-	 * @return {@link BundleValidator} for the given {@link IValidationSupport}
+	 * @param packageWithDependencies
+	 *            not <code>null</code>
+	 * @return {@link BundleValidator} for the given {@link IValidationSupport} and
+	 *         {@link ValidationPackageWithDepedencies}
 	 */
-	BundleValidator createBundleValidator(IValidationSupport validationSupport);
+	BundleValidator createBundleValidator(IValidationSupport validationSupport,
+			ValidationPackageWithDepedencies packageWithDependencies);
 
 	/**
 	 * Downloads the given FHIR package and all its dependencies. Will try to generate snapshots for all