Backport r4 changes to stu3

.editorconfig

@@ -175,7 +175,6 @@ dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion
 dotnet_naming_rule.types_should_be_pascal_case.symbols = types
 dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case
 
-dotnet_naming_rule.private_method_should_be_camelcase.severity = suggestion
 dotnet_naming_rule.private_method_should_be_camelcase.symbols = private_method
 dotnet_naming_rule.private_method_should_be_camelcase.style = camelcase
 
@@ -242,4 +241,4 @@ dotnet_naming_style._camelcase.capitalization = camel_case
 dotnet_naming_style.allupper.required_prefix =
 dotnet_naming_style.allupper.required_suffix =
 dotnet_naming_style.allupper.word_separator =
-dotnet_naming_style.allupper.capitalization = all_upper
+dotnet_naming_style.allupper.capitalization = all_upper

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: nuget
+    directory: "/"
+    schedule:
+      interval: daily
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: daily

.github/workflows/docker_image_linux.yml

@@ -1,4 +1,4 @@
-name: Docker Release
+name: Docker Release
 
 on:
   release:
@@ -8,7 +8,7 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Login to DockerHub Registry
         run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
       - name: Get the version

.github/workflows/integration_tests.yml

@@ -1,4 +1,4 @@
-name: Integration Tests
+name: Integration Tests
 
 on:
   workflow_dispatch:
@@ -13,7 +13,7 @@ jobs:
     steps:
       - 
         name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - 
         name: Build the latest Spark Docker image
         run: docker build . --file .docker/linux/Spark.Dockerfile
@@ -37,35 +37,35 @@ jobs:
       - 
         name: Attach test results
         if: github.event_name != 'pull_request'
-        uses: yuzutech/annotations-action@v0.1.0
+        uses: yuzutech/annotations-action@v0.4.0
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
           input: tests/integration-tests/annotations.json
       - 
         name: Archive logs
         if: ${{ always() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: logs-stu3-${{ github.sha }}
           path: tests/integration-tests/logs/*.log*
       - 
         name: Archive test reports
         if: ${{ always() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: html_summaries-stu3-${{ github.sha }}
           path: tests/integration-tests/html_summaries/**/*.html
       - 
         name: Archive JSON results
         if: ${{ always() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: json_results-stu3-${{ github.sha }}
           path: tests/integration-tests/json_results/**/*.json
       - 
         name: Archive annotations file
         if: ${{ always() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: annotations-stu3-${{ github.sha }}
           path: tests/integration-tests/annotations.json

.github/workflows/nuget_deploy.yml

@@ -8,7 +8,7 @@ jobs:
   build:
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup .NET Core
         uses: actions/setup-dotnet@v3
         with:

.github/workflows/run_tests.yaml

@@ -1,4 +1,4 @@
-name: Tests
+name: Tests
 
 on:
   push:
@@ -12,7 +12,7 @@ jobs:
     runs-on: windows-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup .NET Core
         uses: actions/setup-dotnet@v3
         with:

Spark.sln

@@ -68,6 +68,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{C6DB166C-A
 		docs\InstallingSpark.md = docs\InstallingSpark.md
 		docs\RunningSparkInDocker.md = docs\RunningSparkInDocker.md
 		docs\UsingSpark.md = docs\UsingSpark.md
+		docs\MigrateFromv1Tov2.md = docs\MigrateFromv1Tov2.md
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{977F2FE5-C232-4F93-AA0A-73DB85A20879}"

docs/MigrateFromv1Tov2.md

@@ -0,0 +1,10 @@
+## Migrate from 1.0 to 2.0
+
+### New IGenerator implementation
+MongoIdGenerator is replaced by GuidGenerator and is the new default identity generator for resource's.
+See issue https://github.com/FirelyTeam/spark/issues/572 for more information.
+
+### Rebuild indexes
+Better resolve handling for search parameters of type reference has been added, for this to take effect on existing
+resources the index has to be rebuilt. This can be done by by using the Admin UI that comes with Spark.Web or wire up
+IndexRebuildService in your implementation.

docs/Quickstart.md

@@ -10,9 +10,8 @@ Sample code for this tutorial is located here: [spark-example](https://github.co
 ## Setting up the ASP.NET core application
 
 First create an empty ASP.NET Core project:
-
 ```bash
-dotnet new web --framework netcoreapp3.1
+dotnet new web --name spark-example
 ```
 
 Add the core package:

src/Spark-Legacy/packages.config

@@ -1,7 +1,7 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="Antlr" version="3.5.0.2" targetFramework="net472" />
-  <package id="Autofac" version="6.5.0" targetFramework="net472" />
+  <package id="Autofac" version="7.1.0" targetFramework="net472" />
   <package id="Autofac.Mvc5" version="6.1.0" targetFramework="net472" />
   <package id="Autofac.SignalR2" version="6.0.0" targetFramework="net472" />
   <package id="Autofac.WebApi2" version="6.1.1" targetFramework="net472" />
@@ -11,7 +11,7 @@
   <package id="Microsoft.Extensions.Logging.Abstractions" version="6.0.3" targetFramework="net472" />
   <package id="Snappier" version="1.0.0" targetFramework="net472" />
   <package id="ZstdSharp.Port" version="0.6.5" targetFramework="net472" />
-  <package id="bootstrap" version="5.2.2" targetFramework="net472" />
+  <package id="bootstrap" version="5.3.1" targetFramework="net472" />
   <package id="Crc32C.NET" version="1.0.5.0" targetFramework="net472" />
   <package id="DnsClient" version="1.7.0" targetFramework="net472" />
   <package id="EnterpriseLibrary.SemanticLogging" version="2.0.1406.1" targetFramework="net472" />
@@ -23,7 +23,7 @@
   <package id="Hl7.Fhir.Support" version="4.3.0" targetFramework="net472" />
   <package id="Hl7.Fhir.Support.Poco" version="4.3.0" targetFramework="net472" />
   <package id="Hl7.FhirPath" version="4.3.0" targetFramework="net472" />
-  <package id="jQuery" version="3.6.1" targetFramework="net472" />
+  <package id="jQuery" version="3.7.1" targetFramework="net472" />
   <package id="Microsoft.AspNet.Cors" version="5.2.9" targetFramework="net472" />
   <package id="Microsoft.AspNet.Mvc" version="5.2.9" targetFramework="net472" />
   <package id="Microsoft.AspNet.Razor" version="3.2.9" targetFramework="net472" />

src/Spark.Engine.Test/Service/IndexServiceTests.cs

@@ -54,16 +54,18 @@ public void TestInitialize()
             };
             var searchParameters = new List<SearchParamDefinition> { spPatientName, spMiddleName };
             var resources = new Dictionary<Type, string> { { typeof(Patient), "Patient" }, { typeof(HumanName), "HumanName" } };
+
+            var resourceResolver = new ResourceResolver();
 
             // For this test setup we want a limited available types and search parameters.
             IFhirModel limitedFhirModel = new FhirModel(resources, searchParameters);
             ElementIndexer limitedElementIndexer = new ElementIndexer(limitedFhirModel);
-            _limitedIndexService = new IndexService(limitedFhirModel, indexStoreMock.Object, limitedElementIndexer);
+            _limitedIndexService = new IndexService(limitedFhirModel, indexStoreMock.Object, limitedElementIndexer, resourceResolver);
 
             // For this test setup we want all available types and search parameters.
             IFhirModel fullFhirModel = new FhirModel();
             ElementIndexer fullElementIndexer = new ElementIndexer(fullFhirModel);
-            _fullIndexService = new IndexService(fullFhirModel, indexStoreMock.Object, fullElementIndexer);
+            _fullIndexService = new IndexService(fullFhirModel, indexStoreMock.Object, fullElementIndexer, resourceResolver);
         }
 
         [TestMethod]

src/Spark.Engine.Test/Spark.Engine.Test.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netcoreapp3.1;net6.0</TargetFrameworks>
+    <TargetFrameworks>net6.0</TargetFrameworks>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
@@ -53,4 +53,4 @@
     </None>
   </ItemGroup>
 
-</Project>
+</Project>

src/Spark.Engine/Extensions/BundleExtensions.cs

@@ -1,4 +1,4 @@
-/* 
+/*
  * Copyright (c) 2014, Furore (info@furore.com) and contributors
  * See the file CONTRIBUTORS for details.
  * 
@@ -12,7 +12,7 @@
 
 namespace Spark.Engine.Extensions
 {
-    public static class FhirModelExtensions
+    public static class BundleExtensions
     {
         public static void Append(this Bundle bundle, Resource resource)
         {

src/Spark.Engine/Extensions/NetCore/IServiceCollectionExtensions.cs

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021, Incendi (info@incendi.no) and contributors
+ * Copyright (c) 2019-2023, Incendi (info@incendi.no) and contributors
  * See the file CONTRIBUTORS for details.
  *
  * This file is licensed under the BSD 3-Clause license
@@ -141,6 +141,8 @@ public static IMvcCoreBuilder AddFhir(this IServiceCollection services, SparkSet
 
             services.TryAddTransient<IReferenceNormalizationService, ReferenceNormalizationService>();
 
+            services.TryAddSingleton<ResourceResolver>();
+
             services.TryAddTransient<IIndexService, IndexService>();
             services.TryAddTransient<ILocalhost>((provider) => new Localhost(settings.Endpoint));
             services.TryAddTransient<IFhirModel>((provider) => new FhirModel(ModelInfo.SearchParameters));

src/Spark.Engine/Search/ResourceResolver.cs

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2023, Incendi (info@incendi.no) and contributors
+ * See the file CONTRIBUTORS for details.
+ *
+ * This file is licensed under the BSD 3-Clause license
+ * available at https://raw.githubusercontent.com/FirelyTeam/spark/stu3/master/LICENSE
+ */
+
+using Hl7.Fhir.ElementModel;
+using Hl7.Fhir.Model;
+using Hl7.Fhir.Serialization;
+using Hl7.Fhir.Specification;
+using Newtonsoft.Json.Linq;
+using System.Text.RegularExpressions;
+
+namespace Spark.Engine.Search
+{
+    public class ResourceResolver
+    {
+        private const string RESOURCE_TYPE_CAPTURE = "resourceType";
+        private const string RESOURCE_ID_CAPTURE = "resourceId";
+
+        private readonly Regex _referenceRegex;
+        private readonly IStructureDefinitionSummaryProvider _structureDefinitionSummaryProvider;
+
+        public ResourceResolver()
+        {
+            var resourceTypesPattern = string.Join("|", ModelInfo.SupportedResources);
+            var referenceCaptureRegexPattern = $@"(?<{RESOURCE_TYPE_CAPTURE}>{resourceTypesPattern})\/(?<{RESOURCE_ID_CAPTURE}>[A-Za-z0-9\-\.]{{1,64}})(\/_history\/[A-Za-z0-9\-\.]{{1,64}})?";
+            _referenceRegex = new Regex(referenceCaptureRegexPattern, RegexOptions.Singleline | RegexOptions.Compiled | RegexOptions.ExplicitCapture);
+
+            _structureDefinitionSummaryProvider = new PocoStructureDefinitionSummaryProvider();
+        }
+
+        public ITypedElement Resolve(string reference)
+        {
+            if (string.IsNullOrWhiteSpace(reference))
+            {
+                return null;
+            }
+
+            var match = _referenceRegex.Match(reference);
+            if (!match.Success)
+            {
+                return null;
+            }
+
+            string resourceTypeInString = match.Groups[RESOURCE_TYPE_CAPTURE].Value;
+            string resourceId = match.Groups[RESOURCE_ID_CAPTURE].Value;
+            ISourceNode node = FhirJsonNode.Create(
+                JObject.FromObject(
+                    new
+                    {
+                        resourceType = resourceTypeInString,
+                        id = resourceId,
+                    }));
+
+            return node.ToTypedElement(_structureDefinitionSummaryProvider);
+        }
+    }
+}

src/Spark.Engine/Service/FhirServiceExtensions/IndexService.cs

@@ -1,13 +1,15 @@
-/* 
- * Copyright (c) 2021, Incendi (info@incendi.no) and contributors
+/*
+ * Copyright (c) 2021-2023, Incendi (info@incendi.no) and contributors
  * See the file CONTRIBUTORS for details.
- * 
+ *
  * This file is licensed under the BSD 3-Clause license
  * available at https://raw.githubusercontent.com/FirelyTeam/spark/stu3/master/LICENSE
  */
 
+using Hl7.Fhir.ElementModel;
 using Hl7.Fhir.FhirPath;
 using Hl7.Fhir.Model;
+using Hl7.FhirPath;
 using Spark.Engine.Core;
 using Spark.Engine.Extensions;
 using Spark.Engine.Model;
@@ -28,12 +30,14 @@ public class IndexService : IIndexService
         private readonly IFhirModel _fhirModel;
         private readonly IIndexStore _indexStore;
         private readonly ElementIndexer _elementIndexer;
+        private readonly ResourceResolver _elementResolver;
 
-        public IndexService(IFhirModel fhirModel, IIndexStore indexStore, ElementIndexer elementIndexer)
+        public IndexService(IFhirModel fhirModel, IIndexStore indexStore, ElementIndexer elementIndexer, ResourceResolver elementResolver)
         {
-            _fhirModel = fhirModel;
-            _indexStore = indexStore;
-            _elementIndexer = elementIndexer;
+            _fhirModel = fhirModel ?? throw new ArgumentNullException(nameof(fhirModel));
+            _indexStore = indexStore ?? throw new ArgumentNullException(nameof(indexStore));
+            _elementIndexer = elementIndexer ?? throw new ArgumentNullException(nameof(elementIndexer));
+            _elementResolver = elementResolver ?? throw new ArgumentNullException(nameof(elementResolver));
         }
 
         public void Process(Entry entry)
@@ -84,6 +88,14 @@ public async Task<IndexValue> IndexResourceAsync(Resource resource, IKey key)
             return indexValue;
         }
 
+        private EvaluationContext GetEvaluationContext(Func<string, ITypedElement> elementResolver = null)
+        {
+            return new FhirEvaluationContext
+            {
+                ElementResolver = elementResolver,
+            };
+        }
+
         private IndexValue IndexResourceRecursively(Resource resource, IKey key, string rootPartName = "root")
         {
             IEnumerable<SearchParameter> searchParameters = _fhirModel.FindSearchParameters(resource.GetType());
@@ -104,8 +116,19 @@ private IndexValue IndexResourceRecursively(Resource resource, IKey key, string
                 if (searchParameter.Type == Hl7.Fhir.Model.SearchParamType.Composite) continue;
 
                 var indexValue = new IndexValue(searchParameter.Code);
-                var resolvedValues = resource.SelectNew(searchParameter.Expression);
-                foreach(var value in resolvedValues)
+                IEnumerable<Base> resolvedValues;
+                // HACK: Ignoring search parameter expressions which the FhirPath engine does not yet have support for
+
+                try
+                {
+                    resolvedValues = resource.SelectNew(searchParameter.Expression, new FhirEvaluationContext { ElementResolver = _elementResolver.Resolve });
+                }
+                catch (Exception)
+                {
+                    // TODO: log error!
+                    resolvedValues = new List<Base>();
+                }
+                foreach (var value in resolvedValues)
                 {
                     if (!(value is Element element)) continue;
 

src/Spark.Engine/Spark.Engine.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
         <TargetFrameworks>netstandard2.0;net6.0;net462</TargetFrameworks>
@@ -9,7 +9,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Fhir.Metrics" Version="1.2.1" />
+        <PackageReference Include="Fhir.Metrics" Version="1.2.2" />
         <PackageReference Include="Hl7.Fhir.STU3" Version="4.3.0" />
         <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="6.0.0" />
         <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />