Adds support for entity hierarchies in compare

src/NLU.DevOps.Core.Tests/JsonLabeledUtteranceConverterTests.cs

@@ -4,9 +4,9 @@
 namespace NLU.DevOps.Core.Tests
 {
     using System;
-    using System.Collections.Generic;
     using System.Linq;
     using FluentAssertions;
+    using FluentAssertions.Json;
     using Newtonsoft.Json;
     using Newtonsoft.Json.Linq;
     using Newtonsoft.Json.Serialization;
@@ -87,6 +87,63 @@ public static void ConvertsUtteranceWithStartPosAndEndPosEntity()
             actual.Entities[0].MatchIndex.Should().Be(2);
         }
 
+        [Test]
+        public static void ConvertsUtteranceWithNestedEntities()
+        {
+            var text = "foo bar baz";
+
+            var leafEntity = new JObject
+            {
+                { "entity", "baz" },
+                { "startPos", 8 },
+                { "endPos", 10 },
+                { "foo", new JArray(42) },
+                { "bar", null },
+                { "baz", 42 },
+                { "qux", JValue.CreateUndefined() },
+            };
+
+            var midEntity = new JObject
+            {
+                { "entityType", "bar" },
+                { "matchText", "bar baz" },
+                { "children", new JArray { leafEntity } },
+                { "entityValue", new JObject { { "bar", "qux" } } },
+            };
+
+            var entity = new JObject
+            {
+                { "entity", "foo" },
+                { "startPos", 0 },
+                { "endPos", 10 },
+                { "children", new JArray { midEntity } },
+            };
+
+            var json = new JObject
+            {
+                { "text", text },
+                { "entities", new JArray { entity } },
+            };
+
+            var serializer = CreateSerializer();
+            var actual = json.ToObject<JsonLabeledUtterance>(serializer);
+            actual.Text.Should().Be(text);
+            actual.Entities.Count.Should().Be(3);
+            actual.Entities[0].EntityType.Should().Be("foo");
+            actual.Entities[0].MatchText.Should().Be(text);
+            actual.Entities[1].EntityType.Should().Be("foo::bar");
+            actual.Entities[1].MatchText.Should().Be("bar baz");
+            actual.Entities[1].EntityValue.Should().BeEquivalentTo(new JObject { { "bar", "qux" } });
+            actual.Entities[2].EntityType.Should().Be("foo::bar::baz");
+            actual.Entities[2].MatchText.Should().Be("baz");
+
+            var additionalProperties = actual.Entities[2].As<Entity>().AdditionalProperties;
+            additionalProperties["foo"].As<JToken>().Should().BeEquivalentTo(new JArray(42));
+            additionalProperties["bar"].Should().BeNull();
+            additionalProperties["baz"].Should().Be(42);
+            additionalProperties["qux"].Should().BeNull();
+        }
+
         private static JsonSerializer CreateSerializer()
         {
             var serializer = JsonSerializer.CreateDefault();

src/NLU.DevOps.Core.Tests/NLU.DevOps.Core.Tests.csproj

@@ -15,7 +15,8 @@
     <PackageReference Include="nunit" Version="3.12.0" />
     <PackageReference Include="NUnit3TestAdapter" Version="3.13.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
-    <PackageReference Include="FluentAssertions" Version="5.7.0" />
+    <PackageReference Include="FluentAssertions" Version="5.5.3" />
+    <PackageReference Include="FluentAssertions.Json" Version="5.0.0" />
   </ItemGroup>
 
   <ItemGroup>

src/NLU.DevOps.Core/EntityConverter.cs

@@ -4,6 +4,8 @@
 namespace NLU.DevOps.Core
 {
     using System;
+    using System.Collections.Generic;
+    using System.Diagnostics;
     using Newtonsoft.Json;
     using Newtonsoft.Json.Linq;
 
@@ -16,42 +18,33 @@ public EntityConverter(string utterance)
 
         private string Utterance { get; }
 
+        private string Prefix { get; set; } = string.Empty;
+
         public override Entity ReadJson(JsonReader reader, Type objectType, Entity existingValue, bool hasExistingValue, JsonSerializer serializer)
         {
+            Debug.Assert(!hasExistingValue, "Entity instance can only be constructor initialized.");
+
             var jsonObject = JObject.Load(reader);
+            return typeof(HierarchicalEntity).IsAssignableFrom(objectType)
+                ? this.ReadHierarchicalEntity(jsonObject, serializer)
+                : this.ReadEntity(jsonObject, objectType, serializer);
+        }
+
+        public override void WriteJson(JsonWriter writer, Entity value, JsonSerializer serializer)
+        {
+            throw new NotImplementedException();
+        }
+
+        private Entity ReadEntity(JObject jsonObject, Type objectType, JsonSerializer serializer)
+        {
             var matchText = jsonObject.Value<string>("matchText");
+            var matchIndex = jsonObject.Value<int>("matchIndex");
             var startPosOrNull = jsonObject.Value<int?>("startPos");
             var endPosOrNull = jsonObject.Value<int?>("endPos");
-            if (matchText == null && startPosOrNull != null && endPosOrNull != null)
+            if (matchText == null && startPosOrNull.HasValue && endPosOrNull.HasValue)
             {
-                var startPos = startPosOrNull.Value;
-                var endPos = endPosOrNull.Value;
-                var length = endPos - startPos + 1;
-                if (!this.IsValid(startPos, endPos))
-                {
-                    throw new InvalidOperationException(
-                        $"Invalid start position '{startPos}' or end position '{endPos}' for utterance '{this.Utterance}'.");
-                }
-
-                matchText = this.Utterance.Substring(startPos, length);
+                (matchText, matchIndex) = this.GetMatchInfo(startPosOrNull.Value, endPosOrNull.Value);
                 jsonObject.Add("matchText", matchText);
-                var matchIndex = 0;
-                var currentPos = 0;
-                while (true)
-                {
-                    currentPos = this.Utterance.IndexOf(matchText, currentPos, StringComparison.InvariantCulture);
-
-                    // Because 'matchText' is derived from the utterance from 'startPos' and 'endPos',
-                    // we are guaranteed to find a match at with index 'startPos'.
-                    if (currentPos == startPos)
-                    {
-                        break;
-                    }
-
-                    currentPos += length;
-                    matchIndex++;
-                }
-
                 jsonObject.Add("matchIndex", matchIndex);
                 jsonObject.Remove("startPos");
                 jsonObject.Remove("endPos");
@@ -76,9 +69,86 @@ public override Entity ReadJson(JsonReader reader, Type objectType, Entity exist
             }
         }
 
-        public override void WriteJson(JsonWriter writer, Entity value, JsonSerializer serializer)
+        private HierarchicalEntity ReadHierarchicalEntity(JObject jsonObject, JsonSerializer serializer)
         {
-            throw new NotImplementedException();
+            var matchText = jsonObject.Value<string>("matchText");
+            var matchIndex = jsonObject.Value<int>("matchIndex");
+            var startPosOrNull = jsonObject.Value<int?>("startPos");
+            var endPosOrNull = jsonObject.Value<int?>("endPos");
+            if (matchText == null && startPosOrNull.HasValue && endPosOrNull.HasValue)
+            {
+                (matchText, matchIndex) = this.GetMatchInfo(startPosOrNull.Value, endPosOrNull.Value);
+            }
+
+            var entityType = jsonObject.Value<string>("entityType") ?? jsonObject.Value<string>("entity");
+            var childrenJson = jsonObject["children"];
+            var children = default(IEnumerable<HierarchicalEntity>);
+            if (childrenJson != null)
+            {
+                var prefix = $"{entityType}::";
+                this.Prefix += prefix;
+                try
+                {
+                    children = childrenJson.ToObject<IEnumerable<HierarchicalEntity>>(serializer);
+                }
+                finally
+                {
+                    this.Prefix = this.Prefix.Substring(0, this.Prefix.Length - prefix.Length);
+                }
+            }
+
+            var entity = new HierarchicalEntity($"{this.Prefix}{entityType}", jsonObject["entityValue"], matchText, matchIndex, children);
+            foreach (var property in jsonObject)
+            {
+                switch (property.Key)
+                {
+                    case "children":
+                    case "endPos":
+                    case "entity":
+                    case "entityType":
+                    case "entityValue":
+                    case "matchText":
+                    case "matchIndex":
+                    case "startPos":
+                        break;
+                    default:
+                        var value = property.Value is JValue jsonValue ? jsonValue.Value : property.Value;
+                        entity.AdditionalProperties.Add(property.Key, value);
+                        break;
+                }
+            }
+
+            return entity;
+        }
+
+        private Tuple<string, int> GetMatchInfo(int startPos, int endPos)
+        {
+            if (!this.IsValid(startPos, endPos))
+            {
+                throw new InvalidOperationException(
+                    $"Invalid start position '{startPos}' or end position '{endPos}' for utterance '{this.Utterance}'.");
+            }
+
+            var length = endPos - startPos + 1;
+            var matchText = this.Utterance.Substring(startPos, length);
+            var matchIndex = 0;
+            var currentPos = 0;
+            while (true)
+            {
+                currentPos = this.Utterance.IndexOf(matchText, currentPos, StringComparison.InvariantCulture);
+
+                // Because 'matchText' is derived from the utterance from 'startPos' and 'endPos',
+                // we are guaranteed to find a match at with index 'startPos'.
+                if (currentPos == startPos)
+                {
+                    break;
+                }
+
+                currentPos += length;
+                matchIndex++;
+            }
+
+            return Tuple.Create(matchText, matchIndex);
         }
 
         private bool IsValid(int startPos, int endPos)

src/NLU.DevOps.Core/HierarchicalEntity.cs

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace NLU.DevOps.Core
+{
+    using System.Collections.Generic;
+    using Newtonsoft.Json.Linq;
+
+    /// <summary>
+    /// Entity appearing in utterance.
+    /// </summary>
+    public sealed class HierarchicalEntity : Entity, IHierarchicalEntity
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="HierarchicalEntity"/> class.
+        /// </summary>
+        /// <param name="entityType">Entity type name.</param>
+        /// <param name="entityValue">Entity value, generally a canonical form of the entity.</param>
+        /// <param name="matchText">Matching text in the utterance.</param>
+        /// <param name="matchIndex">Occurrence index of matching token in the utterance.</param>
+        /// <param name="children">Children entities.</param>
+        public HierarchicalEntity(string entityType, JToken entityValue, string matchText, int matchIndex, IEnumerable<HierarchicalEntity> children)
+            : base(entityType, entityValue, matchText, matchIndex)
+        {
+            this.Children = children;
+        }
+
+        /// <inheritdoc />
+        public IEnumerable<IHierarchicalEntity> Children { get; }
+    }
+}

src/NLU.DevOps.Core/IHierarchicalEntity.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace NLU.DevOps.Core
+{
+    using System.Collections.Generic;
+    using Models;
+
+    /// <summary>
+    /// Entity with nested children.
+    /// </summary>
+    public interface IHierarchicalEntity : IEntity
+    {
+        /// <summary>
+        /// Gets the child entities.
+        /// </summary>
+        IEnumerable<IHierarchicalEntity> Children { get; }
+    }
+}

src/NLU.DevOps.Core/JsonEntities.cs

@@ -3,7 +3,10 @@
 
 namespace NLU.DevOps.Core
 {
+    using System;
     using System.Collections.Generic;
+    using System.Linq;
+    using Models;
     using Newtonsoft.Json;
 
     /// <summary>
@@ -15,20 +18,44 @@ public class JsonEntities
         /// Initializes a new instance of the <see cref="JsonEntities"/> class.
         /// </summary>
         /// <param name="entities">Entities referenced in the utterance.</param>
-        public JsonEntities(IReadOnlyList<Entity> entities)
+        public JsonEntities(IEnumerable<HierarchicalEntity> entities)
         {
-            this.Entities = entities;
+            this.Entities = FlattenChildren(entities)?.ToArray();
         }
 
         /// <summary>
         /// Gets the entities referenced in the utterance.
         /// </summary>
-        public IReadOnlyList<Entity> Entities { get; }
+        public IReadOnlyList<IEntity> Entities { get; }
 
         /// <summary>
         /// Gets the additional properties.
         /// </summary>
         [JsonExtensionData]
         public IDictionary<string, object> AdditionalProperties { get; } = new Dictionary<string, object>();
+
+        private static IEnumerable<IEntity> FlattenChildren(IEnumerable<IHierarchicalEntity> entities, string prefix = "")
+        {
+            if (entities == null)
+            {
+                return null;
+            }
+
+            IEnumerable<IEntity> getChildren(IHierarchicalEntity entity)
+            {
+                yield return entity;
+
+                var children = FlattenChildren(entity.Children, $"{prefix}{entity.EntityType}::");
+                if (children != null)
+                {
+                    foreach (var child in children)
+                    {
+                        yield return child;
+                    }
+                }
+            }
+
+            return entities.SelectMany(getChildren);
+        }
     }
 }

src/NLU.DevOps.Core/JsonLabeledUtteranceConverter.cs

@@ -4,6 +4,7 @@
 namespace NLU.DevOps.Core
 {
     using System;
+    using System.Diagnostics;
     using Newtonsoft.Json;
     using Newtonsoft.Json.Linq;
 
@@ -18,6 +19,8 @@ public class JsonLabeledUtteranceConverter : JsonConverter<JsonLabeledUtterance>
         /// <inheritdoc />
         public override JsonLabeledUtterance ReadJson(JsonReader reader, Type objectType, JsonLabeledUtterance existingValue, bool hasExistingValue, JsonSerializer serializer)
         {
+            Debug.Assert(!hasExistingValue, "Utterance instance can only be constructor initialized.");
+
             var jsonObject = JObject.Load(reader);
             var utterance = jsonObject.Value<string>("text") ?? jsonObject.Value<string>("query");
             var entityConverter = new EntityConverter(utterance);

src/NLU.DevOps.LuisV3.Tests/LuisNLUTestClientTests.cs

@@ -402,19 +402,19 @@ public static async Task UtteranceWithNestedMLEntity()
                 result.Text.Should().Be(test);
                 result.Intent.Should().Be("RequestVacation");
                 result.Entities.Count.Should().Be(7);
-                result.Entities[0].EntityType.Should().Be("leave-type");
+                result.Entities[0].EntityType.Should().Be("vacation-request::leave-type");
                 result.Entities[0].EntityValue.Should().BeEquivalentTo(@"[ ""sick"" ]");
                 result.Entities[0].MatchText.Should().Be("sick leave");
                 result.Entities[0].MatchIndex.Should().Be(0);
-                result.Entities[1].EntityType.Should().Be("days-number");
+                result.Entities[1].EntityType.Should().Be("vacation-request::days-duration::days-number");
                 result.Entities[1].EntityValue.Should().BeEquivalentTo("6");
                 result.Entities[1].MatchText.Should().Be("6");
                 result.Entities[1].MatchIndex.Should().Be(0);
-                result.Entities[2].EntityType.Should().Be("days-duration");
+                result.Entities[2].EntityType.Should().Be("vacation-request::days-duration");
                 result.Entities[2].EntityValue.Should().BeEquivalentTo(@"{ ""days-number"": [ 6 ] }");
                 result.Entities[2].MatchText.Should().Be("6 days");
                 result.Entities[2].MatchIndex.Should().Be(0);
-                result.Entities[3].EntityType.Should().Be("start-date");
+                result.Entities[3].EntityType.Should().Be("vacation-request::start-date");
                 result.Entities[3].MatchText.Should().Be("starting march 5");
                 result.Entities[3].MatchIndex.Should().Be(0);
                 result.Entities[4].EntityType.Should().Be("vacation-request");