Skip to content

Commit

Permalink
Add support for composable index templates (opensearch-project#437)
Browse files Browse the repository at this point in the history 
* Allow renaming URL path parts in generator

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Generate {delete,exists,get,put}_index_template as *ComposableTemplate

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Add resp/req bodies for *ComposableIndexTemplate

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Fix tests

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Add ComposableIndexTemplateCrudTests

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Add ComposableIndexTemplateExists tests

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Add DeleteComposableIndexTemplate tests

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Add GetComposableIndexTemplate tests

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Add PutComposableIndexTemplate tests

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Non-overlapping templates

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Fix tests

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Test data_stream template mapping serialization

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Update guide and add sample

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Add changelog entry

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Tidy generated code

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Review feedback

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

---------

Signed-off-by: Thomas Farr <tsfarr@amazon.com>
  • Loading branch information
@Xtansia
Xtansia committed Dec 3, 2023
1 parent 74f9b6c commit ef15768
Show file tree
Hide file tree
Showing 8 changed files with 102 additions and 85 deletions.
10 changes: 5 additions & 5 deletions src/ApiGenerator/Configuration/CodeConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ public static class CodeConfiguration
new("cluster.*_component_template"),

new("dangling_indices.*"),

new("indices.{delete,exists,get,put}_index_template"),

new("ingest.*"),
new("nodes.*"),
new("snapshot.*"),
Expand All @@ -62,11 +65,8 @@ public static class CodeConfiguration
/// <summary>
/// Map API default names for API's we are only supporting on the low level client first
/// </summary>
private static readonly Dictionary<string, string> LowLevelApiNameMapping = new Dictionary<string, string>
{
{ "indices.delete_index_template", "DeleteIndexTemplateV2" },
{ "indices.get_index_template", "GetIndexTemplateV2" },
{ "indices.put_index_template", "PutIndexTemplateV2" }
private static readonly Dictionary<string, string> LowLevelApiNameMapping = new()
{
};

/// <summary>
Expand Down
2 changes: 2 additions & 0 deletions src/ApiGenerator/Configuration/Overrides/EndpointOverridesBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ namespace ApiGenerator.Configuration.Overrides
{
public abstract class EndpointOverridesBase : IEndpointOverrides
{
public virtual IDictionary<string, string> RenameUrlParts { get; } = new SortedDictionary<string, string>();

public virtual IDictionary<string, string> ObsoleteQueryStringParams { get; set; } = new SortedDictionary<string, string>();

public virtual IDictionary<string, string> RenameQueryStringParams { get; } = new SortedDictionary<string, string>();
Expand Down
6 changes: 5 additions & 1 deletion src/ApiGenerator/Configuration/Overrides/GlobalOverrides.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,11 @@
namespace ApiGenerator.Configuration.Overrides
{
public class GlobalOverrides : EndpointOverridesBase
{
{
public static readonly GlobalOverrides Instance = new();

private GlobalOverrides() { }

public IDictionary<string, Dictionary<string, string>> ObsoleteEnumMembers { get; set; } = new Dictionary<string, Dictionary<string, string>>()
{
{ "VersionType", new Dictionary<string, string>() { { "force", "Force is no longer accepted by the server as of 7.5.0 and will result in an error when used" } } }
Expand Down
7 changes: 6 additions & 1 deletion src/ApiGenerator/Configuration/Overrides/IEndpointOverrides.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,15 @@ namespace ApiGenerator.Configuration.Overrides
/// </summary>
public interface IEndpointOverrides
{
/// <summary>
/// Override how the url part is exposed to the client.
/// </summary>
IDictionary<string, string> RenameUrlParts { get; }

/// <summary>
/// A map of key -> obsolete message for properties in the spec that should not be used any longer
/// </summary>
IDictionary<string, string> ObsoleteQueryStringParams { get; set; }
IDictionary<string, string> ObsoleteQueryStringParams { get; }

/// <summary>
/// Override how the query param name is exposed to the client.
Expand Down
63 changes: 38 additions & 25 deletions ...rator/Domain/ApiQueryParametersPatcher.cs → ...tor/Domain/ApiRequestParametersPatcher.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,23 +34,37 @@

namespace ApiGenerator.Domain
{
public static class ApiQueryParametersPatcher
public static class ApiRequestParametersPatcher
{
public static SortedDictionary<string, QueryParameters> Patch(
public static void PatchUrlPaths(string endpointName, IList<UrlPath> source, IEndpointOverrides overrides)
{
var declaredKeys = source.SelectMany(p => p.Parts).Select(p => p.Name).ToHashSet();
var renameLookup = CreateUrlPartRenameLookup(overrides, declaredKeys);

foreach (var path in source)
{
foreach (var part in path.Parts)
{
if (!renameLookup.TryGetValue(part.Name, out var newName)) continue;

path.Path = path.Path.Replace($"{{{part.Name}}}", $"{{{newName}}}");
part.Name = newName;
}
}
}

public static SortedDictionary<string, QueryParameters> PatchQueryParameters(
string endpointName,
IDictionary<string, QueryParameters> source,
IEndpointOverrides overrides
)
{
if (source == null) return null;

var globalOverrides = new GlobalOverrides();
var declaredKeys = source.Keys;
var skipList = CreateSkipList(globalOverrides, overrides, declaredKeys);
var partialList = CreatePartialList(globalOverrides, overrides, declaredKeys);
var skipList = CreateSkipList(overrides, declaredKeys);
var partialList = CreatePartialList(overrides, declaredKeys);

var renameLookup = CreateRenameLookup(globalOverrides, overrides, declaredKeys);
var obsoleteLookup = CreateObsoleteLookup(globalOverrides, overrides, declaredKeys);
var renameLookup = CreateRenameLookup(overrides, declaredKeys);
var obsoleteLookup = CreateObsoleteLookup(overrides, declaredKeys);

var patchedParams = new SortedDictionary<string, QueryParameters>();
foreach (var (queryStringKey, value) in source)
Expand Down Expand Up @@ -96,18 +110,18 @@ private static string CreateCSharpName(string queryStringKey, string endpointNam
}
}

private static IList<string> CreateSkipList(IEndpointOverrides global, IEndpointOverrides local, ICollection<string> declaredKeys) =>
CreateList(global, local, "skip", e => e.SkipQueryStringParams, declaredKeys);
private static IList<string> CreateSkipList(IEndpointOverrides local, ICollection<string> declaredKeys) =>
CreateList(local, "skip", e => e.SkipQueryStringParams, declaredKeys);

private static IList<string> CreatePartialList(IEndpointOverrides global, IEndpointOverrides local, ICollection<string> declaredKeys) =>
CreateList(global, local, "partial", e => e.RenderPartial, declaredKeys);
private static IList<string> CreatePartialList(IEndpointOverrides local, ICollection<string> declaredKeys) =>
CreateList(local, "partial", e => e.RenderPartial, declaredKeys);

private static IDictionary<string, string> CreateLookup(IEndpointOverrides global, IEndpointOverrides local, string type,
private static IDictionary<string, string> CreateLookup(IEndpointOverrides local, string type,
Func<IEndpointOverrides, IDictionary<string, string>> @from, ICollection<string> declaredKeys
)
{
var d = new SortedDictionary<string, string>();
foreach (var kv in from(global)) d[kv.Key] = kv.Value;
foreach (var kv in from(GlobalOverrides.Instance)) d[kv.Key] = kv.Value;

if (local == null) return d;

Expand All @@ -121,12 +135,12 @@ Func<IEndpointOverrides, IDictionary<string, string>> @from, ICollection<string>
return d;
}

private static IList<string> CreateList(IEndpointOverrides global, IEndpointOverrides local, string type,
private static IList<string> CreateList(IEndpointOverrides local, string type,
Func<IEndpointOverrides, IEnumerable<string>> @from, ICollection<string> declaredKeys
)
{
var list = new List<string>();
if (global != null) list.AddRange(from(global));
list.AddRange(from(GlobalOverrides.Instance));
if (local != null)
{
var localList = from(local).ToList();
Expand All @@ -138,14 +152,13 @@ Func<IEndpointOverrides, IEnumerable<string>> @from, ICollection<string> declare
return list.Distinct().ToList();
}

private static IDictionary<string, string> CreateRenameLookup(IEndpointOverrides global, IEndpointOverrides local,
ICollection<string> declaredKeys
) =>
CreateLookup(global, local, "rename", e => e.RenameQueryStringParams, declaredKeys);
private static IDictionary<string, string> CreateUrlPartRenameLookup(IEndpointOverrides local, ICollection<string> declaredKeys) =>
CreateLookup(local, "url_part_rename", e => e.RenameUrlParts, declaredKeys);

private static IDictionary<string, string> CreateRenameLookup(IEndpointOverrides local, ICollection<string> declaredKeys) =>
CreateLookup(local, "rename", e => e.RenameQueryStringParams, declaredKeys);

private static IDictionary<string, string> CreateObsoleteLookup(IEndpointOverrides global, IEndpointOverrides local,
ICollection<string> declaredKeys
) =>
CreateLookup(global, local, "obsolete", e => e.ObsoleteQueryStringParams, declaredKeys);
private static IDictionary<string, string> CreateObsoleteLookup(IEndpointOverrides local, ICollection<string> declaredKeys) =>
CreateLookup(local, "obsolete", e => e.ObsoleteQueryStringParams, declaredKeys);
}
}
2 changes: 1 addition & 1 deletion src/ApiGenerator/Domain/Specification/UrlPath.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ namespace ApiGenerator.Domain.Specification
public class UrlPath
{
private readonly IList<UrlPart> _additionalPartsForConstructor;
public string Path { get; }
public string Path { get; set; }
public Deprecation Deprecation { get; }
public Version VersionAdded { get; }
public IList<UrlPart> Parts { get; }
Expand Down
82 changes: 38 additions & 44 deletions src/ApiGenerator/Generator/ApiEndpointFactory.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,8 @@

using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Net.Mime;
using System.Text.RegularExpressions;
using ApiGenerator.Configuration;
using ApiGenerator.Configuration.Overrides;
using ApiGenerator.Domain;
Expand All @@ -40,7 +38,6 @@
using NJsonSchema;
using NJsonSchema.References;
using NSwag;
using SemanticVersioning;
using Version = SemanticVersioning.Version;

namespace ApiGenerator.Generator
Expand All @@ -52,6 +49,8 @@ public static ApiEndpoint From(string name, List<(string HttpPath, OpenApiPathIt
var tokens = name.Split(".");
var methodName = tokens[^1];
var ns = tokens.Length > 1 ? tokens[0] : null;
var names = new CsharpNames(name, methodName, ns);
var overrides = LoadOverrides(name, names.MethodName);

HashSet<string> requiredPathParts = null;
var allParts = new Dictionary<string, UrlPart>();
Expand Down Expand Up @@ -126,6 +125,9 @@ public static ApiEndpoint From(string name, List<(string HttpPath, OpenApiPathIt
.Where(p => !canonicalPaths.ContainsKey(p.Key))
.Select(p => p.Value))
.ToList();

ApiRequestParametersPatcher.PatchUrlPaths(name, paths, overrides);

paths.Sort((p1, p2) => p1.Parts
.Zip(p2.Parts)
.Select(t => string.Compare(t.First.Name, t.Second.Name, StringComparison.Ordinal))
Expand All @@ -151,58 +153,52 @@ public static ApiEndpoint From(string name, List<(string HttpPath, OpenApiPathIt

foreach (var partName in requiredPathParts ?? Enumerable.Empty<string>()) allParts[partName].Required = true;

var endpoint = new ApiEndpoint
{
Name = name,
Namespace = ns,
MethodName = methodName,
CsharpNames = new CsharpNames(name, methodName, ns),
Stability = Stability.Stable, // TODO: for realsies
OfficialDocumentationLink = new Documentation
{
Description = variants[0].Operation.Description,
Url = variants[0].Operation.ExternalDocumentation?.Url
},
Url = new UrlInformation
IDictionary<string, QueryParameters> queryParams = variants.SelectMany(v => v.Path.Parameters.Concat(v.Operation.Parameters))
.Where(p => p.Kind == OpenApiParameterKind.Query)
.DistinctBy(p => p.Name)
.ToDictionary(p => p.Name, BuildQueryParam);
queryParams = ApiRequestParametersPatcher.PatchQueryParameters(name, queryParams, overrides);

return new ApiEndpoint
{
Name = name,
Namespace = ns,
MethodName = methodName,
CsharpNames = names,
Overrides = overrides,
Stability = Stability.Stable, // TODO: for realsies
OfficialDocumentationLink = new Documentation
{
Description = variants[0].Operation.Description,
Url = variants[0].Operation.ExternalDocumentation?.Url
},
Url = new UrlInformation
{
AllPaths = paths,
Params = variants.SelectMany(v => v.Path.Parameters.Concat(v.Operation.Parameters))
.Where(p => p.Kind == OpenApiParameterKind.Query)
.DistinctBy(p => p.Name)
.ToImmutableSortedDictionary(p => p.Name, BuildQueryParam)
Params = queryParams
},
Body = variants
Body = variants
.Select(v => v.Operation.RequestBody)
.FirstOrDefault(b => b != null) is { } reqBody
? new Body { Description = GetDescription(reqBody), Required = reqBody.IsRequired }
: null,
HttpMethods = variants.Select(v => v.HttpMethod.ToString().ToUpper()).Distinct().ToList(),
};

LoadOverridesOnEndpoint(endpoint);
PatchRequestParameters(endpoint);

return endpoint;
HttpMethods = variants.Select(v => v.HttpMethod.ToString().ToUpper()).Distinct().ToList(),
};
}

private static void LoadOverridesOnEndpoint(ApiEndpoint endpoint)
private static IEndpointOverrides LoadOverrides(string endpointName, string methodName)
{
var method = endpoint.CsharpNames.MethodName;
if (CodeConfiguration.ApiNameMapping.TryGetValue(endpoint.Name, out var mapsApiMethodName))
method = mapsApiMethodName;
if (CodeConfiguration.ApiNameMapping.TryGetValue(endpointName, out var mapsApiMethodName))
methodName = mapsApiMethodName;

var namespacePrefix = $"{typeof(GlobalOverrides).Namespace}.Endpoints.";
var typeName = $"{namespacePrefix}{method}Overrides";
var typeName = $"{namespacePrefix}{methodName}Overrides";
var type = GeneratorLocations.Assembly.GetType(typeName);
if (type != null && Activator.CreateInstance(type) is IEndpointOverrides overrides)
endpoint.Overrides = overrides;
}

private static void PatchRequestParameters(ApiEndpoint endpoint) =>
endpoint.Url.Params = ApiQueryParametersPatcher.Patch(endpoint.Name, endpoint.Url.Params, endpoint.Overrides)
?? throw new ArgumentNullException("ApiQueryParametersPatcher.Patch(endpoint.Name, endpoint.Url.Params, endpoint.Overrides)");
return type != null && Activator.CreateInstance(type) is IEndpointOverrides overrides ? overrides : null;
}

private static QueryParameters BuildQueryParam(OpenApiParameter p)
private static QueryParameters BuildQueryParam(OpenApiParameter p)
{
var param = new QueryParameters
{
Expand All @@ -214,11 +210,9 @@ private static QueryParameters BuildQueryParam(OpenApiParameter p)
};

if (param.Type == "enum" && p.Schema.HasReference)
{
param.ClsName = ((IJsonReference)p.Schema).ReferencePath.Split('/').Last();
}
param.ClsName = ((IJsonReference)p.Schema).ReferencePath.Split('/').Last();

return param;
return param;
}

private static string GetOpenSearchType(JsonSchema schema)
Expand Down
15 changes: 7 additions & 8 deletions src/ApiGenerator/Views/LowLevel/Enums.cshtml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,12 @@
@functions {
private const string RawSize = "Raw";
private const string SizeEnum = "Size";
private static GlobalOverrides GlobalOverrides = new GlobalOverrides();

private string CreateEnum(string enumName, string value, int? i)
{
var enumValue = (enumName == SizeEnum && value == string.Empty) ? RawSize : value.ToPascalCase(true);
var enumCsharp = string.Format("[EnumMember(Value = \"{0}\")] {1}{2}", value, enumValue, i.HasValue ? " = 1 << " + i.Value : null);
if (GlobalOverrides.ObsoleteEnumMembers.TryGetValue(enumName, out var d) && d.TryGetValue(value, out var obsolete))
if (GlobalOverrides.Instance.ObsoleteEnumMembers.TryGetValue(enumName, out var d) && d.TryGetValue(value, out var obsolete))
{
return string.Format("[Obsolete(\"{0}\")]{2}\t\t{1}", obsolete, enumCsharp, Environment.NewLine);
}
Expand All @@ -23,7 +22,7 @@
private string CreateCase(string e, string o)
{
var enumValue = GetEnumValue(e, o);
var isObsolete = GlobalOverrides.ObsoleteEnumMembers.TryGetValue(e, out var d) && d.TryGetValue(o, out _);
var isObsolete = GlobalOverrides.Instance.ObsoleteEnumMembers.TryGetValue(e, out var d) && d.TryGetValue(o, out _);
var sb = new StringBuilder();
if (isObsolete) sb.AppendLine("#pragma warning disable 618");
sb.Append(string.Format("case {0}.{1}: return \"{2}\";", e, enumValue, o));
Expand All @@ -37,8 +36,8 @@

private string GetEnumValue(string enumName, string value)
{
return enumName == SizeEnum && value == string.Empty
? RawSize
return enumName == SizeEnum && value == string.Empty
? RawSize
: value.ToPascalCase(true);
}
}
Expand Down Expand Up @@ -91,20 +90,20 @@ namespace OpenSearch.Net
}
<text> var list = new @(Raw("List<string>()"));
</text>
var g = GlobalOverrides.ObsoleteEnumMembers.TryGetValue(e.Name, out var d);
var g = GlobalOverrides.Instance.ObsoleteEnumMembers.TryGetValue(e.Name, out var d);
foreach (var option in e.Options.Where(o => o != "_all"))
{
var value = GetEnumValue(e.Name, option);
if (g && d.TryGetValue(option, out var _)) {
<text>#pragma warning disable 618
if ((enumValue & @(e.Name).@(value)) != 0) list.Add("@(option)");
#pragma warning restore 618
</text>
</text>
}
else {
<text> if ((enumValue & @(e.Name).@(value)) != 0) list.Add("@(option)");
</text>
}
}
}
<text> return string.Join(",", list);
}</text>
Expand Down

0 comments on commit ef15768

Please sign in to comment.