Revert "Remove ApiGenerator (#156)"

OpenSearch.sln

@@ -61,6 +61,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{3EA11364-051
 		src\_PublishArtifacts.Build.props = src\_PublishArtifacts.Build.props
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ApiGenerator", "src\ApiGenerator\ApiGenerator.csproj", "{CA508E92-50AE-4858-BD94-8637E88A8FAC}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.Net.VirtualizedCluster", "src\OpenSearch.Net.VirtualizedCluster\OpenSearch.Net.VirtualizedCluster.csproj", "{CFE97627-8DD3-470B-B7CF-78B62E1D305D}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{29E53C13-34F7-4F0D-8D28-41EF768793E7}"
@@ -120,6 +122,7 @@ Global
 		{81473437-5722-4829-A5CD-125B17CCA238} = {6C4A2627-AF22-4388-9DF7-7A9AEACFD635}
 		{E97CCF40-0BA6-43FE-9F2D-58D454134088} = {3EA11364-0513-44B7-AD6D-A675485E7448}
 		{072BA7DA-7B60-407D-8B6E-95E3186BE70C} = {3EA11364-0513-44B7-AD6D-A675485E7448}
+		{CA508E92-50AE-4858-BD94-8637E88A8FAC} = {3EA11364-0513-44B7-AD6D-A675485E7448}
 		{CFE97627-8DD3-470B-B7CF-78B62E1D305D} = {3EA11364-0513-44B7-AD6D-A675485E7448}
 		{D6997ADC-E933-418E-831C-DE1A78897493} = {29E53C13-34F7-4F0D-8D28-41EF768793E7}
 		{432D5575-2347-4D3C-BF8C-3E38410C46CA} = {29E53C13-34F7-4F0D-8D28-41EF768793E7}
@@ -184,6 +187,10 @@ Global
 		{81473437-5722-4829-A5CD-125B17CCA238}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{81473437-5722-4829-A5CD-125B17CCA238}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{81473437-5722-4829-A5CD-125B17CCA238}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CA508E92-50AE-4858-BD94-8637E88A8FAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CA508E92-50AE-4858-BD94-8637E88A8FAC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CA508E92-50AE-4858-BD94-8637E88A8FAC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CA508E92-50AE-4858-BD94-8637E88A8FAC}.Release|Any CPU.Build.0 = Release|Any CPU
 		{CFE97627-8DD3-470B-B7CF-78B62E1D305D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{CFE97627-8DD3-470B-B7CF-78B62E1D305D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CFE97627-8DD3-470B-B7CF-78B62E1D305D}.Release|Any CPU.ActiveCfg = Release|Any CPU

build/scripts/Commandline.fs

@@ -62,6 +62,8 @@ Targets:
 * benchmark [non-interactive] [url] [username] [password] 
   - Runs a benchmark from Tests.Benchmarking and indexes the results to [url] when provided.
     If non-interactive runs all benchmarks without prompting
+* codegen
+  - runs the code generator interactively
 
 NOTE: both the `test` and `integrate` targets can be suffixed with `-all` to force the tests against all suported TFM's
 
@@ -192,6 +194,7 @@ Execution hints can be provided anywhere on the command line
         | ["build"]
         | ["clean"]
         | ["benchmark"]
+        | ["codegen"; ] 
         | ["profile"] -> parsed
         | "rest-spec-tests" :: tail -> { parsed with RemainingArguments = tail }
 

build/scripts/ReposTooling.fs

@@ -52,6 +52,14 @@ module ReposTooling =
 
         Shell.deleteDir tempDir
 
+    let GenerateApi () =
+        //TODO allow branch name to be passed for CI
+        let folder = Path.getDirectory (Paths.ProjFile "ApiGenerator")
+        let timeout = TimeSpan.FromMinutes(120.)
+        // Building to make sure XML docs files are there, faster then relying on the ApiGenerator to emit these
+        // from a compilation unit
+        Tooling.DotNet.ExecInWithTimeout folder ["run"; "-c"; " Release"; ] timeout  |> ignore
+
     let RestSpecTests args =
         let folder = Path.getDirectory (Paths.TestProjFile "Tests.YamlRunner")
         let timeout = TimeSpan.FromMinutes(120.)

build/scripts/Targets.fs

@@ -130,6 +130,8 @@ module Main =
         command "cluster" [ "restore"; "full-build" ] <| fun _ ->
             ReposTooling.LaunchCluster parsed
 
+        command "codegen" [ ] ReposTooling.GenerateApi
+
         command "rest-spec-tests" [ ] <| fun _ ->
             ReposTooling.RestSpecTests parsed.RemainingArguments
 

src/ApiGenerator/ApiGenerator.csproj

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <IsPackable>false</IsPackable>
+
+    <NoWarn>CS1591;NU1701</NoWarn>
+    <PreserveCompilationContext>true</PreserveCompilationContext>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.2.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageReference Include="ShellProgressBar" Version="5.2.0" />
+    <PackageReference Include="CsQuery.Core" Version="2.0.1" />
+    <PackageReference Include="Spectre.Console" Version="0.30.0" />
+    <PackageReference Include="System.CommandLine.DragonFruit" Version="0.3.0-alpha.20371.2" />
+    <PackageReference Include="RazorLight" Version="2.1.0" />
+    <PackageReference Include="System.Text.Encodings.Web" Version="7.0.0" />
+  </ItemGroup>
+  <ItemGroup>
+    <Folder Include="RestSpecification\Core" />
+  </ItemGroup>
+</Project>

src/ApiGenerator/App.config

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <startup>
+    <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
+  </startup>
+</configuration>

src/ApiGenerator/CodeTemplatePage.cs

@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+/*
+* Modifications Copyright OpenSearch Contributors. See
+* GitHub history for details.
+*
+*  Licensed to Elasticsearch B.V. under one or more contributor
+*  license agreements. See the NOTICE file distributed with
+*  this work for additional information regarding copyright
+*  ownership. Elasticsearch B.V. licenses this file to you under
+*  the Apache License, Version 2.0 (the "License"); you may
+*  not use this file except in compliance with the License.
+*  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+*  Unless required by applicable law or agreed to in writing,
+*  software distributed under the License is distributed on an
+*  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+*  KIND, either express or implied.  See the License for the
+*  specific language governing permissions and limitations
+*  under the License.
+*/
+
+using System;
+using System.Threading.Tasks;
+using RazorLight;
+
+namespace ApiGenerator 
+{
+    /// <summary> This only exists to make the IDE tooling happy, not actually used to render the templates </summary>
+    public class CodeTemplatePage<TModel> : TemplatePage<TModel>
+    {
+        public override Task ExecuteAsync() => throw new NotImplementedException();
+
+        public Task Execute() => Task.CompletedTask;
+    }
+}

src/ApiGenerator/Configuration/CodeConfiguration.cs

@@ -0,0 +1,248 @@
+/* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+/*
+* Modifications Copyright OpenSearch Contributors. See
+* GitHub history for details.
+*
+*  Licensed to Elasticsearch B.V. under one or more contributor
+*  license agreements. See the NOTICE file distributed with
+*  this work for additional information regarding copyright
+*  ownership. Elasticsearch B.V. licenses this file to you under
+*  the Apache License, Version 2.0 (the "License"); you may
+*  not use this file except in compliance with the License.
+*  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+*  Unless required by applicable law or agreed to in writing,
+*  software distributed under the License is distributed on an
+*  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+*  KIND, either express or implied.  See the License for the
+*  specific language governing permissions and limitations
+*  under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+
+namespace ApiGenerator.Configuration
+{
+    public static class CodeConfiguration
+    {
+        /// <summary> These APIs are not implemented yet in the low and high level client</summary>
+        public static string[] IgnoredApis { get; } =
+        {
+            // To be removed
+            "indices.upgrade.json",
+            "indices.get_upgrade.json",
+        };
+
+        private static string[] IgnoredApisHighLevel { get; } =
+        {
+            "indices.delete_index_template.json",
+            "indices.exists_index_template.json",
+            "indices.get_index_template.json",
+            "indices.put_index_template.json",
+            "indices.simulate_index_template.json",
+            "indices.simulate_template.json",
+
+            "get_script_context.json", // 7.7 experimental
+            "get_script_languages.json", // 7.7 experimental
+
+            "indices.exist_type.json", // already removed on client
+
+            "rank_eval.json", // 7.7 experimental
+            "scripts_painless_context.json", // 7.7 experimental
+            "cluster.delete_component_template.json", // 7.8 experimental
+            "cluster.get_component_template.json", // 7.8 experimental
+            "cluster.put_component_template.json", // 7.8 experimental
+            "cluster.exists_component_template.json", // 7.8 experimental
+        };
+
+        /// <summary>
+        /// Map API default names for API's we are only supporting on the low level client first
+        /// </summary>
+        private static readonly Dictionary<string, string> LowLevelApiNameMapping = new Dictionary<string, string>
+        {
+            { "indices.delete_index_template", "DeleteIndexTemplateV2" },
+            { "indices.get_index_template", "GetIndexTemplateV2" },
+            { "indices.put_index_template", "PutIndexTemplateV2" }
+        };
+
+        /// <summary>
+        /// Scan all OSC source code files for Requests and look for the [MapsApi(filename)] attribute.
+        /// The class name minus Request is used as the canonical .NET name for the API.
+        /// </summary>
+        public static readonly Dictionary<string, string> HighLevelApiNameMapping =
+            (from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*.cs", SearchOption.AllDirectories)
+                let contents = File.ReadAllText(f.FullName)
+                let c = Regex.Replace(contents, @"^.+\[MapsApi\(""([^ \r\n]+)""\)\].*$", "$1", RegexOptions.Singleline)
+                where !c.Contains(" ") //filter results that did not match
+                select new { Value = f.Name.Replace("Request", ""), Key = c.Replace(".json", "") })
+            .DistinctBy(v => v.Key)
+            .ToDictionary(k => k.Key, v => v.Value.Replace(".cs", ""));
+
+        public static readonly HashSet<string> EnableHighLevelCodeGen = new HashSet<string>();
+
+        public static bool IsNewHighLevelApi(string apiFileName) =>
+            // if its explicitly ignored we know about it.
+            !IgnoredApis.Contains(apiFileName)
+            && !IgnoredApisHighLevel.Contains(apiFileName)
+            // no requests with [MapsApi("filename.json")] found
+            && !HighLevelApiNameMapping.ContainsKey(apiFileName.Replace(".json", ""));
+
+        public static bool IgnoreHighLevelApi(string apiFileName)
+        {
+            //explicitly ignored
+            if (IgnoredApis.Contains(apiFileName) || IgnoredApisHighLevel.Contains(apiFileName)) return true;
+
+            //always generate already mapped requests
+
+            if (HighLevelApiNameMapping.ContainsKey(apiFileName.Replace(".json", ""))) return false;
+
+            return !EnableHighLevelCodeGen.Contains(apiFileName);
+        }
+
+        private static Dictionary<string, string> _apiNameMapping;
+
+        public static Dictionary<string, string> ApiNameMapping
+        {
+            get
+            {
+                if (_apiNameMapping != null) return _apiNameMapping;
+                lock (LowLevelApiNameMapping)
+                {
+                    if (_apiNameMapping == null)
+                    {
+                        var mapping = new Dictionary<string, string>(HighLevelApiNameMapping);
+                        foreach (var (k, v) in LowLevelApiNameMapping)
+                            mapping[k] = v;
+                        _apiNameMapping = mapping;
+                    }
+                    return _apiNameMapping;
+                }
+            }
+        }
+
+        private static readonly string ResponseBuilderAttributeRegex = @"^.+\[ResponseBuilderWithGeneric\(""([^ \r\n]+)""\)\].*$";
+        /// <summary>
+        /// Scan all OSC source code files for Requests and look for the [MapsApi(filename)] attribute.
+        /// The class name minus Request is used as the canonical .NET name for the API.
+        /// </summary>
+        public static readonly Dictionary<string, string> ResponseBuilderInClientCalls =
+            (from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*.cs", SearchOption.AllDirectories)
+                from l in File.ReadLines(f.FullName)
+                where Regex.IsMatch(l, ResponseBuilderAttributeRegex)
+                let c = Regex.Replace(l, @"^.+\[ResponseBuilderWithGeneric\(""([^ \r\n]+)""\)\].*$", "$1", RegexOptions.Singleline)
+                select new { Key = f.Name.Replace(".cs", ""), Value = c })
+            .DistinctBy(v => v.Key)
+            .ToDictionary(k => k.Key, v => v.Value);
+
+        public static readonly Dictionary<string, string> DescriptorGenericsLookup =
+            (from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*Request.cs", SearchOption.AllDirectories)
+                let name = Path.GetFileNameWithoutExtension(f.Name).Replace("Request", "")
+                let contents = File.ReadAllText(f.FullName)
+                let c = Regex.Replace(contents, $@"^.+class ({name}Descriptor(?:<[^>\r\n]+>)?[^ \r\n]*).*$", "$1", RegexOptions.Singleline)
+                let key = $"{name}Descriptor"
+                select new { Key = key, Value = Regex.Replace(c, @"^.*?(?:(\<.+>).*?)?$", "$1") })
+            .DistinctBy(v => v.Key)
+            .OrderBy(v => v.Key)
+            .ToDictionary(k => k.Key, v => v.Value);
+
+        /// <summary> Scan all OSC files for request interfaces and note any generics declared on them </summary>
+        private static readonly List<Tuple<string, string>> AllKnownRequestInterfaces = (
+            // find all files in OSC ending with Request.cs
+            from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*Request.cs", SearchOption.AllDirectories)
+            from l in File.ReadLines(f.FullName)
+            // attempt to locate all Request interfaces lines
+            where Regex.IsMatch(l, @"^.+interface [^ \r\n]+Request")
+            //grab the interface name including any generics declared on it
+            let c = Regex.Replace(l, @"^.+interface ([^ \r\n]+Request(?:<[^>\r\n]+>)?[^ \r\n]*).*$", "$1", RegexOptions.Singleline)
+            where c.StartsWith("I") && c.Contains("Request")
+            let request = Regex.Replace(c, "<.*$", "")
+            let generics = Regex.Replace(c, @"^.*?(?:(\<.+>).*?)?$", "$1")
+            select Tuple.Create(request,  generics)
+            )
+            .OrderBy(v=>v.Item1)
+            .ToList();
+
+        public static readonly HashSet<string> GenericOnlyInterfaces = new HashSet<string>(AllKnownRequestInterfaces
+            .GroupBy(v => v.Item1)
+            .Where(g => g.All(v => !string.IsNullOrEmpty(v.Item2)))
+            .Select(g => g.Key)
+            .ToList());
+
+        public static readonly HashSet<string> DocumentRequests = new HashSet<string>((
+            // find all files in OSC ending with Request.cs
+            from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*Request.cs", SearchOption.AllDirectories)
+            from l in File.ReadLines(f.FullName)
+            // attempt to locate all Request interfaces lines
+            where Regex.IsMatch(l, @"^.+interface [^ \r\n]+Request")
+            where l.Contains("IDocumentRequest")
+            let c = Regex.Replace(l, @"^.+interface ([^ \r\n]+Request(?:<[^>\r\n]+>)?[^ \r\n]*).*$", "$1", RegexOptions.Singleline)
+            //grab the interface name including any generics declared on it
+            let request = Regex.Replace(c, "<.*$", "")
+            select request
+            )
+            .ToList());
+
+        public static readonly Dictionary<string, string> DescriptorConstructors = (
+            // find all files in OSC ending with Request.cs
+            from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*Request.cs", SearchOption.AllDirectories)
+            let descriptor = Path.GetFileNameWithoutExtension(f.Name).Replace("Request", "Descriptor")
+            let re = $@"^.+public {descriptor}\(([^\r\n\)]+?)\).*$"
+            from l in File.ReadLines(f.FullName)
+            where Regex.IsMatch(l, re)
+            let args = Regex.Replace(l, re, "$1", RegexOptions.Singleline)
+            where !string.IsNullOrWhiteSpace(args) && !args.Contains(": base")
+            select (Descriptor: descriptor, Args: args)
+            )
+            .ToDictionary(r => r.Descriptor, r => r.Args);
+
+        public static readonly Dictionary<string, string> RequestInterfaceGenericsLookup =
+            AllKnownRequestInterfaces
+            .GroupBy(v=>v.Item1)
+            .Select(g=>g.Last())
+            .ToDictionary(k => k.Item1, v => v.Item2);
+
+        /// <summary>
+        /// Some API's reuse response this is a hardcoded map of these cases
+        /// </summary>
+        private static Dictionary<string, (string, string)> ResponseReroute = new Dictionary<string, (string, string)>
+        {
+            {"UpdateByQueryRethrottleResponse", ("ListTasksResponse", "")},
+            {"DeleteByQueryRethrottleResponse", ("ListTasksResponse", "")},
+            {"MultiSearchTemplateResponse", ("MultiSearchResponse", "")},
+            {"ScrollResponse", ("SearchResponse", "<TDocument>")},
+            {"SearchTemplateResponse", ("SearchResponse", "<TDocument>")},
+
+        };
+
+
+        /// <summary> Create a dictionary lookup of all responses and their generics </summary>
+        public static readonly SortedDictionary<string, (string, string)> ResponseLookup = new SortedDictionary<string, (string, string)>(
+        (
+            // find all files in OSC ending with Request.cs
+            from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*Response.cs", SearchOption.AllDirectories)
+            from l in File.ReadLines(f.FullName)
+            // attempt to locate all Response class lines
+            where Regex.IsMatch(l, @"^.+public class [^ \r\n]+Response")
+            //grab the response name including any generics declared on it
+            let c = Regex.Replace(l, @"^.+public class ([^ \r\n]+Response(?:<[^>\r\n]+>)?[^ \r\n]*).*$", "$1", RegexOptions.Singleline)
+            where c.Contains("Response")
+            let response = Regex.Replace(c, "<.*$", "")
+            let generics = Regex.Replace(c, @"^.*?(?:(\<.+>).*?)?$", "$1")
+            select (response,  (response, generics))
+        )
+            .Concat(ResponseReroute.Select(kv=>(kv.Key, (kv.Value.Item1, kv.Value.Item2))))
+            .ToDictionary(t=>t.Item1, t=>t.Item2));
+
+    }
+}