Add support for composable index templates (opensearch-project#437)

* Allow renaming URL path parts in generator Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Generate {delete,exists,get,put}_index_template as *ComposableTemplate Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add resp/req bodies for *ComposableIndexTemplate Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Fix tests Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add ComposableIndexTemplateCrudTests Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add ComposableIndexTemplateExists tests Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add DeleteComposableIndexTemplate tests Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add GetComposableIndexTemplate tests Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add PutComposableIndexTemplate tests Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Non-overlapping templates Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Fix tests Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Test data_stream template mapping serialization Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Update guide and add sample Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add changelog entry Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Tidy generated code Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Review feedback Signed-off-by: Thomas Farr <tsfarr@amazon.com> --------- Signed-off-by: Thomas Farr <tsfarr@amazon.com> (cherry picked from commit dc8612c)
Xtansia · Nov 21, 2023 · 9b8c5ca · 9b8c5ca
1 parent c4a6ac9
commit 9b8c5ca
Show file tree

Hide file tree

Showing 59 changed files with 3,001 additions and 525 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -16,8 +16,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # 1.0.1 is skipped because it doesn't have a tag in OpenSearch repo, we can't check out to compile plugins
-        version: 
+        version:
+          - 2.9.0
+          - 2.8.0
           - 2.7.0
           - 2.6.0
           - 2.5.0
@@ -26,10 +27,9 @@ jobs:
           - 2.2.1
           - 2.1.0
           - 2.0.1
-          - 1.3.10
+          - 1.3.11
           - 1.2.4
           - 1.1.0
-          - 1.0.0
 
     steps:
       - name: Checkout Client
@@ -50,35 +50,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-nuget-
 
-        # Due to https://github.com/opensearch-project/project-website/issues/737
-        # No plugins released for versions 1.0.x , so we need to compile plugins on our own to run tests properly
-        # Compiled plugins are copied to ${OPENSEARCH_PLUGINS_DIRECTORY} where picked up by the test framework
-        # Versions of OpenSearch starting 1.1.0 are able to download plugins from the Internet
-      - name: Restore or Build OpenSearch Plugins
-        uses: ./client/.github/actions/cached-git-build
-        if: matrix.version == '1.0.0'
-        with:
-          repository: opensearch-project/OpenSearch
-          ref: ${{ matrix.version }}
-          path: opensearch
-          cached_paths: |
-            ./opensearch/plugins/*/build/distributions/*.zip
-          build_script: |
-            PluginList=("analysis-icu"  "analysis-kuromoji"  "analysis-nori"  "analysis-phonetic"  "ingest-attachment"  "mapper-murmur3")
-            for plugin in ${PluginList[*]}; do
-              ./gradlew :plugins:$plugin:assemble -Dbuild.snapshot=false
-            done
-      
-      - name: Copy OpenSearch Plugins
-        if: matrix.version == '1.0.0'
-        run: |
-          mkdir -p ${{ env.OPENSEARCH_PLUGINS_DIRECTORY }}
-          cp opensearch/plugins/*/build/distributions/*-${{ matrix.version }}.zip ${{ env.OPENSEARCH_PLUGINS_DIRECTORY }}
-      
       - run: "./build.sh integrate ${{ matrix.version }} readonly,writable random:test_only_one --report"
         name: Integration Tests
         working-directory: client
-        
+
       - name: Upload test report
         if: failure()
         uses: actions/upload-artifact@v3
@@ -87,12 +62,13 @@ jobs:
           path: client/build/output/*
 
   integration-opensearch-unreleased:
+    if: false # TODO: Temporarily disabled due to failures building & running OpenSearch from source, pending investigation & fixes (https://github.com/opensearch-project/opensearch-net/issues/268)
     name: Integration OpenSearch Unreleased
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        opensearch_ref: 
+        opensearch_ref:
           - '1.x'
           - '2.x'
           - 'main'
@@ -102,7 +78,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           path: client
-          
+
       - uses: actions/setup-dotnet@v3
         with:
           dotnet-version: |
@@ -115,82 +91,22 @@ jobs:
           key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.?sproj') }}
           restore-keys: |
             ${{ runner.os }}-nuget-
-            
       - name: Restore or Build OpenSearch
-        uses: ./client/.github/actions/cached-git-build
-        with:
-          repository: opensearch-project/OpenSearch
-          ref: ${{ matrix.opensearch_ref }}
-          path: opensearch
-          cached_paths: |
-            ./opensearch/distribution/archives/linux-tar/build/distributions/opensearch-*.tar.gz
-            ./opensearch/plugins/*/build/distributions/*.zip
-          build_script: |
-            ./gradlew :distribution:archives:linux-tar:assemble
-          
-            PluginList=("analysis-icu"  "analysis-kuromoji"  "analysis-nori"  "analysis-phonetic"  "ingest-attachment"  "mapper-murmur3")
-            for plugin in ${PluginList[*]}; do
-              ./gradlew :plugins:$plugin:assemble
-            done
-            
-      - name: Determine OpenSearch distribution path and version
-        shell: bash -eo pipefail {0}
-        run: |
-          distribution=`ls -1 $PWD/opensearch/distribution/archives/linux-tar/build/distributions/opensearch-*.tar.gz | head -1`
-          version=`basename $distribution | cut -d'-' -f3,4`
-          echo "OPENSEARCH_DISTRIBUTION=$distribution" >> $GITHUB_ENV
-          echo "OPENSEARCH_VERSION=$version" >> $GITHUB_ENV
-        
-      - name: Restore or Build OpenSearch Security
-        uses: ./client/.github/actions/cached-git-build
-        if: matrix.opensearch_ref == '1.x'
+        id: opensearch
+        uses: ./client/.github/actions/build-opensearch
         with:
-          repository: opensearch-project/security
           ref: ${{ matrix.opensearch_ref }}
-          path: opensearch-security
-          cached_paths: |
-            ./opensearch-security/build/distributions/opensearch-security-*-SNAPSHOT.zip
-          build_script: ./gradlew assemble -Dopensearch.version=$OPENSEARCH_VERSION
-
-      - name: Restore or Build OpenSearch k-NN
-        uses: ./client/.github/actions/cached-git-build
-        with:
-          repository: opensearch-project/k-NN
-          ref: ${{ matrix.opensearch_ref }}
-          path: opensearch-knn
-          cached_paths: |
-            ./opensearch-knn/build/distributions/opensearch-knn-*-SNAPSHOT.zip
-          build_script: |
-            sudo apt-get install -y libopenblas-dev libomp-dev
-            ./gradlew buildJniLib assemble -Dopensearch.version=$OPENSEARCH_VERSION
-            distributions=./build/distributions
-            lib_dir=$distributions/lib
-            mkdir $lib_dir
-            cp -v $(ldconfig -p | grep libgomp | cut -d ' ' -f 4) $lib_dir
-            cp -v ./jni/release/libopensearchknn_* $lib_dir
-            ls -l $lib_dir
-            cd $distributions
-            zip -ur opensearch-knn-*.zip lib
-        
-      - name: Copy OpenSearch plugins
-        shell: bash -eo pipefail {0}
-        run: |
-          mkdir -p $OPENSEARCH_PLUGINS_DIRECTORY
-          cp -v ./opensearch/plugins/*/build/distributions/*-$OPENSEARCH_VERSION.zip $OPENSEARCH_PLUGINS_DIRECTORY/
-          
-          plugins=("opensearch-knn" "opensearch-security")
-          for plugin in ${plugins[*]}; do
-            if [[ -d "./$plugin" ]]; then
-              cp -v ./$plugin/build/distributions/$plugin-*-SNAPSHOT.zip $OPENSEARCH_PLUGINS_DIRECTORY/
-            fi
-          done
-          
-          ls -l $OPENSEARCH_PLUGINS_DIRECTORY
-      
-      - run: "./build.sh integrate ${{ env.OPENSEARCH_VERSION }} readonly,writable random:test_only_one --report"
+          security_plugin: ${{ matrix.opensearch_ref == '1.x' }}
+          knn_plugin: true
+          plugins_output_directory: ${{ env.OPENSEARCH_PLUGINS_DIRECTORY }}
+
+      - run: "./build.sh integrate $OPENSEARCH_VERSION readonly,writable random:test_only_one --report"
         name: Integration Tests
         working-directory: client
-
+        env:
+          OPENSEARCH_VERSION: ${{ steps.opensearch.outputs.version }}
+          OPENSEARCH_DISTRIBUTION: ${{ steps.opensearch.outputs.distribution }}
+
       - name: Upload test report
         if: failure()
         uses: actions/upload-artifact@v3

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ### Added
 - Added support for point-in-time search and associated APIs ([#405](https://github.com/opensearch-project/opensearch-net/pull/405))
 - Added support for the component template APIs ([#411](https://github.com/opensearch-project/opensearch-net/pull/411))
+- Added support for the composable index template APIs ([#437](https://github.com/opensearch-project/opensearch-net/pull/437))
 
 ### Dependencies
 - Bumps `FSharp.Data` from 6.2.0 to 6.3.0

diff --git a/OpenSearch.sln.DotSettings b/OpenSearch.sln.DotSettings
@@ -14,33 +14,6 @@
   &amp;lt;inspection_tool class="UnnecessaryReturnJS" enabled="false" level="WARNING" enabled_by_default="false" /&amp;gt;
   &amp;lt;inspection_tool class="WrongPropertyKeyValueDelimiter" enabled="false" level="WEAK WARNING" enabled_by_default="false" /&amp;gt;
 &amp;lt;/profile&amp;gt;&lt;/IDEA_SETTINGS&gt;&lt;VBReformatCode&gt;True&lt;/VBReformatCode&gt;&lt;HtmlReformatCode&gt;True&lt;/HtmlReformatCode&gt;&lt;CSUpdateFileHeader&gt;True&lt;/CSUpdateFileHeader&gt;&lt;CSReorderTypeMembers&gt;True&lt;/CSReorderTypeMembers&gt;&lt;CSReformatCode&gt;True&lt;/CSReformatCode&gt;&lt;/Profile&gt;</s:String>
-	<s:String x:Key="/Default/CodeStyle/FileHeader/FileHeaderText/@EntryValue">/* SPDX-License-Identifier: Apache-2.0
-*
-* The OpenSearch Contributors require contributions made to
-* this file be licensed under the Apache-2.0 license or a
-* compatible open source license.
-*
-* Modifications Copyright OpenSearch Contributors. See
-* GitHub history for details.
-*
-*  Licensed to Elasticsearch B.V. under one or more contributor
-*  license agreements. See the NOTICE file distributed with
-*  this work for additional information regarding copyright
-*  ownership. Elasticsearch B.V. licenses this file to you under
-*  the Apache License, Version 2.0 (the "License"); you may
-*  not use this file except in compliance with the License.
-*  You may obtain a copy of the License at
-*
-* 	http://www.apache.org/licenses/LICENSE-2.0
-*
-*  Unless required by applicable law or agreed to in writing,
-*  software distributed under the License is distributed on an
-*  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-*  KIND, either express or implied.  See the License for the
-*  specific language governing permissions and limitations
-*  under the License.
-*/
-</s:String>
 	<s:Boolean x:Key="/Default/CodeEditing/Intellisense/CodeCompletion/IntelliSenseCompletingCharacters/CSharpCompletingCharacters/UpgradedFromVSSettings/@EntryValue">True</s:Boolean>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/GeneratedFilesAndFolders/=56A87048_002D9065_002D459B_002D826D_002D3DF68B409845_002Fd_003AViews/@EntryIndexedValue">56A87048-9065-459B-826D-3DF68B409845/d:Views</s:String>
 	<s:String x:Key="/Default/CodeInspection/GeneratedCode/GeneratedFileMasks/=_002A_002Edoc_002Ecs/@EntryIndexedValue"></s:String>