Fix synonyms CI tests timeout (#114641)

muted-tests.yml

@@ -259,15 +259,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.DefaultElserIT
   method: testInferCreatesDefaultElser
   issue: https://github.com/elastic/elasticsearch/issues/114503
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=synonyms/60_synonym_rule_get/Synonym set not found}
-  issue: https://github.com/elastic/elasticsearch/issues/114432
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=synonyms/60_synonym_rule_get/Get a synonym rule}
-  issue: https://github.com/elastic/elasticsearch/issues/114443
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=synonyms/60_synonym_rule_get/Synonym rule not found}
-  issue: https://github.com/elastic/elasticsearch/issues/114444
 - class: org.elasticsearch.xpack.inference.integration.ModelRegistryIT
   method: testGetModel
   issue: https://github.com/elastic/elasticsearch/issues/114657

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml

@@ -15,6 +15,10 @@ setup:
 
   - match: { result: "created" }
 
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
   - do:
       synonyms.get_synonym:
         id: test-update-synonyms
@@ -58,6 +62,10 @@ setup:
 
   - match: { result: "created" }
 
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
   - do:
       synonyms.get_synonym:
         id: test-empty-synonyms

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml

@@ -11,6 +11,11 @@ setup:
           synonyms_set:
             synonyms: "foo => bar, baz"
 
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
   - do:
       indices.create:
         index: test_index

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml

@@ -14,6 +14,10 @@ setup:
             - synonyms: "test => check"
               id: "test-id-3"
 
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
 
 ---
 "Get synonyms set":
@@ -31,7 +35,6 @@ setup:
           id: "test-id-2"
         - synonyms: "test => check"
           id: "test-id-3"
-
 ---
 "Get synonyms set - not found":
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml

@@ -12,6 +12,10 @@ setup:
             - synonyms: "bye => goodbye"
               id: "test-id-2"
 
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
 ---
 "Delete synonyms set":
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml

@@ -9,6 +9,12 @@ setup:
           synonyms_set:
             - synonyms: "hello, hi"
             - synonyms: "goodbye, bye"
+
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
   - do:
       synonyms.put_synonym:
         id: test-synonyms-1
@@ -23,21 +29,8 @@ setup:
         body:
           synonyms_set:
             - synonyms: "pc, computer"
-  # set logging to debug for issue: https://github.com/elastic/elasticsearch/issues/102261
-  - do:
-      cluster.put_settings:
-        body:
-          persistent:
-            logger.org.elasticsearch.synonyms: DEBUG
 
 ---
-teardown:
-  - do:
-      cluster.put_settings:
-        body:
-          persistent:
-            logger.org.elasticsearch.synonyms: null
----
 "List synonyms set":
   - do:
       synonyms.get_synonyms_sets: { }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml

@@ -14,7 +14,10 @@ setup:
             - synonyms: "test => check"
               id: "test-id-3"
 
-
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
 ---
 "Update a synonyms rule":
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml

@@ -13,11 +13,12 @@ setup:
               id: "test-id-2"
             - synonyms: "test => check"
               id: "test-id-3"
+
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
   - do:
       cluster.health:
-        index: .synonyms
-        timeout: 1m
-        wait_for_status: green
+        wait_for_no_initializing_shards: true
+
 
 ---
 "Get a synonym rule":

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml

@@ -14,6 +14,11 @@ setup:
             - synonyms: "test => check"
               id: "test-id-3"
 
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
 ---
 "Delete synonym rule":
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml

@@ -2,7 +2,6 @@ setup:
   - requires:
       cluster_features: ["gte_v8.10.0"]
       reason: Loading synonyms from index is introduced in 8.10.0
-
   # Create a new synonyms set
   - do:
       synonyms.put_synonym:
@@ -14,6 +13,11 @@ setup:
             - synonyms: "bye => goodbye"
               id: "synonym-rule-2"
 
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
   # Create an index with synonym_filter that uses that synonyms set
   - do:
       indices.create:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml

@@ -3,7 +3,6 @@
   - requires:
       cluster_features: ["gte_v8.10.0"]
       reason: Reloading analyzers for specific synonym set is introduced in 8.10.0
-
   # Create synonyms_set1
   - do:
       synonyms.put_synonym:
@@ -26,6 +25,11 @@
             - synonyms: "bye => goodbye"
               id: "synonym-rule-2"
 
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
   # Create my_index1 with synonym_filter that uses synonyms_set1
   - do:
       indices.create: