From adc8549bcd807d9981a49e16263c7dc7d588416c Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <danielmitterdorfer@users.noreply.github.com>
Date: Tue, 3 Mar 2020 11:29:21 +0100
Subject: [PATCH] Ensure segments are merged (#104)

With this commit we change all tracks to increase the request timeout of the
`force-merge` task to ensure the force merge finishes before we move on. We also
include another task that waits until all ongoing segment merges finish. Older
Rally versions will entirely rely on the increased request timeout for force
merges but newer Rally versions (every version including elastic/rally#925) will
evaluate the condition and wait accordingly until the system is quiet.

Relates elastic/rally#925
---
 eventdata/challenges/default.json     | 19 ++++++-
 geonames/challenges/default.json      | 76 +++++++++++++++++++++++---
 geopoint/challenges/default.json      | 57 ++++++++++++++++----
 geopointshape/challenges/default.json | 57 +++++++++++++++++---
 http_logs/challenges/default.json     | 78 +++++++++++++++++++++++----
 nested/challenges/default.json        | 38 +++++++++++--
 noaa/challenges/default.json          | 38 +++++++++++--
 nyc_taxis/challenges/default.json     | 37 +++++++++++--
 percolator/challenges/default.json    | 19 ++++++-
 pmc/challenges/default.json           | 76 +++++++++++++++++++++++---
 so/challenges/default.json            | 19 ++++++-
 11 files changed, 457 insertions(+), 57 deletions(-)

diff --git a/eventdata/challenges/default.json b/eventdata/challenges/default.json
index cb2afd0c..dd7e8ffb 100644
--- a/eventdata/challenges/default.json
+++ b/eventdata/challenges/default.json
@@ -29,8 +29,23 @@
           "clients": {{bulk_indexing_clients | default(8)}}
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     }
diff --git a/geonames/challenges/default.json b/geonames/challenges/default.json
index 04ff6b13..6e1e39ed 100644
--- a/geonames/challenges/default.json
+++ b/geonames/challenges/default.json
@@ -34,14 +34,29 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
         },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
+        },
         {
           "operation": "index-stats",
           "clients": 1,
@@ -173,8 +188,23 @@
           "clients": {{bulk_indexing_clients | default(8)}}
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     },
@@ -211,8 +241,23 @@
           "clients": {{bulk_indexing_clients | default(8)}}
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     },
@@ -250,8 +295,23 @@
           "clients": {{bulk_indexing_clients | default(8)}}
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     }
diff --git a/geopoint/challenges/default.json b/geopoint/challenges/default.json
index 49f5f091..e5f28daf 100644
--- a/geopoint/challenges/default.json
+++ b/geopoint/challenges/default.json
@@ -35,17 +35,30 @@
         },
         {
           "operation": {
-            "operation-type": "force-merge"{%- if max_num_segments is defined %},
+            "operation-type": "force-merge",
+            "request-timeout": 7200{%- if max_num_segments is defined %},
             "max-num-segments": {{max_num_segments}}
              {%- endif %}
-          },
-          "clients": 1
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
         },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
+        },
         {
           "operation": "polygon",
           "clients": 1,
@@ -112,16 +125,29 @@
         },
         {
           "operation": {
-            "operation-type": "force-merge"{%- if max_num_segments is defined %},
+            "operation-type": "force-merge",
+            "request-timeout": 7200{%- if max_num_segments is defined %},
             "max-num-segments": {{max_num_segments}}
              {%- endif %}
-          },
-          "clients": 1
+          }
         },
         {
         "name": "refresh-after-force-merge",
         "operation": "refresh",
         "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     },
@@ -165,16 +191,29 @@
         },
         {
           "operation": {
-            "operation-type": "force-merge"{%- if max_num_segments is defined %},
+            "operation-type": "force-merge",
+            "request-timeout": 7200{%- if max_num_segments is defined %},
             "max-num-segments": {{max_num_segments}}
              {%- endif %}
-          },
-          "clients": 1
+          }
         },
         {
         "name": "refresh-after-force-merge",
         "operation": "refresh",
         "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     }
\ No newline at end of file
diff --git a/geopointshape/challenges/default.json b/geopointshape/challenges/default.json
index e8848586..d56d4338 100644
--- a/geopointshape/challenges/default.json
+++ b/geopointshape/challenges/default.json
@@ -34,14 +34,29 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
         },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
+        },
         {
           "operation": "polygon",
           "clients": 1,
@@ -93,13 +108,28 @@
         "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
         "name": "refresh-after-force-merge",
         "operation": "refresh",
         "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     },
@@ -142,13 +172,28 @@
         "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
         "name": "refresh-after-force-merge",
         "operation": "refresh",
         "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     }
diff --git a/http_logs/challenges/default.json b/http_logs/challenges/default.json
index 2fc4f46c..c3349589 100644
--- a/http_logs/challenges/default.json
+++ b/http_logs/challenges/default.json
@@ -34,14 +34,29 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
         },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
+        },
         {
           "operation": "default",
           "clients": 1,
@@ -115,13 +130,28 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     },
@@ -163,13 +193,28 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     },
@@ -211,13 +256,28 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
-    }
\ No newline at end of file
+    }
diff --git a/nested/challenges/default.json b/nested/challenges/default.json
index de2e8e3a..548844eb 100644
--- a/nested/challenges/default.json
+++ b/nested/challenges/default.json
@@ -35,14 +35,29 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
         },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
+        },
         {
           "operation": "randomized-nested-queries",
           "clients": 2,
@@ -130,13 +145,28 @@
         "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     }
\ No newline at end of file
diff --git a/noaa/challenges/default.json b/noaa/challenges/default.json
index 026fec4e..770674fb 100644
--- a/noaa/challenges/default.json
+++ b/noaa/challenges/default.json
@@ -35,14 +35,29 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
         },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
+        },
         {
           "operation": "range_field_big_range",
           "clients": 1,
@@ -137,13 +152,28 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     }
diff --git a/nyc_taxis/challenges/default.json b/nyc_taxis/challenges/default.json
index 4ad52ea0..0018c77f 100644
--- a/nyc_taxis/challenges/default.json
+++ b/nyc_taxis/challenges/default.json
@@ -194,13 +194,28 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     },
@@ -333,11 +348,27 @@
           "operation": "refresh"
         },
         {
-          "operation": "force-merge"
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh"
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         },
 	    {
           "operation": {
diff --git a/percolator/challenges/default.json b/percolator/challenges/default.json
index 7a3e7ed6..39141722 100644
--- a/percolator/challenges/default.json
+++ b/percolator/challenges/default.json
@@ -35,14 +35,29 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
         },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
+        },
         {
           "operation": "percolator_with_content_president_bush",
           "clients": 1,
diff --git a/pmc/challenges/default.json b/pmc/challenges/default.json
index d745a35d..42d34b27 100644
--- a/pmc/challenges/default.json
+++ b/pmc/challenges/default.json
@@ -44,14 +44,29 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
         },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
+        },
         {
           "operation": "default",
           "clients": 1,
@@ -131,13 +146,28 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     },
@@ -179,13 +209,28 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     },
@@ -228,13 +273,28 @@
           "clients": 1
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh",
           "clients": 1
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     }
\ No newline at end of file
diff --git a/so/challenges/default.json b/so/challenges/default.json
index 7ba4624f..b02f23c8 100644
--- a/so/challenges/default.json
+++ b/so/challenges/default.json
@@ -29,8 +29,23 @@
           "clients": {{bulk_indexing_clients | default(8)}}
         },
         {
-          "operation": "force-merge",
-          "clients": 1
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200
+          }
+        },
+        {
+          "name": "wait-until-merges-finish",
+          "operation": {
+            "operation-type": "index-stats",
+            "index": "_all",
+            "condition": {
+              "path": "_all.total.merges.current",
+              "expected-value": 0
+            },
+            "retry-until-success": true,
+            "include-in-reporting": false
+          }
         }
       ]
     }