Skip to content

Commit

Permalink
Fix race condition with old ES processes in IT tests (#886)
Browse files Browse the repository at this point in the history
Occasionally IT tests fail in a very subtle way when Rally spins up ES
shortly after a previous test: while the previous ES process gets
killed, the socket lingers and when Rally spins up the new ES process
for the test, ES detects the used port and binds itself to the
next one (e.g. 39201, for the HTTP API).

Wait until the socket used by ES HTTP API is really closed across IT
tests.
  • Loading branch information
dliappis authored Jan 31, 2020
1 parent 2771063 commit 949107e
Showing 1 changed file with 11 additions and 0 deletions.
11 changes: 11 additions & 0 deletions integration-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@ function stop_and_clean_docker_container {
docker rm ${1} > /dev/null || true
}

function wait_for_free_es_port {
es_port=39200
while nc -z localhost ${es_port}; do sleep 1; echo "Port ${es_port} occupied waiting"; done
}

function kill_rally_processes {
# kill all lingering Rally instances that might still be hanging
set +e
Expand Down Expand Up @@ -287,9 +292,12 @@ function test_sources {
# build Elasticsearch and a core plugin
info "test sources [--configuration-name=${cfg}], [--revision=latest], [--track=geonames], [--challenge=append-no-conflicts], [--car=4gheap] [--elasticsearch-plugins=analysis-icu]"
kill_rally_processes
wait_for_free_es_port
esrally --configuration-name="${cfg}" --on-error=abort --revision=latest --track=geonames --test-mode --challenge=append-no-conflicts --car=4gheap --elasticsearch-plugins=analysis-icu

info "test sources [--configuration-name=${cfg}], [--pipeline=from-sources-skip-build], [--track=geonames], [--challenge=append-no-conflicts-index-only], [--car=4gheap,ea] "
kill_rally_processes
wait_for_free_es_port
esrally --configuration-name="${cfg}" --on-error=abort --pipeline=from-sources-skip-build --track=geonames --test-mode --challenge=append-no-conflicts-index-only --car="4gheap,ea"
}

Expand All @@ -303,6 +311,7 @@ function test_distributions {
random_configuration cfg
info "test distributions [--configuration-name=${cfg}], [--distribution-version=${dist}], [--track=${track}], [--car=4gheap]"
kill_rally_processes
wait_for_free_es_port
esrally --configuration-name="${cfg}" --on-error=abort --distribution-version="${dist}" --track="${track}" --test-mode --car=4gheap
done
done
Expand Down Expand Up @@ -341,6 +350,7 @@ function test_distribution_fails_with_wrong_track_params {

info "test distribution [--configuration-name=${cfg}], [--distribution-version=${dist}], [--track=${track}], [--track-params=${track_params}], [--car=4gheap]"
kill_rally_processes
wait_for_free_es_port

backup_rally_log
set +e
Expand Down Expand Up @@ -515,6 +525,7 @@ function test_node_management_commands {

info "test install [--configuration-name=${cfg}] [--build-type=${build_type}]"
kill_rally_processes
wait_for_free_es_port

raw_install_id=$(esrally install --quiet --configuration-name="${cfg}" --distribution-version="${dist}" --build-type="${build_type}" --node-name="rally-node-0" --master-nodes="rally-node-0" --network-host="127.0.0.1" --http-port=39200 --seed-hosts="127.0.0.1:39300")
install_id=$(echo "${raw_install_id}" | grep installation-id | cut -d '"' -f4)
Expand Down

0 comments on commit 949107e

Please sign in to comment.