From 6aafcfdff3730cad6179ca7266ff8bfc4858dea7 Mon Sep 17 00:00:00 2001 From: emily Date: Mon, 28 Feb 2022 23:46:44 -0800 Subject: [PATCH] [BEAM-9150] Fix beam_PostRelease_Python_Candidate (python RC validation scripts) (#16955) --- .../python_release_automation_utils.sh | 35 ++++++++++++++++--- ..._release_candidate_python_mobile_gaming.sh | 10 ++++-- ...run_release_candidate_python_quickstart.sh | 4 +-- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/release/src/main/python-release/python_release_automation_utils.sh b/release/src/main/python-release/python_release_automation_utils.sh index 5d0e1c6eff762..b328c428a7973 100644 --- a/release/src/main/python-release/python_release_automation_utils.sh +++ b/release/src/main/python-release/python_release_automation_utils.sh @@ -228,7 +228,7 @@ function cleanup_pubsub() { # $2 - pid: the pid of running pipeline # $3 - running_job (DataflowRunner only): the job id of streaming pipeline running on DataflowRunner ####################################### -function verify_steaming_result() { +function verify_streaming_result() { retry=3 should_see="Python: " while(( $retry > 0 )); do @@ -295,9 +295,11 @@ function verify_user_score() { function verify_hourly_team_score() { retry=3 should_see='AntiqueBrassPlatypus' + runner=$1 + while(( $retry >= 0 )); do if [[ $retry > 0 ]]; then - bq_pull_result=$(bq head -n 500 $DATASET.hourly_team_score_python_$1) + bq_pull_result=$(bq head -n 500 ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}) if [[ $bq_pull_result = *"$should_see"* ]]; then echo "SUCCEED: hourly_team_score example successful run on $1-runner" break @@ -307,14 +309,38 @@ function verify_hourly_team_score() { sleep 15 fi else - echo "FAILED: HourlyTeamScore example failed running on $1-runner. \ - Did not found scores of team $should_see in $DATASET.leader_board" + echo "FAILED: HourlyTeamScore example failed running on $runner runner. \ + Did not found scores of team $should_see in ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}" complete "FAILED" exit 1 fi done } +function cleanup_hourly_team_score() { + retry=3 + runner=$1 + + echo "Removing previously created table ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}..." + bq rm -q -f -t "${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}" + + while(( $retry >= 0 )); do + if [[ $retry > 0 ]]; then + bq_ls_result=$(bq ls $DATASET) + if [[ $bq_ls_result = *"${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}"* ]]; then + retry=$(($retry-1)) + echo "${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner} not cleaned up yet, waiting" + sleep 1000 + else + echo "Confirmed ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner} removed before running new test." + break + fi + else + echo "WARNING: Unable to clean up table ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}. \ + You may need to manually run 'bq rm -r -t ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}'." + fi + done +} # Python RC configurations VERSION=$(get_version) @@ -336,4 +362,5 @@ PUBSUB_SUBSCRIPTION='wordstream-python-sub2' # Mobile Gaming Configurations DATASET='beam_postrelease_mobile_gaming' USERSCORE_OUTPUT_PREFIX='python-userscore_result' +HOURLY_TEAM_SCORE_TABLE_PREFIX='hourly_team_score_python' GAME_INPUT_DATA='gs://dataflow-samples/game/5000_gaming_data.csv' diff --git a/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh b/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh index 30d8cea2bd951..efd903da48b84 100755 --- a/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh +++ b/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh @@ -100,12 +100,15 @@ function verify_userscore_dataflow() { ####################################### function verify_hourlyteamscore_direct() { print_separator "Running HourlyTeamScore example with DirectRunner" + # Clean up old bq tables + cleanup_hourly_team_score "direct" + python -m apache_beam.examples.complete.game.hourly_team_score \ --project=$PROJECT_ID \ --dataset=$DATASET \ --input=$GAME_INPUT_DATA \ --temp_location=gs://$BUCKET_NAME/temp/ \ - --table="hourly_team_score_python_direct" + --table="${HOURLY_TEAM_SCORE_TABLE_PREFIX}_direct" verify_hourly_team_score "direct" } @@ -121,6 +124,9 @@ function verify_hourlyteamscore_direct() { ####################################### function verify_hourlyteamscore_dataflow() { print_separator "Running HourlyTeamScore example with DataflowRunner" + # Clean up old bq tables + cleanup_hourly_team_score "dataflow" + python -m apache_beam.examples.complete.game.hourly_team_score \ --project=$PROJECT_ID \ --region=$REGION_ID \ @@ -129,7 +135,7 @@ function verify_hourlyteamscore_dataflow() { --temp_location=gs://$BUCKET_NAME/temp/ \ --sdk_location $BEAM_PYTHON_SDK \ --input=$GAME_INPUT_DATA \ - --table="hourly_team_score_python_dataflow" + --table="${HOURLY_TEAM_SCORE_TABLE_PREFIX}_dataflow" verify_hourly_team_score "dataflow" } diff --git a/release/src/main/python-release/run_release_candidate_python_quickstart.sh b/release/src/main/python-release/run_release_candidate_python_quickstart.sh index 6f1e10da33b38..3af527acfa2e6 100755 --- a/release/src/main/python-release/run_release_candidate_python_quickstart.sh +++ b/release/src/main/python-release/run_release_candidate_python_quickstart.sh @@ -155,7 +155,7 @@ function verify_streaming_wordcount_direct() { # verify result run_pubsub_publish - verify_steaming_result "DirectRunner" $pid + verify_streaming_result "DirectRunner" $pid kill -9 $pid sleep 10 @@ -194,7 +194,7 @@ function verify_streaming_wordcount_dataflow() { # verify result run_pubsub_publish sleep 420 - verify_steaming_result "DataflowRunner" $pid $running_job + verify_streaming_result "DataflowRunner" $pid $running_job kill -9 $pid gcloud dataflow jobs cancel $running_job