diff --git a/terraform/examples/io500/setup_io500.sh b/terraform/examples/io500/setup_io500.sh index 13a9144..b5db49d 100755 --- a/terraform/examples/io500/setup_io500.sh +++ b/terraform/examples/io500/setup_io500.sh @@ -9,6 +9,9 @@ source ./configure.sh # Clean and configure DAOS servers source ./clean.sh +# Copy SSH keys to client nodes +pdcp -w ^hosts -r .ssh ~ + echo "Copy agent config files from server" rm -f .ssh/known_hosts scp ${DAOS_FIRST_SERVER}:/etc/daos/daos_agent.yml . @@ -78,7 +81,7 @@ sed -i "s/^blockSize.*/blockSize = 1000000m/g" temp.ini sed -i "s/^filePerProc.*/filePerProc = TRUE /g" temp.ini sed -i "s/^nproc.*/nproc = $(( ${NUMBER_OF_CLIENTS_INSTANCES} * $(nproc --all) ))/g" temp.ini -# Run IO500 benchmark +echo "# Run IO500 benchmark" mpirun --hostfile hosts -env I_MPI_OFI_PROVIDER="tcp;ofi_rxm" --bind-to socket -np $(( ${NUMBER_OF_CLIENTS_INSTANCES} * $(nproc --all) )) /usr/local/io500/io500 temp.ini echo "Cleaning up after run ..." diff --git a/terraform/examples/io500/start.sh b/terraform/examples/io500/start.sh index 0728140..08d184e 100755 --- a/terraform/examples/io500/start.sh +++ b/terraform/examples/io500/start.sh @@ -26,8 +26,10 @@ terraform plan -out=tfplan -input=false terraform apply -input=false tfplan popd -echo "# Wait for instances" -sleep 10 +echo "# Wait for DAOS client instances" +gcloud compute instance-groups managed wait-until ${TF_VAR_client_template_name} --stable --zone ${TF_VAR_zone} +echo "# Wait for DAOS server instances" +gcloud compute instance-groups managed wait-until ${TF_VAR_server_template_name} --stable --zone ${TF_VAR_zone} echo "# Add external IP to first client, so that it will be accessible over normal SSH" gcloud compute instances add-access-config ${DAOS_FIRST_CLIENT} --zone ${TF_VAR_zone} && sleep 10 @@ -37,18 +39,21 @@ echo "##########################" echo "# Configure SSH access #" echo "##########################" echo "# Prepare SSH key" -rm -f ./id_rsa* ; ssh-keygen -t rsa -b 4096 -C "root" -N '' -f id_rsa +rm -f ./id_rsa* ; ssh-keygen -t rsa -b 4096 -C "${SSH_USER}" -N '' -f id_rsa echo "${SSH_USER}:$(cat id_rsa.pub)" > keys.txt +echo "# Configuring SSH on nodes" for node in $ALL_NODES do - echo "# Configuring SSH on ${node}" # Disable OSLogin to be able to connect with SSH keys uploaded in next command - gcloud compute instances add-metadata ${node} --metadata enable-oslogin=FALSE + gcloud compute instances add-metadata ${node} --metadata enable-oslogin=FALSE && \ # Upload SSH key to instance, so that you could login to instance over SSH - gcloud compute instances add-metadata ${node} --metadata-from-file ssh-keys=keys.txt + gcloud compute instances add-metadata ${node} --metadata-from-file ssh-keys=keys.txt & done +# Wait for SSH configuring tasks to finish +wait + echo "# Copy SSH key to first DAOS client" scp -i id_rsa -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ id_rsa \