From 158c14fbd9da1496cfe63ad6f2f974e40d332352 Mon Sep 17 00:00:00 2001 From: Leonid Chernin Date: Wed, 11 Sep 2024 07:07:35 +0000 Subject: [PATCH 1/3] take the spdk change that fixes change of ns load balance group Signed-off-by: Leonid Chernin --- .gitmodules | 2 +- spdk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index d39e8591..619b8c21 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "spdk"] path = spdk url = https://github.com/ceph/spdk.git - branch = ceph-nvmeof-v24.05 + branch = branch = leonid-ceph-nvmeof-v24.05 diff --git a/spdk b/spdk index 0d9430fa..fa696ecb 160000 --- a/spdk +++ b/spdk @@ -1 +1 @@ -Subproject commit 0d9430fab9b368761b13dc73a65b0ed13d6c6bc6 +Subproject commit fa696ecb2402d84c7249373b4480083221aec21c From e16319fe2cfe2ce78f8effef06cc8023a06dbc62 Mon Sep 17 00:00:00 2001 From: Leonid Chernin Date: Wed, 11 Sep 2024 13:13:23 +0000 Subject: [PATCH 2/3] test of ns change lb group Signed-off-by: Leonid Chernin --- .github/workflows/build-container.yml | 2 +- tests/ha/ns_lb_change.sh | 121 ++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) create mode 100755 tests/ha/ns_lb_change.sh diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 9c629cee..76cde126 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -737,7 +737,7 @@ jobs: strategy: fail-fast: false matrix: - test: ["sanity", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status"] + test: ["sanity", "ns_lb_change", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status"] runs-on: ubuntu-latest env: HUGEPAGES: 1024 # 4 spdk instances diff --git a/tests/ha/ns_lb_change.sh b/tests/ha/ns_lb_change.sh new file mode 100755 index 00000000..077c9394 --- /dev/null +++ b/tests/ha/ns_lb_change.sh @@ -0,0 +1,121 @@ +#!/bin/bash +set -xe +# See +# - https://github.com/spdk/spdk/blob/master/doc/jsonrpc.md +# - https://spdk.io/doc/nvmf_multipath_howto.html + +GW1_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /1/ {print $1}') +GW2_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /2/ {print $1}') + +ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$GW1_NAME")" +ip2="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$GW2_NAME")" + +calc_written_bytes_in_sec() +{ + IP=$1 + num_bytes=$(docker compose run --rm nvmeof-cli --server-address $IP --server-port 5500 --output stdio --format json namespace get_io_stats -n nqn.2016-06.io.spdk:cnode1 --nsid 1 | jq '.bytes_written'| sed 's/[^0-9]*//g'); + + sleep 1; + num_bytes1=$(docker compose run --rm nvmeof-cli --server-address $IP --server-port 5500 --output stdio --format json namespace get_io_stats -n nqn.2016-06.io.spdk:cnode1 --nsid 1 | jq '.bytes_written'| sed 's/[^0-9]*//g'); + + res=$(expr $num_bytes1 - $num_bytes ); + #echo "Bytes written in sec: $res"; + if [ "$res" -gt 0 ]; then + # limit values to boolean for simplify futher analysis + res=1; + else + res=0; + fi; + echo "$res"; +} + + +echo -n "ℹ️ Starting bdevperf container" +docker compose up -d bdevperf +sleep 10 +echo "ℹ️ bdevperf start up logs" +make logs SVC=bdevperf +BDEVPERF_SOCKET=/tmp/bdevperf.sock +NVMEOF_DISC_PORT=8009 + + +echo "ℹ️ Using discovery service in gateway $GW1 ip $ip" +rpc="/usr/libexec/spdk/scripts/rpc.py" +echo "ℹ️ bdevperf bdev_nvme_set_options" +make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1" +echo "ℹ️ bdevperf start discovery ip: $ip port: $NVMEOF_DISC_PORT" +make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_start_discovery -b Nvme0 -t tcp -a $ip -s $NVMEOF_DISC_PORT -f ipv4 -w" +echo "ℹ️ bdevperf bdev_nvme_get_discovery_info" +make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_get_discovery_info" +echo "ℹ️ bdevperf perform_tests" +eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' ) + +timeout=$(expr $BDEVPERF_TEST_DURATION \* 2) + +echo $timeout +bdevperf="/usr/libexec/spdk/scripts/bdevperf.py" +echo "run io test" +make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests" & +#test write ios for the ns1 + +( +sleep 8; + +lb_group=1; +docker compose run -T --rm nvmeof-cli --server-address $ip --server-port 5500 namespace change_load_balancing_group -n nqn.2016-06.io.spdk:cnode1 --nsid 1 --load-balancing-group $lb_group ; + + +priv_res1=$(calc_written_bytes_in_sec $ip) ; + +echo "ℹ️ written bytes through $ip $priv_res1 "; + +priv_res2=$(calc_written_bytes_in_sec $ip2); + +echo "ℹ️ written bytes through $ip2 $priv_res2 "; + + +for i in $(seq 6); do + if [ $lb_group -eq 1 ]; then + lb_group=2 + else + lb_group=1 + fi; + + echo "ℹ️ ℹ️ Change lb group of ns 1 to $lb_group :" ; + docker compose run -T --rm nvmeof-cli --server-address $ip --server-port 5500 namespace change_load_balancing_group -n nqn.2016-06.io.spdk:cnode1 --nsid 1 --load-balancing-group $lb_group ; + sleep 4; + + res1=$(calc_written_bytes_in_sec $ip) ; + + echo "ℹ️ written bytes through $ip ?: $res1"; + + res2=$(calc_written_bytes_in_sec $ip2) ; + + echo "ℹ️ written bytes through $ip2 ?: $res2 "; + echo "ℹ️ ℹ️ ℹ️ DEBUG iteration $i : priv_res1 and res1 : $priv_res1 , $res1 , priv_res2 and res2 : $priv_res2 , $res2 "; + + #check that io is switched each iteration to different Gateway + if [ $res1 -eq $res2 ]; then + echo " ℹ️ ℹ️ ℹ️ res1 and res2 : $res1 $res2 "; + exit 1 #both eq 0 - no traffic at all + fi; + + if [ $res1 -ne $priv_res1 ] && [ $res2 -ne $priv_res2 ]; then + echo " ℹ️ ℹ️ Valid traffic results"; + else + echo "ℹ️ ℹ️ ℹ️ Not valid checks !!! : priv_res1 and res1 : $priv_res1 $res1 , priv_res2 and res2 : $priv_res2 $res2 "; + exit 1; + fi; + + priv_res1=$res1; + priv_res2=$res2; + +done; + + +echo "wait for join"; + +) & + +wait +exit 0 From a59c5eb64461007918d4eab63919609b89c54f8f Mon Sep 17 00:00:00 2001 From: Leonid Chernin Date: Mon, 16 Sep 2024 07:00:42 +0000 Subject: [PATCH 3/3] take latest spdk version ceph-nvmeof-v24.05 Signed-off-by: Leonid Chernin --- .env | 2 +- .gitmodules | 2 +- spdk | 2 +- tests/atom/clusterBuildTestsRun.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 1eb5d8f2..edb3c425 100644 --- a/.env +++ b/.env @@ -71,7 +71,7 @@ CEPH_SHA=latest CEPH_DEVEL_MGR_PATH=../ceph # Atom -ATOM_SHA=be72d1c50eb7d0c5ad9274795c1b38b3ee261878 +ATOM_SHA=af604da197c260403c5cefccc2665c6758e558d5 # Demo settings RBD_POOL=rbd diff --git a/.gitmodules b/.gitmodules index 619b8c21..f2611fc3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "spdk"] path = spdk url = https://github.com/ceph/spdk.git - branch = branch = leonid-ceph-nvmeof-v24.05 + branch = ceph-nvmeof-v24.05 diff --git a/spdk b/spdk index fa696ecb..6140a67f 160000 --- a/spdk +++ b/spdk @@ -1 +1 @@ -Subproject commit fa696ecb2402d84c7249373b4480083221aec21c +Subproject commit 6140a67ffec9d231a103577ea139c082a6e470ff diff --git a/tests/atom/clusterBuildTestsRun.sh b/tests/atom/clusterBuildTestsRun.sh index 9b5c6ed9..4cf5d863 100755 --- a/tests/atom/clusterBuildTestsRun.sh +++ b/tests/atom/clusterBuildTestsRun.sh @@ -38,7 +38,7 @@ sudo docker run \ --rmNvmeofDaemon \ --gitHubActionDeployment \ --dontUseMTLS \ - --skipLbalancingTest \ + --skiplbTest \ --journalctlToConsole \ --dontPowerOffCloudVMs noKey noKey \ --multiIBMCloudServers_m2