fix(regression-test): Sentinel test stabilization (#826)

* Ditch docker whcih is complex on CI in favour of local redis binary Signed-off-by: ashotland <ari@dragonflydb.io> * Fix typo Signed-off-by: ashotland <ari@dragonflydb.io> * Wait for sentinel termination Signed-off-by: ashotland <ari@dragonflydb.io> * fix(regression-tests): sentinel test increase timeout waiting for key to exist in replica * debug sentinel test Signed-off-by: ashotland <ari@dragonflydb.io> * add pytest repeat - tmp comment out failure notficicaiton Signed-off-by: ashotland <ari@dragonflydb.io> * fix typo Signed-off-by: ashotland <ari@dragonflydb.io> * repeat 100 Signed-off-by: ashotland <ari@dragonflydb.io> * Increase timeout for debuging Signed-off-by: ashotland <ari@dragonflydb.io> * Debug prints Signed-off-by: ashotland <ari@dragonflydb.io> * fix Signed-off-by: ashotland <ari@dragonflydb.io> * increase overall timeout Signed-off-by: ashotland <ari@dragonflydb.io> * Debug Signed-off-by: ashotland <ari@dragonflydb.io> * cleanup Signed-off-by: ashotland <ari@dragonflydb.io> --------- Signed-off-by: ashotland <ari@dragonflydb.io>
dragonflydb · Feb 20, 2023 · 1f82f9a · 1f82f9a
1 parent 03e99a5
commit 1f82f9a
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 8 deletions.
diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml
@@ -53,7 +53,6 @@ jobs:
           export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/build/dragonfly" # used by PyTests
 
           pytest -sxvr dragonfly --ignore=dragonfly/replication_test.py
-
       - name: Run PyTests replication test
         timeout-minutes: 15
         if:  ${{ inputs.run_replication }}

diff --git a/tests/dragonfly/requirements.txt b/tests/dragonfly/requirements.txt
@@ -12,4 +12,4 @@ tomli==2.0.1
 wrapt==1.14.1
 aioredis==2.0.1
 pytest-asyncio==0.20.1
-
+pytest-repeat==0.9.1
diff --git a/tests/dragonfly/sentinel_test.py b/tests/dragonfly/sentinel_test.py
@@ -127,6 +127,7 @@ async def test_failover(df_local_factory, sentinel):
 
     master_client = aioredis.Redis(port=master.port)
     replica_client = aioredis.Redis(port=replica.port)
+    print("master: " + str(master.port) + " replica: " + str(replica.port), flush=True)
 
     await replica_client.execute_command("REPLICAOF localhost " + str(master.port))
 
@@ -149,12 +150,27 @@ async def test_failover(df_local_factory, sentinel):
     assert sentinel.slaves()[0]["port"] == str(master.port)
 
     # Verify we can now write to replica and read replicated value from master.
-    await replica_client.set("key", "value")
-    await await_for(
-        lambda: master_client.get("key"),
-        lambda val: val == b"value",
-        10, "Timeout waiting for key to exist in replica."
-    )
+    assert await replica_client.set("key", "value"), "Failed to set key promoted replica."
+    try:
+        await await_for(
+            lambda: master_client.get("key"),
+            lambda val: val == b"value",
+            10, "Timeout waiting for key to exist in replica."
+        )
+    except AssertionError:
+        syncid, r_offset = await master_client.execute_command("DEBUG REPLICA OFFSET")
+        replicaoffset_cmd = "DFLY REPLICAOFFSET " + syncid.decode()
+        m_offset = await replica_client.execute_command(replicaoffset_cmd)
+        print(syncid.decode(),  r_offset, m_offset)
+        print("replica client role:")
+        print(await replica_client.execute_command("role"))
+        print("master client role:")
+        print(await master_client.execute_command("role"))
+        print("replica client info:")
+        print(await replica_client.info())
+        print("master client info:")
+        print(await master_client.info())
+        raise
 
 
 @pytest.mark.asyncio