From 9c3b19870237855a1efcf8c25d2cde32503308ea Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Tue, 15 Oct 2024 13:19:26 +0200 Subject: [PATCH 01/37] Migrate to client-testing image - Use clients-testing image for standalone and cluster - Remove hardcoded TLS certificates and keys - Remove stunnel - Remove Cluster docker and configs --- docker-compose.yml | 50 +++++++++++--------------- dockers/Dockerfile.cluster | 7 ---- dockers/cluster.redis.conf | 2 -- dockers/create_cluster.sh | 47 ------------------------- dockers/stunnel/README | 1 - dockers/stunnel/conf/redis.conf | 6 ---- dockers/stunnel/create_certs.sh | 45 ------------------------ dockers/stunnel/keys/ca-cert.pem | 19 ---------- dockers/stunnel/keys/ca-key.pem | 28 --------------- dockers/stunnel/keys/client-cert.pem | 17 --------- dockers/stunnel/keys/client-key.pem | 28 --------------- dockers/stunnel/keys/client-req.pem | 15 -------- dockers/stunnel/keys/server-cert.pem | 17 --------- dockers/stunnel/keys/server-key.pem | 28 --------------- dockers/stunnel/keys/server-req.pem | 15 -------- tasks.py | 5 +-- tests/conftest.py | 19 +++++----- tests/ssl_utils.py | 43 +++++++++++++++++++---- tests/test_asyncio/test_cluster.py | 6 ++-- tests/test_asyncio/test_connect.py | 15 ++++---- tests/test_connect.py | 13 +++---- tests/test_ssl.py | 52 ++++++++++++++-------------- 22 files changed, 107 insertions(+), 371 deletions(-) delete mode 100644 dockers/Dockerfile.cluster delete mode 100644 dockers/cluster.redis.conf delete mode 100644 dockers/create_cluster.sh delete mode 100644 dockers/stunnel/README delete mode 100644 dockers/stunnel/conf/redis.conf delete mode 100755 dockers/stunnel/create_certs.sh delete mode 100644 dockers/stunnel/keys/ca-cert.pem delete mode 100644 dockers/stunnel/keys/ca-key.pem delete mode 100644 dockers/stunnel/keys/client-cert.pem delete mode 100644 dockers/stunnel/keys/client-key.pem delete mode 100644 dockers/stunnel/keys/client-req.pem delete mode 100644 dockers/stunnel/keys/server-cert.pem delete mode 100644 dockers/stunnel/keys/server-key.pem delete mode 100644 dockers/stunnel/keys/server-req.pem diff --git a/docker-compose.yml b/docker-compose.yml index c8528a7d58..2d326d964a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,20 @@ --- -version: "3.8" - services: redis: - image: ${REDIS_IMAGE:-redis:latest} + image: redislabs/client-libs-test:8.0-M01 container_name: redis-standalone - command: redis-server --enable-debug-command yes --protected-mode no + environment: + - REDIS_TLS_ENABLED=yes + - PORT=6379 + - TLS_PORT=6666 + - REDIS_DIRECTIVES=--enable-debug-command yes ports: - 6379:6379 + - 6666:6666 # TLS port + volumes: + - "./dockers/standalone:/redis/work" profiles: - standalone - sentinel @@ -29,39 +34,24 @@ services: - all cluster: + image: redislabs/client-libs-test:8.0-M01 container_name: redis-cluster - build: - context: . - dockerfile: dockers/Dockerfile.cluster - args: - REDIS_IMAGE: ${REDIS_IMAGE:-redis:latest} + environment: + - REDIS_CLUSTER=yes + - NODES=6 + - TLS_ENABLED=yes + - PORT=16379 + - TLS_PORT=17379 + - REDIS_DIRECTIVES=--tls-auth-clients optional ports: - - 16379:16379 - - 16380:16380 - - 16381:16381 - - 16382:16382 - - 16383:16383 - - 16384:16384 + - "16379-16384:16379-16384" + - "17379-17384:17379-17384" volumes: - - "./dockers/cluster.redis.conf:/redis.conf:ro" + - "./dockers/cluster:/redis/work" profiles: - cluster - all - stunnel: - image: redisfab/stunnel:latest - depends_on: - - redis - ports: - - 6666:6666 - profiles: - - all - - standalone - - ssl - volumes: - - "./dockers/stunnel/conf:/etc/stunnel/conf.d:ro" - - "./dockers/stunnel/keys:/etc/stunnel/keys:ro" - sentinel: image: ${REDIS_IMAGE:-redis:latest} container_name: redis-sentinel diff --git a/dockers/Dockerfile.cluster b/dockers/Dockerfile.cluster deleted file mode 100644 index 4096009f9a..0000000000 --- a/dockers/Dockerfile.cluster +++ /dev/null @@ -1,7 +0,0 @@ -ARG REDIS_IMAGE=redis:latest -FROM $REDIS_IMAGE - -COPY dockers/create_cluster.sh /create_cluster.sh -RUN chmod a+x /create_cluster.sh - -ENTRYPOINT [ "/create_cluster.sh"] diff --git a/dockers/cluster.redis.conf b/dockers/cluster.redis.conf deleted file mode 100644 index e9f7617d09..0000000000 --- a/dockers/cluster.redis.conf +++ /dev/null @@ -1,2 +0,0 @@ -protected-mode no -enable-debug-command yes diff --git a/dockers/create_cluster.sh b/dockers/create_cluster.sh deleted file mode 100644 index dc17c7c4d9..0000000000 --- a/dockers/create_cluster.sh +++ /dev/null @@ -1,47 +0,0 @@ -#! /bin/bash - -mkdir -p /nodes -touch /nodes/nodemap -if [ -z ${START_PORT} ]; then - START_PORT=16379 -fi -if [ -z ${END_PORT} ]; then - END_PORT=16384 -fi -if [ ! -z "$3" ]; then - START_PORT=$2 - START_PORT=$3 -fi -echo "STARTING: ${START_PORT}" -echo "ENDING: ${END_PORT}" - -for PORT in `seq ${START_PORT} ${END_PORT}`; do - mkdir -p /nodes/$PORT - if [[ -e /redis.conf ]]; then - cp /redis.conf /nodes/$PORT/redis.conf - else - touch /nodes/$PORT/redis.conf - fi - cat << EOF >> /nodes/$PORT/redis.conf -port ${PORT} -cluster-enabled yes -daemonize yes -logfile /redis.log -dir /nodes/$PORT -EOF - - set -x - redis-server /nodes/$PORT/redis.conf - sleep 1 - if [ $? -ne 0 ]; then - echo "Redis failed to start, exiting." - continue - fi - echo 127.0.0.1:$PORT >> /nodes/nodemap -done -if [ -z "${REDIS_PASSWORD}" ]; then - echo yes | redis-cli --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1 -else - echo yes | redis-cli -a ${REDIS_PASSWORD} --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1 -fi -tail -f /redis.log diff --git a/dockers/stunnel/README b/dockers/stunnel/README deleted file mode 100644 index e92ae78981..0000000000 --- a/dockers/stunnel/README +++ /dev/null @@ -1 +0,0 @@ - This directory contains a helper script to create ssl certificates for ssl tests. If the certificates are out of date, re-run create_certs and check them in. These are snake oil certificates. diff --git a/dockers/stunnel/conf/redis.conf b/dockers/stunnel/conf/redis.conf deleted file mode 100644 index a150d8b011..0000000000 --- a/dockers/stunnel/conf/redis.conf +++ /dev/null @@ -1,6 +0,0 @@ -[redis] -accept = 6666 -connect = redis:6379 -cert = /etc/stunnel/keys/server-cert.pem -key = /etc/stunnel/keys/server-key.pem -verify = 0 diff --git a/dockers/stunnel/create_certs.sh b/dockers/stunnel/create_certs.sh deleted file mode 100755 index 4065562cfb..0000000000 --- a/dockers/stunnel/create_certs.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -set -e - -DESTDIR=`dirname "$0"`/keys -test -d ${DESTDIR} || mkdir ${DESTDIR} -cd ${DESTDIR} - -which openssl &>/dev/null -if [ $? -ne 0 ]; then - echo "No openssl binary present, exiting." - exit 1 -fi - -openssl genrsa -out ca-key.pem 2048 &>/dev/null - -openssl req -new -x509 -nodes -days 365000 \ - -key ca-key.pem \ - -out ca-cert.pem \ - -subj "/CN=redis-py-ca" &>/dev/null - -openssl req -newkey rsa:2048 -nodes -days 365000 \ - -keyout server-key.pem \ - -out server-req.pem \ - -subj "/CN=redis-py-server" &>/dev/null - -openssl x509 -req -days 365000 -set_serial 01 \ - -in server-req.pem \ - -out server-cert.pem \ - -CA ca-cert.pem \ - -CAkey ca-key.pem &>/dev/null - -openssl req -newkey rsa:2048 -nodes -days 365000 \ - -keyout client-key.pem \ - -out client-req.pem \ - -subj "/CN=redis-py-client" &>/dev/null - -openssl x509 -req -days 365000 -set_serial 01 \ - -in client-req.pem \ - -out client-cert.pem \ - -CA ca-cert.pem \ - -CAkey ca-key.pem &>/dev/null - -echo "Keys generated in ${DESTDIR}:" -ls diff --git a/dockers/stunnel/keys/ca-cert.pem b/dockers/stunnel/keys/ca-cert.pem deleted file mode 100644 index 291cf8e23f..0000000000 --- a/dockers/stunnel/keys/ca-cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDDzCCAfegAwIBAgIUZWdrJiIH/w7FJkNbLTYldxOFEpswDQYJKoZIhvcNAQEL -BQAwFjEUMBIGA1UEAwwLcmVkaXMtcHktY2EwIBcNMjQwNTA5MDcyMDE4WhgPMzAy -MzA5MTAwNzIwMThaMBYxFDASBgNVBAMMC3JlZGlzLXB5LWNhMIIBIjANBgkqhkiG -9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0N9BXLRx3Hxb+ZGuKi5hZabcDWDMEeUGunJG -F1ijxO9XbNWXxYiR127Le2dMkS3TefU3CNiiYJa7eRxMPAS/wGUp6Bb7LrCoeC3F -1bfJSYnzC6SwhMq66m51VhqctjAbJxBBAPYqyNBFB2w2BQZOIkKDNPgPJTDNmF/7 -G/5jmAaOPlhm1GITnT+sSTyfr/JcoRRbV9VTVc9VUaTjk6ytHsW+K2sK+uWrjdig -qdzZDng0gtasTn907QkTDDyR4E/UY9N47aD2Jy5F3XHesy9kEfuppq+A1WYOs8/H -bXgEL53ncayqDNAgjnid5kHvKJ9wTAPSMDqmupHG0l5ADisahwIDAQABo1MwUTAd -BgNVHQ4EFgQUWg70hcbq4zibHXAFlZd8mHVEWzowHwYDVR0jBBgwFoAUWg70hcbq -4zibHXAFlZd8mHVEWzowDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC -AQEAe1qupf8GoqCgtzTwFCwmcDygLibX4vI/EfCMOLVZHMgDacDwQbmYPlM+goJT -Pz8WCklopFcMJ6MSdUGy3g4hjKmJpKttTSuhEd3uZWPZYjhRj2SY8531/aAajg9/ -oezyvlgN/DWXAREG31XWyXLzPU7VLbg99mYB+2+lo2cAciAOCBdIOu6WzqnQax82 -aDSqXIHiTGc/5QYZ6ZIzdVRYiVdddKSxTNKZn9x0hu3L8r2e9ryGLLVKJmZfNZDS -tXYwiY3fE0EwYViIPiPlmBEXiBhHlC2kAQMFK8Qd4LgX6rGki4luL15GYxxKPQbF -EtDS9EqM4EdRWZq3SDjOA1zODA== ------END CERTIFICATE----- diff --git a/dockers/stunnel/keys/ca-key.pem b/dockers/stunnel/keys/ca-key.pem deleted file mode 100644 index 25989d0817..0000000000 --- a/dockers/stunnel/keys/ca-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDQ30FctHHcfFv5 -ka4qLmFlptwNYMwR5Qa6ckYXWKPE71ds1ZfFiJHXbst7Z0yRLdN59TcI2KJglrt5 -HEw8BL/AZSnoFvsusKh4LcXVt8lJifMLpLCEyrrqbnVWGpy2MBsnEEEA9irI0EUH -bDYFBk4iQoM0+A8lMM2YX/sb/mOYBo4+WGbUYhOdP6xJPJ+v8lyhFFtX1VNVz1VR -pOOTrK0exb4rawr65auN2KCp3NkOeDSC1qxOf3TtCRMMPJHgT9Rj03jtoPYnLkXd -cd6zL2QR+6mmr4DVZg6zz8dteAQvnedxrKoM0CCOeJ3mQe8on3BMA9IwOqa6kcbS -XkAOKxqHAgMBAAECggEAB16eh28qcUrF/VPsNDrMtEcjOSmdfv14s6K34bepQkKQ -8BsdLsVhzUXF0jB+iBojfbMZjQCvwf6vgKzEl9LcZ8+/Sca9zWjtmMfsqgdrsmI2 -psYvIDr9m1XoYpsFGnyEs2fPE1dG19eusn4D7et0svVr0bZK5SyypFoGmcyWUP/M -kA990HAP7enGzPfpvcpr++Iu3EwWlTY3rjYgh9a7AiFhtj9zDzb9Sg0+4Xl9+8TZ -dsOvyVsiLu09MZ3vScGg5l+46w+rai+R0IxpgI9QM0sMxAS3AYFY666akrJqn6NU -S0Q5Q9gZ5V9hHxU7IHfo3weygPQuBW07nbwtX6+JCQKBgQDp7+smBlstRD+1/ZHJ -KO4Xhi+yrhtkKzViC+gF2vXpZ1GQ+3plRJFzRMFu+LkBgn1jPfg479Tm7CM4W4vM -cTZo45+hhnpwmLGnltTf3Vw23yXzLdUMenaE2u66PWh3DFPkPHwNqb30QGnx131Q -Mjnp+2EsBdiZ1d8TFF815ucG7QKBgQDkkiz7I4JgGGCbd51AseFryHgUepsrgeaA -DIWKEKBOoxOnfWH7JOxtm0oXcpWHLciQ4M6FaTFNv2vNA9Hrz5yApXFwIkKgXVU9 -+zsok4eWdEYmwxZFwjCNYvzsIDGBBwa1PQeps6C5L+nciOE8IZHYW7egAR96prV3 -E4ZQ6aWkwwKBgQCL/nJXIAiiLyx9SVBb9C1/UGLs57ommKDqmrtv/ZeZ5KVwQL3/ -KihstaGYOinkmGVW5XfNAuECjB+Lk2U2pC1uWYFm1SYiiY4O/3lGup57i9CXFT9g -p0yTtryUITmJvIvbksKeHo05RO7hthYczuHPfwqooJr9fHpxXYiYpiRtBQKBgCp0 -kFBRhyzsOj2GWTokEDfh85PyNhI9vZ+5M7CyZ+RTXBo3KtToRdYSCxAR435JXcCz -UQjswhCr5o0dEYfYdzxZ/pkSdAevbl7l5FYkGQI0NLeMcv2gFT6dzVban/dUY8WU -QXEfAVKEeM7SyetOXPWwC4p3yu4QOxKUGNW8oFzbAoGBAK3WKV51jhmMz3dtCkGW -UZxcDp5q/3uV29/UUF3/CNEhLcVuQLtNOPYRG+S9zMvwo0SNsz4mZJH1nFDSWSNL -xGXg/Ret9Li4JQTWD47kcheBCVLoTtX1bc66D2LlXDKzN5DRBACxKkAJPUjouhMB -mPDd05msnfgzPBMHMwsNjg5W ------END PRIVATE KEY----- diff --git a/dockers/stunnel/keys/client-cert.pem b/dockers/stunnel/keys/client-cert.pem deleted file mode 100644 index 4db466a4f2..0000000000 --- a/dockers/stunnel/keys/client-cert.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICpjCCAY4CAQEwDQYJKoZIhvcNAQELBQAwFjEUMBIGA1UEAwwLcmVkaXMtcHkt -Y2EwIBcNMjQwNTA5MDcyMDE5WhgPMzAyMzA5MTAwNzIwMTlaMBoxGDAWBgNVBAMM -D3JlZGlzLXB5LWNsaWVudDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB -ALOL3znn2vpX8+VHOlETymeFpw8wsCeOfr9fNhK2o5APIG1NhrGjlu+T7ri/DfrM -ZmjF+uDSuuUs044o5SFOECNi7yOwpdC9YVWSPQQ5VrsMENqyjIYyq2BC7fLHztAt -VF1jg0D0zijfFg/4meG2tAOnXLa0O9WUcmwsNlxEgyFzcLvCoTaXpUJbLYJZ2IxW -BoKgJ85acLlIFQIex053CqmgG/odM8Ib8s1YO+IXI4JsJlJFd9we+zYgZ2TRSZ8L -v8A8gXM+WTBZpZXNXYv020dW22X7gu+VH4LHcg/6eF0GtkdrFdlQjCEjwGIoVFTu -fNSp3NvSSYrK/qeJtSNaSw0CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAdA1QqJn/ -d4rcSO8z2L64d3SdO4wLf78Qznh3vTrIlQ/i0sESRQppw1U57PHSyYtAJzc1MV39 -zgn8KvuQToPQl9UoRWD6mVK8L//xplTPxWJB4BqD/kUc+lA9akBNU8Yhx7KbI5zX -z4OgTIeWAtY9R5CH1xbQlVCqAAk+SdDk2raOebNQMpzJrMUdeDTrgoDaBFnHgDbb -XHQCOF9/LrbBlrTlNJh6PHY8YztrJKdDDhSxJ9Tudz7ynUA+NcZ8dF5o/Co+QD5b -gkVdz/nV8LoDeO8QjJXsgsHFD/B+ljWYeEGc5flFe6jWLGOCtgQB5JhImg9lsWFh -X9i921F9Cqox3Q== ------END CERTIFICATE----- diff --git a/dockers/stunnel/keys/client-key.pem b/dockers/stunnel/keys/client-key.pem deleted file mode 100644 index a53cbce0f2..0000000000 --- a/dockers/stunnel/keys/client-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCzi98559r6V/Pl -RzpRE8pnhacPMLAnjn6/XzYStqOQDyBtTYaxo5bvk+64vw36zGZoxfrg0rrlLNOO -KOUhThAjYu8jsKXQvWFVkj0EOVa7DBDasoyGMqtgQu3yx87QLVRdY4NA9M4o3xYP -+JnhtrQDp1y2tDvVlHJsLDZcRIMhc3C7wqE2l6VCWy2CWdiMVgaCoCfOWnC5SBUC -HsdOdwqpoBv6HTPCG/LNWDviFyOCbCZSRXfcHvs2IGdk0UmfC7/APIFzPlkwWaWV -zV2L9NtHVttl+4LvlR+Cx3IP+nhdBrZHaxXZUIwhI8BiKFRU7nzUqdzb0kmKyv6n -ibUjWksNAgMBAAECggEAEelgSZyRwevITxU+AhyhUpaIxgErcabLijfrYw6JXrPD -nmPfjhUt15TAefnFYUHG7ajikE81ietg54u44AuznHQgO0VCJYLfFPRT1foKZvqb -K9YoIrMnWaETr+azAR2kjvSAgZhqgLVQtCMu5s+dQcgOfcOZPINkrtnySl4jXtDE -SOTaj65VjSIkura17rj7nJNUPmDGFwsxwKpeEcXZTfa//ypT/hHVREkRmbSFk5Kw -rf3T3O1pMVF8+SeacK/oyDUf3ISc8wn9Xmwgpv8I74xWtDy3kAs315tfWPMOHe4b -CYk7GD1fu2rVRhtDCvkljiw2NejfeMzKt5+2wLXRmQKBgQD0KeCv8vdw6JBLH6PI -72yE/GRkjAn4KfhmHK+1GZN6m49DV4XAYaA7T6u2Q3gn9gNsVsHC2FCsCHy63BpA -I6ZJfdm2rcJkqgeKKRQpLBRedDMpQLY1WyXjugpV46KmA0ThtgtZeVKilJWvamHs -t/TwSbf/humg0cIcamEnkKVawwKBgQC8QBS1pfMqlSodylbPG0VaJqgdF/yAthp6 -gunVqpgbTMqGLTCpKUfSgPMpzu8znaCNeZN0EK1p7qZ7VE1VHpVoyQHC9Eu8d6PF -HAENaOUcUoCQNtXLoaN4waSjt7i6vYRldT/qrYB1YdpkkVKdj39w2N+uaxtZzDXu -hHu0eixF7wKBgCR3TLN6mjImycYuh4uvFooWF/hcYfDKc+rsReHKXBhnu1HXdIZz -DjdNgtvJ39w4BfLcUjwDiqjm65oM3W7O5Dr9rNJ3yRy3uECOOhCcIL6qpCl5HL2D -S3ljg7+oK9aXjmYXhkJquEjH4EM+pDlykAaDPBPR1nrKWS9dQ/1gwRF5AoGAd+Uo -S3jiIqDWLhsMpuNrjDtKnx0DyMYynwx5+YepUNnbsxFdCKAuCjfupxYQ6wLdmr1v -2GA20l0Y0zuh9TCBYDeFU7Fb+zEHsSZg1TWVljBFiZQjHopYHzTVsx/0G5tQk33V -s5XFVv13ps2XnJokRK8b5254AP067Cqczxlw0SkCgYEA0ito+l4TOa1/DnsbP1Q0 -kgeTb/9wPHpHVJ0Hz6vIXabaDlvvYwgRh151+9xzMmrs/0QCbI2+SHucAzu4RTjM -MAiytSBQtXA+L9deNNU9QqPKsy6/Xq6SsKLRkL9kiUasiUE0v7c/T7L9D81nTFuS -8htCfXw1/Tf8tLb+Rtvvwtw= ------END PRIVATE KEY----- diff --git a/dockers/stunnel/keys/client-req.pem b/dockers/stunnel/keys/client-req.pem deleted file mode 100644 index 62828e1950..0000000000 --- a/dockers/stunnel/keys/client-req.pem +++ /dev/null @@ -1,15 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIICXzCCAUcCAQAwGjEYMBYGA1UEAwwPcmVkaXMtcHktY2xpZW50MIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs4vfOefa+lfz5Uc6URPKZ4WnDzCwJ45+ -v182ErajkA8gbU2GsaOW75PuuL8N+sxmaMX64NK65SzTjijlIU4QI2LvI7Cl0L1h -VZI9BDlWuwwQ2rKMhjKrYELt8sfO0C1UXWODQPTOKN8WD/iZ4ba0A6dctrQ71ZRy -bCw2XESDIXNwu8KhNpelQlstglnYjFYGgqAnzlpwuUgVAh7HTncKqaAb+h0zwhvy -zVg74hcjgmwmUkV33B77NiBnZNFJnwu/wDyBcz5ZMFmllc1di/TbR1bbZfuC75Uf -gsdyD/p4XQa2R2sV2VCMISPAYihUVO581Knc29JJisr+p4m1I1pLDQIDAQABoAAw -DQYJKoZIhvcNAQELBQADggEBAD3H8McA7SmTrswSp0lw1C1UFmtazhKbFYY3/+Ld -ntZimzTy4Y5Ai1UW/blgwVLZxWWzazfkfWPMsRXtWcttuW/pxFGkLlyzFm4OsUQA -hpxtUNlmEwzcYZAin3qNnCA9bQfGL/z+zUcuMuf6HGplAUhtPhTUnvGZ2B7rJ+aC -syyt+/T/JJdnnnY0o4s4OzQa9ow6P7mC6egefHgLrtFbbuB4L/L/NdVj5NBzkXso -kmHLTUwkEtKOiG4gFLRDXsgXCy+sfEEqqWapeFhOQdagENYg+LXSN0jpxGWeR1J/ -vZHMSJT4GK4SgyNpZFu5To2lf7ucw6ywCFfg6jH2EWQeCjk= ------END CERTIFICATE REQUEST----- diff --git a/dockers/stunnel/keys/server-cert.pem b/dockers/stunnel/keys/server-cert.pem deleted file mode 100644 index c17bf9ca0f..0000000000 --- a/dockers/stunnel/keys/server-cert.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICpjCCAY4CAQEwDQYJKoZIhvcNAQELBQAwFjEUMBIGA1UEAwwLcmVkaXMtcHkt -Y2EwIBcNMjQwNTA5MDcyMDE5WhgPMzAyMzA5MTAwNzIwMTlaMBoxGDAWBgNVBAMM -D3JlZGlzLXB5LXNlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB -AOg14yTsgmakeSFuqtvy4fV1rcSgLiGdGKzOBsoytmCZzV++5Jljj7utSpJiYMYk -HOTZtyqAVwmF/0yyZ25lbEHR/N3S3Jj/al4EG9u+K7O3eNZrTQkg4+ifwcT+V1Xo -s6f+L6BRld4y78QVZwdEsTy4SIeSAwGygACymEWYZ6NZBgM2xgp8SInHYxHP3gXh -02wioB79B62DExFVUKwUXjbUhPooyvGf9MMpUrmdFmQFfcosW/urCQF9YI6ZcPnr -ybXJ6kiplmNKeVD4dEyQLYNp09alnT6q+pcJa+NwW6O0eyqEsHQxCJyo9ZA3IW5I -SH+oftVxnZJIIPcsXABuH10CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAdWY0UeR4 -/9hpK3Mhl8VVz0zQwwEfnxCmI/TxpqNw+5lvpit/WvriIAEP9MToWHwYvG24zRrp -zv/LDHNh8UtnX3GILGs0CY/oFDevAEU1tixbmFJPceuMwKsrMtkp/6NyWF4p62o2 -fiQK68l1HSGgaH7kJ6BKYgV4JQK3Fgk9J4KrejwmYXzCFKcEvNtKMG7i0WN+AmK2 -vnxxZ3xx4HPH3OJ5ss6T2gGlvjFnOS7Z0kHtbkzPzxaC9ZVqMySwPRggf84tUUdk -vCwDHiJcbk5BMLug3yI9xTfSG3lMnwgZAWXMOqm/w6c1IIM8R/nKwNfwbG+4eUK0 -t2F8EBCShzAJGg== ------END CERTIFICATE----- diff --git a/dockers/stunnel/keys/server-key.pem b/dockers/stunnel/keys/server-key.pem deleted file mode 100644 index 8dd9a1e21a..0000000000 --- a/dockers/stunnel/keys/server-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDoNeMk7IJmpHkh -bqrb8uH1da3EoC4hnRiszgbKMrZgmc1fvuSZY4+7rUqSYmDGJBzk2bcqgFcJhf9M -smduZWxB0fzd0tyY/2peBBvbviuzt3jWa00JIOPon8HE/ldV6LOn/i+gUZXeMu/E -FWcHRLE8uEiHkgMBsoAAsphFmGejWQYDNsYKfEiJx2MRz94F4dNsIqAe/QetgxMR -VVCsFF421IT6KMrxn/TDKVK5nRZkBX3KLFv7qwkBfWCOmXD568m1yepIqZZjSnlQ -+HRMkC2DadPWpZ0+qvqXCWvjcFujtHsqhLB0MQicqPWQNyFuSEh/qH7VcZ2SSCD3 -LFwAbh9dAgMBAAECggEAI0llDgxeuIhP2/O8RRZAnhNW56VLvVHpGQFp6LoSGtXk -bqNMi76kbemkhmAqwpFkTqaC/hNoporVQ+tsaktBSzNE0NSlLx7JJCZNsXPRokrE -Mxk1KKj12TjFslDQJr7o5iNrS1p6gryM0OhLssAOiuKaKvfWOyDL8M8y8oh5X0ny -1M6IAJMkbpwiWU2OHIH7irkS8fYyCeOz0JMovCwMPwYkovHD7uHKbV4qGKzdOKN1 -QD8qMWAF1lCv/57juuwpzulGY3sSyU7yRZMMxJQ7nbIRj5iuj6+e2m6JhVghIiYG -IObIkGyubCr9QH315byiSS9ma1xzml3EqyM3XQkEhQKBgQDyxGY+60/dkUW9vAAm -g20eVZnflhE8+ooEpX9VPIliL7ADy3HU2poV2oXif8pVauMvRaYla8BHIOPV2qGI -tHTYNvubs6lxEq2Z7gM+8c5qOElXjup8Ch9/XCHXZavW8caWEcA9Z84Z4dCxbaku -EhEL0SduCn7j1tU1+Z9jBs08ewKBgQD03i29kCUeCnW+zEo+aO2Spn6HpdyZkuzG -2az5XurHGUFAgWYLOpShatjD4BY1GONvJTlD/gH2vqEkfY2OGgZ2pbjCFSfhIma/ -cnMuhsO2IlcuETqzlod1HGHcn6gGRM5LvYP343UIdv9nmJaT31nckueWv+yBd8HO -kAx1W2boBwKBgBtM7tqgh8i474jYvYOXQAwrQDSeoa2j1yWSnvEs7541Eqw6ksCH -HNDcVDYWfOCCNq44POj0ZxkYn8aK4aOH96Pg+waVe7aVjSREWeUYOEhFsCnCjqgI -U2Z1K/EXI+32Hoj90gqVw92xQVDSrjXaHkSf7rk3QPHKVQvO2JfAShBFAoGAW5ic -nZNE/ybEgrmicBQKAlh7bjxx95SJM50LYkDKK+3bhcihpkOkg3kXWrYBOJ11vga7 -lB55F5aZaq/4epZroog9Q4RsZX/b1XN3eIj6vq+70sSpI7KEOx+Bz+h9DtNAI/7h -VaHlDmSNB3CBqxDaaXMeZDqouolUmvMxZdjp9pMCgYEA1Y7vhCvMZA62OJwFJ4X8 -9Xg7bPE3jZQ6Tj3FbVCMy+RQdlo8GzYeoNZDjhpSxjJe/1lyk//EBwVNs3E4rRNl -+GcaEOo0X/J7SkPFqM6aFITypIIGeJpFyz/S99i/5tkfsNt9BQtiTS+x1Kj1iREV -bXIoNJRac5m/LLZKtDtHv18= ------END PRIVATE KEY----- diff --git a/dockers/stunnel/keys/server-req.pem b/dockers/stunnel/keys/server-req.pem deleted file mode 100644 index 6d853693fb..0000000000 --- a/dockers/stunnel/keys/server-req.pem +++ /dev/null @@ -1,15 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIICXzCCAUcCAQAwGjEYMBYGA1UEAwwPcmVkaXMtcHktc2VydmVyMIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6DXjJOyCZqR5IW6q2/Lh9XWtxKAuIZ0Y -rM4GyjK2YJnNX77kmWOPu61KkmJgxiQc5Nm3KoBXCYX/TLJnbmVsQdH83dLcmP9q -XgQb274rs7d41mtNCSDj6J/BxP5XVeizp/4voFGV3jLvxBVnB0SxPLhIh5IDAbKA -ALKYRZhno1kGAzbGCnxIicdjEc/eBeHTbCKgHv0HrYMTEVVQrBReNtSE+ijK8Z/0 -wylSuZ0WZAV9yixb+6sJAX1gjplw+evJtcnqSKmWY0p5UPh0TJAtg2nT1qWdPqr6 -lwlr43Bbo7R7KoSwdDEInKj1kDchbkhIf6h+1XGdkkgg9yxcAG4fXQIDAQABoAAw -DQYJKoZIhvcNAQELBQADggEBAGMLI6jfG95L1Kqny8+Fl9sVnJ4ynb5905Hk9vXJ -V/BVc3P6JS6c4qYSeFd6wihHC7/j2EC3wt55Sj6JzYKy93AEjBfDfBb2ZuB6VpPy -iGKXzSGO71ziI2uzz92ltJhptNc6TNUUxwaBhOZiq2sxnLpnIcPZ/txDC75fGYEm -9iSbeeHNNZTSqQyQOzKW0OL6ss+GHhlfJPzx6mSH5dvb6bpKB2SCG1aZaDuOQTl3 -8aDIo1Z/ug6BrqoDMCyRAZTDnTohhC96bbKLRMdm0g3wwDeoWuQy1q9s1/AUYfBm -305LUYORBdFy08n41lFWo1JA4errzBhVTpHNKZ6DyQfMOxA= ------END CERTIFICATE REQUEST----- diff --git a/tasks.py b/tasks.py index 76737b8eff..0b5ce55f78 100644 --- a/tasks.py +++ b/tasks.py @@ -68,13 +68,14 @@ def cluster_tests(c, uvloop=False, protocol=2, profile=False): """Run tests against a redis cluster""" profile_arg = "--profile" if profile else "" cluster_url = "redis://localhost:16379/0" + cluster_tls_url = "rediss://localhost:17379/0" if uvloop: run( - f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_cluster.xml -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --junit-xml=cluster-uvloop-results.xml --uvloop" + f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_cluster.xml -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --redis-ssl-url={cluster_tls_url} --junit-xml=cluster-uvloop-results.xml --uvloop" ) else: run( - f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_clusteclient.xml -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --junit-xml=cluster-results.xml" + f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_clusteclient.xml -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --redis-ssl-url={cluster_tls_url} --junit-xml=cluster-results.xml" ) diff --git a/tests/conftest.py b/tests/conftest.py index 0c98eee4d8..44935dd827 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,7 +21,7 @@ from redis.connection import Connection, ConnectionInterface, SSLConnection, parse_url from redis.exceptions import RedisClusterException from redis.retry import Retry -from tests.ssl_utils import get_ssl_filename +from tests.ssl_utils import get_tls_certificates REDIS_INFO = {} default_redis_url = "redis://localhost:6379/0" @@ -325,6 +325,9 @@ def _get_client( redis_url = request.config.getoption("--redis-url") else: redis_url = from_url + + redis_tls_url = request.config.getoption("--redis-ssl-url") + if "protocol" not in redis_url and kwargs.get("protocol") is None: kwargs["protocol"] = request.config.getoption("--protocol") @@ -335,15 +338,11 @@ def _get_client( connection_class = Connection if ssl: connection_class = SSLConnection - kwargs["ssl_certfile"] = get_ssl_filename("client-cert.pem") - kwargs["ssl_keyfile"] = get_ssl_filename("client-key.pem") - # When you try to assign "required" as single string - # it assigns tuple instead of string. - # Probably some reserved keyword - # I can't explain how does it work -_- - kwargs["ssl_cert_reqs"] = "require" + "d" - kwargs["ssl_ca_certs"] = get_ssl_filename("ca-cert.pem") - kwargs["port"] = 6666 + kwargs["ssl_certfile"], kwargs["ssl_keyfile"], kwargs["ssl_ca_certs"] = ( + get_tls_certificates() + ) + kwargs["ssl_cert_reqs"] = "required" + kwargs["port"] = redis_tls_url.port kwargs["connection_class"] = connection_class url_options.update(kwargs) pool = redis.ConnectionPool(**url_options) diff --git a/tests/ssl_utils.py b/tests/ssl_utils.py index 1de53bbf66..f74e8f814e 100644 --- a/tests/ssl_utils.py +++ b/tests/ssl_utils.py @@ -1,14 +1,45 @@ +import enum import os +from collections import namedtuple -def get_ssl_filename(name): + +CLIENT_CERT_NAME = "client.crt" +CLIENT_KEY_NAME = "client.key" +SERVER_CERT_NAME = "redis.crt" +SERVER_KEY_NAME = "redis.key" +CA_CERT_NAME = "ca.crt" + + +class CertificateType(str, enum.Enum): + client = "client" + server = "server" + + +TLSFiles = namedtuple("TLSFiles", ["certfile", "keyfile", "ca_certfile"]) + + +def get_tls_certificates( + subdir: str = "standalone", + cert_type: CertificateType = CertificateType.client, +): root = os.path.join(os.path.dirname(__file__), "..") - cert_dir = os.path.abspath(os.path.join(root, "dockers", "stunnel", "keys")) + cert_subdir = ("dockers", subdir, "tls") + cert_dir = os.path.abspath(os.path.join(root, *cert_subdir)) if not os.path.isdir(cert_dir): # github actions package validation case - cert_dir = os.path.abspath( - os.path.join(root, "..", "dockers", "stunnel", "keys") - ) + cert_dir = os.path.abspath(os.path.join(root, "..", *cert_subdir)) if not os.path.isdir(cert_dir): raise OSError(f"No SSL certificates found. They should be in {cert_dir}") - return os.path.join(cert_dir, name) + if cert_type == CertificateType.client: + return TLSFiles( + os.path.join(cert_dir, CLIENT_CERT_NAME), + os.path.join(cert_dir, CLIENT_KEY_NAME), + os.path.join(cert_dir, CA_CERT_NAME), + ) + elif cert_type == CertificateType.server: + return TLSFiles( + os.path.join(cert_dir, SERVER_CERT_NAME), + os.path.join(cert_dir, SERVER_KEY_NAME), + os.path.join(cert_dir, CA_CERT_NAME), + ) diff --git a/tests/test_asyncio/test_cluster.py b/tests/test_asyncio/test_cluster.py index f3b76b80c9..5eed044def 100644 --- a/tests/test_asyncio/test_cluster.py +++ b/tests/test_asyncio/test_cluster.py @@ -38,7 +38,7 @@ skip_unless_arch_bits, ) -from ..ssl_utils import get_ssl_filename +from ..ssl_utils import get_tls_certificates from .compat import aclosing, mock pytestmark = pytest.mark.onlycluster @@ -2899,9 +2899,7 @@ class TestSSL: appropriate port. """ - CA_CERT = get_ssl_filename("ca-cert.pem") - CLIENT_CERT = get_ssl_filename("client-cert.pem") - CLIENT_KEY = get_ssl_filename("client-key.pem") + CLIENT_CERT, CLIENT_KEY, CA_CERT = get_tls_certificates("cluster") @pytest_asyncio.fixture() def create_client(self, request: FixtureRequest) -> Callable[..., RedisCluster]: diff --git a/tests/test_asyncio/test_connect.py b/tests/test_asyncio/test_connect.py index 943540c885..2aed218b94 100644 --- a/tests/test_asyncio/test_connect.py +++ b/tests/test_asyncio/test_connect.py @@ -11,7 +11,7 @@ ) from redis.exceptions import ConnectionError -from ..ssl_utils import get_ssl_filename +from ..ssl_utils import get_tls_certificates, CertificateType _CLIENT_NAME = "test-suite-client" _CMD_SEP = b"\r\n" @@ -57,9 +57,7 @@ async def test_uds_connect(uds_address): ) async def test_tcp_ssl_tls12_custom_ciphers(tcp_address, ssl_ciphers): host, port = tcp_address - certfile = get_ssl_filename("client-cert.pem") - keyfile = get_ssl_filename("client-key.pem") - ca_certfile = get_ssl_filename("ca-cert.pem") + certfile, keyfile, ca_certfile = get_tls_certificates() conn = SSLConnection( host=host, port=port, @@ -86,9 +84,9 @@ async def test_tcp_ssl_tls12_custom_ciphers(tcp_address, ssl_ciphers): ) async def test_tcp_ssl_connect(tcp_address, ssl_min_version): host, port = tcp_address - certfile = get_ssl_filename("client-cert.pem") - keyfile = get_ssl_filename("client-key.pem") - ca_certfile = get_ssl_filename("ca-cert.pem") + + certfile, keyfile, ca_certfile = get_tls_certificates() + conn = SSLConnection( host=host, port=port, @@ -105,8 +103,7 @@ async def test_tcp_ssl_connect(tcp_address, ssl_min_version): @pytest.mark.skipif(not ssl.HAS_TLSv1_3, reason="requires TLSv1.3") async def test_tcp_ssl_version_mismatch(tcp_address): host, port = tcp_address - certfile = get_ssl_filename("server-cert.pem") - keyfile = get_ssl_filename("server-key.pem") + certfile, keyfile, _ = get_tls_certificates(cert_type=CertificateType.server) conn = SSLConnection( host=host, port=port, diff --git a/tests/test_connect.py b/tests/test_connect.py index b11c4446e5..f178c62ee9 100644 --- a/tests/test_connect.py +++ b/tests/test_connect.py @@ -8,7 +8,7 @@ from redis.connection import Connection, SSLConnection, UnixDomainSocketConnection from redis.exceptions import RedisError -from .ssl_utils import get_ssl_filename +from .ssl_utils import get_tls_certificates, CertificateType _CLIENT_NAME = "test-suite-client" _CMD_SEP = b"\r\n" @@ -54,9 +54,7 @@ def test_uds_connect(uds_address): ) def test_tcp_ssl_connect(tcp_address, ssl_min_version): host, port = tcp_address - certfile = get_ssl_filename("client-cert.pem") - keyfile = get_ssl_filename("client-key.pem") - ca_certfile = get_ssl_filename("ca-cert.pem") + certfile, keyfile, ca_certfile = get_tls_certificates() conn = SSLConnection( host=host, port=port, @@ -79,9 +77,7 @@ def test_tcp_ssl_connect(tcp_address, ssl_min_version): ) def test_tcp_ssl_tls12_custom_ciphers(tcp_address, ssl_ciphers): host, port = tcp_address - certfile = get_ssl_filename("client-cert.pem") - keyfile = get_ssl_filename("client-key.pem") - ca_certfile = get_ssl_filename("ca-cert.pem") + certfile, keyfile, ca_certfile = get_tls_certificates() conn = SSLConnection( host=host, port=port, @@ -115,8 +111,7 @@ def test_unix_socket_with_timeout(): @pytest.mark.skipif(not ssl.HAS_TLSv1_3, reason="requires TLSv1.3") def test_tcp_ssl_version_mismatch(tcp_address): host, port = tcp_address - certfile = get_ssl_filename("server-cert.pem") - keyfile = get_ssl_filename("server-key.pem") + certfile, keyfile, _ = get_tls_certificates(cert_type=CertificateType.server) conn = SSLConnection( host=host, port=port, diff --git a/tests/test_ssl.py b/tests/test_ssl.py index fc7416dbc7..280879e600 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -7,7 +7,7 @@ from redis.exceptions import ConnectionError, RedisError from .conftest import skip_if_cryptography, skip_if_nocryptography -from .ssl_utils import get_ssl_filename +from .ssl_utils import get_tls_certificates, CertificateType @pytest.mark.ssl @@ -18,10 +18,10 @@ class TestSSL: and connecting to the appropriate port. """ - CA_CERT = get_ssl_filename("ca-cert.pem") - CLIENT_CERT = get_ssl_filename("client-cert.pem") - CLIENT_KEY = get_ssl_filename("client-key.pem") - SERVER_CERT = get_ssl_filename("server-cert.pem") + @pytest.fixture(autouse=True) + def _set_ssl_certs(self): + self.client_certs = get_tls_certificates() + self.server_certs = get_tls_certificates(cert_type=CertificateType.server) def test_ssl_with_invalid_cert(self, request): ssl_url = request.config.option.redis_ssl_url @@ -55,16 +55,16 @@ def test_validating_self_signed_certificate(self, request): host=p[0], port=p[1], ssl=True, - ssl_certfile=self.CLIENT_CERT, - ssl_keyfile=self.CLIENT_KEY, + ssl_certfile=self.client_certs.cert, + ssl_keyfile=self.client_certs.keyfile, ssl_cert_reqs="required", - ssl_ca_certs=self.CA_CERT, + ssl_ca_certs=self.client_certs.ca_certfile, ) assert r.ping() r.close() def test_validating_self_signed_string_certificate(self, request): - with open(self.CA_CERT) as f: + with open(self.client_certs.ca_certfile) as f: cert_data = f.read() ssl_url = request.config.option.redis_ssl_url p = urlparse(ssl_url)[1].split(":") @@ -72,8 +72,8 @@ def test_validating_self_signed_string_certificate(self, request): host=p[0], port=p[1], ssl=True, - ssl_certfile=self.CLIENT_CERT, - ssl_keyfile=self.CLIENT_KEY, + ssl_certfile=self.client_certs.cert, + ssl_keyfile=self.client_certs.keyfile, ssl_cert_reqs="required", ssl_ca_data=cert_data, ) @@ -149,10 +149,10 @@ def _create_oscp_conn(self, request): host=p[0], port=p[1], ssl=True, - ssl_certfile=self.CLIENT_CERT, - ssl_keyfile=self.CLIENT_KEY, + ssl_certfile=self.client_certs.cert, + ssl_keyfile=self.client_certs.keyfile, ssl_cert_reqs="required", - ssl_ca_certs=self.CA_CERT, + ssl_ca_certs=self.client_certs.ca_certfile, ssl_validate_ocsp=True, ) return r @@ -247,10 +247,10 @@ def test_mock_ocsp_staple(self, request): host=p[0], port=p[1], ssl=True, - ssl_certfile=self.CLIENT_CERT, - ssl_keyfile=self.CLIENT_KEY, + ssl_certfile=self.client_certs.cert, + ssl_keyfile=self.client_certs.keyfile, ssl_cert_reqs="required", - ssl_ca_certs=self.CA_CERT, + ssl_ca_certs=self.client_certs.ca_certfile, ssl_validate_ocsp=True, ssl_ocsp_context=p, # just needs to not be none ) @@ -260,19 +260,19 @@ def test_mock_ocsp_staple(self, request): r.close() ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) - ctx.use_certificate_file(self.CLIENT_CERT) - ctx.use_privatekey_file(self.CLIENT_KEY) + ctx.use_certificate_file(self.client_certs.cert) + ctx.use_privatekey_file(self.client_certs.keyfile) r = redis.Redis( host=p[0], port=p[1], ssl=True, - ssl_certfile=self.CLIENT_CERT, - ssl_keyfile=self.CLIENT_KEY, + ssl_certfile=self.client_certs.cert, + ssl_keyfile=self.client_certs.keyfile, ssl_cert_reqs="required", - ssl_ca_certs=self.CA_CERT, + ssl_ca_certs=self.client_certs.ca_certfile, ssl_ocsp_context=ctx, - ssl_ocsp_expected_cert=open(self.SERVER_CERT, "rb").read(), + ssl_ocsp_expected_cert=open(self.server_certs.ca_certfile, "rb").read(), ssl_validate_ocsp_stapled=True, ) @@ -285,10 +285,10 @@ def test_mock_ocsp_staple(self, request): host=p[0], port=p[1], ssl=True, - ssl_certfile=self.CLIENT_CERT, - ssl_keyfile=self.CLIENT_KEY, + ssl_certfile=self.client_certs.cert, + ssl_keyfile=self.client_certs.keyfile, ssl_cert_reqs="required", - ssl_ca_certs=self.CA_CERT, + ssl_ca_certs=self.client_certs.ca_certfile, ssl_validate_ocsp_stapled=True, ) From a0c09d7fedef9bf98328f4165d310a415585e2b8 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Tue, 15 Oct 2024 15:08:43 +0200 Subject: [PATCH 02/37] Fix migration bugs --- .github/workflows/integration.yaml | 2 +- .gitignore | 4 ++++ docker-compose.yml | 7 ++++--- tests/test_asyncio/test_connect.py | 18 ++++++++++++------ tests/test_connect.py | 18 ++++++++++++------ tests/test_ssl.py | 6 +++--- 6 files changed, 36 insertions(+), 19 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index ed969d11f1..5e3de7673f 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -86,7 +86,7 @@ jobs: pip install hiredis fi invoke devenv - sleep 10 # time to settle + sleep 10 # time to settle invoke ${{matrix.test-type}}-tests ls -1 diff --git a/.gitignore b/.gitignore index c6a4efd1ec..ee1bda0fa5 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,7 @@ coverage.xml prof profile_output* docker/stunnel/keys +/dockers/*/node-*/* +/dockers/*/tls/* +/dockers/standalone/ +/dockers/cluster/ diff --git a/docker-compose.yml b/docker-compose.yml index 2d326d964a..a9dcf6b5c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,10 +6,11 @@ services: image: redislabs/client-libs-test:8.0-M01 container_name: redis-standalone environment: - - REDIS_TLS_ENABLED=yes + - TLS_ENABLED=yes + - REDIS_CLUSTER=no - PORT=6379 - TLS_PORT=6666 - - REDIS_DIRECTIVES=--enable-debug-command yes + command: --enable-debug-command yes --tls-auth-clients optional ports: - 6379:6379 - 6666:6666 # TLS port @@ -42,7 +43,7 @@ services: - TLS_ENABLED=yes - PORT=16379 - TLS_PORT=17379 - - REDIS_DIRECTIVES=--tls-auth-clients optional + command: --tls-auth-clients optional ports: - "16379-16384:16379-16384" - "17379-17384:17379-17384" diff --git a/tests/test_asyncio/test_connect.py b/tests/test_asyncio/test_connect.py index 2aed218b94..5d949c9639 100644 --- a/tests/test_asyncio/test_connect.py +++ b/tests/test_asyncio/test_connect.py @@ -57,17 +57,21 @@ async def test_uds_connect(uds_address): ) async def test_tcp_ssl_tls12_custom_ciphers(tcp_address, ssl_ciphers): host, port = tcp_address - certfile, keyfile, ca_certfile = get_tls_certificates() + + server_certs = get_tls_certificates(cert_type=CertificateType.server) + conn = SSLConnection( host=host, port=port, client_name=_CLIENT_NAME, - ssl_ca_certs=ca_certfile, + ssl_ca_certs=server_certs.ca_certfile, socket_timeout=10, ssl_min_version=ssl.TLSVersion.TLSv1_2, ssl_ciphers=ssl_ciphers, ) - await _assert_connect(conn, tcp_address, certfile=certfile, keyfile=keyfile) + await _assert_connect( + conn, tcp_address, certfile=server_certs.certfile, keyfile=server_certs.keyfile + ) await conn.disconnect() @@ -85,17 +89,19 @@ async def test_tcp_ssl_tls12_custom_ciphers(tcp_address, ssl_ciphers): async def test_tcp_ssl_connect(tcp_address, ssl_min_version): host, port = tcp_address - certfile, keyfile, ca_certfile = get_tls_certificates() + server_certs = get_tls_certificates(cert_type=CertificateType.server) conn = SSLConnection( host=host, port=port, client_name=_CLIENT_NAME, - ssl_ca_certs=ca_certfile, + ssl_ca_certs=server_certs.ca_certfile, socket_timeout=10, ssl_min_version=ssl_min_version, ) - await _assert_connect(conn, tcp_address, certfile=certfile, keyfile=keyfile) + await _assert_connect( + conn, tcp_address, certfile=server_certs.certfile, keyfile=server_certs.keyfile + ) await conn.disconnect() diff --git a/tests/test_connect.py b/tests/test_connect.py index f178c62ee9..7a10e361c8 100644 --- a/tests/test_connect.py +++ b/tests/test_connect.py @@ -54,16 +54,18 @@ def test_uds_connect(uds_address): ) def test_tcp_ssl_connect(tcp_address, ssl_min_version): host, port = tcp_address - certfile, keyfile, ca_certfile = get_tls_certificates() + server_certs = get_tls_certificates(cert_type=CertificateType.server) conn = SSLConnection( host=host, port=port, client_name=_CLIENT_NAME, - ssl_ca_certs=ca_certfile, + ssl_ca_certs=server_certs.ca_certfile, socket_timeout=10, ssl_min_version=ssl_min_version, ) - _assert_connect(conn, tcp_address, certfile=certfile, keyfile=keyfile) + _assert_connect( + conn, tcp_address, certfile=server_certs.certfile, keyfile=server_certs.keyfile + ) @pytest.mark.ssl @@ -77,17 +79,21 @@ def test_tcp_ssl_connect(tcp_address, ssl_min_version): ) def test_tcp_ssl_tls12_custom_ciphers(tcp_address, ssl_ciphers): host, port = tcp_address - certfile, keyfile, ca_certfile = get_tls_certificates() + + server_certs = get_tls_certificates(cert_type=CertificateType.server) + conn = SSLConnection( host=host, port=port, client_name=_CLIENT_NAME, - ssl_ca_certs=ca_certfile, + ssl_ca_certs=server_certs.ca_certfile, socket_timeout=10, ssl_min_version=ssl.TLSVersion.TLSv1_2, ssl_ciphers=ssl_ciphers, ) - _assert_connect(conn, tcp_address, certfile=certfile, keyfile=keyfile) + _assert_connect( + conn, tcp_address, certfile=server_certs.certfile, keyfile=server_certs.keyfile + ) """ diff --git a/tests/test_ssl.py b/tests/test_ssl.py index 280879e600..e26c00dc3a 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -55,7 +55,7 @@ def test_validating_self_signed_certificate(self, request): host=p[0], port=p[1], ssl=True, - ssl_certfile=self.client_certs.cert, + ssl_certfile=self.client_certs.certfile, ssl_keyfile=self.client_certs.keyfile, ssl_cert_reqs="required", ssl_ca_certs=self.client_certs.ca_certfile, @@ -72,7 +72,7 @@ def test_validating_self_signed_string_certificate(self, request): host=p[0], port=p[1], ssl=True, - ssl_certfile=self.client_certs.cert, + ssl_certfile=self.client_certs.certfile, ssl_keyfile=self.client_certs.keyfile, ssl_cert_reqs="required", ssl_ca_data=cert_data, @@ -149,7 +149,7 @@ def _create_oscp_conn(self, request): host=p[0], port=p[1], ssl=True, - ssl_certfile=self.client_certs.cert, + ssl_certfile=self.client_certs.certfile, ssl_keyfile=self.client_certs.keyfile, ssl_cert_reqs="required", ssl_ca_certs=self.client_certs.ca_certfile, From bcc489a53b89f98fcd609c210c04f75f6031818f Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 16 Oct 2024 10:55:40 +0200 Subject: [PATCH 03/37] Create reusable action to run tests - Reduce copy paste by using reusable action for running tests - Gain better control of tests matrix Add missing actions checkout More fixes in integration workflow Another attempt to fix matrix --- .github/actions/run-tests/action.yml | 71 +++++++++++++ .github/workflows/integration.yaml | 152 ++++++--------------------- 2 files changed, 102 insertions(+), 121 deletions(-) create mode 100644 .github/actions/run-tests/action.yml diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml new file mode 100644 index 0000000000..495485dbb0 --- /dev/null +++ b/.github/actions/run-tests/action.yml @@ -0,0 +1,71 @@ +name: 'Run redis-py tests' +description: 'Runs redis-py tests against different Redis versions and configurations' +inputs: + python-version: + description: 'Python version to use for running tests' + default: '3.12' + test-type: + description: 'Type of tests to run: standalone or cluster' + required: true + parser-backend: + description: 'Parser backend to use: plain or hiredis' + required: true + resp-protocol: + description: 'RESP protocol version to use: 2 or 3' + required: true + redis-version: + description: 'Redis version to test against' + required: true + hiredis-version: + description: 'hiredis version to test against' + required: false + default: '>3.0.0' + event-loop: + description: 'Event loop to use' + required: false + default: 'asyncio' +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + cache: 'pip' + + - name: Run tests + run: | + pip install -U setuptools wheel + pip install -r requirements.txt + pip install -r dev_requirements.txt + if [ "${{inputs.parser-backend}}" == "hiredis" ]; then + pip install hiredis${{inputs.hiredis-version}} + fi + invoke devenv + sleep 10 # time to settle + + if [ "${{inputs.event-loop}}" == "uvloop" ]; then + invoke ${{inputs.test-type}}-tests --uvloop --protocol=${{inputs.resp-protocol}} + else + invoke ${{inputs.test-type}}-tests --protocol=${{inputs.resp-protocol}} + fi + + ls -1 + shell: bash + + - name: Upload test results and profiling data + uses: actions/upload-artifact@v4 + with: + name: pytest-results-${{inputs.test-type}}-${{inputs.parser-backend}}-${{inputs.python-version}}-${{inputs.event-loop}}-${{inputs.resp-protocol}} + path: | + ${{inputs.test-type}}*-results.xml + prof/** + profile_output* + if-no-files-found: error + retention-days: 10 + + - name: Upload codecov coverage + uses: codecov/codecov-action@v4 + with: + fail_ci_if_error: false diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 5e3de7673f..5a11a424d7 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -56,148 +56,58 @@ jobs: pip install -r dev_requirements.txt invoke linters - resp2-tests: + tests: runs-on: ubuntu-latest timeout-minutes: 60 strategy: max-parallel: 15 fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10'] - test-type: ['standalone', 'cluster'] - connection-type: ['hiredis', 'plain'] - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: true - name: RESP2 ${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.connection-type}} - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - - name: Run tests - run: | - pip install -U setuptools wheel - pip install -r requirements.txt - pip install -r dev_requirements.txt - if [ "${{matrix.connection-type}}" == "hiredis" ]; then - pip install hiredis - fi - invoke devenv - sleep 10 # time to settle - invoke ${{matrix.test-type}}-tests - ls -1 - - - name: Run tests against hiredis < 3.0.0 - if: ${{ matrix.connection-type == 'hiredis' && matrix.python-version == '3.12'}} - run: | - pip uninstall hiredis -y - pip install -U setuptools wheel - pip install -r requirements.txt - pip install -r dev_requirements.txt - if [ "${{matrix.connection-type}}" == "hiredis" ]; then - pip install "hiredis<3.0.0" - fi - invoke devenv - sleep 10 # time to settle - invoke ${{matrix.test-type}}-tests - ls -1 - - - name: Upload test results and profiling data - uses: actions/upload-artifact@v4 - with: - name: pytest-results-${{matrix.test-type}}-${{matrix.connection-type}}-${{matrix.python-version}} - path: | - ${{matrix.test-type}}*-results.xml - prof/** - profile_output* - if-no-files-found: error - retention-days: 10 - - - name: Upload codecov coverage - uses: codecov/codecov-action@v4 - with: - fail_ci_if_error: false - - resp3-tests: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.8', '3.12'] + redis-version: ['8.0.0-M01'] + resp-protocol: ['2', '3'] test-type: ['standalone', 'cluster'] - connection-type: ['hiredis', 'plain'] - event-loop: ['asyncio', 'uvloop'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10'] + parser-backend: ['hiredis', 'plain'] + event-loop: ['asyncio'] + include: + # Run uvloop tests only for resp3 + - resp-protocol: '3' + parser-backend: 'plain' + redis-version: '8.0.0-M01' + event-loop: 'uvloop' exclude: - test-type: 'cluster' - connection-type: 'hiredis' + parser-backend: 'hiredis' + resp-protocol: '3' env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true - name: RESP3 ${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.connection-type}}-${{matrix.event-loop}} + name: Redis ${{ matrix.redis-version }} - RESP${{ matrix.resp-protocol }} ${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.parser-backend}}-${{matrix.event-loop}} steps: - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - name: Run tests - run: | - pip install -U setuptools wheel - pip install -r requirements.txt - pip install -r dev_requirements.txt - if [ "${{matrix.connection-type}}" == "hiredis" ]; then - pip install hiredis - fi - invoke devenv - sleep 10 # time to settle - if [ "${{matrix.event-loop}}" == "uvloop" ]; then - invoke ${{matrix.test-type}}-tests --uvloop --protocol=3 - else - invoke ${{matrix.test-type}}-tests --protocol=3 - fi - - - name: Run tests against hiredis < 3.0.0 - if: ${{ matrix.connection-type == 'hiredis' && matrix.python-version == '3.12'}} - run: | - pip uninstall hiredis -y - pip install -U setuptools wheel - pip install -r requirements.txt - pip install -r dev_requirements.txt - if [ "${{matrix.connection-type}}" == "hiredis" ]; then - pip install "hiredis<3.0.0" - fi - invoke devenv - sleep 10 # time to settle - if [ "${{matrix.event-loop}}" == "uvloop" ]; then - invoke ${{matrix.test-type}}-tests --uvloop --protocol=3 - else - invoke ${{matrix.test-type}}-tests --protocol=3 - fi - - - name: Upload test results and profiling data - uses: actions/upload-artifact@v4 + uses: ./.github/actions/run-tests with: - name: pytest-results-${{matrix.test-type}}-${{matrix.connection-type}}-${{matrix.python-version}}-${{matrix.event-loop}}-resp3 - path: | - ${{matrix.test-type}}*-results.xml - prof/** - profile_output* - if-no-files-found: error - retention-days: 10 + python-version: ${{ matrix.python-version }} + test-type: ${{ matrix.test-type }} + parser-backend: ${{ matrix.parser-backend }} + resp-protocol: ${{ matrix.resp-protocol }} + redis-version: ${{ matrix.redis-version }} - - name: Upload codecov coverage - uses: codecov/codecov-action@v4 + - name: Run tests against hiredis < 3.0.0 + uses: ./.github/actions/run-tests + if: ${{ matrix.parser-backend == 'hiredis' && matrix.python-version == '3.12'}} with: - fail_ci_if_error: false + python-version: ${{ matrix.python-version }} + test-type: ${{ matrix.test-type }} + parser-backend: ${{ matrix.parser-backend }} + resp-protocol: ${{ matrix.resp-protocol }} + redis-version: ${{ matrix.redis-version }} + hiredis-version: '<3.0.0' build-and-test-package: name: Validate building and installing the package runs-on: ubuntu-latest - needs: [resp2-tests, resp3-tests] + needs: [tests] strategy: fail-fast: false matrix: From 1b7bca61626272c003120a6a719485c00f79744d Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 16 Oct 2024 12:14:19 +0200 Subject: [PATCH 04/37] Reorg test matrix --- .github/actions/run-tests/action.yml | 28 +++++++----- .github/workflows/integration.yaml | 65 ++++++++++++++++++++++------ 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 495485dbb0..9425a74ff0 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -10,9 +10,6 @@ inputs: parser-backend: description: 'Parser backend to use: plain or hiredis' required: true - resp-protocol: - description: 'RESP protocol version to use: 2 or 3' - required: true redis-version: description: 'Redis version to test against' required: true @@ -34,7 +31,7 @@ runs: python-version: ${{ inputs.python-version }} cache: 'pip' - - name: Run tests + - name: Setup Test environment run: | pip install -U setuptools wheel pip install -r requirements.txt @@ -43,21 +40,32 @@ runs: pip install hiredis${{inputs.hiredis-version}} fi invoke devenv - sleep 10 # time to settle - + sleep 10 # time to settle + shell: bash + + - name: Run RESP2 tests + run: | if [ "${{inputs.event-loop}}" == "uvloop" ]; then - invoke ${{inputs.test-type}}-tests --uvloop --protocol=${{inputs.resp-protocol}} + invoke ${{inputs.test-type}}-tests --uvloop --protocol=2 else - invoke ${{inputs.test-type}}-tests --protocol=${{inputs.resp-protocol}} + invoke ${{inputs.test-type}}-tests --protocol=2 fi + shell: bash - ls -1 + - name: Run RESP3 tests + if: ${{ inputs.test-type != 'cluster' && inputs.parser-backend != 'hiredis' }} + run: | + if [ "${{inputs.event-loop}}" == "uvloop" ]; then + invoke ${{inputs.test-type}}-tests --uvloop --protocol=3 + else + invoke ${{inputs.test-type}}-tests --protocol=3 + fi shell: bash - name: Upload test results and profiling data uses: actions/upload-artifact@v4 with: - name: pytest-results-${{inputs.test-type}}-${{inputs.parser-backend}}-${{inputs.python-version}}-${{inputs.event-loop}}-${{inputs.resp-protocol}} + name: pytest-results-${{inputs.redis-version}}-${{inputs.test-type}}-${{inputs.parser-backend}}-${{inputs.python-version}}-${{inputs.event-loop}} path: | ${{inputs.test-type}}*-results.xml prof/** diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 5a11a424d7..5222090036 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -64,21 +64,10 @@ jobs: fail-fast: false matrix: redis-version: ['8.0.0-M01'] - resp-protocol: ['2', '3'] test-type: ['standalone', 'cluster'] python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10'] - parser-backend: ['hiredis', 'plain'] + parser-backend: ['plain'] event-loop: ['asyncio'] - include: - # Run uvloop tests only for resp3 - - resp-protocol: '3' - parser-backend: 'plain' - redis-version: '8.0.0-M01' - event-loop: 'uvloop' - exclude: - - test-type: 'cluster' - parser-backend: 'hiredis' - resp-protocol: '3' env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true name: Redis ${{ matrix.redis-version }} - RESP${{ matrix.resp-protocol }} ${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.parser-backend}}-${{matrix.event-loop}} @@ -90,9 +79,33 @@ jobs: python-version: ${{ matrix.python-version }} test-type: ${{ matrix.test-type }} parser-backend: ${{ matrix.parser-backend }} - resp-protocol: ${{ matrix.resp-protocol }} redis-version: ${{ matrix.redis-version }} + hiredis-tests: + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + max-parallel: 15 + fail-fast: false + matrix: + redis-version: [ '8.0.0-M01' ] + test-type: [ 'standalone', 'cluster' ] + python-version: [ '3.8', '3.12'] + parser-backend: [ 'hiredis' ] + event-loop: [ 'asyncio' ] + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + name: Redis ${{ matrix.redis-version }} - RESP${{ matrix.resp-protocol }} ${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.parser-backend}}-${{matrix.event-loop}} + steps: + - uses: actions/checkout@v4 + - name: Run tests + uses: ./.github/actions/run-tests + with: + python-version: ${{ matrix.python-version }} + test-type: ${{ matrix.test-type }} + parser-backend: ${{ matrix.parser-backend }} + redis-version: ${{ matrix.redis-version }} + - name: Run tests against hiredis < 3.0.0 uses: ./.github/actions/run-tests if: ${{ matrix.parser-backend == 'hiredis' && matrix.python-version == '3.12'}} @@ -100,10 +113,34 @@ jobs: python-version: ${{ matrix.python-version }} test-type: ${{ matrix.test-type }} parser-backend: ${{ matrix.parser-backend }} - resp-protocol: ${{ matrix.resp-protocol }} redis-version: ${{ matrix.redis-version }} hiredis-version: '<3.0.0' + uvloop-tests: + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + max-parallel: 15 + fail-fast: false + matrix: + redis-version: [ '8.0.0-M01' ] + test-type: [ 'standalone', 'cluster' ] + python-version: [ '3.8', '3.12' ] + parser-backend: [ 'plain' ] + event-loop: [ 'uvloop' ] + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + name: Redis ${{ matrix.redis-version }} - RESP${{ matrix.resp-protocol }} ${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.parser-backend}}-${{matrix.event-loop}} + steps: + - uses: actions/checkout@v4 + - name: Run tests + uses: ./.github/actions/run-tests + with: + python-version: ${{ matrix.python-version }} + test-type: ${{ matrix.test-type }} + parser-backend: ${{ matrix.parser-backend }} + redis-version: ${{ matrix.redis-version }} + build-and-test-package: name: Validate building and installing the package runs-on: ubuntu-latest From 639b167f72443de6330f7e023405e425c8892aec Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 16 Oct 2024 12:18:32 +0200 Subject: [PATCH 05/37] Fix jobs names and execution order --- .github/workflows/integration.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 5222090036..0c270bd757 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -70,7 +70,7 @@ jobs: event-loop: ['asyncio'] env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true - name: Redis ${{ matrix.redis-version }} - RESP${{ matrix.resp-protocol }} ${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.parser-backend}}-${{matrix.event-loop}} + name: Redis ${{ matrix.redis-version }} (${{matrix.test-type}}); Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}}; EL:${{matrix.event-loop}} steps: - uses: actions/checkout@v4 - name: Run tests @@ -83,6 +83,7 @@ jobs: hiredis-tests: runs-on: ubuntu-latest + needs: [tests] timeout-minutes: 60 strategy: max-parallel: 15 @@ -95,7 +96,7 @@ jobs: event-loop: [ 'asyncio' ] env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true - name: Redis ${{ matrix.redis-version }} - RESP${{ matrix.resp-protocol }} ${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.parser-backend}}-${{matrix.event-loop}} + name: Redis ${{ matrix.redis-version }} (${{matrix.test-type}}); Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}}; EL:${{matrix.event-loop}} steps: - uses: actions/checkout@v4 - name: Run tests @@ -118,6 +119,7 @@ jobs: uvloop-tests: runs-on: ubuntu-latest + needs: [tests] timeout-minutes: 60 strategy: max-parallel: 15 @@ -130,7 +132,7 @@ jobs: event-loop: [ 'uvloop' ] env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true - name: Redis ${{ matrix.redis-version }} - RESP${{ matrix.resp-protocol }} ${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.parser-backend}}-${{matrix.event-loop}} + name: Redis ${{ matrix.redis-version }} (${{matrix.test-type}}); Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}}; EL:${{matrix.event-loop}} steps: - uses: actions/checkout@v4 - name: Run tests From 34cd6791eea63710d123d3b6fb4c207ce421fd09 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 16 Oct 2024 12:58:59 +0200 Subject: [PATCH 06/37] Execute standalone and cluster test simultaneously --- .github/actions/run-tests/action.yml | 28 ++++++++++++++++++---------- .github/workflows/integration.yaml | 11 +++-------- tasks.py | 12 ++++++------ 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 9425a74ff0..ffad0bf483 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -4,9 +4,6 @@ inputs: python-version: description: 'Python version to use for running tests' default: '3.12' - test-type: - description: 'Type of tests to run: standalone or cluster' - required: true parser-backend: description: 'Parser backend to use: plain or hiredis' required: true @@ -46,28 +43,39 @@ runs: - name: Run RESP2 tests run: | if [ "${{inputs.event-loop}}" == "uvloop" ]; then - invoke ${{inputs.test-type}}-tests --uvloop --protocol=2 + invoke standalone-tests --uvloop --protocol=2 + invoke cluster-tests --uvloop --protocol=2 else - invoke ${{inputs.test-type}}-tests --protocol=2 + invoke standalone-tests --protocol=2 + invoke cluster-tests --protocol=2 fi shell: bash - name: Run RESP3 tests - if: ${{ inputs.test-type != 'cluster' && inputs.parser-backend != 'hiredis' }} run: | if [ "${{inputs.event-loop}}" == "uvloop" ]; then - invoke ${{inputs.test-type}}-tests --uvloop --protocol=3 + invoke standalone-tests --uvloop --protocol=3 + + # hiredis does not support RESP3 on cluster + if [ "${{inputs.parser-backend}}" != 'hiredis' ]; then + invoke cluster-tests --uvloop --protocol=3 + fi else - invoke ${{inputs.test-type}}-tests --protocol=3 + invoke standalone-tests --protocol=3 + + # hiredis does not support RESP3 on cluster + if [ "${{inputs.parser-backend}}" != 'hiredis' ]; then + invoke cluster-tests --uvloop --protocol=3 + fi fi shell: bash - name: Upload test results and profiling data uses: actions/upload-artifact@v4 with: - name: pytest-results-${{inputs.redis-version}}-${{inputs.test-type}}-${{inputs.parser-backend}}-${{inputs.python-version}}-${{inputs.event-loop}} + name: pytest-results-${{inputs.redis-version}}-${{inputs.parser-backend}}-${{inputs.python-version}}-${{inputs.event-loop}} path: | - ${{inputs.test-type}}*-results.xml + *-results.xml prof/** profile_output* if-no-files-found: error diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 0c270bd757..82353b9dfe 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -64,13 +64,12 @@ jobs: fail-fast: false matrix: redis-version: ['8.0.0-M01'] - test-type: ['standalone', 'cluster'] python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10'] parser-backend: ['plain'] event-loop: ['asyncio'] env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true - name: Redis ${{ matrix.redis-version }} (${{matrix.test-type}}); Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}}; EL:${{matrix.event-loop}} + name: Redis ${{ matrix.redis-version }}; Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}}; EL:${{matrix.event-loop}} steps: - uses: actions/checkout@v4 - name: Run tests @@ -90,20 +89,18 @@ jobs: fail-fast: false matrix: redis-version: [ '8.0.0-M01' ] - test-type: [ 'standalone', 'cluster' ] python-version: [ '3.8', '3.12'] parser-backend: [ 'hiredis' ] event-loop: [ 'asyncio' ] env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true - name: Redis ${{ matrix.redis-version }} (${{matrix.test-type}}); Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}}; EL:${{matrix.event-loop}} + name: Redis ${{ matrix.redis-version }}; Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}}; EL:${{matrix.event-loop}} steps: - uses: actions/checkout@v4 - name: Run tests uses: ./.github/actions/run-tests with: python-version: ${{ matrix.python-version }} - test-type: ${{ matrix.test-type }} parser-backend: ${{ matrix.parser-backend }} redis-version: ${{ matrix.redis-version }} @@ -112,7 +109,6 @@ jobs: if: ${{ matrix.parser-backend == 'hiredis' && matrix.python-version == '3.12'}} with: python-version: ${{ matrix.python-version }} - test-type: ${{ matrix.test-type }} parser-backend: ${{ matrix.parser-backend }} redis-version: ${{ matrix.redis-version }} hiredis-version: '<3.0.0' @@ -132,14 +128,13 @@ jobs: event-loop: [ 'uvloop' ] env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true - name: Redis ${{ matrix.redis-version }} (${{matrix.test-type}}); Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}}; EL:${{matrix.event-loop}} + name: Redis ${{ matrix.redis-version }}; Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}}; EL:${{matrix.event-loop}} steps: - uses: actions/checkout@v4 - name: Run tests uses: ./.github/actions/run-tests with: python-version: ${{ matrix.python-version }} - test-type: ${{ matrix.test-type }} parser-backend: ${{ matrix.parser-backend }} redis-version: ${{ matrix.redis-version }} diff --git a/tasks.py b/tasks.py index 0b5ce55f78..0c9b766144 100644 --- a/tasks.py +++ b/tasks.py @@ -10,10 +10,10 @@ @task -def devenv(c): +def devenv(c, endpoints="all"): """Brings up the test environment, by wrapping docker compose.""" clean(c) - cmd = "docker compose --profile all up -d --build" + cmd = f"docker compose --profile {endpoints} up -d --build" run(cmd) @@ -55,11 +55,11 @@ def standalone_tests(c, uvloop=False, protocol=2, profile=False): profile_arg = "--profile" if profile else "" if uvloop: run( - f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_redis.xml -m 'not onlycluster' --uvloop --junit-xml=standalone-uvloop-results.xml" + f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_resp{protocol}_uvloop.xml -m 'not onlycluster' --uvloop --junit-xml=standalone-resp{protocol}-uvloop-results.xml" ) else: run( - f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_redis.xml -m 'not onlycluster' --junit-xml=standalone-results.xml" + f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_resp{protocol}.xml -m 'not onlycluster' --junit-xml=standalone-resp{protocol}-results.xml" ) @@ -71,11 +71,11 @@ def cluster_tests(c, uvloop=False, protocol=2, profile=False): cluster_tls_url = "rediss://localhost:17379/0" if uvloop: run( - f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_cluster.xml -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --redis-ssl-url={cluster_tls_url} --junit-xml=cluster-uvloop-results.xml --uvloop" + f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_cluster_resp{protocol}_uvloop.xml -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --redis-ssl-url={cluster_tls_url} --junit-xml=cluster-resp{protocol}-uvloop-results.xml --uvloop" ) else: run( - f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_clusteclient.xml -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --redis-ssl-url={cluster_tls_url} --junit-xml=cluster-results.xml" + f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_cluster_resp{protocol}.xml -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --redis-ssl-url={cluster_tls_url} --junit-xml=cluster-resp{protocol}-results.xml" ) From 46b83ede1884752697f27510d9c9e22a7789e489 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 16 Oct 2024 18:41:48 +0200 Subject: [PATCH 07/37] Streamline test execution - Automatically map Redis version to Redis Stack version and use it for testing module commands - Remove Graph commands from execution by default - Include more Redis versions to the test matrix --- .github/actions/run-tests/action.yml | 41 ++++++++++++++++++++++--- .github/workflows/integration.yaml | 4 +-- docker-compose.yml | 15 ++++++--- pytest.ini | 1 + tasks.py | 12 +++++--- tests/conftest.py | 18 ++++++----- tests/test_asyncio/test_graph.py | 40 ++++++++++++------------ tests/test_graph.py | 46 ++++++++++++++-------------- tests/test_graph_utils/test_edge.py | 8 ++--- tests/test_graph_utils/test_node.py | 6 ++-- tests/test_graph_utils/test_path.py | 10 +++--- 11 files changed, 120 insertions(+), 81 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index ffad0bf483..02c1fe695b 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -29,24 +29,55 @@ runs: cache: 'pip' - name: Setup Test environment + env: + REDIS_VERSION: ${{ inputs.redis-version }} + REDIS_IMAGE: "redis:${{ inputs.redis-version }}" + CLIENT_LIBS_TEST_IMAGE: "redislabs/client-libs-test:${{ inputs.redis-version }}" run: | + set -e pip install -U setuptools wheel pip install -r requirements.txt pip install -r dev_requirements.txt if [ "${{inputs.parser-backend}}" == "hiredis" ]; then pip install hiredis${{inputs.hiredis-version}} fi - invoke devenv + + redis_major_version=$(echo "$REDIS_VERSION" | grep -oP '^\d+') + + if (( redis_major_version < 8 )); then + echo "Using redis-stack for module tests" + + # Mapping of redis version to stack version + declare -A redis_stack_version_mapping=( + ["7.4.1"]="7.4.0-v1" + ["7.2.6"]="7.2.0-v13" + ["6.2.16"]="6.2.6-v17" + ) + + if [[ -v version_mapping[$REDIS_VERSION] ]]; then + export REDIS_STACK_IMAGE="redis/redis-stack-server:${version_mapping[$version]}" + echo "REDIS_MOD_URL=redis://127.0.0.1:6479/0" >> $GITHUB_ENV + else + echo "Version not found in the mapping." + exit 1 + fi + invoke devenv --endpoints=all-stack + else + echo "Using redis CE for module tests" + echo "REDIS_MOD_URL=redis://127.0.0.1:6379" >> $GITHUB_ENV + invoke devenv --endpoints all + fi + sleep 10 # time to settle shell: bash - name: Run RESP2 tests run: | if [ "${{inputs.event-loop}}" == "uvloop" ]; then - invoke standalone-tests --uvloop --protocol=2 + invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} --uvloop --protocol=2 invoke cluster-tests --uvloop --protocol=2 else - invoke standalone-tests --protocol=2 + invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} --protocol=2 invoke cluster-tests --protocol=2 fi shell: bash @@ -54,14 +85,14 @@ runs: - name: Run RESP3 tests run: | if [ "${{inputs.event-loop}}" == "uvloop" ]; then - invoke standalone-tests --uvloop --protocol=3 + invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} --uvloop --protocol=3 # hiredis does not support RESP3 on cluster if [ "${{inputs.parser-backend}}" != 'hiredis' ]; then invoke cluster-tests --uvloop --protocol=3 fi else - invoke standalone-tests --protocol=3 + invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} --protocol=3 # hiredis does not support RESP3 on cluster if [ "${{inputs.parser-backend}}" != 'hiredis' ]; then diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 82353b9dfe..7878fe459f 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -63,7 +63,7 @@ jobs: max-parallel: 15 fail-fast: false matrix: - redis-version: ['8.0.0-M01'] + redis-version: ['8.0.0-M01', '7.4.1', '7.2.6', '6.2.16'] python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10'] parser-backend: ['plain'] event-loop: ['asyncio'] @@ -76,7 +76,6 @@ jobs: uses: ./.github/actions/run-tests with: python-version: ${{ matrix.python-version }} - test-type: ${{ matrix.test-type }} parser-backend: ${{ matrix.parser-backend }} redis-version: ${{ matrix.redis-version }} @@ -122,7 +121,6 @@ jobs: fail-fast: false matrix: redis-version: [ '8.0.0-M01' ] - test-type: [ 'standalone', 'cluster' ] python-version: [ '3.8', '3.12' ] parser-backend: [ 'plain' ] event-loop: [ 'uvloop' ] diff --git a/docker-compose.yml b/docker-compose.yml index a9dcf6b5c5..73b7f32aad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: redis: - image: redislabs/client-libs-test:8.0-M01 + image: ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.0-M01} container_name: redis-standalone environment: - TLS_ENABLED=yes @@ -20,6 +20,7 @@ services: - standalone - sentinel - replica + - all-stack - all replica: @@ -32,10 +33,11 @@ services: - 6380:6379 profiles: - replica + - all-stack - all cluster: - image: redislabs/client-libs-test:8.0-M01 + image: ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.0-M01} container_name: redis-cluster environment: - REDIS_CLUSTER=yes @@ -51,6 +53,7 @@ services: - "./dockers/cluster:/redis/work" profiles: - cluster + - all-stack - all sentinel: @@ -65,6 +68,7 @@ services: - "./dockers/sentinel.conf:/redis.conf" profiles: - sentinel + - all-stack - all sentinel2: @@ -79,6 +83,7 @@ services: - "./dockers/sentinel.conf:/redis.conf" profiles: - sentinel + - all-stack - all sentinel3: @@ -93,6 +98,7 @@ services: - "./dockers/sentinel.conf:/redis.conf" profiles: - sentinel + - all-stack - all redis-stack: @@ -104,7 +110,7 @@ services: - "REDIS_ARGS=--enable-debug-command yes --enable-module-command yes" profiles: - standalone - - all + - all-stack redis-stack-graph: image: redis/redis-stack-server:6.2.6-v15 @@ -112,5 +118,4 @@ services: ports: - 6480:6379 profiles: - - standalone - - all + - graph diff --git a/pytest.ini b/pytest.ini index 990968d6f9..bbb8d420c4 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,6 +2,7 @@ addopts = -s markers = redismod: run only the redis module tests + graph: run only the redisgraph tests pipeline: pipeline tests onlycluster: marks tests to be run only with cluster mode redis onlynoncluster: marks tests to be run only with standalone redis diff --git a/tasks.py b/tasks.py index 0c9b766144..d488793e65 100644 --- a/tasks.py +++ b/tasks.py @@ -50,16 +50,18 @@ def tests(c, uvloop=False, protocol=2, profile=False): @task -def standalone_tests(c, uvloop=False, protocol=2, profile=False): +def standalone_tests(c, uvloop=False, protocol=2, profile=False, redis_mod_url=None): """Run tests against a standalone redis instance""" profile_arg = "--profile" if profile else "" + redis_mod_url = f"--redis-mod-url={redis_mod_url}" if redis_mod_url else "" + if uvloop: run( - f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_resp{protocol}_uvloop.xml -m 'not onlycluster' --uvloop --junit-xml=standalone-resp{protocol}-uvloop-results.xml" + f"pytest {profile_arg} --protocol={protocol} {redis_mod_url} --cov=./ --cov-report=xml:coverage_resp{protocol}_uvloop.xml -m 'not onlycluster and not graph' --uvloop --junit-xml=standalone-resp{protocol}-uvloop-results.xml" ) else: run( - f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_resp{protocol}.xml -m 'not onlycluster' --junit-xml=standalone-resp{protocol}-results.xml" + f"pytest {profile_arg} --protocol={protocol} {redis_mod_url} --cov=./ --cov-report=xml:coverage_resp{protocol}.xml -m 'not onlycluster and not graph' --junit-xml=standalone-resp{protocol}-results.xml" ) @@ -71,11 +73,11 @@ def cluster_tests(c, uvloop=False, protocol=2, profile=False): cluster_tls_url = "rediss://localhost:17379/0" if uvloop: run( - f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_cluster_resp{protocol}_uvloop.xml -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --redis-ssl-url={cluster_tls_url} --junit-xml=cluster-resp{protocol}-uvloop-results.xml --uvloop" + f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_cluster_resp{protocol}_uvloop.xml -m 'not onlynoncluster and not redismod and not graph' --redis-url={cluster_url} --redis-ssl-url={cluster_tls_url} --junit-xml=cluster-resp{protocol}-uvloop-results.xml --uvloop" ) else: run( - f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_cluster_resp{protocol}.xml -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --redis-ssl-url={cluster_tls_url} --junit-xml=cluster-resp{protocol}-results.xml" + f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_cluster_resp{protocol}.xml -m 'not onlynoncluster and not redismod and not graph' --redis-url={cluster_url} --redis-ssl-url={cluster_tls_url} --junit-xml=cluster-resp{protocol}-results.xml" ) diff --git a/tests/conftest.py b/tests/conftest.py index 44935dd827..ff0cf5f6a3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -103,6 +103,13 @@ def pytest_addoption(parser): help="Redis connection string, defaults to `%(default)s`", ) + parser.addoption( + "--redis-mod-url", + default=default_redismod_url, + action="store", + help="Redis with modules connection string, defaults to `%(default)s`", + ) + parser.addoption( "--protocol", default=default_protocol, @@ -182,9 +189,8 @@ def pytest_sessionstart(session): session.config.REDIS_INFO = REDIS_INFO # module info - stack_url = redis_url - if stack_url == default_redis_url: - stack_url = default_redismod_url + stack_url = session.config.getoption("--redis-mod-url") + try: stack_info = _get_info(stack_url) REDIS_INFO["modules"] = stack_info["modules"] @@ -392,11 +398,7 @@ def r(request): @pytest.fixture() def stack_url(request): - stack_url = request.config.getoption("--redis-url", default=default_redismod_url) - if stack_url == default_redis_url: - return default_redismod_url - else: - return stack_url + return request.config.getoption("--redis-mod-url", default=default_redismod_url) @pytest.fixture() diff --git a/tests/test_asyncio/test_graph.py b/tests/test_asyncio/test_graph.py index 2014ea38b6..2a506d5e22 100644 --- a/tests/test_asyncio/test_graph.py +++ b/tests/test_asyncio/test_graph.py @@ -12,7 +12,7 @@ async def decoded_r(create_redis, stack_url): return await create_redis(decode_responses=True, url="redis://localhost:6480") -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_bulk(decoded_r): with pytest.raises(NotImplementedError): @@ -20,7 +20,7 @@ async def test_bulk(decoded_r): await decoded_r.graph().bulk(foo="bar!") -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_graph_creation(decoded_r: redis.Redis): graph = decoded_r.graph() @@ -66,7 +66,7 @@ async def test_graph_creation(decoded_r: redis.Redis): await graph.delete() -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_array_functions(decoded_r: redis.Redis): graph = decoded_r.graph() @@ -90,7 +90,7 @@ async def test_array_functions(decoded_r: redis.Redis): assert [a] == result.result_set[0][0] -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_path(decoded_r: redis.Redis): node0 = Node(node_id=0, label="L1") @@ -111,7 +111,7 @@ async def test_path(decoded_r: redis.Redis): assert expected_results == result.result_set -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_param(decoded_r: redis.Redis): params = [1, 2.3, "str", True, False, None, [0, 1, 2]] @@ -122,7 +122,7 @@ async def test_param(decoded_r: redis.Redis): assert expected_results == result.result_set -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_map(decoded_r: redis.Redis): query = "RETURN {a:1, b:'str', c:NULL, d:[1,2,3], e:True, f:{x:1, y:2}}" @@ -140,7 +140,7 @@ async def test_map(decoded_r: redis.Redis): assert actual == expected -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_point(decoded_r: redis.Redis): query = "RETURN point({latitude: 32.070794860, longitude: 34.820751118})" @@ -158,7 +158,7 @@ async def test_point(decoded_r: redis.Redis): assert abs(actual["longitude"] - expected_lon) < 0.001 -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_index_response(decoded_r: redis.Redis): result_set = await decoded_r.graph().query("CREATE INDEX ON :person(age)") @@ -174,7 +174,7 @@ async def test_index_response(decoded_r: redis.Redis): await decoded_r.graph().query("DROP INDEX ON :person(age)") -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_stringify_query_result(decoded_r: redis.Redis): graph = decoded_r.graph() @@ -229,7 +229,7 @@ async def test_stringify_query_result(decoded_r: redis.Redis): await graph.delete() -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_optional_match(decoded_r: redis.Redis): # Build a graph of form (a)-[R]->(b) @@ -255,7 +255,7 @@ async def test_optional_match(decoded_r: redis.Redis): await graph.delete() -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_cached_execution(decoded_r: redis.Redis): await decoded_r.graph().query("CREATE ()") @@ -276,7 +276,7 @@ async def test_cached_execution(decoded_r: redis.Redis): assert cached_result.cached_execution -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_slowlog(decoded_r: redis.Redis): create_query = """CREATE @@ -291,7 +291,7 @@ async def test_slowlog(decoded_r: redis.Redis): @pytest.mark.xfail(strict=False) -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_query_timeout(decoded_r: redis.Redis): # Build a sample graph with 1000 nodes. @@ -306,7 +306,7 @@ async def test_query_timeout(decoded_r: redis.Redis): assert False is False -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_read_only_query(decoded_r: redis.Redis): with pytest.raises(Exception): @@ -316,7 +316,7 @@ async def test_read_only_query(decoded_r: redis.Redis): assert False is False -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_profile(decoded_r: redis.Redis): q = """UNWIND range(1, 3) AS x CREATE (p:Person {v:x})""" @@ -333,7 +333,7 @@ async def test_profile(decoded_r: redis.Redis): @skip_if_redis_enterprise() -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_config(decoded_r: redis.Redis): config_name = "RESULTSET_SIZE" @@ -366,7 +366,7 @@ async def test_config(decoded_r: redis.Redis): @pytest.mark.onlynoncluster -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_list_keys(decoded_r: redis.Redis): result = await decoded_r.graph().list_keys() @@ -390,7 +390,7 @@ async def test_list_keys(decoded_r: redis.Redis): assert result == [] -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_multi_label(decoded_r: redis.Redis): redis_graph = decoded_r.graph("g") @@ -417,7 +417,7 @@ async def test_multi_label(decoded_r: redis.Redis): assert True -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_execution_plan(decoded_r: redis.Redis): redis_graph = decoded_r.graph("execution_plan") @@ -437,7 +437,7 @@ async def test_execution_plan(decoded_r: redis.Redis): await redis_graph.delete() -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) async def test_explain(decoded_r: redis.Redis): redis_graph = decoded_r.graph("execution_plan") diff --git a/tests/test_graph.py b/tests/test_graph.py index c82c5030c8..efb10dada7 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -32,7 +32,7 @@ def client(request, stack_url): return r -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_bulk(client): with pytest.raises(NotImplementedError): @@ -40,7 +40,7 @@ def test_bulk(client): client.graph().bulk(foo="bar!") -@pytest.mark.redismod +@pytest.mark.graph def test_graph_creation_throws_deprecation_warning(client): """Verify that a DeprecationWarning is raised when creating a Graph instance.""" @@ -49,7 +49,7 @@ def test_graph_creation_throws_deprecation_warning(client): client.graph() -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_graph_creation(client): graph = client.graph() @@ -95,7 +95,7 @@ def test_graph_creation(client): graph.delete() -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_array_functions(client): query = """CREATE (p:person{name:'a',age:32, array:[0,1,2]})""" @@ -117,7 +117,7 @@ def test_array_functions(client): assert [a] == result.result_set[0][0] -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_path(client): node0 = Node(node_id=0, label="L1") @@ -138,7 +138,7 @@ def test_path(client): assert expected_results == result.result_set -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_param(client): params = [1, 2.3, "str", True, False, None, [0, 1, 2], r"\" RETURN 1337 //"] @@ -149,7 +149,7 @@ def test_param(client): assert expected_results == result.result_set -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_map(client): query = "RETURN {a:1, b:'str', c:NULL, d:[1,2,3], e:True, f:{x:1, y:2}}" @@ -167,7 +167,7 @@ def test_map(client): assert actual == expected -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_point(client): query = "RETURN point({latitude: 32.070794860, longitude: 34.820751118})" @@ -185,7 +185,7 @@ def test_point(client): assert abs(actual["longitude"] - expected_lon) < 0.001 -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_index_response(client): result_set = client.graph().query("CREATE INDEX ON :person(age)") @@ -201,7 +201,7 @@ def test_index_response(client): client.graph().query("DROP INDEX ON :person(age)") -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_stringify_query_result(client): graph = client.graph() @@ -256,7 +256,7 @@ def test_stringify_query_result(client): graph.delete() -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_optional_match(client): # Build a graph of form (a)-[R]->(b) @@ -282,7 +282,7 @@ def test_optional_match(client): graph.delete() -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_cached_execution(client): client.graph().query("CREATE ()") @@ -301,7 +301,7 @@ def test_cached_execution(client): assert cached_result.cached_execution -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_slowlog(client): create_query = """CREATE (:Rider @@ -315,7 +315,7 @@ def test_slowlog(client): assert results[0][2] == create_query -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) @pytest.mark.xfail(strict=False) def test_query_timeout(client): @@ -331,7 +331,7 @@ def test_query_timeout(client): assert False is False -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_read_only_query(client): with pytest.raises(Exception): @@ -341,7 +341,7 @@ def test_read_only_query(client): assert False is False -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_profile(client): q = """UNWIND range(1, 3) AS x CREATE (p:Person {v:x})""" @@ -357,7 +357,7 @@ def test_profile(client): assert "Node By Label Scan | (p:Person) | Records produced: 3" in profile -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) @skip_if_redis_enterprise() def test_config(client): @@ -391,7 +391,7 @@ def test_config(client): @pytest.mark.onlynoncluster -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_list_keys(client): result = client.graph().list_keys() @@ -415,7 +415,7 @@ def test_list_keys(client): assert result == [] -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_multi_label(client): redis_graph = client.graph("g") @@ -442,7 +442,7 @@ def test_multi_label(client): assert True -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_cache_sync(client): pass @@ -516,7 +516,7 @@ def test_cache_sync(client): assert A._relationship_types[1] == "R" -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_execution_plan(client): redis_graph = client.graph("execution_plan") @@ -536,7 +536,7 @@ def test_execution_plan(client): redis_graph.delete() -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_explain(client): redis_graph = client.graph("execution_plan") @@ -626,7 +626,7 @@ def test_explain(client): redis_graph.delete() -@pytest.mark.redismod +@pytest.mark.graph @skip_if_resp_version(3) def test_resultset_statistics(client): with patch.object(target=QueryResult, attribute="_get_stat") as mock_get_stats: diff --git a/tests/test_graph_utils/test_edge.py b/tests/test_graph_utils/test_edge.py index 1918a6ff44..09e6fa08ed 100644 --- a/tests/test_graph_utils/test_edge.py +++ b/tests/test_graph_utils/test_edge.py @@ -2,7 +2,7 @@ from redis.commands.graph import edge, node -@pytest.mark.redismod +@pytest.mark.graph def test_init(): with pytest.raises(AssertionError): edge.Edge(None, None, None) @@ -14,7 +14,7 @@ def test_init(): ) -@pytest.mark.redismod +@pytest.mark.graph def test_to_string(): props_result = edge.Edge( node.Node(), None, node.Node(), properties={"a": "a", "b": 10} @@ -27,7 +27,7 @@ def test_to_string(): assert no_props_result == "" -@pytest.mark.redismod +@pytest.mark.graph def test_stringify(): john = node.Node( alias="a", @@ -60,7 +60,7 @@ def test_stringify(): ) -@pytest.mark.redismod +@pytest.mark.graph def test_comparison(): node1 = node.Node(node_id=1) node2 = node.Node(node_id=2) diff --git a/tests/test_graph_utils/test_node.py b/tests/test_graph_utils/test_node.py index 22e6d59414..e9b8a54f43 100644 --- a/tests/test_graph_utils/test_node.py +++ b/tests/test_graph_utils/test_node.py @@ -12,7 +12,7 @@ def fixture(): return no_args, no_props, props_only, no_label, multi_label -@pytest.mark.redismod +@pytest.mark.graph def test_to_string(fixture): no_args, no_props, props_only, no_label, multi_label = fixture assert no_args.to_string() == "" @@ -22,7 +22,7 @@ def test_to_string(fixture): assert multi_label.to_string() == "" -@pytest.mark.redismod +@pytest.mark.graph def test_stringify(fixture): no_args, no_props, props_only, no_label, multi_label = fixture assert str(no_args) == "()" @@ -32,7 +32,7 @@ def test_stringify(fixture): assert str(multi_label) == "(alias:l:ll)" -@pytest.mark.redismod +@pytest.mark.graph def test_comparison(fixture): no_args, no_props, props_only, no_label, multi_label = fixture diff --git a/tests/test_graph_utils/test_path.py b/tests/test_graph_utils/test_path.py index 1bd38efab4..33ca041cfa 100644 --- a/tests/test_graph_utils/test_path.py +++ b/tests/test_graph_utils/test_path.py @@ -2,7 +2,7 @@ from redis.commands.graph import edge, node, path -@pytest.mark.redismod +@pytest.mark.graph def test_init(): with pytest.raises(TypeError): path.Path(None, None) @@ -12,7 +12,7 @@ def test_init(): assert isinstance(path.Path([], []), path.Path) -@pytest.mark.redismod +@pytest.mark.graph def test_new_empty_path(): new_empty_path = path.Path.new_empty_path() assert isinstance(new_empty_path, path.Path) @@ -20,7 +20,7 @@ def test_new_empty_path(): assert new_empty_path._edges == [] -@pytest.mark.redismod +@pytest.mark.graph def test_wrong_flows(): node_1 = node.Node(node_id=1) node_2 = node.Node(node_id=2) @@ -42,7 +42,7 @@ def test_wrong_flows(): p.add_edge(edge_2) -@pytest.mark.redismod +@pytest.mark.graph def test_nodes_and_edges(): node_1 = node.Node(node_id=1) node_2 = node.Node(node_id=2) @@ -69,7 +69,7 @@ def test_nodes_and_edges(): assert 2 == p.nodes_count() -@pytest.mark.redismod +@pytest.mark.graph def test_compare(): node_1 = node.Node(node_id=1) node_2 = node.Node(node_id=2) From 3379ab376aaf25a33f588b034f1afb74afbacf4e Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 16 Oct 2024 19:06:37 +0200 Subject: [PATCH 08/37] More fixes to integration job --- .github/actions/run-tests/action.yml | 4 ++-- .github/workflows/integration.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 02c1fe695b..907bc32642 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -54,8 +54,8 @@ runs: ["6.2.16"]="6.2.6-v17" ) - if [[ -v version_mapping[$REDIS_VERSION] ]]; then - export REDIS_STACK_IMAGE="redis/redis-stack-server:${version_mapping[$version]}" + if [[ -v redis_stack_version_mapping[$REDIS_VERSION] ]]; then + export REDIS_STACK_IMAGE="redis/redis-stack-server:${redis_stack_version_mapping[$REDIS_VERSION]}" echo "REDIS_MOD_URL=redis://127.0.0.1:6479/0" >> $GITHUB_ENV else echo "Version not found in the mapping." diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 7878fe459f..c099d775a9 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -63,7 +63,7 @@ jobs: max-parallel: 15 fail-fast: false matrix: - redis-version: ['8.0.0-M01', '7.4.1', '7.2.6', '6.2.16'] + redis-version: ['8.0-M01', '7.4.1', '7.2.6', '6.2.16'] python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10'] parser-backend: ['plain'] event-loop: ['asyncio'] From 7dcb2f616ef34e4b65a452236393abeea386ad96 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 17 Oct 2024 09:42:10 +0200 Subject: [PATCH 09/37] Move python compatibility tests to a separate task --- .github/workflows/integration.yaml | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index c099d775a9..eb0451396f 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -64,7 +64,7 @@ jobs: fail-fast: false matrix: redis-version: ['8.0-M01', '7.4.1', '7.2.6', '6.2.16'] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10'] + python-version: ['3.8', '3.12'] parser-backend: ['plain'] event-loop: ['asyncio'] env: @@ -79,6 +79,30 @@ jobs: parser-backend: ${{ matrix.parser-backend }} redis-version: ${{ matrix.redis-version }} + python-compatibility-tests: + runs-on: ubuntu-latest + needs: [ tests ] + timeout-minutes: 60 + strategy: + max-parallel: 15 + fail-fast: false + matrix: + redis-version: [ '8.0-M01' ] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10'] + parser-backend: [ 'plain' ] + event-loop: [ 'asyncio' ] + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + name: Redis ${{ matrix.redis-version }}; Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}}; EL:${{matrix.event-loop}} + steps: + - uses: actions/checkout@v4 + - name: Run tests + uses: ./.github/actions/run-tests + with: + python-version: ${{ matrix.python-version }} + parser-backend: ${{ matrix.parser-backend }} + redis-version: ${{ matrix.redis-version }} + hiredis-tests: runs-on: ubuntu-latest needs: [tests] @@ -87,7 +111,7 @@ jobs: max-parallel: 15 fail-fast: false matrix: - redis-version: [ '8.0.0-M01' ] + redis-version: [ '8.0-M01' ] python-version: [ '3.8', '3.12'] parser-backend: [ 'hiredis' ] event-loop: [ 'asyncio' ] @@ -120,7 +144,7 @@ jobs: max-parallel: 15 fail-fast: false matrix: - redis-version: [ '8.0.0-M01' ] + redis-version: [ '8.0-M01' ] python-version: [ '3.8', '3.12' ] parser-backend: [ 'plain' ] event-loop: [ 'uvloop' ] From b98850c994faade55dcd4105bc149038c980a6f1 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 17 Oct 2024 10:21:22 +0200 Subject: [PATCH 10/37] Improve run-tests action --- .github/actions/run-tests/action.yml | 49 ++++++++++++++-------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 907bc32642..f9f026fe8e 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -35,13 +35,17 @@ runs: CLIENT_LIBS_TEST_IMAGE: "redislabs/client-libs-test:${{ inputs.redis-version }}" run: | set -e + + echo "::group::Installing dependencies" pip install -U setuptools wheel pip install -r requirements.txt pip install -r dev_requirements.txt if [ "${{inputs.parser-backend}}" == "hiredis" ]; then pip install hiredis${{inputs.hiredis-version}} fi + echo "::endgroup::" + echo "::group::Starting Redis servers" redis_major_version=$(echo "$REDIS_VERSION" | grep -oP '^\d+') if (( redis_major_version < 8 )); then @@ -69,36 +73,33 @@ runs: fi sleep 10 # time to settle + echo "::endgroup::" shell: bash - name: Run RESP2 tests - run: | - if [ "${{inputs.event-loop}}" == "uvloop" ]; then - invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} --uvloop --protocol=2 - invoke cluster-tests --uvloop --protocol=2 - else - invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} --protocol=2 - invoke cluster-tests --protocol=2 - fi - shell: bash - - - name: Run RESP3 tests run: | - if [ "${{inputs.event-loop}}" == "uvloop" ]; then - invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} --uvloop --protocol=3 - - # hiredis does not support RESP3 on cluster - if [ "${{inputs.parser-backend}}" != 'hiredis' ]; then - invoke cluster-tests --uvloop --protocol=3 + run_tests() { + local protocol=$1 + local eventloop="" + + if [ "${{inputs.event-loop}}" == "uvloop" ]; then + eventloop="--uvloop" fi - else - invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} --protocol=3 - - # hiredis does not support RESP3 on cluster - if [ "${{inputs.parser-backend}}" != 'hiredis' ]; then - invoke cluster-tests --uvloop --protocol=3 + + echo "::group::RESP${protocol} standalone tests" + echo "REDIS_MOD_URL=${REDIS_MOD_URL}" + invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} $eventloop --protocol=${protocol} + echo "::endgroup::" + + if [ "$protocol" == "2" ] || [ "${{inputs.parser-backend}}" != 'hiredis' ]; then + echo "::group::RESP${protocol} cluster tests" + invoke cluster-tests $eventloop --protocol=${protocol} + echo "::endgroup::" fi - fi + } + + run_tests 2 + run_tests 3 shell: bash - name: Upload test results and profiling data From 4881c1b8392efb2dc609315f3144c44f84a97cc1 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 17 Oct 2024 10:21:47 +0200 Subject: [PATCH 11/37] Add missing pytest marks for TS tests --- tests/test_asyncio/test_timeseries.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_asyncio/test_timeseries.py b/tests/test_asyncio/test_timeseries.py index 5a1597f2d0..b21f7d0ac8 100644 --- a/tests/test_asyncio/test_timeseries.py +++ b/tests/test_asyncio/test_timeseries.py @@ -765,6 +765,7 @@ async def test_uncompressed(decoded_r: redis.Redis): assert compressed_info["memoryUsage"] != uncompressed_info["memoryUsage"] +@pytest.mark.redismod @skip_ifmodversion_lt("1.12.0", "timeseries") async def test_create_with_insertion_filters(decoded_r: redis.Redis): await decoded_r.ts().create( @@ -788,6 +789,7 @@ async def test_create_with_insertion_filters(decoded_r: redis.Redis): ) +@pytest.mark.redismod @skip_ifmodversion_lt("1.12.0", "timeseries") async def test_alter_with_insertion_filters(decoded_r: redis.Redis): assert 1000 == await decoded_r.ts().add("time-series-1", 1000, 1.0) @@ -812,6 +814,7 @@ async def test_alter_with_insertion_filters(decoded_r: redis.Redis): ) +@pytest.mark.redismod @skip_ifmodversion_lt("1.12.0", "timeseries") async def test_add_with_insertion_filters(decoded_r: redis.Redis): assert 1000 == await decoded_r.ts().add( @@ -829,6 +832,7 @@ async def test_add_with_insertion_filters(decoded_r: redis.Redis): assert_resp_response(decoded_r, data_points, [(1000, 1.0)], [[1000, 1.0]]) +@pytest.mark.redismod @skip_ifmodversion_lt("1.12.0", "timeseries") async def test_incrby_with_insertion_filters(decoded_r: redis.Redis): assert 1000 == await decoded_r.ts().incrby( @@ -851,6 +855,7 @@ async def test_incrby_with_insertion_filters(decoded_r: redis.Redis): assert_resp_response(decoded_r, data_points, [(1000, 11.1)], [[1000, 11.1]]) +@pytest.mark.redismod @skip_ifmodversion_lt("1.12.0", "timeseries") async def test_decrby_with_insertion_filters(decoded_r: redis.Redis): assert 1000 == await decoded_r.ts().decrby( From 8ed1c16e8a0f77941fe475f0ee9106cc45c72b02 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 17 Oct 2024 11:45:33 +0200 Subject: [PATCH 12/37] Fix cluster configuration --- docker-compose.yml | 7 ++++--- tests/test_asyncio/test_cluster.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 73b7f32aad..3b3307ad07 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,13 +42,14 @@ services: environment: - REDIS_CLUSTER=yes - NODES=6 + - REPLICAS=1 - TLS_ENABLED=yes - PORT=16379 - - TLS_PORT=17379 - command: --tls-auth-clients optional + - TLS_PORT=27379 + command: --enable-debug-command yes --tls-auth-clients optional ports: - "16379-16384:16379-16384" - - "17379-17384:17379-17384" + - "27379-27384:27379-27384" volumes: - "./dockers/cluster:/redis/work" profiles: diff --git a/tests/test_asyncio/test_cluster.py b/tests/test_asyncio/test_cluster.py index 5eed044def..9a5e8315b0 100644 --- a/tests/test_asyncio/test_cluster.py +++ b/tests/test_asyncio/test_cluster.py @@ -493,8 +493,8 @@ async def test_execute_command_node_flag_replicas(self, r: RedisCluster) -> None Test command execution with nodes flag REPLICAS """ replicas = r.get_replicas() - if not replicas: - r = await get_mocked_redis_client(default_host, default_port) + assert len(replicas) != 0, "This test requires Cluster with 1 replica" + primaries = r.get_primaries() mock_all_nodes_resp(r, "PONG") assert await r.ping(target_nodes=RedisCluster.REPLICAS) is True From d0e12a3406a6b361b9b825887e0024061f24eb2b Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 17 Oct 2024 12:09:14 +0200 Subject: [PATCH 13/37] Debug cluster tests --- .github/actions/run-tests/action.yml | 9 ++++++++- .github/workflows/integration.yaml | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index f9f026fe8e..96a5ec7352 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -76,7 +76,7 @@ runs: echo "::endgroup::" shell: bash - - name: Run RESP2 tests + - name: Run tests run: | run_tests() { local protocol=$1 @@ -102,6 +102,13 @@ runs: run_tests 3 shell: bash + - name: Debug + if: failure() + run: | + echo "Docker Containers:" + docker ps + shell: bash + - name: Upload test results and profiling data uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index eb0451396f..3c2b96ad6a 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -63,7 +63,7 @@ jobs: max-parallel: 15 fail-fast: false matrix: - redis-version: ['8.0-M01', '7.4.1', '7.2.6', '6.2.16'] + redis-version: ['7.4.1'] #, '8.0-M01', '7.2.6', '6.2.16'] python-version: ['3.8', '3.12'] parser-backend: ['plain'] event-loop: ['asyncio'] From f15b0f0af61e0253b10e13e1b147d3e16e3da3e4 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 17 Oct 2024 12:21:02 +0200 Subject: [PATCH 14/37] Fix Cluster TLS port --- tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index d488793e65..c84fc92127 100644 --- a/tasks.py +++ b/tasks.py @@ -70,7 +70,7 @@ def cluster_tests(c, uvloop=False, protocol=2, profile=False): """Run tests against a redis cluster""" profile_arg = "--profile" if profile else "" cluster_url = "redis://localhost:16379/0" - cluster_tls_url = "rediss://localhost:17379/0" + cluster_tls_url = "rediss://localhost:27379/0" if uvloop: run( f"pytest {profile_arg} --protocol={protocol} --cov=./ --cov-report=xml:coverage_cluster_resp{protocol}_uvloop.xml -m 'not onlynoncluster and not redismod and not graph' --redis-url={cluster_url} --redis-ssl-url={cluster_tls_url} --junit-xml=cluster-resp{protocol}-uvloop-results.xml --uvloop" From dd3f6508499f5cc77d84c17dfdc3032a727be9a6 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Fri, 18 Oct 2024 15:23:20 +0200 Subject: [PATCH 15/37] Move current redis version to env var --- .github/workflows/integration.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 3c2b96ad6a..102cf7f30f 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -29,6 +29,7 @@ env: COVERAGE_CORE: sysmon REDIS_IMAGE: redis:latest REDIS_STACK_IMAGE: redis/redis-stack-server:latest + CURRENT_REDIS_VERSION: '7.4.1' jobs: dependency-audit: @@ -63,7 +64,7 @@ jobs: max-parallel: 15 fail-fast: false matrix: - redis-version: ['7.4.1'] #, '8.0-M01', '7.2.6', '6.2.16'] + redis-version: [${{ env.CURRENT_REDIS_VERSION }}] #, '8.0-M01', '7.2.6', '6.2.16'] python-version: ['3.8', '3.12'] parser-backend: ['plain'] event-loop: ['asyncio'] @@ -87,7 +88,7 @@ jobs: max-parallel: 15 fail-fast: false matrix: - redis-version: [ '8.0-M01' ] + redis-version: [ ${{ env.CURRENT_REDIS_VERSION }}] python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10'] parser-backend: [ 'plain' ] event-loop: [ 'asyncio' ] @@ -111,7 +112,7 @@ jobs: max-parallel: 15 fail-fast: false matrix: - redis-version: [ '8.0-M01' ] + redis-version: [ ${{ env.CURRENT_REDIS_VERSION }} ] python-version: [ '3.8', '3.12'] parser-backend: [ 'hiredis' ] event-loop: [ 'asyncio' ] @@ -144,7 +145,7 @@ jobs: max-parallel: 15 fail-fast: false matrix: - redis-version: [ '8.0-M01' ] + redis-version: [ ${{ env.CURRENT_REDIS_VERSION }} ] python-version: [ '3.8', '3.12' ] parser-backend: [ 'plain' ] event-loop: [ 'uvloop' ] From d6792b5c44bd3ce6abf7a31474040159611fec03 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Fri, 18 Oct 2024 15:23:36 +0200 Subject: [PATCH 16/37] Fix ssl tests --- tests/conftest.py | 1 + tests/test_ssl.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ff0cf5f6a3..bbbc0e4c05 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -184,6 +184,7 @@ def pytest_sessionstart(session): REDIS_INFO["version"] = version REDIS_INFO["arch_bits"] = arch_bits REDIS_INFO["cluster_enabled"] = cluster_enabled + REDIS_INFO["tls_cert_subdir"] = "cluster" if cluster_enabled else "standalone" REDIS_INFO["enterprise"] = enterprise # store REDIS_INFO in config so that it is available from "condition strings" session.config.REDIS_INFO = REDIS_INFO diff --git a/tests/test_ssl.py b/tests/test_ssl.py index e26c00dc3a..ac50d6eeab 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -19,9 +19,12 @@ class TestSSL: """ @pytest.fixture(autouse=True) - def _set_ssl_certs(self): - self.client_certs = get_tls_certificates() - self.server_certs = get_tls_certificates(cert_type=CertificateType.server) + def _set_ssl_certs(self, request): + tls_cert_subdir = request.session.config.REDIS_INFO["tls_cert_subdir"] + self.client_certs = get_tls_certificates(tls_cert_subdir) + self.server_certs = get_tls_certificates( + tls_cert_subdir, cert_type=CertificateType.server + ) def test_ssl_with_invalid_cert(self, request): ssl_url = request.config.option.redis_ssl_url From b682e113463aa9b4e8161ddbbee5ef7f9a4f3505 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Fri, 18 Oct 2024 15:24:10 +0200 Subject: [PATCH 17/37] Show CLUSTER NODES on fail --- .github/actions/run-tests/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 96a5ec7352..7ebd3fd38b 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -107,6 +107,7 @@ runs: run: | echo "Docker Containers:" docker ps + redis-cli -p 16379 CLUSTER NODES shell: bash - name: Upload test results and profiling data From 1768d0ae33a738ddbe7ba2f07410b8a7f7792ebd Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Fri, 18 Oct 2024 15:27:15 +0200 Subject: [PATCH 18/37] Fix integration workflow bugs --- .github/actions/run-tests/action.yml | 16 +++++++++++++--- .github/workflows/integration.yaml | 24 +++++++++++++++++------- docker-compose.yml | 8 ++++---- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 7ebd3fd38b..67c4688b91 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -78,6 +78,8 @@ runs: - name: Run tests run: | + set -e + run_tests() { local protocol=$1 local eventloop="" @@ -88,7 +90,7 @@ runs: echo "::group::RESP${protocol} standalone tests" echo "REDIS_MOD_URL=${REDIS_MOD_URL}" - invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} $eventloop --protocol=${protocol} + invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} $eventloop --protocol="${protocol}" echo "::endgroup::" if [ "$protocol" == "2" ] || [ "${{inputs.parser-backend}}" != 'hiredis' ]; then @@ -98,18 +100,26 @@ runs: fi } - run_tests 2 - run_tests 3 + run_tests 2 "${{inputs.event-loop}}" + run_tests 3 "${{inputs.event-loop}}" shell: bash - name: Debug if: failure() run: | + sudo apt-get install -y redis-tools echo "Docker Containers:" docker ps redis-cli -p 16379 CLUSTER NODES shell: bash + - name: Setup upterm session + if: failure() + uses: lhotari/action-upterm@v1 + with: + limit-access-to-actor: true + limit-access-to-users: uglide + - name: Upload test results and profiling data uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 102cf7f30f..f162b2308d 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -57,14 +57,24 @@ jobs: pip install -r dev_requirements.txt invoke linters + redis_version: + runs-on: ubuntu-latest + outputs: + CURRENT: ${{ env.CURRENT_REDIS_VERSION }} + steps: + - name: Compute outputs + run: | + echo "CURRENT=${{ env.CURRENT_REDIS_VERSION }}" >> $GITHUB_OUTPUT + tests: runs-on: ubuntu-latest timeout-minutes: 60 + needs: redis_version strategy: max-parallel: 15 fail-fast: false matrix: - redis-version: [${{ env.CURRENT_REDIS_VERSION }}] #, '8.0-M01', '7.2.6', '6.2.16'] + redis-version: [ '${{ needs.redis_version.outputs.CURRENT }}' ] #, '8.0-M01', '7.2.6', '6.2.16'] python-version: ['3.8', '3.12'] parser-backend: ['plain'] event-loop: ['asyncio'] @@ -82,13 +92,13 @@ jobs: python-compatibility-tests: runs-on: ubuntu-latest - needs: [ tests ] + needs: [ redis_version, tests ] timeout-minutes: 60 strategy: max-parallel: 15 fail-fast: false matrix: - redis-version: [ ${{ env.CURRENT_REDIS_VERSION }}] + redis-version: [ '${{ needs.redis_version.outputs.CURRENT }}' ] python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10'] parser-backend: [ 'plain' ] event-loop: [ 'asyncio' ] @@ -106,13 +116,13 @@ jobs: hiredis-tests: runs-on: ubuntu-latest - needs: [tests] + needs: [redis_version, tests] timeout-minutes: 60 strategy: max-parallel: 15 fail-fast: false matrix: - redis-version: [ ${{ env.CURRENT_REDIS_VERSION }} ] + redis-version: [ '${{ needs.redis_version.outputs.CURRENT }}' ] python-version: [ '3.8', '3.12'] parser-backend: [ 'hiredis' ] event-loop: [ 'asyncio' ] @@ -139,13 +149,13 @@ jobs: uvloop-tests: runs-on: ubuntu-latest - needs: [tests] + needs: [redis_version, tests] timeout-minutes: 60 strategy: max-parallel: 15 fail-fast: false matrix: - redis-version: [ ${{ env.CURRENT_REDIS_VERSION }} ] + redis-version: [ '${{ needs.redis_version.outputs.CURRENT }}' ] python-version: [ '3.8', '3.12' ] parser-backend: [ 'plain' ] event-loop: [ 'uvloop' ] diff --git a/docker-compose.yml b/docker-compose.yml index 3b3307ad07..1d80e97b1d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - REDIS_CLUSTER=no - PORT=6379 - TLS_PORT=6666 - command: --enable-debug-command yes --tls-auth-clients optional + command: --enable-debug-command yes --tls-auth-clients optional --save "" ports: - 6379:6379 - 6666:6666 # TLS port @@ -28,7 +28,7 @@ services: container_name: redis-replica depends_on: - redis - command: redis-server --replicaof redis 6379 --protected-mode no + command: redis-server --replicaof redis 6379 --protected-mode no --save "" ports: - 6380:6379 profiles: @@ -46,7 +46,7 @@ services: - TLS_ENABLED=yes - PORT=16379 - TLS_PORT=27379 - command: --enable-debug-command yes --tls-auth-clients optional + command: --enable-debug-command yes --tls-auth-clients optional --save "" ports: - "16379-16384:16379-16384" - "27379-27384:27379-27384" @@ -108,7 +108,7 @@ services: ports: - 6479:6379 environment: - - "REDIS_ARGS=--enable-debug-command yes --enable-module-command yes" + - "REDIS_ARGS=--enable-debug-command yes --enable-module-command yes --save ''" profiles: - standalone - all-stack From f469f26cf8588b564a258f9e0ec15da59713f978 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Tue, 22 Oct 2024 15:57:34 +0200 Subject: [PATCH 19/37] Add workarounds for IPv6 bug in tests --- tests/test_asyncio/test_cluster.py | 18 +++++++++--------- tests/test_cluster.py | 13 +++++++------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/test_asyncio/test_cluster.py b/tests/test_asyncio/test_cluster.py index 9a5e8315b0..502fdb604e 100644 --- a/tests/test_asyncio/test_cluster.py +++ b/tests/test_asyncio/test_cluster.py @@ -380,20 +380,22 @@ async def test_cluster_retry_object(self, request: FixtureRequest) -> None: async with RedisCluster.from_url(url) as rc_default: # Test default retry retry = rc_default.connection_kwargs.get("retry") + + # FIXME: Workaround for https://github.com/redis/redis-py/issues/3030 + host = rc_default.get_default_node().host + assert isinstance(retry, Retry) assert retry._retries == 3 assert isinstance(retry._backoff, type(default_backoff())) - assert rc_default.get_node("127.0.0.1", 16379).connection_kwargs.get( + assert rc_default.get_node(host, 16379).connection_kwargs.get( "retry" - ) == rc_default.get_node("127.0.0.1", 16380).connection_kwargs.get("retry") + ) == rc_default.get_node(host, 16380).connection_kwargs.get("retry") retry = Retry(ExponentialBackoff(10, 5), 5) async with RedisCluster.from_url(url, retry=retry) as rc_custom_retry: # Test custom retry assert ( - rc_custom_retry.get_node("127.0.0.1", 16379).connection_kwargs.get( - "retry" - ) + rc_custom_retry.get_node(host, 16379).connection_kwargs.get("retry") == retry ) @@ -402,9 +404,7 @@ async def test_cluster_retry_object(self, request: FixtureRequest) -> None: ) as rc_no_retries: # Test no connection retries assert ( - rc_no_retries.get_node("127.0.0.1", 16379).connection_kwargs.get( - "retry" - ) + rc_no_retries.get_node(host, 16379).connection_kwargs.get("retry") is None ) @@ -412,7 +412,7 @@ async def test_cluster_retry_object(self, request: FixtureRequest) -> None: url, retry=Retry(NoBackoff(), 0) ) as rc_no_retries: assert ( - rc_no_retries.get_node("127.0.0.1", 16379) + rc_no_retries.get_node(host, 16379) .connection_kwargs.get("retry") ._retries == 0 diff --git a/tests/test_cluster.py b/tests/test_cluster.py index fe5852d1fb..1b9b9969c5 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -862,21 +862,22 @@ def test_cluster_get_set_retry_object(self, request): def test_cluster_retry_object(self, r) -> None: # Test default retry + # FIXME: Workaround for https://github.com/redis/redis-py/issues/3030 + host = r.get_default_node().host + retry = r.get_connection_kwargs().get("retry") assert isinstance(retry, Retry) assert retry._retries == 0 assert isinstance(retry._backoff, type(default_backoff())) - node1 = r.get_node("127.0.0.1", 16379).redis_connection - node2 = r.get_node("127.0.0.1", 16380).redis_connection + node1 = r.get_node(host, 16379).redis_connection + node2 = r.get_node(host, 16380).redis_connection assert node1.get_retry()._retries == node2.get_retry()._retries # Test custom retry retry = Retry(ExponentialBackoff(10, 5), 5) - rc_custom_retry = RedisCluster("127.0.0.1", 16379, retry=retry) + rc_custom_retry = RedisCluster(host, 16379, retry=retry) assert ( - rc_custom_retry.get_node("127.0.0.1", 16379) - .redis_connection.get_retry() - ._retries + rc_custom_retry.get_node(host, 16379).redis_connection.get_retry()._retries == retry._retries ) From 83ceb684a5365265bfbecc7e62177ccb6d74a33c Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Tue, 22 Oct 2024 17:55:53 +0200 Subject: [PATCH 20/37] Use hostname instead of hardcoded IPv4 loopback --- tests/test_asyncio/test_cwe_404.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_asyncio/test_cwe_404.py b/tests/test_asyncio/test_cwe_404.py index e920a3fb98..677e165fc6 100644 --- a/tests/test_asyncio/test_cwe_404.py +++ b/tests/test_asyncio/test_cwe_404.py @@ -208,7 +208,7 @@ def remap(address): port = cluster_port + i remapped = remap_base + i forward_addr = hostname, port - proxy = DelayProxy(addr=("127.0.0.1", remapped), redis_addr=forward_addr) + proxy = DelayProxy(addr=(hostname, remapped), redis_addr=forward_addr) proxies.append(proxy) def all_clear(): @@ -233,7 +233,7 @@ def set_delay(delay: float): await stack.enter_async_context(p) r = RedisCluster.from_url( - f"redis://127.0.0.1:{remap_base}", address_remap=remap + f"redis://{hostname}:{remap_base}", address_remap=remap ) try: await r.initialize() From 0daf73b28bfd8a65d418a57a1e5d15cf0df92e60 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Tue, 22 Oct 2024 18:12:16 +0200 Subject: [PATCH 21/37] Fix bug in _get_client --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index bbbc0e4c05..7c65898856 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -349,7 +349,7 @@ def _get_client( get_tls_certificates() ) kwargs["ssl_cert_reqs"] = "required" - kwargs["port"] = redis_tls_url.port + kwargs["port"] = urlparse(redis_tls_url).port kwargs["connection_class"] = connection_class url_options.update(kwargs) pool = redis.ConnectionPool(**url_options) From 6b79e789c7d342b8a84a568f4597ee5ef5032541 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 23 Oct 2024 13:13:20 +0200 Subject: [PATCH 22/37] Fix run-tests action --- .github/actions/run-tests/action.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 67c4688b91..42f7174663 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -35,13 +35,16 @@ runs: CLIENT_LIBS_TEST_IMAGE: "redislabs/client-libs-test:${{ inputs.redis-version }}" run: | set -e - + echo "::group::Installing dependencies" pip install -U setuptools wheel pip install -r requirements.txt pip install -r dev_requirements.txt if [ "${{inputs.parser-backend}}" == "hiredis" ]; then - pip install hiredis${{inputs.hiredis-version}} + pip install hiredis${{inputs.hiredis-version}} + echo "PARSER_BACKEND=${{inputs.parser-backend}}_${"${{inputs.hiredis-version}}"//[^a-zA-Z0-9]/_}" >> $GITHUB_ENV + else + echo "PARSER_BACKEND=${{inputs.parser-backend}}" >> $GITHUB_ENV fi echo "::endgroup::" @@ -113,17 +116,10 @@ runs: redis-cli -p 16379 CLUSTER NODES shell: bash - - name: Setup upterm session - if: failure() - uses: lhotari/action-upterm@v1 - with: - limit-access-to-actor: true - limit-access-to-users: uglide - - name: Upload test results and profiling data uses: actions/upload-artifact@v4 with: - name: pytest-results-${{inputs.redis-version}}-${{inputs.parser-backend}}-${{inputs.python-version}}-${{inputs.event-loop}} + name: pytest-results-redis_${{inputs.redis-version}}-python_${{inputs.python-version}}-parser_${{env.PARSER_BACKEND}}-el_${{inputs.event-loop}} path: | *-results.xml prof/** From 8f91c577d3a366c9ba35bcc9450fa15eb09d0bb3 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 23 Oct 2024 13:15:11 +0200 Subject: [PATCH 23/37] Fix imports --- tests/ssl_utils.py | 2 -- tests/test_asyncio/test_connect.py | 2 +- tests/test_connect.py | 2 +- tests/test_ssl.py | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/ssl_utils.py b/tests/ssl_utils.py index f74e8f814e..612c0d5aca 100644 --- a/tests/ssl_utils.py +++ b/tests/ssl_utils.py @@ -1,9 +1,7 @@ import enum import os - from collections import namedtuple - CLIENT_CERT_NAME = "client.crt" CLIENT_KEY_NAME = "client.key" SERVER_CERT_NAME = "redis.crt" diff --git a/tests/test_asyncio/test_connect.py b/tests/test_asyncio/test_connect.py index 5d949c9639..6c4b3c33d7 100644 --- a/tests/test_asyncio/test_connect.py +++ b/tests/test_asyncio/test_connect.py @@ -11,7 +11,7 @@ ) from redis.exceptions import ConnectionError -from ..ssl_utils import get_tls_certificates, CertificateType +from ..ssl_utils import CertificateType, get_tls_certificates _CLIENT_NAME = "test-suite-client" _CMD_SEP = b"\r\n" diff --git a/tests/test_connect.py b/tests/test_connect.py index 7a10e361c8..f3c02b330f 100644 --- a/tests/test_connect.py +++ b/tests/test_connect.py @@ -8,7 +8,7 @@ from redis.connection import Connection, SSLConnection, UnixDomainSocketConnection from redis.exceptions import RedisError -from .ssl_utils import get_tls_certificates, CertificateType +from .ssl_utils import CertificateType, get_tls_certificates _CLIENT_NAME = "test-suite-client" _CMD_SEP = b"\r\n" diff --git a/tests/test_ssl.py b/tests/test_ssl.py index ac50d6eeab..2a945ac287 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -7,7 +7,7 @@ from redis.exceptions import ConnectionError, RedisError from .conftest import skip_if_cryptography, skip_if_nocryptography -from .ssl_utils import get_tls_certificates, CertificateType +from .ssl_utils import CertificateType, get_tls_certificates @pytest.mark.ssl From 75cf787096a0cf8461bc67cb1bd6b2a1c179c4c8 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 23 Oct 2024 13:33:26 +0200 Subject: [PATCH 24/37] Add missing version guards in search tests --- tests/test_search.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_search.py b/tests/test_search.py index 0f0e7bb309..c6e9a3717f 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -30,6 +30,7 @@ is_resp2_connection, skip_if_redis_enterprise, skip_if_resp_version, + skip_if_server_version_lt, skip_ifmodversion_lt, ) @@ -932,6 +933,8 @@ def test_phonetic_matcher(client): @pytest.mark.redismod @pytest.mark.onlynoncluster +# NOTE(imalinovskyi): This test contains hardcoded scores valid only for RediSearch 2.8+ +@skip_ifmodversion_lt("2.8.0", "search") def test_scorer(client): client.ft().create_index((TextField("description"),)) @@ -2239,6 +2242,8 @@ def test_geo_params(client): @pytest.mark.redismod +@skip_if_server_version_lt("7.4.0") +@skip_ifmodversion_lt("2.10.0", "search") def test_geoshapes_query_intersects_and_disjoint(client): client.ft().create_index((GeoShapeField("g", coord_system=GeoShapeField.FLAT))) client.hset("doc_point1", mapping={"g": "POINT (10 10)"}) @@ -2442,6 +2447,8 @@ def test_query_timeout(r: redis.Redis): @pytest.mark.redismod +@skip_if_server_version_lt("7.2.0") +@skip_ifmodversion_lt("2.8.4", "search") def test_geoshape(client: redis.Redis): client.ft().create_index(GeoShapeField("geom", GeoShapeField.FLAT)) waitForIndex(client, getattr(client.ft(), "index_name", "idx")) @@ -2458,6 +2465,8 @@ def test_geoshape(client: redis.Redis): @pytest.mark.redismod +@skip_if_server_version_lt("7.4.0") +@skip_ifmodversion_lt("2.10.0", "search") def test_search_missing_fields(client): definition = IndexDefinition(prefix=["property:"], index_type=IndexType.HASH) @@ -2525,6 +2534,8 @@ def test_search_missing_fields(client): @pytest.mark.redismod +@skip_if_server_version_lt("7.4.0") +@skip_ifmodversion_lt("2.10.0", "search") def test_search_empty_fields(client): definition = IndexDefinition(prefix=["property:"], index_type=IndexType.HASH) @@ -2596,6 +2607,8 @@ def test_search_empty_fields(client): @pytest.mark.redismod +@skip_if_server_version_lt("7.4.0") +@skip_ifmodversion_lt("2.10.0", "search") def test_special_characters_in_fields(client): definition = IndexDefinition(prefix=["resource:"], index_type=IndexType.HASH) From 0ea5dfc11e983a297134309cd25d8ea10881c0f3 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 23 Oct 2024 14:30:26 +0200 Subject: [PATCH 25/37] Add compatibility for Redis < 7 --- .github/actions/run-tests/action.yml | 8 +++++++- .github/workflows/integration.yaml | 4 ++-- docker-compose.yml | 6 +++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 42f7174663..46bcc265ae 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -42,7 +42,7 @@ runs: pip install -r dev_requirements.txt if [ "${{inputs.parser-backend}}" == "hiredis" ]; then pip install hiredis${{inputs.hiredis-version}} - echo "PARSER_BACKEND=${{inputs.parser-backend}}_${"${{inputs.hiredis-version}}"//[^a-zA-Z0-9]/_}" >> $GITHUB_ENV + echo "PARSER_BACKEND=$(echo "${{inputs.parser-backend}}_${{inputs.hiredis-version}}" | sed 's/[^a-zA-Z0-9]/_/g') >> $GITHUB_ENV else echo "PARSER_BACKEND=${{inputs.parser-backend}}" >> $GITHUB_ENV fi @@ -68,6 +68,12 @@ runs: echo "Version not found in the mapping." exit 1 fi + + if (( redis_major_version < 6 )); then + export REDIS_STACK_EXTRA_ARGS="--save ''" + export REDIS_EXTRA_ARGS="--save ''" + fi + invoke devenv --endpoints=all-stack else echo "Using redis CE for module tests" diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index f162b2308d..7fe5bb8129 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -74,7 +74,7 @@ jobs: max-parallel: 15 fail-fast: false matrix: - redis-version: [ '${{ needs.redis_version.outputs.CURRENT }}' ] #, '8.0-M01', '7.2.6', '6.2.16'] + redis-version: [ '${{ needs.redis_version.outputs.CURRENT }}', '7.2.6', '6.2.16'] python-version: ['3.8', '3.12'] parser-backend: ['plain'] event-loop: ['asyncio'] @@ -99,7 +99,7 @@ jobs: fail-fast: false matrix: redis-version: [ '${{ needs.redis_version.outputs.CURRENT }}' ] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10'] + python-version: ['3.9', '3.10', '3.11', 'pypy-3.9', 'pypy-3.10'] parser-backend: [ 'plain' ] event-loop: [ 'asyncio' ] env: diff --git a/docker-compose.yml b/docker-compose.yml index 1d80e97b1d..472941aaf8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - REDIS_CLUSTER=no - PORT=6379 - TLS_PORT=6666 - command: --enable-debug-command yes --tls-auth-clients optional --save "" + command: ${REDIS_EXTRA_ARGS:---enable-debug-command yes --enable-module-command yes --save ""} ports: - 6379:6379 - 6666:6666 # TLS port @@ -46,7 +46,7 @@ services: - TLS_ENABLED=yes - PORT=16379 - TLS_PORT=27379 - command: --enable-debug-command yes --tls-auth-clients optional --save "" + command: ${REDIS_EXTRA_ARGS:---enable-debug-command yes --enable-module-command yes --save ""} ports: - "16379-16384:16379-16384" - "27379-27384:27379-27384" @@ -108,7 +108,7 @@ services: ports: - 6479:6379 environment: - - "REDIS_ARGS=--enable-debug-command yes --enable-module-command yes --save ''" + - "REDIS_ARGS=${REDIS_STACK_EXTRA_ARGS:---enable-debug-command yes --enable-module-command yes --save ''}" profiles: - standalone - all-stack From da974b2511c1dfe329969318405f02716fd5b382 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 23 Oct 2024 14:47:38 +0200 Subject: [PATCH 26/37] Add missing version guard in search tests --- tests/test_asyncio/test_search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_asyncio/test_search.py b/tests/test_asyncio/test_search.py index fb813b0bc7..5260605039 100644 --- a/tests/test_asyncio/test_search.py +++ b/tests/test_asyncio/test_search.py @@ -1025,6 +1025,8 @@ async def test_phonetic_matcher(decoded_r: redis.Redis): @pytest.mark.redismod @pytest.mark.onlynoncluster +# NOTE(imalinovskyi): This test contains hardcoded scores valid only for RediSearch 2.8+ +@skip_ifmodversion_lt("2.8.0", "search") async def test_scorer(decoded_r: redis.Redis): await decoded_r.ft().create_index((TextField("description"),)) From b85e69d9dd45ea1091e09b6d584a6acfab5dbd03 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 23 Oct 2024 14:50:44 +0200 Subject: [PATCH 27/37] Fix run-tests --- .github/actions/run-tests/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 46bcc265ae..7f957f09f9 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -42,7 +42,7 @@ runs: pip install -r dev_requirements.txt if [ "${{inputs.parser-backend}}" == "hiredis" ]; then pip install hiredis${{inputs.hiredis-version}} - echo "PARSER_BACKEND=$(echo "${{inputs.parser-backend}}_${{inputs.hiredis-version}}" | sed 's/[^a-zA-Z0-9]/_/g') >> $GITHUB_ENV + echo "PARSER_BACKEND=$(echo "${{inputs.parser-backend}}_${{inputs.hiredis-version}}" | sed 's/[^a-zA-Z0-9]/_/g')" >> $GITHUB_ENV else echo "PARSER_BACKEND=${{inputs.parser-backend}}" >> $GITHUB_ENV fi From 1e757beb613b8c04095b5188c5a515c1eda87e2d Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 23 Oct 2024 15:03:40 +0200 Subject: [PATCH 28/37] Add missing tls-auth-clients option --- .github/actions/run-tests/action.yml | 6 +++--- docker-compose.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 7f957f09f9..16be40c809 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -69,9 +69,9 @@ runs: exit 1 fi - if (( redis_major_version < 6 )); then - export REDIS_STACK_EXTRA_ARGS="--save ''" - export REDIS_EXTRA_ARGS="--save ''" + if (( redis_major_version < 7 )); then + export REDIS_STACK_EXTRA_ARGS="--tls-auth-clients optional --save ''" + export REDIS_EXTRA_ARGS="--tls-auth-clients optional --save ''" fi invoke devenv --endpoints=all-stack diff --git a/docker-compose.yml b/docker-compose.yml index 472941aaf8..10819a3ff8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - REDIS_CLUSTER=no - PORT=6379 - TLS_PORT=6666 - command: ${REDIS_EXTRA_ARGS:---enable-debug-command yes --enable-module-command yes --save ""} + command: ${REDIS_EXTRA_ARGS:---enable-debug-command yes --enable-module-command yes --tls-auth-clients optional --save ""} ports: - 6379:6379 - 6666:6666 # TLS port @@ -46,7 +46,7 @@ services: - TLS_ENABLED=yes - PORT=16379 - TLS_PORT=27379 - command: ${REDIS_EXTRA_ARGS:---enable-debug-command yes --enable-module-command yes --save ""} + command: ${REDIS_EXTRA_ARGS:---enable-debug-command yes --enable-module-command yes --tls-auth-clients optional --save ""} ports: - "16379-16384:16379-16384" - "27379-27384:27379-27384" From 6f63bd6dfafb740ad404fb1e57aaa0c645ffbe2a Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Wed, 23 Oct 2024 19:01:02 +0200 Subject: [PATCH 29/37] Skip module tests when Redis < 7 and RESP3 is enabled --- .github/actions/run-tests/action.yml | 10 +++++++++- tasks.py | 9 ++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 16be40c809..564b689c7e 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -72,6 +72,7 @@ runs: if (( redis_major_version < 7 )); then export REDIS_STACK_EXTRA_ARGS="--tls-auth-clients optional --save ''" export REDIS_EXTRA_ARGS="--tls-auth-clients optional --save ''" + echo "REDIS_MAJOR_VERSION=${redis_major_version}" >> $GITHUB_ENV fi invoke devenv --endpoints=all-stack @@ -99,7 +100,14 @@ runs: echo "::group::RESP${protocol} standalone tests" echo "REDIS_MOD_URL=${REDIS_MOD_URL}" - invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} $eventloop --protocol="${protocol}" + + if (( $REDIS_MAJOR_VERSION < 7 )) && [ "$protocol" == "3" ]; then + echo "Skipping module tests: Modules doesn't support RESP3 for Redis versions < 7" + invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} $eventloop --protocol="${protocol}" --extra-markers="not redismod" + else + invoke standalone-tests --redis-mod-url=${REDIS_MOD_URL} $eventloop --protocol="${protocol}" + fi + echo "::endgroup::" if [ "$protocol" == "2" ] || [ "${{inputs.parser-backend}}" != 'hiredis' ]; then diff --git a/tasks.py b/tasks.py index c84fc92127..f7b728aed4 100644 --- a/tasks.py +++ b/tasks.py @@ -50,18 +50,21 @@ def tests(c, uvloop=False, protocol=2, profile=False): @task -def standalone_tests(c, uvloop=False, protocol=2, profile=False, redis_mod_url=None): +def standalone_tests( + c, uvloop=False, protocol=2, profile=False, redis_mod_url=None, extra_markers="" +): """Run tests against a standalone redis instance""" profile_arg = "--profile" if profile else "" redis_mod_url = f"--redis-mod-url={redis_mod_url}" if redis_mod_url else "" + extra_markers = f" and {extra_markers}" if extra_markers else "" if uvloop: run( - f"pytest {profile_arg} --protocol={protocol} {redis_mod_url} --cov=./ --cov-report=xml:coverage_resp{protocol}_uvloop.xml -m 'not onlycluster and not graph' --uvloop --junit-xml=standalone-resp{protocol}-uvloop-results.xml" + f"pytest {profile_arg} --protocol={protocol} {redis_mod_url} --cov=./ --cov-report=xml:coverage_resp{protocol}_uvloop.xml -m 'not onlycluster and not graph{extra_markers}' --uvloop --junit-xml=standalone-resp{protocol}-uvloop-results.xml" ) else: run( - f"pytest {profile_arg} --protocol={protocol} {redis_mod_url} --cov=./ --cov-report=xml:coverage_resp{protocol}.xml -m 'not onlycluster and not graph' --junit-xml=standalone-resp{protocol}-results.xml" + f"pytest {profile_arg} --protocol={protocol} {redis_mod_url} --cov=./ --cov-report=xml:coverage_resp{protocol}.xml -m 'not onlycluster and not graph{extra_markers}' --junit-xml=standalone-resp{protocol}-results.xml" ) From 61677eb72d882e4bb1c9f5c97bf5403bfc13c462 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 24 Oct 2024 14:23:34 +0200 Subject: [PATCH 30/37] Fix async test_moved_redirection_on_slave_with_default The test was broken for a while after migrating to all-in-one container with Cluster --- tests/test_asyncio/test_cluster.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_asyncio/test_cluster.py b/tests/test_asyncio/test_cluster.py index 502fdb604e..bb497407b8 100644 --- a/tests/test_asyncio/test_cluster.py +++ b/tests/test_asyncio/test_cluster.py @@ -2804,7 +2804,6 @@ async def test_asking_error(self, r: RedisCluster) -> None: assert ask_node._free.pop().read_response.await_count assert res == ["MOCK_OK"] - @skip_if_server_version_gte("7.0.0") async def test_moved_redirection_on_slave_with_default( self, r: RedisCluster ) -> None: @@ -2814,6 +2813,7 @@ async def test_moved_redirection_on_slave_with_default( # set read_from_replicas to True r.read_from_replicas = True primary = r.get_node_from_key(key, False) + replica = r.get_node_from_key(key, True) moved_error = f"{r.keyslot(key)} {primary.host}:{primary.port}" parse_response_orig = primary.parse_response @@ -2824,11 +2824,7 @@ async def test_moved_redirection_on_slave_with_default( async def parse_response( self, connection: Connection, command: str, **kwargs: Any ) -> Any: - if ( - command == "GET" - and self.host != primary.host - and self.port != primary.port - ): + if command == "GET" and self.port != primary.port: raise MovedError(moved_error) return await parse_response_orig(connection, command, **kwargs) From fbabc6dea351068297375a41ad8494eba9495bd8 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 24 Oct 2024 14:26:55 +0200 Subject: [PATCH 31/37] Cleanup test after debugging --- tests/test_asyncio/test_cluster.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_asyncio/test_cluster.py b/tests/test_asyncio/test_cluster.py index bb497407b8..477397dd5f 100644 --- a/tests/test_asyncio/test_cluster.py +++ b/tests/test_asyncio/test_cluster.py @@ -33,7 +33,6 @@ assert_resp_response, is_resp2_connection, skip_if_redis_enterprise, - skip_if_server_version_gte, skip_if_server_version_lt, skip_unless_arch_bits, ) @@ -2813,7 +2812,6 @@ async def test_moved_redirection_on_slave_with_default( # set read_from_replicas to True r.read_from_replicas = True primary = r.get_node_from_key(key, False) - replica = r.get_node_from_key(key, True) moved_error = f"{r.keyslot(key)} {primary.host}:{primary.port}" parse_response_orig = primary.parse_response From 3f2926c7ab38a4bbe29f954b6c2643b6a6f4271f Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 24 Oct 2024 15:08:56 +0200 Subject: [PATCH 32/37] Use correct profile in install_and_test.sh --- .github/workflows/install_and_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/install_and_test.sh b/.github/workflows/install_and_test.sh index 33a1edb1e7..5d13dacec8 100755 --- a/.github/workflows/install_and_test.sh +++ b/.github/workflows/install_and_test.sh @@ -21,7 +21,7 @@ python -m venv ${DESTENV} source ${DESTENV}/bin/activate pip install --upgrade --quiet pip pip install --quiet -r dev_requirements.txt -invoke devenv +invoke devenv --endpoints=all-stack invoke package # find packages @@ -39,7 +39,7 @@ cd ${TESTDIR} # install, run tests pip install ${PKG} # Redis tests -pytest -m 'not onlycluster' +pytest -m 'not onlycluster and not graph' # RedisCluster tests CLUSTER_URL="redis://localhost:16379/0" pytest -m 'not onlynoncluster and not redismod and not ssl' --redis-url=${CLUSTER_URL} From fb50dbdd44cd174ff11f3f211f2cdf2140c3920b Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 24 Oct 2024 15:09:21 +0200 Subject: [PATCH 33/37] Use matrix to execute hiredis<=3.0.0 tests --- .github/workflows/integration.yaml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 7fe5bb8129..a953c93af3 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -125,6 +125,7 @@ jobs: redis-version: [ '${{ needs.redis_version.outputs.CURRENT }}' ] python-version: [ '3.8', '3.12'] parser-backend: [ 'hiredis' ] + hiredis-version: [ '>=3.0.0', '<3.0.0' ] event-loop: [ 'asyncio' ] env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true @@ -137,15 +138,7 @@ jobs: python-version: ${{ matrix.python-version }} parser-backend: ${{ matrix.parser-backend }} redis-version: ${{ matrix.redis-version }} - - - name: Run tests against hiredis < 3.0.0 - uses: ./.github/actions/run-tests - if: ${{ matrix.parser-backend == 'hiredis' && matrix.python-version == '3.12'}} - with: - python-version: ${{ matrix.python-version }} - parser-backend: ${{ matrix.parser-backend }} - redis-version: ${{ matrix.redis-version }} - hiredis-version: '<3.0.0' + hiredis-version: ${{ matrix.hiredis-version }} uvloop-tests: runs-on: ubuntu-latest From 46cfc93cc448e9ffd92df853a5c93cc612531fbb Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 24 Oct 2024 16:02:37 +0200 Subject: [PATCH 34/37] Fix hiredis job --- .github/actions/run-tests/action.yml | 2 +- .github/workflows/integration.yaml | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 564b689c7e..64d31969bd 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -41,7 +41,7 @@ runs: pip install -r requirements.txt pip install -r dev_requirements.txt if [ "${{inputs.parser-backend}}" == "hiredis" ]; then - pip install hiredis${{inputs.hiredis-version}} + pip install "hiredis${{inputs.hiredis-version}}" echo "PARSER_BACKEND=$(echo "${{inputs.parser-backend}}_${{inputs.hiredis-version}}" | sed 's/[^a-zA-Z0-9]/_/g')" >> $GITHUB_ENV else echo "PARSER_BACKEND=${{inputs.parser-backend}}" >> $GITHUB_ENV diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index a953c93af3..4a45d8f586 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -129,7 +129,7 @@ jobs: event-loop: [ 'asyncio' ] env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true - name: Redis ${{ matrix.redis-version }}; Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}}; EL:${{matrix.event-loop}} + name: Redis ${{ matrix.redis-version }}; Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}} (${{ matrix.hiredis-version }}); EL:${{matrix.event-loop}} steps: - uses: actions/checkout@v4 - name: Run tests @@ -178,6 +178,10 @@ jobs: with: python-version: 3.9 - name: Run installed unit tests + env: + REDIS_VERSION: ${{ env.CURRENT_REDIS_VERSION }} + REDIS_IMAGE: "redis:${{ env.CURRENT_REDIS_VERSION }}" + CLIENT_LIBS_TEST_IMAGE: "redislabs/client-libs-test:${{ env.CURRENT_REDIS_VERSION }}" run: | bash .github/workflows/install_and_test.sh ${{ matrix.extension }} From 077a525a7d1c8b213b501e80807fe861e2cb6879 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 24 Oct 2024 17:01:13 +0200 Subject: [PATCH 35/37] Fix pytest command in install_and_test.sh --- .github/workflows/install_and_test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/install_and_test.sh b/.github/workflows/install_and_test.sh index 5d13dacec8..5c879c1b3a 100755 --- a/.github/workflows/install_and_test.sh +++ b/.github/workflows/install_and_test.sh @@ -42,4 +42,6 @@ pip install ${PKG} pytest -m 'not onlycluster and not graph' # RedisCluster tests CLUSTER_URL="redis://localhost:16379/0" -pytest -m 'not onlynoncluster and not redismod and not ssl' --redis-url=${CLUSTER_URL} +CLUSTER_SSL_URL="rediss://localhost:27379/0" +pytest -m 'not onlynoncluster and not redismod and not ssl and not graph' \ + --redis-url="${CLUSTER_URL}" --redis-ssl-url="${CLUSTER_SSL_URL}" From 1638bbbe824c3b56bd54ee67f8f8653be5703a78 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 24 Oct 2024 17:13:26 +0200 Subject: [PATCH 36/37] Use 7.4.1 as default version in docker-compose.yml --- docker-compose.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 10819a3ff8..7804f09c8a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: redis: - image: ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.0-M01} + image: ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:7.4.1} container_name: redis-standalone environment: - TLS_ENABLED=yes @@ -24,7 +24,7 @@ services: - all replica: - image: ${REDIS_IMAGE:-redis:latest} + image: ${REDIS_IMAGE:-redis:7.4.1} container_name: redis-replica depends_on: - redis @@ -37,7 +37,7 @@ services: - all cluster: - image: ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.0-M01} + image: ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:7.4.1} container_name: redis-cluster environment: - REDIS_CLUSTER=yes @@ -58,7 +58,7 @@ services: - all sentinel: - image: ${REDIS_IMAGE:-redis:latest} + image: ${REDIS_IMAGE:-redis:7.4.1} container_name: redis-sentinel depends_on: - redis @@ -73,7 +73,7 @@ services: - all sentinel2: - image: ${REDIS_IMAGE:-redis:latest} + image: ${REDIS_IMAGE:-redis:7.4.1} container_name: redis-sentinel2 depends_on: - redis @@ -88,7 +88,7 @@ services: - all sentinel3: - image: ${REDIS_IMAGE:-redis:latest} + image: ${REDIS_IMAGE:-redis:7.4.1} container_name: redis-sentinel3 depends_on: - redis From f04991dfb621d5a47f0d4fc9f7c223a1554901c9 Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Thu, 24 Oct 2024 17:18:30 +0200 Subject: [PATCH 37/37] Fix uvloop-tests --- .github/workflows/integration.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 4a45d8f586..7c74de5290 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -163,6 +163,7 @@ jobs: python-version: ${{ matrix.python-version }} parser-backend: ${{ matrix.parser-backend }} redis-version: ${{ matrix.redis-version }} + event-loop: ${{ matrix.event-loop }} build-and-test-package: name: Validate building and installing the package