From 3fda9651ccbcb7b23f843515297683bca091253b Mon Sep 17 00:00:00 2001 From: Nadav Har'El Date: Tue, 29 Oct 2024 12:26:21 +0200 Subject: [PATCH] test/alternator: option to run alternator tests against specific release We recently added a "--release " option to test/cql-pytest/run to run a cql-pytest test against a released version of Scylla, downloaded automatically from ScyllaDB's precompiled binary repository. This patch adds the same capability also to test/alternator/run - allowing to run a current test/alternator test on older releases of Scylla. The implementation in this patch reuses the same implementation from the cql-pytest patch. Here is an example use case: the pull request #19941 claimed that a certain bug fix was backported to release 6.0. Was it? Let's run the test reproducing that bug on two releases: test/alternator/run --release 6.0 test_streams.py::test_stream_list_tables test/alternator/run --release 6.1 test_streams.py::test_stream_list_tables It shows that the test passes on 6.1 (so the bug is fixed there) but the test fails 6.0. It turns out that although the fix was backported to branch-6.0, this happened shortly after 6.0.4 was released and no later 6.0 minor release came afterwards! So the bug wasn't actually fixed on any official release of 6.0. Signed-off-by: Nadav Har'El Closes scylladb/scylladb#21343 --- test/alternator/README.md | 19 +++++++++++++++++++ test/alternator/run | 18 +++++++++++++++++- test/cqlpy/fetch_scylla.py | 1 + 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/test/alternator/README.md b/test/alternator/README.md index 24cc1426262b..ec50c2c77767 100644 --- a/test/alternator/README.md +++ b/test/alternator/README.md @@ -57,6 +57,25 @@ region = us-east-1 You don't need these files to run the tests against a local installation of ScyllaDB. +The "run" script also has an ability to run tests against a specific old +release of Scylla downloaded (pre-compiled) from ScyllaDB's official +release collection. For example: + +``` +test/alternator/run --release 6.0 test_streams.py::test_stream_list_tables +test/alternator/run --release 6.1 test_streams.py::test_stream_list_tables +``` + +can demonstrate that a bug (reproduced by that test) was fixed in the latest +official release of 6.1, but not in the latest 6.0 release. The `--release` +option (which must be the first option to "run") downloads the requested +official release and caches it in the `build/` directory (e.g., +`build/6.0.4`), and then runs the requested tests against that version. +The `--release` option supports various version specifiers, such as 5.4.7 +(a specific version), 5.4 (asking for the latest version in the 5.4 branch), +5.4.0~rc2 (a pre-release), or Enterprise releases such as 2021.1.9 or 2023.1 +(the latest in that branch). + ## HTTPS support In order to run tests over HTTPS instead of the default HTTP, run diff --git a/test/alternator/run b/test/alternator/run index c0033f5c282c..64131dad6de7 100755 --- a/test/alternator/run +++ b/test/alternator/run @@ -21,6 +21,22 @@ if '--aws' in sys.argv: import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +# If the first option is "--release", download that release (see +# fetch_scylla.py for supported release numbers), and use that. +# The downloaded Scylla will be cached in the directory build/, +# where is the specific release downloaded - E.g., if the user +# asks "--release 2022.1" and the downloaded release is 2022.1.9, it +# will be stored in build/2022.1.9. +if len(sys.argv) > 1 and sys.argv[1] == '--release': + release = sys.argv[2] + exe = run.download_precompiled_scylla(release) + run_scylla_cmd = lambda pid, dir: run.run_precompiled_scylla_cmd(exe, pid, dir) + sys.argv = sys.argv[0:1] + sys.argv[3:] + os.environ['SCYLLA'] = exe # so find_scylla() prints the right one +else: + run_scylla_cmd = run.run_scylla_cmd + print('Scylla is: ' + run.find_scylla() + '.') extra_scylla_options = [] @@ -46,7 +62,7 @@ run.omit_scylla_output = "--omit-scylla-output" in sys.argv # parameters, so in particular both CQL and Alternator will be enabled. # This is useful for us because we need CQL to setup authentication. def run_alternator_cmd(pid, dir): - (cmd, env) = run.run_scylla_cmd(pid, dir) + (cmd, env) = run_scylla_cmd(pid, dir) ip = run.pid_to_ip(pid) cmd += [ '--alternator-address', ip, diff --git a/test/cqlpy/fetch_scylla.py b/test/cqlpy/fetch_scylla.py index c36f6fa2daa0..eecc06184f13 100755 --- a/test/cqlpy/fetch_scylla.py +++ b/test/cqlpy/fetch_scylla.py @@ -27,6 +27,7 @@ def parse_version(version): def s3_download(s3, bucket, object_key, out): print(f'Downloading {object_key}... ', end='') if not sys.stdout.isatty(): + sys.stdout.flush() # show the "Downloading" message at least bucket.download_file(object_key, out) else: # When interactive, show download progress: