From b35f24c3c54671d78961237b02ef3002c3349490 Mon Sep 17 00:00:00 2001 From: Israel Fruchter Date: Thu, 20 Jun 2024 11:02:26 +0300 Subject: [PATCH 1/5] CI: add integration tests with python 3.13 since we want to start test the alpha/beta version of python refactoring a bit the integration test workflow the action for pyenv we are using isn't really getting updates too much, and trying to switch back to the offical use python action that can now have prerelease python versions --- .github/workflows/integration-tests.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 8c364e93a..625026532 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -16,23 +16,27 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8.17", "3.11.4", "3.12.0b4"] + python-version: ["3.8", "3.11", "3.12", "3.13"] event_loop_manager: ["libev", "asyncio", "asyncore"] exclude: - - python-version: "3.12.0b4" + - python-version: "3.12" + event_loop_manager: "asyncore" + - python-version: "3.13" event_loop_manager: "asyncore" steps: - - uses: actions/checkout@v3 - - name: setup pyenv ${{ matrix.python-version }} - uses: "gabrielfalcao/pyenv-action@v16" + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + name: Install Python with: - default: 2.7.14 - versions: ${{ matrix.python-version }} + python-version: "${{ matrix.python_version }}" + allow-prereleases: true + - name: Test with pytest run: | export EVENT_LOOP_MANAGER=${{ matrix.event_loop_manager }} - export SCYLLA_VERSION='release:5.1' + export SCYLLA_VERSION='release:5.4' ./ci/run_integration_test.sh tests/integration/standard/ tests/integration/cqlengine/ - name: Test tablets From aeb8fe6486b1a761422706aa1eabd0e36c5f0066 Mon Sep 17 00:00:00 2001 From: Israel Fruchter Date: Tue, 2 Jul 2024 22:12:50 +0300 Subject: [PATCH 2/5] cqlengine/management.py: handle more options of views names so far the code assumed view can be name like: ```python possible_names = [name, f'values({name})'] ``` we have now one more option, that need to be considered: ```python f'keys({name})' ``` --- cassandra/cqlengine/management.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cassandra/cqlengine/management.py b/cassandra/cqlengine/management.py index 5e49fb54e..aa4c8cb42 100644 --- a/cassandra/cqlengine/management.py +++ b/cassandra/cqlengine/management.py @@ -154,9 +154,10 @@ def _get_index_name_by_column(table, column_name): Find the index name for a given table and column. """ protected_name = metadata.protect_name(column_name) - possible_index_values = [protected_name, "values(%s)" % protected_name] + possible_index_values = [protected_name, "values(%s)" % protected_name, "keys(%s)" % protected_name] for index_metadata in table.indexes.values(): options = dict(index_metadata.index_options) + if options.get('target') in possible_index_values: return index_metadata.name From 4a15353565cc402798490b4881ee51a9fdce5ff7 Mon Sep 17 00:00:00 2001 From: Israel Fruchter Date: Tue, 2 Jul 2024 22:16:05 +0300 Subject: [PATCH 3/5] tests/integration: introduced @xfail_scylla_version some test that was marked with @xfail_scylla, are now passing with newer release, i.e. some issue were fixed. so this new decorator can xfail up to a certion scylla_version note that it support both a oss version, and enterprise version --- tests/integration/__init__.py | 11 +++++++++++ tests/integration/standard/test_cluster.py | 5 +++-- tests/integration/standard/test_metadata.py | 7 ++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 7826f4bcf..c99d3f8b0 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -390,6 +390,17 @@ def _id_and_mark(f): requires_custom_payload = pytest.mark.skipif(SCYLLA_VERSION is not None or PROTOCOL_VERSION < 4, reason='Scylla does not support custom payloads. Cassandra requires native protocol v4.0+') xfail_scylla = lambda reason, *args, **kwargs: pytest.mark.xfail(SCYLLA_VERSION is not None, reason=reason, *args, **kwargs) + + +def xfail_scylla_version(reason, oss_scylla_version, ent_scylla_version, *args, **kwargs): + current_version = Version(get_scylla_version(SCYLLA_VERSION) if SCYLLA_VERSION is not None else '0.0.0') + if current_version > Version("2018.1"): + lt_scylla_version = Version(ent_scylla_version) + else: + lt_scylla_version = Version(oss_scylla_version) + return pytest.mark.xfail(current_version < lt_scylla_version, + reason=reason, *args, **kwargs) + incorrect_test = lambda reason='This test seems to be incorrect and should be fixed', *args, **kwargs: pytest.mark.xfail(reason=reason, *args, **kwargs) pypy = unittest.skipUnless(platform.python_implementation() == "PyPy", "Test is skipped unless it's on PyPy") diff --git a/tests/integration/standard/test_cluster.py b/tests/integration/standard/test_cluster.py index 43356dbd8..d5e563140 100644 --- a/tests/integration/standard/test_cluster.py +++ b/tests/integration/standard/test_cluster.py @@ -42,7 +42,7 @@ from tests.integration import use_cluster, get_server_versions, CASSANDRA_VERSION, \ execute_until_pass, execute_with_long_wait_retry, get_node, MockLoggingHandler, get_unsupported_lower_protocol, \ get_unsupported_upper_protocol, lessthanprotocolv3, protocolv6, local, CASSANDRA_IP, greaterthanorequalcass30, \ - lessthanorequalcass40, DSE_VERSION, TestCluster, PROTOCOL_VERSION, xfail_scylla, incorrect_test + lessthanorequalcass40, DSE_VERSION, TestCluster, PROTOCOL_VERSION, xfail_scylla_version, incorrect_test from tests.integration.util import assert_quiescent_pool_state import sys @@ -288,7 +288,8 @@ def test_protocol_negotiation(self): cluster.shutdown() - @xfail_scylla("Failing with scylla because there is option to create a cluster with 'lower bound' protocol") + @xfail_scylla_version("Failing with scylla because there is option to create a cluster with 'lower bound' protocol", + oss_scylla_version="5.2", ent_scylla_version="2023.1") def test_invalid_protocol_negotation(self): """ Test for protocol negotiation when explicit versions are set diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index c561491ab..972e5a66e 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -41,7 +41,7 @@ greaterthancass21, assert_startswith, greaterthanorequalcass40, greaterthanorequaldse67, lessthancass40, TestCluster, DSE_VERSION, requires_java_udf, requires_composite_type, - requires_collection_indexes, xfail_scylla) + requires_collection_indexes, xfail_scylla, xfail_scylla_version) from tests.util import wait_until @@ -502,7 +502,7 @@ def test_indexes(self): self.assertIn('CREATE INDEX e_index', statement) @greaterthancass21 - @requires_collection_indexes + @xfail_scylla_version('failing cause of scylladb/scylladb#19278', oss_scylla_version='6.1', ent_scylla_version='2025.1') def test_collection_indexes(self): self.session.execute("CREATE TABLE %s.%s (a int PRIMARY KEY, b map)" @@ -1210,7 +1210,8 @@ def test_export_keyspace_schema_udts(self): cluster.shutdown() @greaterthancass21 - @pytest.mark.xfail(reason='Column name in CREATE INDEX is not quoted. It\'s a bug in driver or in Scylla') + @xfail_scylla_version(reason='Column name in CREATE INDEX is not quoted. It\'s a bug in driver or in Scylla', + oss_scylla_version="5.2", ent_scylla_version="2023.1") def test_case_sensitivity(self): """ Test that names that need to be escaped in CREATE statements are From c15ba8e8bfb4a13cf9563a128e57a1838e13a570 Mon Sep 17 00:00:00 2001 From: Israel Fruchter Date: Tue, 2 Jul 2024 22:17:30 +0300 Subject: [PATCH 4/5] tests/*cqlengine/management: fix parameters passed to compaction at some point scylla started validation of the arguments to compaction and newer versions rejects the values passed from this test chnaged it to something within the values expected --- .../cqlengine/management/test_compaction_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cqlengine/management/test_compaction_settings.py b/tests/integration/cqlengine/management/test_compaction_settings.py index 63161643f..1633bcec5 100644 --- a/tests/integration/cqlengine/management/test_compaction_settings.py +++ b/tests/integration/cqlengine/management/test_compaction_settings.py @@ -61,7 +61,7 @@ class SizeTieredCompactionChangesDetectionTest(Model): __options__ = {'compaction': {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'bucket_high': '20', - 'bucket_low': '10', + 'bucket_low': '0.5', 'max_threshold': '200', 'min_threshold': '100', 'min_sstable_size': '1000', From 9b4ef7f82bb7833aa43763c2d24619cc54e07b0c Mon Sep 17 00:00:00 2001 From: Israel Fruchter Date: Thu, 4 Jul 2024 21:36:51 +0300 Subject: [PATCH 5/5] tests: xfail `test_order_by_success_case` for scylla test is failing with the following: ``` cassandra.InvalidRequest: Error from server: code=2200 [Invalid query] message="Order by currently only supports the ordering of columns following their declared order in the PRIMARY KEY" ``` seems like scylla doesn't support ordering by clustering keys Ref: scylladb/python-driver#343 --- tests/integration/cqlengine/query/test_queryset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/cqlengine/query/test_queryset.py b/tests/integration/cqlengine/query/test_queryset.py index a2f9f23d4..038d403c4 100644 --- a/tests/integration/cqlengine/query/test_queryset.py +++ b/tests/integration/cqlengine/query/test_queryset.py @@ -39,7 +39,7 @@ from cassandra.util import uuid_from_time from cassandra.cqlengine.connection import get_session from tests.integration import PROTOCOL_VERSION, CASSANDRA_VERSION, greaterthancass20, greaterthancass21, \ - greaterthanorequalcass30, TestCluster, requires_collection_indexes + greaterthanorequalcass30, TestCluster, requires_collection_indexes, xfail_scylla from tests.integration.cqlengine import execute_count, DEFAULT_KEYSPACE @@ -599,6 +599,7 @@ def test_distinct_with_explicit_count(self): @requires_collection_indexes class TestQuerySetOrdering(BaseQuerySetUsage): + @xfail_scylla(reason="Scylla does not support ordering on non-primary key columns: https://github.com/scylladb/python-driver/issues/343") @execute_count(2) def test_order_by_success_case(self): q = TestModel.objects(test_id=0).order_by('attempt_id')