Merge pull request #23 from aiven/aiven-4.0.13-test

Apply Aiven-specific changes on top of upstream 4.0.13

.github/workflows/build.yml

@@ -0,0 +1,53 @@
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#
+name: Build cassandra
+
+# Default to read-only access to all APIs.
+permissions: read-all
+
+on:
+  push:
+    branches:
+      - aiven-4.0.3
+    tags:
+      - '**'
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+    steps:
+      - name: checkout-code
+        uses: actions/checkout@v3
+      - name: setup jdk 8
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+      - name: test
+        continue-on-error: true
+        run: |
+          ant test
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        if: always()
+        with:
+          files: "build/test/output/*.xml"
+

CHANGES.txt

@@ -84,6 +84,8 @@ Merged from 3.0:
  * Backport CASSANDRA-10508: Remove hard-coded SSL cipher suites (CASSANDRA-18575)
  * Suppress CVE-2023-2976 (CASSANDRA-18562)
  * Remove dh_python use in Debian packaging (CASSANDRA-18558)
+Backported from 4.1:
+ * Add broadcast_rpc_address to system.local (CASSANDRA-11181)
 
 
 4.0.10

pylib/cqlshlib/saferscanner.py

@@ -53,23 +53,6 @@ def scrub_sub(cls, sub, flags):
         return re.sre_parse.SubPattern(sub.pattern, scrubbedsub)
 
 
-class Py2SaferScanner(SaferScannerBase):
-
-    def __init__(self, lexicon, flags=0):
-        self.lexicon = lexicon
-        p = []
-        s = re.sre_parse.Pattern()
-        s.flags = flags
-        for phrase, action in lexicon:
-            p.append(re.sre_parse.SubPattern(s, [
-                (SUBPATTERN, (len(p) + 1, self.subpat(phrase, flags))),
-            ]))
-        s.groups = len(p) + 1
-        p = re.sre_parse.SubPattern(s, [(BRANCH, (None, p))])
-        self.p = p
-        self.scanner = re.sre_compile.compile(p)
-
-
 class Py36SaferScanner(SaferScannerBase):
 
     def __init__(self, lexicon, flags=0):
@@ -118,7 +101,7 @@ def __init__(self, lexicon, flags=0):
         self.scanner = re._compiler.compile(p)
 
 
-SaferScanner = Py36SaferScanner if six.PY3 else Py2SaferScanner
+SaferScanner = Py36SaferScanner
 if version_info >= (3, 11):
     SaferScanner = Py311SaferScanner
 elif version_info >= (3, 8):

redhat/cassandra.spec

@@ -27,14 +27,19 @@
 %define __python /usr/bin/python3
 
 %global username cassandra
+%global uid_offset 100
 
 # input of ~alphaN, ~betaN, ~rcN package versions need to retain upstream '-alphaN, etc' version for sources
 %define upstream_version %(echo %{version} | sed -r 's/~/-/g')
 %define relname apache-cassandra-%{upstream_version}
 
-Name:          cassandra
+%define cassandra_major_version 4
+%define cassandraX cassandra%{cassandra_major_version}
+
+Name:          %{cassandraX}
 Version:       %{version}
 Release:       %{revision}
+Epoch:         %{epoch}
 Summary:       Cassandra is a highly scalable, eventually consistent, distributed, structured key-value store.
 
 Group:         Development/Libraries
@@ -52,6 +57,7 @@ Requires:      procps-ng >= 3.3
 Requires(pre): user(cassandra)
 Requires(pre): group(cassandra)
 Requires(pre): shadow-utils
+Conflicts:     cassandra
 Provides:      user(cassandra)
 Provides:      group(cassandra)
 
@@ -69,25 +75,14 @@ Cassandra is a distributed (peer-to-peer) system for the management and storage
 %build
 export LANG=en_US.UTF-8
 export JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF-8"
-ant clean jar -Dversion=%{upstream_version}
+ant clean jar -Dversion=%{upstream_version} -Drat.skip=true
 
 %install
 %{__rm} -rf %{buildroot}
-mkdir -p %{buildroot}/%{_sysconfdir}/%{username}
-mkdir -p %{buildroot}/usr/share/%{username}
-mkdir -p %{buildroot}/usr/share/%{username}/lib
-mkdir -p %{buildroot}/%{_sysconfdir}/%{username}/default.conf
-mkdir -p %{buildroot}/%{_sysconfdir}/rc.d/init.d
-mkdir -p %{buildroot}/%{_sysconfdir}/security/limits.d
-mkdir -p %{buildroot}/%{_sysconfdir}/default
-mkdir -p %{buildroot}/usr/sbin
-mkdir -p %{buildroot}/usr/bin
-mkdir -p %{buildroot}/var/lib/%{username}/commitlog
-mkdir -p %{buildroot}/var/lib/%{username}/data
-mkdir -p %{buildroot}/var/lib/%{username}/saved_caches
-mkdir -p %{buildroot}/var/lib/%{username}/hints
-mkdir -p %{buildroot}/var/run/%{username}
-mkdir -p %{buildroot}/var/log/%{username}
+mkdir -p %{buildroot}/usr/share/%{cassandraX}
+mkdir -p %{buildroot}/usr/share/%{cassandraX}/lib
+mkdir -p %{buildroot}/usr/%{cassandraX}/sbin
+mkdir -p %{buildroot}/usr/%{cassandraX}/bin
 ( cd pylib && %{__python} setup.py install --no-compile --root %{buildroot}; )
 
 # patches for data and log paths
@@ -101,108 +96,80 @@ rm -f bin/stop-server
 rm -f bin/*.orig
 rm -f bin/cassandra.in.sh
 rm -f lib/sigar-bin/*winnt*  # strip segfaults on dll..
+rm -f lib/sigar-bin/*sparc*
+rm -f lib/sigar-bin/*s390x*
+rm -f lib/sigar-bin/*ppc*
+rm -f lib/sigar-bin/*ia64*
 rm -f tools/bin/cassandra.in.sh
 
-# copy default configs
-cp -pr conf/* %{buildroot}/%{_sysconfdir}/%{username}/default.conf/
-
-# step on default config with our redhat one
-cp -p redhat/%{username}.in.sh %{buildroot}/usr/share/%{username}/%{username}.in.sh
-cp -p redhat/%{username} %{buildroot}/%{_sysconfdir}/rc.d/init.d/%{username}
-cp -p redhat/%{username}.conf %{buildroot}/%{_sysconfdir}/security/limits.d/
-cp -p redhat/default %{buildroot}/%{_sysconfdir}/default/%{username}
-
 # copy cassandra bundled libs
-cp -pr lib/* %{buildroot}/usr/share/%{username}/lib/
+cp -pr lib/* %{buildroot}/usr/share/%{cassandraX}/lib/
 
 # copy stress jar
-cp -p build/tools/lib/stress.jar %{buildroot}/usr/share/%{username}/
+cp -p build/tools/lib/stress.jar %{buildroot}/usr/share/%{cassandraX}/
 
 # copy fqltool jar
-cp -p build/tools/lib/fqltool.jar %{buildroot}/usr/share/%{username}/
+cp -p build/tools/lib/fqltool.jar %{buildroot}/usr/share/%{cassandraX}/
 
 # copy binaries
-mv bin/cassandra %{buildroot}/usr/sbin/
-cp -p bin/* %{buildroot}/usr/bin/
-cp -p tools/bin/* %{buildroot}/usr/bin/
+mv bin/cassandra %{buildroot}/usr/%{cassandraX}/sbin/
+cp -p bin/* %{buildroot}/usr/%{cassandraX}/bin/
+cp -p tools/bin/* %{buildroot}/usr/%{cassandraX}/bin/
 
 # copy cassandra jar
-cp build/apache-cassandra-%{upstream_version}.jar %{buildroot}/usr/share/%{username}/
+cp build/apache-cassandra-%{upstream_version}.jar %{buildroot}/usr/share/%{cassandraX}/
 
 %clean
 %{__rm} -rf %{buildroot}
 
-%pre
-getent group %{username} >/dev/null || groupadd -r %{username}
-getent passwd %{username} >/dev/null || \
-useradd -d /var/lib/%{username} -g %{username} -M -r %{username}
-exit 0
 
 %files
 %defattr(0644,root,root,0755)
 %doc CHANGES.txt LICENSE.txt README.asc NEWS.txt NOTICE.txt CASSANDRA-14092.txt
-%attr(755,root,root) %{_bindir}/auditlogviewer
-%attr(755,root,root) %{_bindir}/jmxtool
-%attr(755,root,root) %{_bindir}/cassandra-stress
-%attr(755,root,root) %{_bindir}/cqlsh
-%attr(755,root,root) %{_bindir}/cqlsh.py
-%attr(755,root,root) %{_bindir}/debug-cql
-%attr(755,root,root) %{_bindir}/fqltool
-%attr(755,root,root) %{_bindir}/generatetokens
-%attr(755,root,root) %{_bindir}/nodetool
-%attr(755,root,root) %{_bindir}/sstableloader
-%attr(755,root,root) %{_bindir}/sstablescrub
-%attr(755,root,root) %{_bindir}/sstableupgrade
-%attr(755,root,root) %{_bindir}/sstableutil
-%attr(755,root,root) %{_bindir}/sstableverify
-%attr(755,root,root) %{_sbindir}/cassandra
-%attr(755,root,root) /%{_sysconfdir}/rc.d/init.d/%{username}
-%{_sysconfdir}/default/%{username}
-%{_sysconfdir}/security/limits.d/%{username}.conf
-/usr/share/%{username}*
-%config(noreplace) /%{_sysconfdir}/%{username}
-%attr(755,%{username},%{username}) %config(noreplace) /var/lib/%{username}/*
-%attr(755,%{username},%{username}) /var/log/%{username}*
-%attr(755,%{username},%{username}) /var/run/%{username}*
+%attr(755,root,root) /usr/%{cassandraX}/bin/auditlogviewer
+%attr(755,root,root) /usr/%{cassandraX}/bin/jmxtool
+%attr(755,root,root) /usr/%{cassandraX}/bin/cassandra-stress
+%attr(755,root,root) /usr/%{cassandraX}/bin/cqlsh
+%attr(755,root,root) /usr/%{cassandraX}/bin/cqlsh.py
+%attr(755,root,root) /usr/%{cassandraX}/bin/debug-cql
+%attr(755,root,root) /usr/%{cassandraX}/bin/fqltool
+%attr(755,root,root) /usr/%{cassandraX}/bin/generatetokens
+%attr(755,root,root) /usr/%{cassandraX}/bin/nodetool
+%attr(755,root,root) /usr/%{cassandraX}/bin/sstableloader
+%attr(755,root,root) /usr/%{cassandraX}/bin/sstablescrub
+%attr(755,root,root) /usr/%{cassandraX}/bin/sstableupgrade
+%attr(755,root,root) /usr/%{cassandraX}/bin/sstableutil
+%attr(755,root,root) /usr/%{cassandraX}/bin/sstableverify
+%attr(755,root,root) /usr/%{cassandraX}/sbin/
+/usr/share/%{cassandraX}*
 %{python_sitelib}/cqlshlib/
 %{python_sitelib}/cassandra_pylib*.egg-info
 
-%post
-alternatives --install /%{_sysconfdir}/%{username}/conf %{username} /%{_sysconfdir}/%{username}/default.conf/ 0
-exit 0
-
-%preun
-# only delete alternative on removal, not upgrade
-if [ "$1" = "0" ]; then
-    alternatives --remove %{username} /%{_sysconfdir}/%{username}/default.conf/
-fi
-exit 0
-
-
 %package tools
 Summary:       Extra tools for Cassandra. Cassandra is a highly scalable, eventually consistent, distributed, structured key-value store.
 Group:         Development/Libraries
-Requires:      cassandra = %{version}-%{revision}
+Requires:      cassandra4 = %{epoch}:%{version}-%{revision}
+Conflicts:     cassandra-tools
 
 %description tools
 Cassandra is a distributed (peer-to-peer) system for the management and storage of structured data.
 .
 This package contains extra tools for working with Cassandra clusters.
 
 %files tools
-%attr(755,root,root) %{_bindir}/sstabledump
-%attr(755,root,root) %{_bindir}/cassandra-stressd
-%attr(755,root,root) %{_bindir}/compaction-stress
-%attr(755,root,root) %{_bindir}/sstableexpiredblockers
-%attr(755,root,root) %{_bindir}/sstablelevelreset
-%attr(755,root,root) %{_bindir}/sstablemetadata
-%attr(755,root,root) %{_bindir}/sstableofflinerelevel
-%attr(755,root,root) %{_bindir}/sstablerepairedset
-%attr(755,root,root) %{_bindir}/sstablesplit
-%attr(755,root,root) %{_bindir}/auditlogviewer
-%attr(755,root,root) %{_bindir}/jmxtool
-%attr(755,root,root) %{_bindir}/fqltool
-%attr(755,root,root) %{_bindir}/generatetokens
+%attr(755,root,root) /usr/%{cassandraX}/bin/sstabledump
+%attr(755,root,root) /usr/%{cassandraX}/bin/cassandra-stressd
+%attr(755,root,root) /usr/%{cassandraX}/bin/compaction-stress
+%attr(755,root,root) /usr/%{cassandraX}/bin/sstableexpiredblockers
+%attr(755,root,root) /usr/%{cassandraX}/bin/sstablelevelreset
+%attr(755,root,root) /usr/%{cassandraX}/bin/sstablemetadata
+%attr(755,root,root) /usr/%{cassandraX}/bin/sstableofflinerelevel
+%attr(755,root,root) /usr/%{cassandraX}/bin/sstablerepairedset
+%attr(755,root,root) /usr/%{cassandraX}/bin/sstablesplit
+%attr(755,root,root) /usr/%{cassandraX}/bin/auditlogviewer
+%attr(755,root,root) /usr/%{cassandraX}/bin/jmxtool
+%attr(755,root,root) /usr/%{cassandraX}/bin/fqltool
+%attr(755,root,root) /usr/%{cassandraX}/bin/generatetokens
 
 
 %changelog

src/java/org/apache/cassandra/config/Config.java

@@ -91,6 +91,9 @@ public class Config
     /** Triggers automatic allocation of tokens if set, based on the provided replica count for a datacenter */
     public Integer allocate_tokens_for_local_replication_factor = null;
 
+    public boolean skip_bootstrap_streaming = false;
+    public String replace_address_first_boot = null;
+
     public long native_transport_idle_timeout_in_ms = 0L;
 
     public volatile long request_timeout_in_ms = 10000L;
@@ -596,6 +599,8 @@ public static void setOverrideLoadConfig(Supplier<Config> loadConfig)
         overrideLoadConfig = loadConfig;
     }
 
+    public volatile boolean replication_factor_uptuning_enabled = true;
+
     public enum CommitLogSync
     {
         periodic,

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -1571,6 +1571,8 @@ public static InetAddressAndPort getReplaceAddress()
                 return InetAddressAndPort.getByName(System.getProperty(Config.PROPERTY_PREFIX + "replace_address", null));
             else if (System.getProperty(Config.PROPERTY_PREFIX + "replace_address_first_boot", null) != null)
                 return InetAddressAndPort.getByName(System.getProperty(Config.PROPERTY_PREFIX + "replace_address_first_boot", null));
+            else if (conf.replace_address_first_boot != null)
+                return InetAddressAndPort.getByName(conf.replace_address_first_boot);
             return null;
         }
         catch (UnknownHostException e)
@@ -1579,6 +1581,16 @@ else if (System.getProperty(Config.PROPERTY_PREFIX + "replace_address_first_boot
         }
     }
 
+    public static boolean skipBootstrapStreaming()
+    {
+        return conf.skip_bootstrap_streaming;
+    }
+
+    public static boolean replaceOnFirstBootRequested()
+    {
+        return System.getProperty("cassandra.replace_address_first_boot", null) != null || conf.replace_address_first_boot != null;
+    }
+
     public static Collection<String> getReplaceTokens()
     {
         return tokensFromString(System.getProperty(Config.PROPERTY_PREFIX + "replace_token", null));
@@ -3486,4 +3498,12 @@ public static void setForceNewPreparedStatementBehaviour(boolean value)
             conf.force_new_prepared_statement_behaviour = value;
         }
     }
+
+    public static boolean uptuningEnabled() {
+        return conf.replication_factor_uptuning_enabled;
+    }
+
+    public static void setUptuningEnabled(boolean uptuningEnabled) {
+        conf.replication_factor_uptuning_enabled = uptuningEnabled;
+    }
 }

src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java

@@ -19,10 +19,13 @@
 
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import org.slf4j.helpers.NOPLogger;
+
 import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
@@ -39,6 +42,7 @@
 import org.apache.cassandra.locator.ReplicationFactor;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceMetadata.KeyspaceDiff;
+import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
@@ -68,7 +72,11 @@ public Keyspaces apply(Keyspaces schema)
         if (null == keyspace)
             throw ire("Keyspace '%s' doesn't exist", keyspaceName);
 
-        KeyspaceMetadata newKeyspace = keyspace.withSwapped(attrs.asAlteredKeyspaceParams(keyspace.params));
+        final KeyspaceParams inputKeyspaceParams = attrs.asAlteredKeyspaceParams(keyspace.params);
+        final Map<String, String> tunedReplicationParams = TuneUpReplicationFactor.instance.apply(NOPLogger.NOP_LOGGER, keyspaceName, clientWarnings, inputKeyspaceParams);
+        final KeyspaceParams tunedKeyspaceParams = KeyspaceParams.create(inputKeyspaceParams.durableWrites, tunedReplicationParams);
+
+        KeyspaceMetadata newKeyspace = keyspace.withSwapped(tunedKeyspaceParams);
 
         if (newKeyspace.params.replication.klass.equals(LocalStrategy.class))
             throw ire("Unable to use given strategy class: LocalStrategy is reserved for internal use.");