CI Summary

ccmlib.node.TimeoutError: 24 Nov 2024 17:13:51 [node3] after 20.05/20 seconds Missing: ['Replacing a node without bootstrapping risks invalidating consistency guarantees'] not found in system.log:
 Head: INFO  [main] 2024-11-24 17:13:34,390 YamlConfigura
 Tail: ...3 ColumnFamilyStore.java:1060 - Enqueuing flush of system.local_metadata_log, Reason: INTERNALLY_FORCED, Usage: 364B (0%) on-heap, 311B (0%) off-heap-self = 

    @since('3.0')
    def test_fail_when_seed(self):
        """
        When a node is a seed replace should fail
        @jira_ticket CASSANDRA-14463
        """
        self.fixture_dtest_setup.ignore_log_patterns = list(self.fixture_dtest_setup.ignore_log_patterns) + [
            r'Couldn\'t find table with id 5bc52802-de25-35ed-aeab-188eecebb090', # system_auth
            r'Exception encountered during startup',
            r'Unknown endpoint'
        ]
    
        # see CASSANDRA-17611
        if self.cluster.version() < '4.0':
            self.fixture_dtest_setup.ignore_log_patterns = list(self.fixture_dtest_setup.ignore_log_patterns) + [
                r'RejectedExecutionException']
    
        self._setup(n=3)
        node1, node2, node3 = self.cluster.nodelist()
        self.cluster.seeds.append(node3.address())
    
        node3.stop(gently=False)
        mark = node3.mark_log()
    
        for d in chain([os.path.join(node3.get_path(), "commitlogs")],
                       [os.path.join(node3.get_path(), "saved_caches")],
                       node3.data_directories()):
            if os.path.exists(d):
                rmtree(d)
    
        node3.start(jvm_args=["-Dcassandra.replace_address=" + node3.address()], wait_other_notice=False)
>       node3.watch_log_for('Replacing a node without bootstrapping risks invalidating consistency guarantees', from_mark=mark, timeout=20)

replace_address_test.py:432: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:610: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1732468411.8734467, timeout = 20
msg = "Missing: ['Replacing a node without bootstrapping risks invalidating consistency guarantees'] not found in system.log...nqueuing flush of system.local_metadata_log, Reason: INTERNALLY_FORCED, Usage: 364B (0%) on-heap, 311B (0%) off-heap\n"
node = 'node3'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 24 Nov 2024 17:13:51 [node3] after 20.05/20 seconds Missing: ['Replacing a node without bootstrapping risks invalidating consistency guarantees'] not found in system.log:
E            Head: INFO  [main] 2024-11-24 17:13:34,390 YamlConfigura
E            Tail: ...3 ColumnFamilyStore.java:1060 - Enqueuing flush of system.local_metadata_log, Reason: INTERNALLY_FORCED, Usage: 364B (0%) on-heap, 311B (0%) off-heap

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

AssertionError: Commitlog was written with durable writes disabled
assert 163840 == 90112
  +163840
  -90112-self = 

    @pytest.mark.timeout(60*30)
    def test_change_durable_writes(self):
        """
        @jira_ticket CASSANDRA-9560
    
        Test that changes to the DURABLE_WRITES option on keyspaces is
        respected in subsequent writes.
    
        This test starts by writing a dataset to a cluster and asserting that
        the commitlogs have been written to. The subsequent test depends on
        the assumption that this dataset triggers an fsync.
    
        After checking this assumption, the test destroys the cluster and
        creates a fresh one. Then it tests that DURABLE_WRITES is respected by:
    
        - creating a keyspace with DURABLE_WRITES set to false,
        - using ALTER KEYSPACE to set its DURABLE_WRITES option to true,
        - writing a dataset to this keyspace that is known to trigger a commitlog fsync,
        - asserting that the commitlog has grown in size since the data was written.
        """
        cluster = self.cluster
        cluster.set_batch_commitlog(enabled=True, use_batch_window = cluster.version() < '5.0')
        cluster.set_configuration_options(values={'commitlog_segment_size_in_mb': 1})
    
        cluster.populate(1).start()
        durable_node = cluster.nodelist()[0]
    
        durable_init_size = commitlog_size(durable_node)
        durable_session = self.patient_exclusive_cql_connection(durable_node)
    
        # test assumption that write_to_trigger_fsync actually triggers a commitlog fsync
        durable_session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1} "
                                "AND DURABLE_WRITES = true")
        durable_session.execute('CREATE TABLE ks.tab (key int PRIMARY KEY, a int, b int, c int)')
        logger.debug('commitlog size diff = ' + str(commitlog_size(durable_node) - durable_init_size))
        write_to_trigger_fsync(durable_session, 'ks', 'tab')
        logger.debug('commitlog size diff = ' + str(commitlog_size(durable_node) - durable_init_size))
    
        assert commitlog_size(durable_node) > durable_init_size, \
            "This test will not work in this environment; write_to_trigger_fsync does not trigger fsync."
    
        durable_session.shutdown()
        cluster.stop()
        cluster.clear()
    
        cluster.set_batch_commitlog(enabled=True, use_batch_window = cluster.version() < '5.0')
        cluster.set_configuration_options(values={'commitlog_segment_size_in_mb': 1})
        cluster.start()
        node = cluster.nodelist()[0]
        session = self.patient_exclusive_cql_connection(node)
    
        # set up a keyspace without durable writes, then alter it to use them
        session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1} "
                        "AND DURABLE_WRITES = false")
        session.execute('CREATE TABLE ks.tab (key int PRIMARY KEY, a int, b int, c int)')
        init_size = commitlog_size(node)
        write_to_trigger_fsync(session, 'ks', 'tab')
>       assert commitlog_size(node) == init_size, "Commitlog was written with durable writes disabled"
E       AssertionError: Commitlog was written with durable writes disabled
E       assert 163840 == 90112
E         +163840
E         -90112

configuration_test.py:104: AssertionError

cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042-self = 

    def test_bulk_round_trip_default(self):
        """
        Test bulk import with default stress import (one row per operation)
    
        @jira_ticket CASSANDRA-9302
        """
>       self._test_bulk_round_trip(nodes=3, partitioner="murmur3", num_operations=100000)

cqlsh_tests/test_cqlsh_copy.py:2466: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cqlsh_tests/test_cqlsh_copy.py:2436: in _test_bulk_round_trip
    num_records = create_records()
cqlsh_tests/test_cqlsh_copy.py:2409: in create_records
    ret = rows_to_list(self.session.execute(count_statement))[0][0]
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

cassandra.OperationTimedOut: errors={'127.0.0.2:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.2:9042-self = 

    def test_bulk_round_trip_with_single_core(self):
        """
        Perform a round trip on a simulated single core machine. When determining the number of cores,
        copyutil.py will return the number carried by the environment variable CQLSH_COPY_TEST_NUM_CORES if it has
        been set.
    
        @jira_ticket CASSANDRA-11053
        """
        os.environ['CQLSH_COPY_TEST_NUM_CORES'] = '1'
>       ret = self._test_bulk_round_trip(nodes=3, partitioner="murmur3", num_operations=100000)

cqlsh_tests/test_cqlsh_copy.py:2539: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cqlsh_tests/test_cqlsh_copy.py:2436: in _test_bulk_round_trip
    num_records = create_records()
cqlsh_tests/test_cqlsh_copy.py:2409: in create_records
    ret = rows_to_list(self.session.execute(count_statement))[0][0]
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.2:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.2:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

failed on teardown with "Unexpected error found in node logs (see stdout for full details). Errors: [[node1] 'ERROR [ViewBuildExecutor:1] 2024-11-24 17:39:36,907 JVMStabilityInspector.java:70 - Exception in thread Thread[ViewBuildExecutor:1,5,ViewBuildExecutor]\njava.lang.NullPointerException: Cannot invoke "org.apache.cassandra.schema.TableMetadata.partitionKeyColumns()" because "this.viewMetadata" is null\n\tat org.apache.cassandra.db.view.ViewUpdateGenerator.(ViewUpdateGenerator.java:99)\n\tat org.apache.cassandra.db.view.TableViews.generateViewUpdates(TableViews.java:229)\n\tat org.apache.cassandra.db.view.ViewBuilderTask.buildKey(ViewBuilderTask.java:114)\n\tat org.apache.cassandra.db.view.ViewBuilderTask.call(ViewBuilderTask.java:154)\n\tat org.apache.cassandra.db.compaction.CompactionManager.lambda$submitViewBuilder$11(CompactionManager.java:2020)\n\tat org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)\n\tat org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:840)', [node1] 'ERROR [GlobalLogFollower] 2024-11-24 17:39:37,044 LocalLog.java:542 - Could not process the entry\njava.lang.NullPointerException: Cannot invoke "org.apache.cassandra.schema.TableMetadata.partitionKeyColumns()" because "this.viewMetadata" is null\n\tat org.apache.cassandra.db.view.ViewUpdateGenerator.(ViewUpdateGenerator.java:99)\n\tat org.apache.cassandra.db.view.TableViews.generateViewUpdates(TableViews.java:229)\n\tat org.apache.cassandra.db.view.ViewBuilderTask.buildKey(ViewBuilderTask.java:114)\n\tat org.apache.cassandra.db.view.ViewBuilderTask.call(ViewBuilderTask.java:154)\n\tat org.apache.cassandra.db.compaction.CompactionManager.lambda$submitViewBuilder$11(CompactionManager.java:2020)\n\tat org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)\n\tat org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:840)']"-Unexpected error found in node logs (see stdout for full details). Errors: [[node1] 'ERROR [ViewBuildExecutor:1] 2024-11-24 17:39:36,907 JVMStabilityInspector.java:70 - Exception in thread Thread[ViewBuildExecutor:1,5,ViewBuildExecutor]\njava.lang.NullPointerException: Cannot invoke "org.apache.cassandra.schema.TableMetadata.partitionKeyColumns()" because "this.viewMetadata" is null\n\tat org.apache.cassandra.db.view.ViewUpdateGenerator.(ViewUpdateGenerator.java:99)\n\tat org.apache.cassandra.db.view.TableViews.generateViewUpdates(TableViews.java:229)\n\tat org.apache.cassandra.db.view.ViewBuilderTask.buildKey(ViewBuilderTask.java:114)\n\tat org.apache.cassandra.db.view.ViewBuilderTask.call(ViewBuilderTask.java:154)\n\tat org.apache.cassandra.db.compaction.CompactionManager.lambda$submitViewBuilder$11(CompactionManager.java:2020)\n\tat org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)\n\tat org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:840)', [node1] 'ERROR [GlobalLogFollower] 2024-11-24 17:39:37,044 LocalLog.java:542 - Could not process the entry\njava.lang.NullPointerException: Cannot invoke "org.apache.cassandra.schema.TableMetadata.partitionKeyColumns()" because "this.viewMetadata" is null\n\tat org.apache.cassandra.db.view.ViewUpdateGenerator.(ViewUpdateGenerator.java:99)\n\tat org.apache.cassandra.db.view.TableViews.generateViewUpdates(TableViews.java:229)\n\tat org.apache.cassandra.db.view.ViewBuilderTask.buildKey(ViewBuilderTask.java:114)\n\tat org.apache.cassandra.db.view.ViewBuilderTask.call(ViewBuilderTask.java:154)\n\tat org.apache.cassandra.db.compaction.CompactionManager.lambda$submitViewBuilder$11(CompactionManager.java:2020)\n\tat org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)\n\tat org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:840)']

Failed: Timeout >900.0s-self = 

    def test_functional(self):
        cluster = self.cluster
        cluster.populate(3)
        node1 = cluster.nodelist()[0]
    
        for keycache_size in (0, 10):
            for rowcache_size in (0, 10):
                cluster.stop()
                logger.debug("Testing with keycache size of %d MB, rowcache size of %d MB " %
                      (keycache_size, rowcache_size))
                keyspace_name = 'ks_%d_%d' % (keycache_size, rowcache_size)
    
                # make the caches save every five seconds
                cluster.set_configuration_options(values={
                    'key_cache_size_in_mb': keycache_size,
                    'row_cache_size_in_mb': rowcache_size,
                    'row_cache_save_period': 5,
                    'key_cache_save_period': 5,
                })
    
                cluster.start()
                session = self.patient_cql_connection(node1)
                create_ks(session, keyspace_name, rf=3)
    
                session.set_keyspace(keyspace_name)
                create_cf_simple(session, 'test', "CREATE TABLE test (k int PRIMARY KEY, v1 int, v2 int)")
                create_cf_simple(session, 'test_clustering',
                                 "CREATE TABLE test_clustering (k int, v1 int, v2 int, PRIMARY KEY (k, v1))")
                create_cf_simple(session, 'test_counter', "CREATE TABLE test_counter (k int PRIMARY KEY, v1 counter)")
                create_cf_simple(session, 'test_counter_clustering',
                                 "CREATE TABLE test_counter_clustering (k int, v1 int, v2 counter, PRIMARY KEY (k, v1))")
    
                # insert 100 rows into each table
                for cf in ('test', 'test_clustering'):
                    execute_concurrent_with_args(
                        session, session.prepare("INSERT INTO %s (k, v1, v2) VALUES (?, ?, ?)" % (cf,)),
                        [(i, i, i) for i in range(100)])
    
                execute_concurrent_with_args(
                    session, session.prepare("UPDATE test_counter SET v1 = v1 + ? WHERE k = ?"),
                    [(i, i) for i in range(100)],
                    concurrency=2)
    
                execute_concurrent_with_args(
                    session, session.prepare("UPDATE test_counter_clustering SET v2 = v2 + ? WHERE k = ? AND v1 = ?"),
                    [(i, i, i) for i in range(100)],
                    concurrency=2)
    
                # flush everything to get it into sstables
                for node in cluster.nodelist():
                    node.flush()
    
                # update the first 10 rows in every table
                # on non-counter tables, delete the first (remaining) row each round
                num_updates = 10
                for validation_round in range(3):
                    session.execute("DELETE FROM test WHERE k = %s", (validation_round,))
                    execute_concurrent_with_args(
                        session, session.prepare("UPDATE test SET v1 = ?, v2 = ? WHERE k = ?"),
                        [(i, validation_round, i) for i in range(validation_round + 1, num_updates)])
    
                    session.execute("DELETE FROM test_clustering WHERE k = %s AND v1 = %s", (validation_round, validation_round))
                    execute_concurrent_with_args(
                        session, session.prepare("UPDATE test_clustering SET v2 = ? WHERE k = ? AND v1 = ?"),
                        [(validation_round, i, i) for i in range(validation_round + 1, num_updates)])
    
                    execute_concurrent_with_args(
                        session, session.prepare("UPDATE test_counter SET v1 = v1 + ? WHERE k = ?"),
                        [(1, i) for i in range(num_updates)],
                        concurrency=2)
    
                    execute_concurrent_with_args(
                        session, session.prepare("UPDATE test_counter_clustering SET v2 = v2 + ? WHERE k = ? AND v1 = ?"),
                        [(1, i, i) for i in range(num_updates)],
                        concurrency=2)
    
                    self._validate_values(session, num_updates, validation_round)
    
                session.shutdown()
    
                # let the data be written to the row/key caches.
                logger.debug("Letting caches be saved to disk")
                time.sleep(10)
                logger.debug("Stopping cluster")
>               cluster.stop()

global_row_key_cache_test.py:104: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/cluster.py:578: in stop
    if not node.stop(wait=wait, signal_event=signal_event, **kwargs):
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = , wait = True
wait_other_notice = False, signal_event = , kwargs = {}
still_running = True, wait_time_sec = 4, i = 2

    def stop(self, wait=True, wait_other_notice=False, signal_event=signal.SIGTERM, **kwargs):
        """
        Stop the node.
          - wait: if True (the default), wait for the Cassandra process to be
            really dead. Otherwise return after having sent the kill signal.
          - wait_other_notice: return only when the other live nodes of the
            cluster have marked this node has dead.
          - signal_event: Signal event to send to Cassandra; default is to
            let Cassandra clean up and shut down properly (SIGTERM [15])
          - Optional:
             + gently: Let Cassandra clean up and shut down properly; unless
                       false perform a 'kill -9' which shuts down faster.
        """
        if self.is_running():
            if wait_other_notice:
                marks = [(node, node.mark_log()) for node in list(self.cluster.nodes.values()) if node.is_live() and node is not self]
    
            if common.is_win():
                # Just taskkill the instance, don't bother trying to shut it down gracefully.
                # Node recovery should prevent data loss from hard shutdown.
                # We have recurring issues with nodes not stopping / releasing files in the CI
                # environment so it makes more sense just to murder it hard since there's
                # really little downside.
    
                # We want the node to flush its data before shutdown as some tests rely on small writes being present.
                # The default Periodic sync at 10 ms may not have flushed data yet, causing tests to fail.
                # This is not a hard requirement, however, so we swallow any exceptions this may throw and kill anyway.
                if signal_event is signal.SIGTERM:
                    try:
                        self.flush()
                    except:
                        common.warning("Failed to flush node: {0} on shutdown.".format(self.name))
                        pass
    
                os.system("taskkill /F /PID " + str(self.pid))
                if self._find_pid_on_windows():
                    common.warning("Failed to terminate node: {0} with pid: {1}".format(self.name, self.pid))
            else:
                # Determine if the signal event should be updated to keep API compatibility
                if 'gently' in kwargs and kwargs['gently'] is False:
                    signal_event = signal.SIGKILL
    
                os.kill(self.pid, signal_event)
    
            if wait_other_notice:
                for node, mark in marks:
                    node.watch_log_for_death(self, from_mark=mark)
            else:
                time.sleep(.1)
    
            still_running = self.is_running()
            if still_running and wait:
                wait_time_sec = 1
                for i in xrange(0, 7):
                    # we'll double the wait time each try and cassandra should
                    # not take more than 1 minute to shutdown
>                   time.sleep(wait_time_sec)
E                   Failed: Timeout >900.0s

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:997: Failed

ccmlib.node.TimeoutError: 24 Nov 2024 19:50:38 [node1] after 90.18/90 seconds Missing: ['Starting listening for CQL clients'] not found in system.log:
 Head: INFO  [main] 2024-11-24 19:49:13,426 YamlConfigura
 Tail: ...4 19:50:36,616 IndexSummaryManager.java:105 - Initializing index summary manager with a memory pool size of 51 MB and a resize interval of 60 minutes-self = 

    def test_bootstrap_multidc(self):
        # try and add a new node
        # multi dc, 2 nodes in each dc
        self.prepare()
        cluster = self.cluster
    
        if cluster.version() >= '5.0':
            cluster.set_configuration_options({'user_defined_functions_threads_enabled': 'true',
                                               'scripted_user_defined_functions_enabled': 'false'})
        elif cluster.version() >= '3.0':
            cluster.set_configuration_options({'enable_user_defined_functions': 'true',
                                               'enable_scripted_user_defined_functions': 'true'})
        elif cluster.version() >= '2.2':
            cluster.set_configuration_options({'enable_user_defined_functions': 'true'})
    
        cluster.populate([2, 2])
        [node.start(use_jna=True, wait_for_binary_proto=True) for node in self.cluster.nodelist()]
        self._multidc_schema_create()
>       self.upgrade_scenario(populate=False, create_schema=False, after_upgrade_call=(self._bootstrap_new_node_multidc,))

upgrade_tests/upgrade_through_versions_test.py:865: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
upgrade_tests/upgrade_through_versions_test.py:493: in upgrade_scenario
    self.upgrade_to_version(version_meta, internode_ssl=internode_ssl)
upgrade_tests/upgrade_through_versions_test.py:581: in upgrade_to_version
    node.start(wait_other_notice=400, wait_for_binary_proto=True,
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:928: in start
    self.wait_for_binary_interface(from_mark=self.mark)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:711: in wait_for_binary_interface
    self.watch_log_for("Starting listening for CQL clients", **kwargs)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:610: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1732477747.9405704, timeout = 90
msg = "Missing: ['Starting listening for CQL clients'] not found in system.log:\n Head: INFO  [main] 2024-11-24 19:49:13,426...r.java:105 - Initializing index summary manager with a memory pool size of 51 MB and a resize interval of 60 minutes\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 24 Nov 2024 19:50:38 [node1] after 90.18/90 seconds Missing: ['Starting listening for CQL clients'] not found in system.log:
E            Head: INFO  [main] 2024-11-24 19:49:13,426 YamlConfigura
E            Tail: ...4 19:50:36,616 IndexSummaryManager.java:105 - Initializing index summary manager with a memory pool size of 51 MB and a resize interval of 60 minutes

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

cassandra.cluster.NoHostAvailable: ('Unable to connect to any servers', {'127.0.0.1:9042': ConnectionRefusedError(111, "Tried connecting to [('127.0.0.1', 9042)]. Last error: Connection refused")})-self = 

    def test_commitlog_replay_on_startup(self):
        """
        Test commit log replay
        """
        node1 = self.node1
        node1.set_batch_commitlog(enabled=True, use_batch_window = self.cluster.version() < '5.0')
        node1.start()
    
        logger.debug("Insert data")
>       session = self.patient_cql_connection(node1)

commitlog_test.py:242: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
dtest_setup.py:284: in patient_cql_connection
    session = retry_till_success(
dtest_setup.py:39: in retry_till_success
    return fun(*args, **kwargs)
dtest_setup.py:222: in cql_connection
    return self._create_session(node, keyspace, user, password, compression,
dtest_setup.py:262: in _create_session
    session = cluster.connect(wait_for_all_pools=True)
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:1690: in connect
    self.control_connection.connect()
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:3488: in connect
    self._set_new_connection(self._reconnect_internal())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def _reconnect_internal(self):
        """
        Tries to connect to each host in the query plan until one succeeds
        or every attempt fails. If successful, a new Connection will be
        returned.  Otherwise, :exc:`NoHostAvailable` will be raised
        with an "errors" arg that is a dict mapping host addresses
        to the exception that was raised when an attempt was made to open
        a connection to that host.
        """
        errors = {}
        lbp = (
            self._cluster.load_balancing_policy
            if self._cluster._config_mode == _ConfigMode.LEGACY else
            self._cluster._default_load_balancing_policy
        )
    
        for host in lbp.make_query_plan():
            try:
                return self._try_connect(host)
            except ConnectionException as exc:
                errors[str(host.endpoint)] = exc
                log.warning("[control connection] Error connecting to %s:", host, exc_info=True)
                self._cluster.signal_connection_failure(host, exc, is_host_addition=False)
            except Exception as exc:
                errors[str(host.endpoint)] = exc
                log.warning("[control connection] Error connecting to %s:", host, exc_info=True)
            if self._is_shutdown:
                raise DriverException("[control connection] Reconnection in progress during shutdown")
    
>       raise NoHostAvailable("Unable to connect to any servers", errors)
E       cassandra.cluster.NoHostAvailable: ('Unable to connect to any servers', {'127.0.0.1:9042': ConnectionRefusedError(111, "Tried connecting to [('127.0.0.1', 9042)]. Last error: Connection refused")})

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:3533: NoHostAvailable

junit.framework.AssertionFailedError
	at org.apache.cassandra.distributed.test.guardrails.GuardrailTester.assertNotWarnsOnSSTableWrite(GuardrailTester.java:127)
	at org.apache.cassandra.distributed.test.guardrails.GuardrailTester.lambda$assertNotWarnsOnSSTableWrite$0(GuardrailTester.java:118)
	at java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1625)
	at java.base/java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:762)
	at org.apache.cassandra.distributed.test.guardrails.GuardrailTester.assertNotWarnsOnSSTableWrite(GuardrailTester.java:118)
	at org.apache.cassandra.distributed.test.guardrails.GuardrailTester.assertNotWarnedOnFlush(GuardrailTester.java:108)
	at org.apache.cassandra.distributed.test.guardrails.GuardrailCollectionSizeOnSSTableWriteTest.testListSizeWithUpdates(GuardrailCollectionSizeOnSSTableWriteTest.java:221)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

Property error detected:
Seed = 3443888726639793107
Examples = 1
Pure = true
Error: Uncaught exceptions were thrown during test
Steps: 20
Values:
	State: Yaml Config:
		---
		commitlog_sync: batch
		commitlog_disk_access_mode: legacy
		memtable_allocation_type: offheap_buffers
		memtable:
		  configurations:
		    TrieMemtable_1:
		      inherits: TrieMemtable_0
		    TrieMemtable_base:
		      class_name: TrieMemtable
		      parameters:
		        shards: 59
		    TrieMemtable_0:
		      inherits: TrieMemtable_base
		    TrieMemtable_2:
		      inherits: TrieMemtable_1
		    SkipListMemtable_0:
		      inherits: SkipListMemtable_base
		    SkipListMemtable_1:
		      inherits: SkipListMemtable_0
		    SkipListMemtable_base:
		      class_name: SkipListMemtable
		    default:
		      inherits: ShardedSkipListMemtable
		    ShardedSkipListMemtable_base:
		      class_name: ShardedSkipListMemtable
		      parameters:
		        serialize_writes: true
		    ShardedSkipListMemtable:
		      inherits: ShardedSkipListMemtable_0
		    TrieMemtable:
		      inherits: TrieMemtable_2
		    ShardedSkipListMemtable_0:
		      inherits: ShardedSkipListMemtable_base
		    SkipListMemtable_2:
		      inherits: SkipListMemtable_1
		    SkipListMemtable:
		      inherits: SkipListMemtable_2
		sstable:
		  selected_format: bti
		disk_access_mode: mmap
		
		Topology:
		
			Node1: status=Up, tokens=[-2127851176]
			Node2: status=Up, tokens=[743515505]
		CMS Voting Group: [1, 2]: org.apache.cassandra.fuzz.topology.HarryTopologyMixupTest.HarryState
	History:
-junit.framework.AssertionFailedError: Property error detected:
Seed = 3443888726639793107
Examples = 1
Pure = true
Error: Uncaught exceptions were thrown during test
Steps: 20
Values:
	State: Yaml Config:
		---
		commitlog_sync: batch
		commitlog_disk_access_mode: legacy
		memtable_allocation_type: offheap_buffers
		memtable:
		  configurations:
		    TrieMemtable_1:
		      inherits: TrieMemtable_0
		    TrieMemtable_base:
		      class_name: TrieMemtable
		      parameters:
		        shards: 59
		    TrieMemtable_0:
		      inherits: TrieMemtable_base
		    TrieMemtable_2:
		      inherits: TrieMemtable_1
		    SkipListMemtable_0:
		      inherits: SkipListMemtable_base
		    SkipListMemtable_1:
		      inherits: SkipListMemtable_0
		    SkipListMemtable_base:
		      class_name: SkipListMemtable
		    default:
		      inherits: ShardedSkipListMemtable
		    ShardedSkipListMemtable_base:
		      class_name: ShardedSkipListMemtable
		      parameters:
		        serialize_writes: true
		    ShardedSkipListMemtable:
		      inherits: ShardedSkipListMemtable_0
		    TrieMemtable:
		      inherits: TrieMemtable_2
		    ShardedSkipListMemtable_0:
		      inherits: ShardedSkipListMemtable_base
		    SkipListMemtable_2:
		      inherits: SkipListMemtable_1
		    SkipListMemtable:
		      inherits: SkipListMemtable_2
		sstable:
		  selected_format: bti
		disk_access_mode: mmap
		
		Topology:
		
			Node1: status=Up, tokens=[-2127851176]
			Node2: status=Up, tokens=[743515505]
		CMS Voting Group: [1, 2]: org.apache.cassandra.fuzz.topology.HarryTopologyMixupTest.HarryState
	History:

	at accord.utils.Property$StatefulBuilder.check(Property.java:487)
	at org.apache.cassandra.fuzz.topology.TopologyMixupTestBase.test(TopologyMixupTestBase.java:288)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
Caused by: org.apache.cassandra.distributed.shared.ShutdownException: Uncaught exceptions were thrown during test
	at org.apache.cassandra.distributed.impl.AbstractCluster.checkAndResetUncaughtExceptions(AbstractCluster.java:1147)
	at org.apache.cassandra.fuzz.topology.TopologyMixupTestBase$State.lambda$new$3(TopologyMixupTestBase.java:452)
	at java.base/java.util.concurrent.CopyOnWriteArrayList.forEach(CopyOnWriteArrayList.java:807)
	at org.apache.cassandra.fuzz.topology.TopologyMixupTestBase.lambda$test$16(TopologyMixupTestBase.java:289)
	at accord.utils.Property$CommandsBuilder$2.commands(Property.java:886)
	at accord.utils.Property$StatefulBuilder.check(Property.java:446)
	Suppressed: java.lang.RuntimeException: java.io.EOFException
		at org.apache.cassandra.tcm.log.LogReader.getLogState(LogReader.java:116)
		at org.apache.cassandra.tcm.log.LogReader.getLogState(LogReader.java:64)
		at org.apache.cassandra.tcm.FetchPeerLog$Handler.doVerb(FetchPeerLog.java:85)
		at org.apache.cassandra.net.InboundSink.lambda$new$0(InboundSink.java:102)
		at org.apache.cassandra.net.InboundSink$Filtered.accept(InboundSink.java:70)
		at org.apache.cassandra.net.InboundSink$Filtered.accept(InboundSink.java:56)
		at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:122)
		at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:51)
		at org.apache.cassandra.net.InboundMessageHandler$ProcessMessage.run(InboundMessageHandler.java:448)
		at org.apache.cassandra.concurrent.ExecutionFailure$1.run(ExecutionFailure.java:133)
		at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
		at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
		at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
		at java.base/java.lang.Thread.run(Thread.java:829)
	Caused by: java.io.EOFException
		at org.apache.cassandra.io.util.RebufferingInputStream.readByte(RebufferingInputStream.java:180)
		at org.apache.cassandra.io.util.RebufferingInputStream.readBoolean(RebufferingInputStream.java:170)
		at org.apache.cassandra.tcm.transformations.cms.PreInitialize$Serializer.deserialize(PreInitialize.java:116)
		at org.apache.cassandra.tcm.transformations.cms.PreInitialize$Serializer.deserialize(PreInitialize.java:102)
		at org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer.deserialize(VerboseMetadataSerializer.java:43)
		at org.apache.cassandra.tcm.Transformation$Kind.fromVersionedBytes(Transformation.java:269)
		at org.apache.cassandra.tcm.log.SystemKeyspaceStorage.toEntryHolder(SystemKeyspaceStorage.java:153)
		at org.apache.cassandra.tcm.log.SystemKeyspaceStorage.getEntries(SystemKeyspaceStorage.java:130)
		at org.apache.cassandra.tcm.log.LogReader.getLogState(LogReader.java:79)

Timeout occurred. Please note the time in the report does not reflect the time until the timeout.-junit.framework.AssertionFailedError: Timeout occurred. Please note the time in the report does not reflect the time until the timeout.
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1394)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1394)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1394)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at org.apache.cassandra.anttasks.TestHelper.execute(TestHelper.java:53)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1394)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)