CI Summary

AssertionError: too many reads hit the degraded node: 3
assert 3 <= 1-self = 

    @pytest.mark.resource_intensive
    @since('3.10')
    def test_multidatacenter_local_quorum(self):
        '''
        @jira_ticket CASSANDRA-13074
    
        If we do only local datacenters reads in a multidatacenter DES setup,
        DES should take effect and route around a degraded node
        '''
    
        def no_cross_dc(scores, cross_dc_nodes):
            return all('/' + k.address() not in scores for k in cross_dc_nodes)
    
        def snitchable(scores_before, scores_after, needed_nodes):
            return all('/' + k.address() in scores_before and '/' + k.address()
                       in scores_after for k in needed_nodes)
    
        cluster = self.cluster
        cluster.populate([3, 3])
        coordinator_node, healthy_node, degraded_node, node4, node5, node6 = cluster.nodelist()
        # increase DES reset/update interval so we clear any cross-DC startup reads faster
        cluster.set_configuration_options(values={'dynamic_snitch_reset_interval_in_ms': 10000,
                                                  'dynamic_snitch_update_interval_in_ms': 50,
                                                  'phi_convict_threshold': 12})
        # Delay reads on the degraded node by 50 milliseconds
        degraded_node.start(jvm_args=['-Dcassandra.test.read_iteration_delay_ms=50',
                                      '-Dcassandra.allow_unsafe_join=true'])
        cluster.start()
    
        des = make_mbean('db', type='DynamicEndpointSnitch')
        read_stage = make_mbean('metrics', type='ThreadPools', path='request',
                                scope='ReadStage', name='CompletedTasks')
        session = self.patient_exclusive_cql_connection(coordinator_node)
        session.execute("CREATE KEYSPACE snitchtestks WITH replication = {'class': 'NetworkTopologyStrategy', 'dc1': 3, 'dc2': 3}")
        session.execute("CREATE TABLE snitchtestks.tbl1 (key int PRIMARY KEY) WITH speculative_retry = 'NONE'")
        read_stmt = session.prepare("SELECT * FROM snitchtestks.tbl1 where key = ?")
        read_stmt.consistency_level = ConsistencyLevel.LOCAL_QUORUM
        insert_stmt = session.prepare("INSERT INTO snitchtestks.tbl1 (key) VALUES (?)")
        insert_stmt.consistency_level = ConsistencyLevel.ALL
        with JolokiaAgent(coordinator_node) as jmx:
            with JolokiaAgent(degraded_node) as bad_jmx:
                for x in range(0, 150):
                    session.execute(insert_stmt, [x])
    
                cleared = False
                # Wait for a snitch reset in case any earlier
                # startup process populated cross-DC read timings
                while not cleared:
                    scores = jmx.read_attribute(des, 'Scores')
                    cleared = ('/127.0.0.1' in scores and (len(scores) == 1)) or not scores
    
                snitchable_count = 0
    
                for x in range(0, 150):
                    degraded_reads_before = bad_jmx.read_attribute(read_stage, 'Value')
                    scores_before = jmx.read_attribute(des, 'Scores')
                    assert no_cross_dc(scores_before, [node4, node5, node6]), "Cross DC scores were present: " + str(scores_before)
                    future = session.execute_async(read_stmt, [x])
                    future.result()
                    scores_after = jmx.read_attribute(des, 'Scores')
                    assert no_cross_dc(scores_after, [node4, node5, node6]), "Cross DC scores were present: " + str(scores_after)
    
                    if snitchable(scores_before, scores_after,
                                  [coordinator_node, healthy_node, degraded_node]):
                        snitchable_count = snitchable_count + 1
                        # If the DES correctly routed the read around the degraded node,
                        # it shouldn't have another completed read request in metrics,
                        # unless there was one right after a reset.
                        degraded_reads = bad_jmx.read_attribute(read_stage, 'Value')
                        difference = abs(degraded_reads_before - degraded_reads)
>                       assert difference <= 1, "too many reads hit the degraded node: %s" % difference
E                       AssertionError: too many reads hit the degraded node: 3
E                       assert 3 <= 1

snitch_test.py:235: AssertionError

AssertionError: Commitlog was written with durable writes disabled
assert 163840 == 90112
  +163840
  -90112-self = 

    @pytest.mark.timeout(60*30)
    def test_change_durable_writes(self):
        """
        @jira_ticket CASSANDRA-9560
    
        Test that changes to the DURABLE_WRITES option on keyspaces is
        respected in subsequent writes.
    
        This test starts by writing a dataset to a cluster and asserting that
        the commitlogs have been written to. The subsequent test depends on
        the assumption that this dataset triggers an fsync.
    
        After checking this assumption, the test destroys the cluster and
        creates a fresh one. Then it tests that DURABLE_WRITES is respected by:
    
        - creating a keyspace with DURABLE_WRITES set to false,
        - using ALTER KEYSPACE to set its DURABLE_WRITES option to true,
        - writing a dataset to this keyspace that is known to trigger a commitlog fsync,
        - asserting that the commitlog has grown in size since the data was written.
        """
        cluster = self.cluster
        cluster.set_batch_commitlog(enabled=True, use_batch_window = cluster.version() < '5.0')
        cluster.set_configuration_options(values={'commitlog_segment_size_in_mb': 1})
    
        cluster.populate(1).start()
        durable_node = cluster.nodelist()[0]
    
        durable_init_size = commitlog_size(durable_node)
        durable_session = self.patient_exclusive_cql_connection(durable_node)
    
        # test assumption that write_to_trigger_fsync actually triggers a commitlog fsync
        durable_session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1} "
                                "AND DURABLE_WRITES = true")
        durable_session.execute('CREATE TABLE ks.tab (key int PRIMARY KEY, a int, b int, c int)')
        logger.debug('commitlog size diff = ' + str(commitlog_size(durable_node) - durable_init_size))
        write_to_trigger_fsync(durable_session, 'ks', 'tab')
        logger.debug('commitlog size diff = ' + str(commitlog_size(durable_node) - durable_init_size))
    
        assert commitlog_size(durable_node) > durable_init_size, \
            "This test will not work in this environment; write_to_trigger_fsync does not trigger fsync."
    
        durable_session.shutdown()
        cluster.stop()
        cluster.clear()
    
        cluster.set_batch_commitlog(enabled=True, use_batch_window = cluster.version() < '5.0')
        cluster.set_configuration_options(values={'commitlog_segment_size_in_mb': 1})
        cluster.start()
        node = cluster.nodelist()[0]
        session = self.patient_exclusive_cql_connection(node)
    
        # set up a keyspace without durable writes, then alter it to use them
        session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1} "
                        "AND DURABLE_WRITES = false")
        session.execute('CREATE TABLE ks.tab (key int PRIMARY KEY, a int, b int, c int)')
        init_size = commitlog_size(node)
        write_to_trigger_fsync(session, 'ks', 'tab')
>       assert commitlog_size(node) == init_size, "Commitlog was written with durable writes disabled"
E       AssertionError: Commitlog was written with durable writes disabled
E       assert 163840 == 90112
E         +163840
E         -90112

configuration_test.py:104: AssertionError

AssertionError: values not within 5.00% of the max: (649.14, 612.08) ()-self = 

    @pytest.mark.no_vnodes
    def test_movement(self):
        cluster = self.cluster
    
        # Create an unbalanced ring
        cluster.populate(3, tokens=[0, 2**48, 2**62]).start()
        node1, node2, node3 = cluster.nodelist()
    
        session = self.patient_cql_connection(node1)
        create_ks(session, 'ks', 1)
        create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})
    
        insert_c1c2(session, n=30000, consistency=ConsistencyLevel.ONE)
    
        cluster.flush()
    
        # Move nodes to balance the cluster
        def move_node(node, token):
            mark = node.mark_log()
            node.move(token)  # can't assume 0 is balanced with m3p
            node.watch_log_for('{} state jump to NORMAL'.format(node.address_for_current_version()), from_mark=mark, timeout=180)
            time.sleep(3)
    
        balancing_tokens = cluster.balanced_tokens(3)
    
        move_node(node1, balancing_tokens[0])
        move_node(node2, balancing_tokens[1])
        move_node(node3, balancing_tokens[2])
    
        time.sleep(1)
        cluster.cleanup()
        for node in cluster.nodelist():
            # after moving nodes we need to relocate any tokens in the wrong places, and after doing that
            # we might have overlapping tokens on the disks, so run a major compaction to get balance even
            if cluster.version() >= '3.2':
                node.nodetool("relocatesstables")
            node.nodetool("compact")
    
        # Check we can get all the keys
        for n in range(0, 30000):
            query_c1c2(session, n, ConsistencyLevel.ONE)
    
        # Now the load should be basically even
        sizes = [node.data_size() for node in [node1, node2, node3]]
    
>       assert_almost_equal(sizes[0], sizes[1], error=0.05)

topology_test.py:322: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args = (649.14, 612.08), kwargs = {'error': 0.05}, error = 0.05, vmax = 649.14
vmin = 612.08, error_message = ''

    def assert_almost_equal(*args, **kwargs):
        """
        Assert variable number of arguments all fall within a margin of error.
        @params *args variable number of numerical arguments to check
        @params error Optional margin of error. Default 0.16
        @params error_message Optional error message to print. Default ''
    
        Examples:
        assert_almost_equal(sizes[2], init_size)
        assert_almost_equal(ttl_session1, ttl_session2[0][0], error=0.005)
        """
        error = kwargs['error'] if 'error' in kwargs else 0.16
        vmax = max(args)
        vmin = min(args)
        error_message = '' if 'error_message' not in kwargs else kwargs['error_message']
>       assert vmin > vmax * (1.0 - error) or vmin == vmax, \
            "values not within {:.2f}% of the max: {} ({})".format(error * 100, args, error_message)
E       AssertionError: values not within 5.00% of the max: (649.14, 612.08) ()

tools/assertions.py:205: AssertionError

Expect batch commitlog shutdown immediately, but took 1005-junit.framework.AssertionFailedError: Expect batch commitlog shutdown immediately, but took 1005
	at org.apache.cassandra.db.commitlog.BatchCommitLogTest.testBatchCLShutDownImmediately(BatchCommitLogTest.java:78)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

junit.framework.AssertionFailedError
	at org.apache.cassandra.db.compaction.CompactionStrategyManagerPendingRepairTest.testFinalizedAndCompactionRace(CompactionStrategyManagerPendingRepairTest.java:351)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

junit.framework.AssertionFailedError
	at org.apache.cassandra.db.compaction.CompactionStrategyManagerPendingRepairTest.testFinalizedAndCompactionRace(CompactionStrategyManagerPendingRepairTest.java:351)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

Unterminated threads detected; active threads:
JNA Cleaner:
	java.base@17.0.12/java.lang.Object.wait(Native Method)
	java.base@17.0.12/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.12/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//com.sun.jna.internal.Cleaner$1.run(Cleaner.java:58)
node2_isolatedExecutor:3:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:7:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_RequestResponseStage-2:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:376)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.awaitUntil(WaitQueue.java:303)
	org.apache.cassandra.utils.concurrent.Awaitable$AsyncAwaitable.awaitUntil(Awaitable.java:314)
	org.apache.cassandra.utils.concurrent.AsyncFuture.awaitUntil(AsyncFuture.java:171)
	org.apache.cassandra.utils.concurrent.Awaitable$Defaults.await(Awaitable.java:114)
	org.apache.cassandra.utils.concurrent.AbstractFuture.await(AbstractFuture.java:482)
	org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:252)
	org.apache.cassandra.tcm.RemoteProcessor.fetchLogAndWait(RemoteProcessor.java:141)
	org.apache.cassandra.tcm.ClusterMetadataService$SwitchableProcessor.fetchLogAndWait(ClusterMetadataService.java:885)
	org.apache.cassandra.tcm.ClusterMetadataService.fetchLogFromCMS(ClusterMetadataService.java:656)
	org.apache.cassandra.tcm.ClusterMetadataService.fetchLogFromPeerOrCMS(ClusterMetadataService.java:766)
	org.apache.cassandra.net.ResponseVerbHandler.maybeFetchLogs(ResponseVerbHandler.java:106)
	org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:73)
	org.apache.cassandra.net.InboundSink.lambda$new$0(InboundSink.java:102)
	org.apache.cassandra.net.InboundSink$$Lambda$10224/0x00007f71c60f1d88.accept(Unknown Source)
	org.apache.cassandra.net.InboundSink$Filtered.accept(InboundSink.java:70)
	org.apache.cassandra.net.InboundSink$Filtered.accept(InboundSink.java:56)
	org.apache.cassandra.net.InboundSink.accept(InboundSink.java:122)
	org.apache.cassandra.net.InboundSink.accept(InboundSink.java:51)
	org.apache.cassandra.net.InboundMessageHandler$ProcessMessage.run(InboundMessageHandler.java:448)
	org.apache.cassandra.concurrent.ExecutionFailure$1.run(ExecutionFailure.java:133)
	org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:143)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:8:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:376)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.awaitUntil(WaitQueue.java:303)
	org.apache.cassandra.utils.concurrent.Awaitable$AsyncAwaitable.awaitUntil(Awaitable.java:314)
	org.apache.cassandra.utils.concurrent.Awaitable$AsyncAwaitable.awaitUntil(Awaitable.java:346)
	org.apache.cassandra.utils.concurrent.Awaitable$Defaults.await(Awaitable.java:114)
	org.apache.cassandra.utils.concurrent.Awaitable$AbstractAwaitable.await(Awaitable.java:210)
	org.apache.cassandra.concurrent.SEPExecutor.awaitTermination(SEPExecutor.java:331)
	org.apache.cassandra.utils.ExecutorUtils.awaitTerminationUntil(ExecutorUtils.java:110)
	org.apache.cassandra.utils.ExecutorUtils.awaitTermination(ExecutorUtils.java:100)
	org.apache.cassandra.concurrent.Stage.shutdownAndWait(Stage.java:195)
	org.apache.cassandra.distributed.impl.Instance.lambda$shutdown$42(Instance.java:959)
	org.apache.cassandra.distributed.impl.Instance$$Lambda$14241/0x00007f71c50d2c30.run(Unknown Source)
	org.apache.cassandra.distributed.impl.Instance.lambda$parallelRun$53(Instance.java:1212)
	org.apache.cassandra.distributed.impl.Instance$$Lambda$14026/0x00007f71c4fed1f8.call(Unknown Source)
	org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)
	org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:1:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.await(WaitQueue.java:290)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.await(WaitQueue.java:283)
	org.apache.cassandra.utils.concurrent.Awaitable$AsyncAwaitable.await(Awaitable.java:306)
	org.apache.cassandra.utils.concurrent.AsyncFuture.await(AsyncFuture.java:165)
	org.apache.cassandra.utils.concurrent.AsyncFuture.await(AsyncFuture.java:46)
	org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:245)
	org.apache.cassandra.distributed.impl.Instance.parallelRun(Instance.java:1225)
	org.apache.cassandra.distributed.impl.Instance.lambda$shutdown$47(Instance.java:957)
	org.apache.cassandra.distributed.impl.Instance$$Lambda$14016/0x00007f71c4fec758.accept(Unknown Source)
	org.apache.cassandra.distributed.impl.IsolatedExecutor.lambda$async$10(IsolatedExecutor.java:156)
	org.apache.cassandra.distributed.impl.IsolatedExecutor$$Lambda$8684/0x00007f71c587a808.run(Unknown Source)
	org.apache.cassandra.concurrent.FutureTask$2.call(FutureTask.java:124)
	org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)
	org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_COMMIT-LOG-ALLOCATOR:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.await(WaitQueue.java:290)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.await(WaitQueue.java:283)
	org.apache.cassandra.utils.concurrent.WaitQueue.waitOnCondition(WaitQueue.java:434)
	org.apache.cassandra.db.commitlog.AbstractCommitLogSegmentManager$AllocatorRunnable.run(AbstractCommitLogSegmentManager.java:225)
	org.apache.cassandra.concurrent.InfiniteLoopExecutor.loop(InfiniteLoopExecutor.java:121)
	org.apache.cassandra.concurrent.InfiniteLoopExecutor$$Lambda$8808/0x00007f71c5cfc000.run(Unknown Source)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:4:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_PERIODIC-COMMIT-LOG-SYNCER:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:717)
	java.base@17.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1074)
	java.base@17.0.12/java.util.concurrent.Semaphore.tryAcquire(Semaphore.java:592)
	org.apache.cassandra.utils.concurrent.Semaphore$Standard.tryAcquireUntil(Semaphore.java:142)
	org.apache.cassandra.db.commitlog.AbstractCommitLogService$SyncRunnable.run(AbstractCommitLogService.java:216)
	org.apache.cassandra.concurrent.InfiniteLoopExecutor.loop(InfiniteLoopExecutor.java:121)
	org.apache.cassandra.concurrent.InfiniteLoopExecutor$$Lambda$8808/0x00007f71c5cfc000.run(Unknown Source)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_RequestResponseStage-1:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:376)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.awaitUntil(WaitQueue.java:303)
	org.apache.cassandra.utils.concurrent.Awaitable$AsyncAwaitable.awaitUntil(Awaitable.java:314)
	org.apache.cassandra.utils.concurrent.AsyncFuture.awaitUntil(AsyncFuture.java:171)
	org.apache.cassandra.utils.concurrent.Awaitable$Defaults.await(Awaitable.java:114)
	org.apache.cassandra.utils.concurrent.AbstractFuture.await(AbstractFuture.java:482)
	org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:252)
	org.apache.cassandra.tcm.RemoteProcessor.fetchLogAndWait(RemoteProcessor.java:141)
	org.apache.cassandra.tcm.ClusterMetadataService$SwitchableProcessor.fetchLogAndWait(ClusterMetadataService.java:885)
	org.apache.cassandra.tcm.ClusterMetadataService.fetchLogFromCMS(ClusterMetadataService.java:656)
	org.apache.cassandra.tcm.ClusterMetadataService.fetchLogFromPeerOrCMS(ClusterMetadataService.java:766)
	org.apache.cassandra.net.ResponseVerbHandler.maybeFetchLogs(ResponseVerbHandler.java:106)
	org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:73)
	org.apache.cassandra.net.InboundSink.lambda$new$0(InboundSink.java:102)
	org.apache.cassandra.net.InboundSink$$Lambda$10224/0x00007f71c60f1d88.accept(Unknown Source)
	org.apache.cassandra.net.InboundSink$Filtered.accept(InboundSink.java:70)
	org.apache.cassandra.net.InboundSink$Filtered.accept(InboundSink.java:56)
	org.apache.cassandra.net.InboundSink.accept(InboundSink.java:122)
	org.apache.cassandra.net.InboundSink.accept(InboundSink.java:51)
	org.apache.cassandra.net.InboundMessageHandler$ProcessMessage.run(InboundMessageHandler.java:448)
	org.apache.cassandra.concurrent.ExecutionFailure$1.run(ExecutionFailure.java:133)
	org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:143)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:5:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:2:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.await(WaitQueue.java:290)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.await(WaitQueue.java:283)
	org.apache.cassandra.utils.concurrent.Awaitable$AsyncAwaitable.await(Awaitable.java:306)
	org.apache.cassandra.utils.concurrent.AsyncFuture.await(AsyncFuture.java:165)
	org.apache.cassandra.utils.concurrent.AsyncFuture.await(AsyncFuture.java:46)
	org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:245)
	org.apache.cassandra.distributed.impl.Instance.lambda$shutdown$48(Instance.java:994)
	org.apache.cassandra.distributed.impl.Instance$$Lambda$14017/0x00007f71c4fec990.call(Unknown Source)
	org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)
	org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:6:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
-java.lang.IllegalStateException: Unterminated threads detected; active threads:
JNA Cleaner:
	java.base@17.0.12/java.lang.Object.wait(Native Method)
	java.base@17.0.12/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.12/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//com.sun.jna.internal.Cleaner$1.run(Cleaner.java:58)
node2_isolatedExecutor:3:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:7:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_RequestResponseStage-2:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:376)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.awaitUntil(WaitQueue.java:303)
	org.apache.cassandra.utils.concurrent.Awaitable$AsyncAwaitable.awaitUntil(Awaitable.java:314)
	org.apache.cassandra.utils.concurrent.AsyncFuture.awaitUntil(AsyncFuture.java:171)
	org.apache.cassandra.utils.concurrent.Awaitable$Defaults.await(Awaitable.java:114)
	org.apache.cassandra.utils.concurrent.AbstractFuture.await(AbstractFuture.java:482)
	org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:252)
	org.apache.cassandra.tcm.RemoteProcessor.fetchLogAndWait(RemoteProcessor.java:141)
	org.apache.cassandra.tcm.ClusterMetadataService$SwitchableProcessor.fetchLogAndWait(ClusterMetadataService.java:885)
	org.apache.cassandra.tcm.ClusterMetadataService.fetchLogFromCMS(ClusterMetadataService.java:656)
	org.apache.cassandra.tcm.ClusterMetadataService.fetchLogFromPeerOrCMS(ClusterMetadataService.java:766)
	org.apache.cassandra.net.ResponseVerbHandler.maybeFetchLogs(ResponseVerbHandler.java:106)
	org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:73)
	org.apache.cassandra.net.InboundSink.lambda$new$0(InboundSink.java:102)
	org.apache.cassandra.net.InboundSink$$Lambda$10224/0x00007f71c60f1d88.accept(Unknown Source)
	org.apache.cassandra.net.InboundSink$Filtered.accept(InboundSink.java:70)
	org.apache.cassandra.net.InboundSink$Filtered.accept(InboundSink.java:56)
	org.apache.cassandra.net.InboundSink.accept(InboundSink.java:122)
	org.apache.cassandra.net.InboundSink.accept(InboundSink.java:51)
	org.apache.cassandra.net.InboundMessageHandler$ProcessMessage.run(InboundMessageHandler.java:448)
	org.apache.cassandra.concurrent.ExecutionFailure$1.run(ExecutionFailure.java:133)
	org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:143)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:8:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:376)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.awaitUntil(WaitQueue.java:303)
	org.apache.cassandra.utils.concurrent.Awaitable$AsyncAwaitable.awaitUntil(Awaitable.java:314)
	org.apache.cassandra.utils.concurrent.Awaitable$AsyncAwaitable.awaitUntil(Awaitable.java:346)
	org.apache.cassandra.utils.concurrent.Awaitable$Defaults.await(Awaitable.java:114)
	org.apache.cassandra.utils.concurrent.Awaitable$AbstractAwaitable.await(Awaitable.java:210)
	org.apache.cassandra.concurrent.SEPExecutor.awaitTermination(SEPExecutor.java:331)
	org.apache.cassandra.utils.ExecutorUtils.awaitTerminationUntil(ExecutorUtils.java:110)
	org.apache.cassandra.utils.ExecutorUtils.awaitTermination(ExecutorUtils.java:100)
	org.apache.cassandra.concurrent.Stage.shutdownAndWait(Stage.java:195)
	org.apache.cassandra.distributed.impl.Instance.lambda$shutdown$42(Instance.java:959)
	org.apache.cassandra.distributed.impl.Instance$$Lambda$14241/0x00007f71c50d2c30.run(Unknown Source)
	org.apache.cassandra.distributed.impl.Instance.lambda$parallelRun$53(Instance.java:1212)
	org.apache.cassandra.distributed.impl.Instance$$Lambda$14026/0x00007f71c4fed1f8.call(Unknown Source)
	org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)
	org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:1:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.await(WaitQueue.java:290)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.await(WaitQueue.java:283)
	org.apache.cassandra.utils.concurrent.Awaitable$AsyncAwaitable.await(Awaitable.java:306)
	org.apache.cassandra.utils.concurrent.AsyncFuture.await(AsyncFuture.java:165)
	org.apache.cassandra.utils.concurrent.AsyncFuture.await(AsyncFuture.java:46)
	org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:245)
	org.apache.cassandra.distributed.impl.Instance.parallelRun(Instance.java:1225)
	org.apache.cassandra.distributed.impl.Instance.lambda$shutdown$47(Instance.java:957)
	org.apache.cassandra.distributed.impl.Instance$$Lambda$14016/0x00007f71c4fec758.accept(Unknown Source)
	org.apache.cassandra.distributed.impl.IsolatedExecutor.lambda$async$10(IsolatedExecutor.java:156)
	org.apache.cassandra.distributed.impl.IsolatedExecutor$$Lambda$8684/0x00007f71c587a808.run(Unknown Source)
	org.apache.cassandra.concurrent.FutureTask$2.call(FutureTask.java:124)
	org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)
	org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_COMMIT-LOG-ALLOCATOR:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.await(WaitQueue.java:290)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.await(WaitQueue.java:283)
	org.apache.cassandra.utils.concurrent.WaitQueue.waitOnCondition(WaitQueue.java:434)
	org.apache.cassandra.db.commitlog.AbstractCommitLogSegmentManager$AllocatorRunnable.run(AbstractCommitLogSegmentManager.java:225)
	org.apache.cassandra.concurrent.InfiniteLoopExecutor.loop(InfiniteLoopExecutor.java:121)
	org.apache.cassandra.concurrent.InfiniteLoopExecutor$$Lambda$8808/0x00007f71c5cfc000.run(Unknown Source)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:4:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_PERIODIC-COMMIT-LOG-SYNCER:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:717)
	java.base@17.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1074)
	java.base@17.0.12/java.util.concurrent.Semaphore.tryAcquire(Semaphore.java:592)
	org.apache.cassandra.utils.concurrent.Semaphore$Standard.tryAcquireUntil(Semaphore.java:142)
	org.apache.cassandra.db.commitlog.AbstractCommitLogService$SyncRunnable.run(AbstractCommitLogService.java:216)
	org.apache.cassandra.concurrent.InfiniteLoopExecutor.loop(InfiniteLoopExecutor.java:121)
	org.apache.cassandra.concurrent.InfiniteLoopExecutor$$Lambda$8808/0x00007f71c5cfc000.run(Unknown Source)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_RequestResponseStage-1:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:376)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.awaitUntil(WaitQueue.java:303)
	org.apache.cassandra.utils.concurrent.Awaitable$AsyncAwaitable.awaitUntil(Awaitable.java:314)
	org.apache.cassandra.utils.concurrent.AsyncFuture.awaitUntil(AsyncFuture.java:171)
	org.apache.cassandra.utils.concurrent.Awaitable$Defaults.await(Awaitable.java:114)
	org.apache.cassandra.utils.concurrent.AbstractFuture.await(AbstractFuture.java:482)
	org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:252)
	org.apache.cassandra.tcm.RemoteProcessor.fetchLogAndWait(RemoteProcessor.java:141)
	org.apache.cassandra.tcm.ClusterMetadataService$SwitchableProcessor.fetchLogAndWait(ClusterMetadataService.java:885)
	org.apache.cassandra.tcm.ClusterMetadataService.fetchLogFromCMS(ClusterMetadataService.java:656)
	org.apache.cassandra.tcm.ClusterMetadataService.fetchLogFromPeerOrCMS(ClusterMetadataService.java:766)
	org.apache.cassandra.net.ResponseVerbHandler.maybeFetchLogs(ResponseVerbHandler.java:106)
	org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:73)
	org.apache.cassandra.net.InboundSink.lambda$new$0(InboundSink.java:102)
	org.apache.cassandra.net.InboundSink$$Lambda$10224/0x00007f71c60f1d88.accept(Unknown Source)
	org.apache.cassandra.net.InboundSink$Filtered.accept(InboundSink.java:70)
	org.apache.cassandra.net.InboundSink$Filtered.accept(InboundSink.java:56)
	org.apache.cassandra.net.InboundSink.accept(InboundSink.java:122)
	org.apache.cassandra.net.InboundSink.accept(InboundSink.java:51)
	org.apache.cassandra.net.InboundMessageHandler$ProcessMessage.run(InboundMessageHandler.java:448)
	org.apache.cassandra.concurrent.ExecutionFailure$1.run(ExecutionFailure.java:133)
	org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:143)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:5:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:2:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.await(WaitQueue.java:290)
	org.apache.cassandra.utils.concurrent.WaitQueue$Standard$AbstractSignal.await(WaitQueue.java:283)
	org.apache.cassandra.utils.concurrent.Awaitable$AsyncAwaitable.await(Awaitable.java:306)
	org.apache.cassandra.utils.concurrent.AsyncFuture.await(AsyncFuture.java:165)
	org.apache.cassandra.utils.concurrent.AsyncFuture.await(AsyncFuture.java:46)
	org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:245)
	org.apache.cassandra.distributed.impl.Instance.lambda$shutdown$48(Instance.java:994)
	org.apache.cassandra.distributed.impl.Instance$$Lambda$14017/0x00007f71c4fec990.call(Unknown Source)
	org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)
	org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)
node2_isolatedExecutor:6:
	java.base@17.0.12/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.12/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.12/java.lang.Thread.run(Thread.java:840)

	at org.apache.cassandra.distributed.impl.AbstractCluster.checkForThreadLeaks(AbstractCluster.java:1171)
	at org.apache.cassandra.distributed.impl.AbstractCluster.close(AbstractCluster.java:1114)
	at org.apache.cassandra.fuzz.ring.ConsistentBootstrapTest.coordinatorIsBehindTest(ConsistentBootstrapTest.java:227)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
Caused by: java.lang.RuntimeException: java.util.concurrent.TimeoutException
	at org.apache.cassandra.utils.Throwables.maybeFail(Throwables.java:79)
	at org.apache.cassandra.utils.FBUtilities.waitOnFutures(FBUtilities.java:541)
	at org.apache.cassandra.distributed.impl.AbstractCluster.close(AbstractCluster.java:1110)
Caused by: java.util.concurrent.TimeoutException
	at org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:253)
	at org.apache.cassandra.utils.FBUtilities.waitOnFutures(FBUtilities.java:533)

expected:<[/127.0.100.1:7012, /127.0.100.2:7012, /127.0.100.3:7012, /127.0.100.4:7012, /127.0.100.5:7012, /127.0.100.6:7012, /127.0.100.7:7012, /127.0.100.8:7012, /127.0.100.9:7012, /127.0.100.10:7012]> but was:<[/127.0.100.1:7012, /127.0.100.3:7012, /127.0.100.5:7012, /127.0.100.6:7012, /127.0.100.7:7012, /127.0.100.8:7012, /127.0.100.9:7012, /127.0.100.10:7012]>-junit.framework.AssertionFailedError: expected:<[/127.0.100.1:7012, /127.0.100.2:7012, /127.0.100.3:7012, /127.0.100.4:7012, /127.0.100.5:7012, /127.0.100.6:7012, /127.0.100.7:7012, /127.0.100.8:7012, /127.0.100.9:7012, /127.0.100.10:7012]> but was:<[/127.0.100.1:7012, /127.0.100.3:7012, /127.0.100.5:7012, /127.0.100.6:7012, /127.0.100.7:7012, /127.0.100.8:7012, /127.0.100.9:7012, /127.0.100.10:7012]>
	at org.apache.cassandra.tcm.DiscoverySimulationTest.discoveryTest(DiscoverySimulationTest.java:105)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)